src/share/vm/opto/macro.cpp
Index Unified diffs Context diffs Sdiffs Patch New Old Previous File Next File 8034812 Sdiff src/share/vm/opto

src/share/vm/opto/macro.cpp

Print this page




  91       }
  92       old_in = new_in;
  93     }
  94     newcall->add_req(old_in);
  95   }
  96 
  97   newcall->set_jvms(oldcall->jvms());
  98   for (JVMState *jvms = newcall->jvms(); jvms != NULL; jvms = jvms->caller()) {
  99     jvms->set_map(newcall);
 100     jvms->set_locoff(jvms->locoff()+jvms_adj);
 101     jvms->set_stkoff(jvms->stkoff()+jvms_adj);
 102     jvms->set_monoff(jvms->monoff()+jvms_adj);
 103     jvms->set_scloff(jvms->scloff()+jvms_adj);
 104     jvms->set_endoff(jvms->endoff()+jvms_adj);
 105   }
 106 }
 107 
 108 Node* PhaseMacroExpand::opt_bits_test(Node* ctrl, Node* region, int edge, Node* word, int mask, int bits, bool return_fast_path) {
 109   Node* cmp;
 110   if (mask != 0) {
 111     Node* and_node = transform_later(new (C) AndXNode(word, MakeConX(mask)));
 112     cmp = transform_later(new (C) CmpXNode(and_node, MakeConX(bits)));
 113   } else {
 114     cmp = word;
 115   }
 116   Node* bol = transform_later(new (C) BoolNode(cmp, BoolTest::ne));
 117   IfNode* iff = new (C) IfNode( ctrl, bol, PROB_MIN, COUNT_UNKNOWN );
 118   transform_later(iff);
 119 
 120   // Fast path taken.
 121   Node *fast_taken = transform_later( new (C) IfFalseNode(iff) );
 122 
 123   // Fast path not-taken, i.e. slow path
 124   Node *slow_taken = transform_later( new (C) IfTrueNode(iff) );
 125 
 126   if (return_fast_path) {
 127     region->init_req(edge, slow_taken); // Capture slow-control
 128     return fast_taken;
 129   } else {
 130     region->init_req(edge, fast_taken); // Capture fast-control
 131     return slow_taken;
 132   }
 133 }
 134 
 135 //--------------------copy_predefined_input_for_runtime_call--------------------
 136 void PhaseMacroExpand::copy_predefined_input_for_runtime_call(Node * ctrl, CallNode* oldcall, CallNode* call) {
 137   // Set fixed predefined input arguments
 138   call->init_req( TypeFunc::Control, ctrl );
 139   call->init_req( TypeFunc::I_O    , oldcall->in( TypeFunc::I_O) );
 140   call->init_req( TypeFunc::Memory , oldcall->in( TypeFunc::Memory ) ); // ?????
 141   call->init_req( TypeFunc::ReturnAdr, oldcall->in( TypeFunc::ReturnAdr ) );
 142   call->init_req( TypeFunc::FramePtr, oldcall->in( TypeFunc::FramePtr ) );
 143 }
 144 
 145 //------------------------------make_slow_call---------------------------------
 146 CallNode* PhaseMacroExpand::make_slow_call(CallNode *oldcall, const TypeFunc* slow_call_type, address slow_call, const char* leaf_name, Node* slow_path, Node* parm0, Node* parm1) {
 147 
 148   // Slow-path call
 149  CallNode *call = leaf_name
 150    ? (CallNode*)new (C) CallLeafNode      ( slow_call_type, slow_call, leaf_name, TypeRawPtr::BOTTOM )
 151    : (CallNode*)new (C) CallStaticJavaNode( slow_call_type, slow_call, OptoRuntime::stub_name(slow_call), oldcall->jvms()->bci(), TypeRawPtr::BOTTOM );
 152 
 153   // Slow path call has no side-effects, uses few values
 154   copy_predefined_input_for_runtime_call(slow_path, oldcall, call );
 155   if (parm0 != NULL)  call->init_req(TypeFunc::Parms+0, parm0);
 156   if (parm1 != NULL)  call->init_req(TypeFunc::Parms+1, parm1);
 157   copy_call_debug_info(oldcall, call);
 158   call->set_cnt(PROB_UNLIKELY_MAG(4));  // Same effect as RC_UNCOMMON.
 159   _igvn.replace_node(oldcall, call);
 160   transform_later(call);
 161 
 162   return call;
 163 }
 164 
 165 void PhaseMacroExpand::extract_call_projections(CallNode *call) {
 166   _fallthroughproj = NULL;
 167   _fallthroughcatchproj = NULL;
 168   _ioproj_fallthrough = NULL;
 169   _ioproj_catchall = NULL;
 170   _catchallcatchproj = NULL;
 171   _memproj_fallthrough = NULL;


 406     if (phi->is_Phi() && phi != mem &&
 407         phi->as_Phi()->is_same_inst_field(phi_type, instance_id, alias_idx, offset)) {
 408       return phi;
 409     }
 410   }
 411   // Check if an appropriate new value phi already exists.
 412   Node* new_phi = value_phis->find(mem->_idx);
 413   if (new_phi != NULL)
 414     return new_phi;
 415 
 416   if (level <= 0) {
 417     return NULL; // Give up: phi tree too deep
 418   }
 419   Node *start_mem = C->start()->proj_out(TypeFunc::Memory);
 420   Node *alloc_mem = alloc->in(TypeFunc::Memory);
 421 
 422   uint length = mem->req();
 423   GrowableArray <Node *> values(length, length, NULL, false);
 424 
 425   // create a new Phi for the value
 426   PhiNode *phi = new (C) PhiNode(mem->in(0), phi_type, NULL, instance_id, alias_idx, offset);
 427   transform_later(phi);
 428   value_phis->push(phi, mem->_idx);
 429 
 430   for (uint j = 1; j < length; j++) {
 431     Node *in = mem->in(j);
 432     if (in == NULL || in->is_top()) {
 433       values.at_put(j, in);
 434     } else  {
 435       Node *val = scan_mem_chain(in, alias_idx, offset, start_mem, alloc, &_igvn);
 436       if (val == start_mem || val == alloc_mem) {
 437         // hit a sentinel, return appropriate 0 value
 438         values.at_put(j, _igvn.zerocon(ft));
 439         continue;
 440       }
 441       if (val->is_Initialize()) {
 442         val = val->as_Initialize()->find_captured_store(offset, type2aelembytes(ft), &_igvn);
 443       }
 444       if (val == NULL) {
 445         return NULL;  // can't find a value on this path
 446       }


 718       // find the array's elements which will be needed for safepoint debug information
 719       nfields = alloc->in(AllocateNode::ALength)->find_int_con(-1);
 720       assert(klass->is_array_klass() && nfields >= 0, "must be an array klass.");
 721       elem_type = klass->as_array_klass()->element_type();
 722       basic_elem_type = elem_type->basic_type();
 723       array_base = arrayOopDesc::base_offset_in_bytes(basic_elem_type);
 724       element_size = type2aelembytes(basic_elem_type);
 725     }
 726   }
 727   //
 728   // Process the safepoint uses
 729   //
 730   while (safepoints.length() > 0) {
 731     SafePointNode* sfpt = safepoints.pop();
 732     Node* mem = sfpt->memory();
 733     assert(sfpt->jvms() != NULL, "missed JVMS");
 734     // Fields of scalar objs are referenced only at the end
 735     // of regular debuginfo at the last (youngest) JVMS.
 736     // Record relative start index.
 737     uint first_ind = (sfpt->req() - sfpt->jvms()->scloff());
 738     SafePointScalarObjectNode* sobj = new (C) SafePointScalarObjectNode(res_type,
 739 #ifdef ASSERT
 740                                                  alloc,
 741 #endif
 742                                                  first_ind, nfields);
 743     sobj->init_req(0, C->root());
 744     transform_later(sobj);
 745 
 746     // Scan object's fields adding an input to the safepoint for each field.
 747     for (int j = 0; j < nfields; j++) {
 748       intptr_t offset;
 749       ciField* field = NULL;
 750       if (iklass != NULL) {
 751         field = iklass->nonstatic_field_at(j);
 752         offset = field->offset();
 753         elem_type = field->type();
 754         basic_elem_type = field->layout_type();
 755       } else {
 756         offset = array_base + j * (intptr_t)element_size;
 757       }
 758 


 826             tty->print(" (alias_idx=%d)", field_idx);
 827           } else { // Array's element
 828             tty->print("=== At SafePoint node %d can't find value of array element [%d]",
 829                        sfpt->_idx, j);
 830           }
 831           tty->print(", which prevents elimination of: ");
 832           if (res == NULL)
 833             alloc->dump();
 834           else
 835             res->dump();
 836         }
 837 #endif
 838         return false;
 839       }
 840       if (UseCompressedOops && field_type->isa_narrowoop()) {
 841         // Enable "DecodeN(EncodeP(Allocate)) --> Allocate" transformation
 842         // to be able scalar replace the allocation.
 843         if (field_val->is_EncodeP()) {
 844           field_val = field_val->in(1);
 845         } else {
 846           field_val = transform_later(new (C) DecodeNNode(field_val, field_val->get_ptr_type()));
 847         }
 848       }
 849       sfpt->add_req(field_val);
 850     }
 851     JVMState *jvms = sfpt->jvms();
 852     jvms->set_endoff(sfpt->req());
 853     // Now make a pass over the debug information replacing any references
 854     // to the allocated object with "sobj"
 855     int start = jvms->debug_start();
 856     int end   = jvms->debug_end();
 857     sfpt->replace_edges_in_range(res, sobj, start, end);
 858     safepoints_done.append_if_missing(sfpt); // keep it for rollback
 859   }
 860   return true;
 861 }
 862 
 863 // Process users of eliminated allocation.
 864 void PhaseMacroExpand::process_users_of_allocation(CallNode *alloc) {
 865   Node* res = alloc->result_cast();
 866   if (res != NULL) {


1052     }
1053     log->tail("eliminate_boxing");
1054   }
1055 
1056   process_users_of_allocation(boxing);
1057 
1058 #ifndef PRODUCT
1059   if (PrintEliminateAllocations) {
1060     tty->print("++++ Eliminated: %d ", boxing->_idx);
1061     boxing->method()->print_short_name(tty);
1062     tty->cr();
1063   }
1064 #endif
1065 
1066   return true;
1067 }
1068 
1069 //---------------------------set_eden_pointers-------------------------
1070 void PhaseMacroExpand::set_eden_pointers(Node* &eden_top_adr, Node* &eden_end_adr) {
1071   if (UseTLAB) {                // Private allocation: load from TLS
1072     Node* thread = transform_later(new (C) ThreadLocalNode());
1073     int tlab_top_offset = in_bytes(JavaThread::tlab_top_offset());
1074     int tlab_end_offset = in_bytes(JavaThread::tlab_end_offset());
1075     eden_top_adr = basic_plus_adr(top()/*not oop*/, thread, tlab_top_offset);
1076     eden_end_adr = basic_plus_adr(top()/*not oop*/, thread, tlab_end_offset);
1077   } else {                      // Shared allocation: load from globals
1078     CollectedHeap* ch = Universe::heap();
1079     address top_adr = (address)ch->top_addr();
1080     address end_adr = (address)ch->end_addr();
1081     eden_top_adr = makecon(TypeRawPtr::make(top_adr));
1082     eden_end_adr = basic_plus_adr(eden_top_adr, end_adr - top_adr);
1083   }
1084 }
1085 
1086 
1087 Node* PhaseMacroExpand::make_load(Node* ctl, Node* mem, Node* base, int offset, const Type* value_type, BasicType bt) {
1088   Node* adr = basic_plus_adr(base, offset);
1089   const TypePtr* adr_type = adr->bottom_type()->is_ptr();
1090   Node* value = LoadNode::make(_igvn, ctl, mem, adr, adr_type, value_type, bt, MemNode::unordered);
1091   transform_later(value);
1092   return value;


1188   } else {
1189     initial_slow_test = BoolNode::make_predicate(initial_slow_test, &_igvn);
1190   }
1191 
1192   if (C->env()->dtrace_alloc_probes() ||
1193       !UseTLAB && (!Universe::heap()->supports_inline_contig_alloc() ||
1194                    (UseConcMarkSweepGC && CMSIncrementalMode))) {
1195     // Force slow-path allocation
1196     always_slow = true;
1197     initial_slow_test = NULL;
1198   }
1199 
1200 
1201   enum { too_big_or_final_path = 1, need_gc_path = 2 };
1202   Node *slow_region = NULL;
1203   Node *toobig_false = ctrl;
1204 
1205   assert (initial_slow_test == NULL || !always_slow, "arguments must be consistent");
1206   // generate the initial test if necessary
1207   if (initial_slow_test != NULL ) {
1208     slow_region = new (C) RegionNode(3);
1209 
1210     // Now make the initial failure test.  Usually a too-big test but
1211     // might be a TRUE for finalizers or a fancy class check for
1212     // newInstance0.
1213     IfNode *toobig_iff = new (C) IfNode(ctrl, initial_slow_test, PROB_MIN, COUNT_UNKNOWN);
1214     transform_later(toobig_iff);
1215     // Plug the failing-too-big test into the slow-path region
1216     Node *toobig_true = new (C) IfTrueNode( toobig_iff );
1217     transform_later(toobig_true);
1218     slow_region    ->init_req( too_big_or_final_path, toobig_true );
1219     toobig_false = new (C) IfFalseNode( toobig_iff );
1220     transform_later(toobig_false);
1221   } else {         // No initial test, just fall into next case
1222     toobig_false = ctrl;
1223     debug_only(slow_region = NodeSentinel);
1224   }
1225 
1226   Node *slow_mem = mem;  // save the current memory state for slow path
1227   // generate the fast allocation code unless we know that the initial test will always go slow
1228   if (!always_slow) {
1229     // Fast path modifies only raw memory.
1230     if (mem->is_MergeMem()) {
1231       mem = mem->as_MergeMem()->memory_at(Compile::AliasIdxRaw);
1232     }
1233 
1234     Node* eden_top_adr;
1235     Node* eden_end_adr;
1236 
1237     set_eden_pointers(eden_top_adr, eden_end_adr);
1238 
1239     // Load Eden::end.  Loop invariant and hoisted.
1240     //
1241     // Note: We set the control input on "eden_end" and "old_eden_top" when using
1242     //       a TLAB to work around a bug where these values were being moved across
1243     //       a safepoint.  These are not oops, so they cannot be include in the oop
1244     //       map, but they can be changed by a GC.   The proper way to fix this would
1245     //       be to set the raw memory state when generating a  SafepointNode.  However
1246     //       this will require extensive changes to the loop optimization in order to
1247     //       prevent a degradation of the optimization.
1248     //       See comment in memnode.hpp, around line 227 in class LoadPNode.
1249     Node *eden_end = make_load(ctrl, mem, eden_end_adr, 0, TypeRawPtr::BOTTOM, T_ADDRESS);
1250 
1251     // allocate the Region and Phi nodes for the result
1252     result_region = new (C) RegionNode(3);
1253     result_phi_rawmem = new (C) PhiNode(result_region, Type::MEMORY, TypeRawPtr::BOTTOM);
1254     result_phi_rawoop = new (C) PhiNode(result_region, TypeRawPtr::BOTTOM);
1255     result_phi_i_o    = new (C) PhiNode(result_region, Type::ABIO); // I/O is used for Prefetch
1256 
1257     // We need a Region for the loop-back contended case.
1258     enum { fall_in_path = 1, contended_loopback_path = 2 };
1259     Node *contended_region;
1260     Node *contended_phi_rawmem;
1261     if (UseTLAB) {
1262       contended_region = toobig_false;
1263       contended_phi_rawmem = mem;
1264     } else {
1265       contended_region = new (C) RegionNode(3);
1266       contended_phi_rawmem = new (C) PhiNode(contended_region, Type::MEMORY, TypeRawPtr::BOTTOM);
1267       // Now handle the passing-too-big test.  We fall into the contended
1268       // loop-back merge point.
1269       contended_region    ->init_req(fall_in_path, toobig_false);
1270       contended_phi_rawmem->init_req(fall_in_path, mem);
1271       transform_later(contended_region);
1272       transform_later(contended_phi_rawmem);
1273     }
1274 
1275     // Load(-locked) the heap top.
1276     // See note above concerning the control input when using a TLAB
1277     Node *old_eden_top = UseTLAB
1278       ? new (C) LoadPNode      (ctrl, contended_phi_rawmem, eden_top_adr, TypeRawPtr::BOTTOM, TypeRawPtr::BOTTOM, MemNode::unordered)
1279       : new (C) LoadPLockedNode(contended_region, contended_phi_rawmem, eden_top_adr, MemNode::acquire);
1280 
1281     transform_later(old_eden_top);
1282     // Add to heap top to get a new heap top
1283     Node *new_eden_top = new (C) AddPNode(top(), old_eden_top, size_in_bytes);
1284     transform_later(new_eden_top);
1285     // Check for needing a GC; compare against heap end
1286     Node *needgc_cmp = new (C) CmpPNode(new_eden_top, eden_end);
1287     transform_later(needgc_cmp);
1288     Node *needgc_bol = new (C) BoolNode(needgc_cmp, BoolTest::ge);
1289     transform_later(needgc_bol);
1290     IfNode *needgc_iff = new (C) IfNode(contended_region, needgc_bol, PROB_UNLIKELY_MAG(4), COUNT_UNKNOWN);
1291     transform_later(needgc_iff);
1292 
1293     // Plug the failing-heap-space-need-gc test into the slow-path region
1294     Node *needgc_true = new (C) IfTrueNode(needgc_iff);
1295     transform_later(needgc_true);
1296     if (initial_slow_test) {
1297       slow_region->init_req(need_gc_path, needgc_true);
1298       // This completes all paths into the slow merge point
1299       transform_later(slow_region);
1300     } else {                      // No initial slow path needed!
1301       // Just fall from the need-GC path straight into the VM call.
1302       slow_region = needgc_true;
1303     }
1304     // No need for a GC.  Setup for the Store-Conditional
1305     Node *needgc_false = new (C) IfFalseNode(needgc_iff);
1306     transform_later(needgc_false);
1307 
1308     // Grab regular I/O before optional prefetch may change it.
1309     // Slow-path does no I/O so just set it to the original I/O.
1310     result_phi_i_o->init_req(slow_result_path, i_o);
1311 
1312     i_o = prefetch_allocation(i_o, needgc_false, contended_phi_rawmem,
1313                               old_eden_top, new_eden_top, length);
1314 
1315     // Name successful fast-path variables
1316     Node* fast_oop = old_eden_top;
1317     Node* fast_oop_ctrl;
1318     Node* fast_oop_rawmem;
1319 
1320     // Store (-conditional) the modified eden top back down.
1321     // StorePConditional produces flags for a test PLUS a modified raw
1322     // memory state.
1323     if (UseTLAB) {
1324       Node* store_eden_top =
1325         new (C) StorePNode(needgc_false, contended_phi_rawmem, eden_top_adr,
1326                               TypeRawPtr::BOTTOM, new_eden_top, MemNode::unordered);
1327       transform_later(store_eden_top);
1328       fast_oop_ctrl = needgc_false; // No contention, so this is the fast path
1329       fast_oop_rawmem = store_eden_top;
1330     } else {
1331       Node* store_eden_top =
1332         new (C) StorePConditionalNode(needgc_false, contended_phi_rawmem, eden_top_adr,
1333                                          new_eden_top, fast_oop/*old_eden_top*/);
1334       transform_later(store_eden_top);
1335       Node *contention_check = new (C) BoolNode(store_eden_top, BoolTest::ne);
1336       transform_later(contention_check);
1337       store_eden_top = new (C) SCMemProjNode(store_eden_top);
1338       transform_later(store_eden_top);
1339 
1340       // If not using TLABs, check to see if there was contention.
1341       IfNode *contention_iff = new (C) IfNode (needgc_false, contention_check, PROB_MIN, COUNT_UNKNOWN);
1342       transform_later(contention_iff);
1343       Node *contention_true = new (C) IfTrueNode(contention_iff);
1344       transform_later(contention_true);
1345       // If contention, loopback and try again.
1346       contended_region->init_req(contended_loopback_path, contention_true);
1347       contended_phi_rawmem->init_req(contended_loopback_path, store_eden_top);
1348 
1349       // Fast-path succeeded with no contention!
1350       Node *contention_false = new (C) IfFalseNode(contention_iff);
1351       transform_later(contention_false);
1352       fast_oop_ctrl = contention_false;
1353 
1354       // Bump total allocated bytes for this thread
1355       Node* thread = new (C) ThreadLocalNode();
1356       transform_later(thread);
1357       Node* alloc_bytes_adr = basic_plus_adr(top()/*not oop*/, thread,
1358                                              in_bytes(JavaThread::allocated_bytes_offset()));
1359       Node* alloc_bytes = make_load(fast_oop_ctrl, store_eden_top, alloc_bytes_adr,
1360                                     0, TypeLong::LONG, T_LONG);
1361 #ifdef _LP64
1362       Node* alloc_size = size_in_bytes;
1363 #else
1364       Node* alloc_size = new (C) ConvI2LNode(size_in_bytes);
1365       transform_later(alloc_size);
1366 #endif
1367       Node* new_alloc_bytes = new (C) AddLNode(alloc_bytes, alloc_size);
1368       transform_later(new_alloc_bytes);
1369       fast_oop_rawmem = make_store(fast_oop_ctrl, store_eden_top, alloc_bytes_adr,
1370                                    0, new_alloc_bytes, T_LONG);
1371     }
1372 
1373     InitializeNode* init = alloc->initialization();
1374     fast_oop_rawmem = initialize_object(alloc,
1375                                         fast_oop_ctrl, fast_oop_rawmem, fast_oop,
1376                                         klass_node, length, size_in_bytes);
1377 
1378     // If initialization is performed by an array copy, any required
1379     // MemBarStoreStore was already added. If the object does not
1380     // escape no need for a MemBarStoreStore. Otherwise we need a
1381     // MemBarStoreStore so that stores that initialize this object
1382     // can't be reordered with a subsequent store that makes this
1383     // object accessible by other threads.
1384     if (init == NULL || (!init->is_complete_with_arraycopy() && !init->does_not_escape())) {
1385       if (init == NULL || init->req() < InitializeNode::RawStores) {
1386         // No InitializeNode or no stores captured by zeroing
1387         // elimination. Simply add the MemBarStoreStore after object
1388         // initialization.
1389         MemBarNode* mb = MemBarNode::make(C, Op_MemBarStoreStore, Compile::AliasIdxBot);
1390         transform_later(mb);
1391 
1392         mb->init_req(TypeFunc::Memory, fast_oop_rawmem);
1393         mb->init_req(TypeFunc::Control, fast_oop_ctrl);
1394         fast_oop_ctrl = new (C) ProjNode(mb,TypeFunc::Control);
1395         transform_later(fast_oop_ctrl);
1396         fast_oop_rawmem = new (C) ProjNode(mb,TypeFunc::Memory);
1397         transform_later(fast_oop_rawmem);
1398       } else {
1399         // Add the MemBarStoreStore after the InitializeNode so that
1400         // all stores performing the initialization that were moved
1401         // before the InitializeNode happen before the storestore
1402         // barrier.
1403 
1404         Node* init_ctrl = init->proj_out(TypeFunc::Control);
1405         Node* init_mem = init->proj_out(TypeFunc::Memory);
1406 
1407         MemBarNode* mb = MemBarNode::make(C, Op_MemBarStoreStore, Compile::AliasIdxBot);
1408         transform_later(mb);
1409 
1410         Node* ctrl = new (C) ProjNode(init,TypeFunc::Control);
1411         transform_later(ctrl);
1412         Node* mem = new (C) ProjNode(init,TypeFunc::Memory);
1413         transform_later(mem);
1414 
1415         // The MemBarStoreStore depends on control and memory coming
1416         // from the InitializeNode
1417         mb->init_req(TypeFunc::Memory, mem);
1418         mb->init_req(TypeFunc::Control, ctrl);
1419 
1420         ctrl = new (C) ProjNode(mb,TypeFunc::Control);
1421         transform_later(ctrl);
1422         mem = new (C) ProjNode(mb,TypeFunc::Memory);
1423         transform_later(mem);
1424 
1425         // All nodes that depended on the InitializeNode for control
1426         // and memory must now depend on the MemBarNode that itself
1427         // depends on the InitializeNode
1428         _igvn.replace_node(init_ctrl, ctrl);
1429         _igvn.replace_node(init_mem, mem);
1430       }
1431     }
1432 
1433     if (C->env()->dtrace_extended_probes()) {
1434       // Slow-path call
1435       int size = TypeFunc::Parms + 2;
1436       CallLeafNode *call = new (C) CallLeafNode(OptoRuntime::dtrace_object_alloc_Type(),
1437                                                 CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_object_alloc_base),
1438                                                 "dtrace_object_alloc",
1439                                                 TypeRawPtr::BOTTOM);
1440 
1441       // Get base of thread-local storage area
1442       Node* thread = new (C) ThreadLocalNode();
1443       transform_later(thread);
1444 
1445       call->init_req(TypeFunc::Parms+0, thread);
1446       call->init_req(TypeFunc::Parms+1, fast_oop);
1447       call->init_req(TypeFunc::Control, fast_oop_ctrl);
1448       call->init_req(TypeFunc::I_O    , top()); // does no i/o
1449       call->init_req(TypeFunc::Memory , fast_oop_rawmem);
1450       call->init_req(TypeFunc::ReturnAdr, alloc->in(TypeFunc::ReturnAdr));
1451       call->init_req(TypeFunc::FramePtr, alloc->in(TypeFunc::FramePtr));
1452       transform_later(call);
1453       fast_oop_ctrl = new (C) ProjNode(call,TypeFunc::Control);
1454       transform_later(fast_oop_ctrl);
1455       fast_oop_rawmem = new (C) ProjNode(call,TypeFunc::Memory);
1456       transform_later(fast_oop_rawmem);
1457     }
1458 
1459     // Plug in the successful fast-path into the result merge point
1460     result_region    ->init_req(fast_result_path, fast_oop_ctrl);
1461     result_phi_rawoop->init_req(fast_result_path, fast_oop);
1462     result_phi_i_o   ->init_req(fast_result_path, i_o);
1463     result_phi_rawmem->init_req(fast_result_path, fast_oop_rawmem);
1464   } else {
1465     slow_region = ctrl;
1466     result_phi_i_o = i_o; // Rename it to use in the following code.
1467   }
1468 
1469   // Generate slow-path call
1470   CallNode *call = new (C) CallStaticJavaNode(slow_call_type, slow_call_address,
1471                                OptoRuntime::stub_name(slow_call_address),
1472                                alloc->jvms()->bci(),
1473                                TypePtr::BOTTOM);
1474   call->init_req( TypeFunc::Control, slow_region );
1475   call->init_req( TypeFunc::I_O    , top() )     ;   // does no i/o
1476   call->init_req( TypeFunc::Memory , slow_mem ); // may gc ptrs
1477   call->init_req( TypeFunc::ReturnAdr, alloc->in(TypeFunc::ReturnAdr) );
1478   call->init_req( TypeFunc::FramePtr, alloc->in(TypeFunc::FramePtr) );
1479 
1480   call->init_req(TypeFunc::Parms+0, klass_node);
1481   if (length != NULL) {
1482     call->init_req(TypeFunc::Parms+1, length);
1483   }
1484 
1485   // Copy debug information and adjust JVMState information, then replace
1486   // allocate node with the call
1487   copy_call_debug_info((CallNode *) alloc,  call);
1488   if (!always_slow) {
1489     call->set_cnt(PROB_UNLIKELY_MAG(4));  // Same effect as RC_UNCOMMON.
1490   } else {


1507   //
1508   extract_call_projections(call);
1509 
1510   // An allocate node has separate memory projections for the uses on
1511   // the control and i_o paths. Replace the control memory projection with
1512   // result_phi_rawmem (unless we are only generating a slow call when
1513   // both memory projections are combined)
1514   if (!always_slow && _memproj_fallthrough != NULL) {
1515     for (DUIterator_Fast imax, i = _memproj_fallthrough->fast_outs(imax); i < imax; i++) {
1516       Node *use = _memproj_fallthrough->fast_out(i);
1517       _igvn.rehash_node_delayed(use);
1518       imax -= replace_input(use, _memproj_fallthrough, result_phi_rawmem);
1519       // back up iterator
1520       --i;
1521     }
1522   }
1523   // Now change uses of _memproj_catchall to use _memproj_fallthrough and delete
1524   // _memproj_catchall so we end up with a call that has only 1 memory projection.
1525   if (_memproj_catchall != NULL ) {
1526     if (_memproj_fallthrough == NULL) {
1527       _memproj_fallthrough = new (C) ProjNode(call, TypeFunc::Memory);
1528       transform_later(_memproj_fallthrough);
1529     }
1530     for (DUIterator_Fast imax, i = _memproj_catchall->fast_outs(imax); i < imax; i++) {
1531       Node *use = _memproj_catchall->fast_out(i);
1532       _igvn.rehash_node_delayed(use);
1533       imax -= replace_input(use, _memproj_catchall, _memproj_fallthrough);
1534       // back up iterator
1535       --i;
1536     }
1537     assert(_memproj_catchall->outcnt() == 0, "all uses must be deleted");
1538     _igvn.remove_dead_node(_memproj_catchall);
1539   }
1540 
1541   // An allocate node has separate i_o projections for the uses on the control
1542   // and i_o paths. Always replace the control i_o projection with result i_o
1543   // otherwise incoming i_o become dead when only a slow call is generated
1544   // (it is different from memory projections where both projections are
1545   // combined in such case).
1546   if (_ioproj_fallthrough != NULL) {
1547     for (DUIterator_Fast imax, i = _ioproj_fallthrough->fast_outs(imax); i < imax; i++) {
1548       Node *use = _ioproj_fallthrough->fast_out(i);
1549       _igvn.rehash_node_delayed(use);
1550       imax -= replace_input(use, _ioproj_fallthrough, result_phi_i_o);
1551       // back up iterator
1552       --i;
1553     }
1554   }
1555   // Now change uses of _ioproj_catchall to use _ioproj_fallthrough and delete
1556   // _ioproj_catchall so we end up with a call that has only 1 i_o projection.
1557   if (_ioproj_catchall != NULL ) {
1558     if (_ioproj_fallthrough == NULL) {
1559       _ioproj_fallthrough = new (C) ProjNode(call, TypeFunc::I_O);
1560       transform_later(_ioproj_fallthrough);
1561     }
1562     for (DUIterator_Fast imax, i = _ioproj_catchall->fast_outs(imax); i < imax; i++) {
1563       Node *use = _ioproj_catchall->fast_out(i);
1564       _igvn.rehash_node_delayed(use);
1565       imax -= replace_input(use, _ioproj_catchall, _ioproj_fallthrough);
1566       // back up iterator
1567       --i;
1568     }
1569     assert(_ioproj_catchall->outcnt() == 0, "all uses must be deleted");
1570     _igvn.remove_dead_node(_ioproj_catchall);
1571   }
1572 
1573   // if we generated only a slow call, we are done
1574   if (always_slow) {
1575     // Now we can unhook i_o.
1576     if (result_phi_i_o->outcnt() > 1) {
1577       call->set_req(TypeFunc::I_O, top());
1578     } else {
1579       assert(result_phi_i_o->unique_ctrl_out() == call, "");


1673     // We have no more use for this link, since the AllocateNode goes away:
1674     init->set_req(InitializeNode::RawAddress, top());
1675     // (If we keep the link, it just confuses the register allocator,
1676     // who thinks he sees a real use of the address by the membar.)
1677   }
1678 
1679   return rawmem;
1680 }
1681 
1682 // Generate prefetch instructions for next allocations.
1683 Node* PhaseMacroExpand::prefetch_allocation(Node* i_o, Node*& needgc_false,
1684                                         Node*& contended_phi_rawmem,
1685                                         Node* old_eden_top, Node* new_eden_top,
1686                                         Node* length) {
1687    enum { fall_in_path = 1, pf_path = 2 };
1688    if( UseTLAB && AllocatePrefetchStyle == 2 ) {
1689       // Generate prefetch allocation with watermark check.
1690       // As an allocation hits the watermark, we will prefetch starting
1691       // at a "distance" away from watermark.
1692 
1693       Node *pf_region = new (C) RegionNode(3);
1694       Node *pf_phi_rawmem = new (C) PhiNode( pf_region, Type::MEMORY,
1695                                                 TypeRawPtr::BOTTOM );
1696       // I/O is used for Prefetch
1697       Node *pf_phi_abio = new (C) PhiNode( pf_region, Type::ABIO );
1698 
1699       Node *thread = new (C) ThreadLocalNode();
1700       transform_later(thread);
1701 
1702       Node *eden_pf_adr = new (C) AddPNode( top()/*not oop*/, thread,
1703                    _igvn.MakeConX(in_bytes(JavaThread::tlab_pf_top_offset())) );
1704       transform_later(eden_pf_adr);
1705 
1706       Node *old_pf_wm = new (C) LoadPNode(needgc_false,
1707                                    contended_phi_rawmem, eden_pf_adr,
1708                                    TypeRawPtr::BOTTOM, TypeRawPtr::BOTTOM,
1709                                    MemNode::unordered);
1710       transform_later(old_pf_wm);
1711 
1712       // check against new_eden_top
1713       Node *need_pf_cmp = new (C) CmpPNode( new_eden_top, old_pf_wm );
1714       transform_later(need_pf_cmp);
1715       Node *need_pf_bol = new (C) BoolNode( need_pf_cmp, BoolTest::ge );
1716       transform_later(need_pf_bol);
1717       IfNode *need_pf_iff = new (C) IfNode( needgc_false, need_pf_bol,
1718                                        PROB_UNLIKELY_MAG(4), COUNT_UNKNOWN );
1719       transform_later(need_pf_iff);
1720 
1721       // true node, add prefetchdistance
1722       Node *need_pf_true = new (C) IfTrueNode( need_pf_iff );
1723       transform_later(need_pf_true);
1724 
1725       Node *need_pf_false = new (C) IfFalseNode( need_pf_iff );
1726       transform_later(need_pf_false);
1727 
1728       Node *new_pf_wmt = new (C) AddPNode( top(), old_pf_wm,
1729                                     _igvn.MakeConX(AllocatePrefetchDistance) );
1730       transform_later(new_pf_wmt );
1731       new_pf_wmt->set_req(0, need_pf_true);
1732 
1733       Node *store_new_wmt = new (C) StorePNode(need_pf_true,
1734                                        contended_phi_rawmem, eden_pf_adr,
1735                                        TypeRawPtr::BOTTOM, new_pf_wmt,
1736                                        MemNode::unordered);
1737       transform_later(store_new_wmt);
1738 
1739       // adding prefetches
1740       pf_phi_abio->init_req( fall_in_path, i_o );
1741 
1742       Node *prefetch_adr;
1743       Node *prefetch;
1744       uint lines = AllocatePrefetchDistance / AllocatePrefetchStepSize;
1745       uint step_size = AllocatePrefetchStepSize;
1746       uint distance = 0;
1747 
1748       for ( uint i = 0; i < lines; i++ ) {
1749         prefetch_adr = new (C) AddPNode( old_pf_wm, new_pf_wmt,
1750                                             _igvn.MakeConX(distance) );
1751         transform_later(prefetch_adr);
1752         prefetch = new (C) PrefetchAllocationNode( i_o, prefetch_adr );
1753         transform_later(prefetch);
1754         distance += step_size;
1755         i_o = prefetch;
1756       }
1757       pf_phi_abio->set_req( pf_path, i_o );
1758 
1759       pf_region->init_req( fall_in_path, need_pf_false );
1760       pf_region->init_req( pf_path, need_pf_true );
1761 
1762       pf_phi_rawmem->init_req( fall_in_path, contended_phi_rawmem );
1763       pf_phi_rawmem->init_req( pf_path, store_new_wmt );
1764 
1765       transform_later(pf_region);
1766       transform_later(pf_phi_rawmem);
1767       transform_later(pf_phi_abio);
1768 
1769       needgc_false = pf_region;
1770       contended_phi_rawmem = pf_phi_rawmem;
1771       i_o = pf_phi_abio;
1772    } else if( UseTLAB && AllocatePrefetchStyle == 3 ) {
1773       // Insert a prefetch for each allocation.
1774       // This code is used for Sparc with BIS.
1775       Node *pf_region = new (C) RegionNode(3);
1776       Node *pf_phi_rawmem = new (C) PhiNode( pf_region, Type::MEMORY,
1777                                              TypeRawPtr::BOTTOM );
1778 
1779       // Generate several prefetch instructions.
1780       uint lines = (length != NULL) ? AllocatePrefetchLines : AllocateInstancePrefetchLines;
1781       uint step_size = AllocatePrefetchStepSize;
1782       uint distance = AllocatePrefetchDistance;
1783 
1784       // Next cache address.
1785       Node *cache_adr = new (C) AddPNode(old_eden_top, old_eden_top,
1786                                             _igvn.MakeConX(distance));
1787       transform_later(cache_adr);
1788       cache_adr = new (C) CastP2XNode(needgc_false, cache_adr);
1789       transform_later(cache_adr);
1790       Node* mask = _igvn.MakeConX(~(intptr_t)(step_size-1));
1791       cache_adr = new (C) AndXNode(cache_adr, mask);
1792       transform_later(cache_adr);
1793       cache_adr = new (C) CastX2PNode(cache_adr);
1794       transform_later(cache_adr);
1795 
1796       // Prefetch
1797       Node *prefetch = new (C) PrefetchAllocationNode( contended_phi_rawmem, cache_adr );
1798       prefetch->set_req(0, needgc_false);
1799       transform_later(prefetch);
1800       contended_phi_rawmem = prefetch;
1801       Node *prefetch_adr;
1802       distance = step_size;
1803       for ( uint i = 1; i < lines; i++ ) {
1804         prefetch_adr = new (C) AddPNode( cache_adr, cache_adr,
1805                                             _igvn.MakeConX(distance) );
1806         transform_later(prefetch_adr);
1807         prefetch = new (C) PrefetchAllocationNode( contended_phi_rawmem, prefetch_adr );
1808         transform_later(prefetch);
1809         distance += step_size;
1810         contended_phi_rawmem = prefetch;
1811       }
1812    } else if( AllocatePrefetchStyle > 0 ) {
1813       // Insert a prefetch for each allocation only on the fast-path
1814       Node *prefetch_adr;
1815       Node *prefetch;
1816       // Generate several prefetch instructions.
1817       uint lines = (length != NULL) ? AllocatePrefetchLines : AllocateInstancePrefetchLines;
1818       uint step_size = AllocatePrefetchStepSize;
1819       uint distance = AllocatePrefetchDistance;
1820       for ( uint i = 0; i < lines; i++ ) {
1821         prefetch_adr = new (C) AddPNode( old_eden_top, new_eden_top,
1822                                             _igvn.MakeConX(distance) );
1823         transform_later(prefetch_adr);
1824         prefetch = new (C) PrefetchAllocationNode( i_o, prefetch_adr );
1825         // Do not let it float too high, since if eden_top == eden_end,
1826         // both might be null.
1827         if( i == 0 ) { // Set control for first prefetch, next follows it
1828           prefetch->init_req(0, needgc_false);
1829         }
1830         transform_later(prefetch);
1831         distance += step_size;
1832         i_o = prefetch;
1833       }
1834    }
1835    return i_o;
1836 }
1837 
1838 
1839 void PhaseMacroExpand::expand_allocate(AllocateNode *alloc) {
1840   expand_allocate_common(alloc, NULL,
1841                          OptoRuntime::new_instance_Type(),
1842                          OptoRuntime::new_instance_Java());
1843 }
1844 


2153      *        // Try to rebias.
2154      *        if( cas(&mark_word, old, new) == 0 ) {
2155      *          // Done.
2156      *        } else {
2157      *          goto slow_path; // Failed.
2158      *        }
2159      *      }
2160      *    }
2161      *  } else {
2162      *    // The object is not biased.
2163      *    cas_lock:
2164      *    if( FastLock(obj) == 0 ) {
2165      *      // Done.
2166      *    } else {
2167      *      slow_path:
2168      *      OptoRuntime::complete_monitor_locking_Java(obj);
2169      *    }
2170      *  }
2171      */
2172 
2173     region  = new (C) RegionNode(5);
2174     // create a Phi for the memory state
2175     mem_phi = new (C) PhiNode( region, Type::MEMORY, TypeRawPtr::BOTTOM);
2176 
2177     Node* fast_lock_region  = new (C) RegionNode(3);
2178     Node* fast_lock_mem_phi = new (C) PhiNode( fast_lock_region, Type::MEMORY, TypeRawPtr::BOTTOM);
2179 
2180     // First, check mark word for the biased lock pattern.
2181     Node* mark_node = make_load(ctrl, mem, obj, oopDesc::mark_offset_in_bytes(), TypeX_X, TypeX_X->basic_type());
2182 
2183     // Get fast path - mark word has the biased lock pattern.
2184     ctrl = opt_bits_test(ctrl, fast_lock_region, 1, mark_node,
2185                          markOopDesc::biased_lock_mask_in_place,
2186                          markOopDesc::biased_lock_pattern, true);
2187     // fast_lock_region->in(1) is set to slow path.
2188     fast_lock_mem_phi->init_req(1, mem);
2189 
2190     // Now check that the lock is biased to the current thread and has
2191     // the same epoch and bias as Klass::_prototype_header.
2192 
2193     // Special-case a fresh allocation to avoid building nodes:
2194     Node* klass_node = AllocateNode::Ideal_klass(obj, &_igvn);
2195     if (klass_node == NULL) {
2196       Node* k_adr = basic_plus_adr(obj, oopDesc::klass_offset_in_bytes());
2197       klass_node = transform_later( LoadKlassNode::make(_igvn, mem, k_adr, _igvn.type(k_adr)->is_ptr()) );
2198 #ifdef _LP64
2199       if (UseCompressedClassPointers && klass_node->is_DecodeNKlass()) {
2200         assert(klass_node->in(1)->Opcode() == Op_LoadNKlass, "sanity");
2201         klass_node->in(1)->init_req(0, ctrl);
2202       } else
2203 #endif
2204       klass_node->init_req(0, ctrl);
2205     }
2206     Node *proto_node = make_load(ctrl, mem, klass_node, in_bytes(Klass::prototype_header_offset()), TypeX_X, TypeX_X->basic_type());
2207 
2208     Node* thread = transform_later(new (C) ThreadLocalNode());
2209     Node* cast_thread = transform_later(new (C) CastP2XNode(ctrl, thread));
2210     Node* o_node = transform_later(new (C) OrXNode(cast_thread, proto_node));
2211     Node* x_node = transform_later(new (C) XorXNode(o_node, mark_node));
2212 
2213     // Get slow path - mark word does NOT match the value.
2214     Node* not_biased_ctrl =  opt_bits_test(ctrl, region, 3, x_node,
2215                                       (~markOopDesc::age_mask_in_place), 0);
2216     // region->in(3) is set to fast path - the object is biased to the current thread.
2217     mem_phi->init_req(3, mem);
2218 
2219 
2220     // Mark word does NOT match the value (thread | Klass::_prototype_header).
2221 
2222 
2223     // First, check biased pattern.
2224     // Get fast path - _prototype_header has the same biased lock pattern.
2225     ctrl =  opt_bits_test(not_biased_ctrl, fast_lock_region, 2, x_node,
2226                           markOopDesc::biased_lock_mask_in_place, 0, true);
2227 
2228     not_biased_ctrl = fast_lock_region->in(2); // Slow path
2229     // fast_lock_region->in(2) - the prototype header is no longer biased
2230     // and we have to revoke the bias on this object.
2231     // We are going to try to reset the mark of this object to the prototype
2232     // value and fall through to the CAS-based locking scheme.
2233     Node* adr = basic_plus_adr(obj, oopDesc::mark_offset_in_bytes());
2234     Node* cas = new (C) StoreXConditionalNode(not_biased_ctrl, mem, adr,
2235                                               proto_node, mark_node);
2236     transform_later(cas);
2237     Node* proj = transform_later( new (C) SCMemProjNode(cas));
2238     fast_lock_mem_phi->init_req(2, proj);
2239 
2240 
2241     // Second, check epoch bits.
2242     Node* rebiased_region  = new (C) RegionNode(3);
2243     Node* old_phi = new (C) PhiNode( rebiased_region, TypeX_X);
2244     Node* new_phi = new (C) PhiNode( rebiased_region, TypeX_X);
2245 
2246     // Get slow path - mark word does NOT match epoch bits.
2247     Node* epoch_ctrl =  opt_bits_test(ctrl, rebiased_region, 1, x_node,
2248                                       markOopDesc::epoch_mask_in_place, 0);
2249     // The epoch of the current bias is not valid, attempt to rebias the object
2250     // toward the current thread.
2251     rebiased_region->init_req(2, epoch_ctrl);
2252     old_phi->init_req(2, mark_node);
2253     new_phi->init_req(2, o_node);
2254 
2255     // rebiased_region->in(1) is set to fast path.
2256     // The epoch of the current bias is still valid but we know
2257     // nothing about the owner; it might be set or it might be clear.
2258     Node* cmask   = MakeConX(markOopDesc::biased_lock_mask_in_place |
2259                              markOopDesc::age_mask_in_place |
2260                              markOopDesc::epoch_mask_in_place);
2261     Node* old = transform_later(new (C) AndXNode(mark_node, cmask));
2262     cast_thread = transform_later(new (C) CastP2XNode(ctrl, thread));
2263     Node* new_mark = transform_later(new (C) OrXNode(cast_thread, old));
2264     old_phi->init_req(1, old);
2265     new_phi->init_req(1, new_mark);
2266 
2267     transform_later(rebiased_region);
2268     transform_later(old_phi);
2269     transform_later(new_phi);
2270 
2271     // Try to acquire the bias of the object using an atomic operation.
2272     // If this fails we will go in to the runtime to revoke the object's bias.
2273     cas = new (C) StoreXConditionalNode(rebiased_region, mem, adr,
2274                                            new_phi, old_phi);
2275     transform_later(cas);
2276     proj = transform_later( new (C) SCMemProjNode(cas));
2277 
2278     // Get slow path - Failed to CAS.
2279     not_biased_ctrl = opt_bits_test(rebiased_region, region, 4, cas, 0, 0);
2280     mem_phi->init_req(4, proj);
2281     // region->in(4) is set to fast path - the object is rebiased to the current thread.
2282 
2283     // Failed to CAS.
2284     slow_path  = new (C) RegionNode(3);
2285     Node *slow_mem = new (C) PhiNode( slow_path, Type::MEMORY, TypeRawPtr::BOTTOM);
2286 
2287     slow_path->init_req(1, not_biased_ctrl); // Capture slow-control
2288     slow_mem->init_req(1, proj);
2289 
2290     // Call CAS-based locking scheme (FastLock node).
2291 
2292     transform_later(fast_lock_region);
2293     transform_later(fast_lock_mem_phi);
2294 
2295     // Get slow path - FastLock failed to lock the object.
2296     ctrl = opt_bits_test(fast_lock_region, region, 2, flock, 0, 0);
2297     mem_phi->init_req(2, fast_lock_mem_phi);
2298     // region->in(2) is set to fast path - the object is locked to the current thread.
2299 
2300     slow_path->init_req(2, ctrl); // Capture slow-control
2301     slow_mem->init_req(2, fast_lock_mem_phi);
2302 
2303     transform_later(slow_path);
2304     transform_later(slow_mem);
2305     // Reset lock's memory edge.
2306     lock->set_req(TypeFunc::Memory, slow_mem);
2307 
2308   } else {
2309     region  = new (C) RegionNode(3);
2310     // create a Phi for the memory state
2311     mem_phi = new (C) PhiNode( region, Type::MEMORY, TypeRawPtr::BOTTOM);
2312 
2313     // Optimize test; set region slot 2
2314     slow_path = opt_bits_test(ctrl, region, 2, flock, 0, 0);
2315     mem_phi->init_req(2, mem);
2316   }
2317 
2318   // Make slow path call
2319   CallNode *call = make_slow_call( (CallNode *) lock, OptoRuntime::complete_monitor_enter_Type(), OptoRuntime::complete_monitor_locking_Java(), NULL, slow_path, obj, box );
2320 
2321   extract_call_projections(call);
2322 
2323   // Slow path can only throw asynchronous exceptions, which are always
2324   // de-opted.  So the compiler thinks the slow-call can never throw an
2325   // exception.  If it DOES throw an exception we would need the debug
2326   // info removed first (since if it throws there is no monitor).
2327   assert ( _ioproj_fallthrough == NULL && _ioproj_catchall == NULL &&
2328            _memproj_catchall == NULL && _catchallcatchproj == NULL, "Unexpected projection from Lock");
2329 
2330   // Capture slow path
2331   // disconnect fall-through projection from call and create a new one
2332   // hook up users of fall-through projection to region
2333   Node *slow_ctrl = _fallthroughproj->clone();
2334   transform_later(slow_ctrl);
2335   _igvn.hash_delete(_fallthroughproj);
2336   _fallthroughproj->disconnect_inputs(NULL, C);
2337   region->init_req(1, slow_ctrl);
2338   // region inputs are now complete
2339   transform_later(region);
2340   _igvn.replace_node(_fallthroughproj, region);
2341 
2342   Node *memproj = transform_later( new(C) ProjNode(call, TypeFunc::Memory) );
2343   mem_phi->init_req(1, memproj );
2344   transform_later(mem_phi);
2345   _igvn.replace_node(_memproj_fallthrough, mem_phi);
2346 }
2347 
2348 //------------------------------expand_unlock_node----------------------
2349 void PhaseMacroExpand::expand_unlock_node(UnlockNode *unlock) {
2350 
2351   Node* ctrl = unlock->in(TypeFunc::Control);
2352   Node* mem = unlock->in(TypeFunc::Memory);
2353   Node* obj = unlock->obj_node();
2354   Node* box = unlock->box_node();
2355 
2356   assert(!box->as_BoxLock()->is_eliminated(), "sanity");
2357 
2358   // No need for a null check on unlock
2359 
2360   // Make the merge point
2361   Node *region;
2362   Node *mem_phi;
2363 
2364   if (UseOptoBiasInlining) {
2365     // Check for biased locking unlock case, which is a no-op.
2366     // See the full description in MacroAssembler::biased_locking_exit().
2367     region  = new (C) RegionNode(4);
2368     // create a Phi for the memory state
2369     mem_phi = new (C) PhiNode( region, Type::MEMORY, TypeRawPtr::BOTTOM);
2370     mem_phi->init_req(3, mem);
2371 
2372     Node* mark_node = make_load(ctrl, mem, obj, oopDesc::mark_offset_in_bytes(), TypeX_X, TypeX_X->basic_type());
2373     ctrl = opt_bits_test(ctrl, region, 3, mark_node,
2374                          markOopDesc::biased_lock_mask_in_place,
2375                          markOopDesc::biased_lock_pattern);
2376   } else {
2377     region  = new (C) RegionNode(3);
2378     // create a Phi for the memory state
2379     mem_phi = new (C) PhiNode( region, Type::MEMORY, TypeRawPtr::BOTTOM);
2380   }
2381 
2382   FastUnlockNode *funlock = new (C) FastUnlockNode( ctrl, obj, box );
2383   funlock = transform_later( funlock )->as_FastUnlock();
2384   // Optimize test; set region slot 2
2385   Node *slow_path = opt_bits_test(ctrl, region, 2, funlock, 0, 0);
2386 
2387   CallNode *call = make_slow_call( (CallNode *) unlock, OptoRuntime::complete_monitor_exit_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::complete_monitor_unlocking_C), "complete_monitor_unlocking_C", slow_path, obj, box );
2388 
2389   extract_call_projections(call);
2390 
2391   assert ( _ioproj_fallthrough == NULL && _ioproj_catchall == NULL &&
2392            _memproj_catchall == NULL && _catchallcatchproj == NULL, "Unexpected projection from Lock");
2393 
2394   // No exceptions for unlocking
2395   // Capture slow path
2396   // disconnect fall-through projection from call and create a new one
2397   // hook up users of fall-through projection to region
2398   Node *slow_ctrl = _fallthroughproj->clone();
2399   transform_later(slow_ctrl);
2400   _igvn.hash_delete(_fallthroughproj);
2401   _fallthroughproj->disconnect_inputs(NULL, C);
2402   region->init_req(1, slow_ctrl);
2403   // region inputs are now complete
2404   transform_later(region);
2405   _igvn.replace_node(_fallthroughproj, region);
2406 
2407   Node *memproj = transform_later( new(C) ProjNode(call, TypeFunc::Memory) );
2408   mem_phi->init_req(1, memproj );
2409   mem_phi->init_req(2, mem);
2410   transform_later(mem_phi);
2411   _igvn.replace_node(_memproj_fallthrough, mem_phi);
2412 }
2413 
2414 //---------------------------eliminate_macro_nodes----------------------
2415 // Eliminate scalar replaced allocations and associated locks.
2416 void PhaseMacroExpand::eliminate_macro_nodes() {
2417   if (C->macro_count() == 0)
2418     return;
2419 
2420   // First, attempt to eliminate locks
2421   int cnt = C->macro_count();
2422   for (int i=0; i < cnt; i++) {
2423     Node *n = C->macro_node(i);
2424     if (n->is_AbstractLock()) { // Lock and Unlock nodes
2425       // Before elimination mark all associated (same box and obj)
2426       // lock and unlock nodes.
2427       mark_eliminated_locking_nodes(n->as_AbstractLock());




  91       }
  92       old_in = new_in;
  93     }
  94     newcall->add_req(old_in);
  95   }
  96 
  97   newcall->set_jvms(oldcall->jvms());
  98   for (JVMState *jvms = newcall->jvms(); jvms != NULL; jvms = jvms->caller()) {
  99     jvms->set_map(newcall);
 100     jvms->set_locoff(jvms->locoff()+jvms_adj);
 101     jvms->set_stkoff(jvms->stkoff()+jvms_adj);
 102     jvms->set_monoff(jvms->monoff()+jvms_adj);
 103     jvms->set_scloff(jvms->scloff()+jvms_adj);
 104     jvms->set_endoff(jvms->endoff()+jvms_adj);
 105   }
 106 }
 107 
 108 Node* PhaseMacroExpand::opt_bits_test(Node* ctrl, Node* region, int edge, Node* word, int mask, int bits, bool return_fast_path) {
 109   Node* cmp;
 110   if (mask != 0) {
 111     Node* and_node = transform_later(new AndXNode(word, MakeConX(mask)));
 112     cmp = transform_later(new CmpXNode(and_node, MakeConX(bits)));
 113   } else {
 114     cmp = word;
 115   }
 116   Node* bol = transform_later(new BoolNode(cmp, BoolTest::ne));
 117   IfNode* iff = new IfNode( ctrl, bol, PROB_MIN, COUNT_UNKNOWN );
 118   transform_later(iff);
 119 
 120   // Fast path taken.
 121   Node *fast_taken = transform_later(new IfFalseNode(iff));
 122 
 123   // Fast path not-taken, i.e. slow path
 124   Node *slow_taken = transform_later(new IfTrueNode(iff));
 125 
 126   if (return_fast_path) {
 127     region->init_req(edge, slow_taken); // Capture slow-control
 128     return fast_taken;
 129   } else {
 130     region->init_req(edge, fast_taken); // Capture fast-control
 131     return slow_taken;
 132   }
 133 }
 134 
 135 //--------------------copy_predefined_input_for_runtime_call--------------------
 136 void PhaseMacroExpand::copy_predefined_input_for_runtime_call(Node * ctrl, CallNode* oldcall, CallNode* call) {
 137   // Set fixed predefined input arguments
 138   call->init_req( TypeFunc::Control, ctrl );
 139   call->init_req( TypeFunc::I_O    , oldcall->in( TypeFunc::I_O) );
 140   call->init_req( TypeFunc::Memory , oldcall->in( TypeFunc::Memory ) ); // ?????
 141   call->init_req( TypeFunc::ReturnAdr, oldcall->in( TypeFunc::ReturnAdr ) );
 142   call->init_req( TypeFunc::FramePtr, oldcall->in( TypeFunc::FramePtr ) );
 143 }
 144 
 145 //------------------------------make_slow_call---------------------------------
 146 CallNode* PhaseMacroExpand::make_slow_call(CallNode *oldcall, const TypeFunc* slow_call_type, address slow_call, const char* leaf_name, Node* slow_path, Node* parm0, Node* parm1) {
 147 
 148   // Slow-path call
 149  CallNode *call = leaf_name
 150    ? (CallNode*)new CallLeafNode      ( slow_call_type, slow_call, leaf_name, TypeRawPtr::BOTTOM )
 151    : (CallNode*)new CallStaticJavaNode( slow_call_type, slow_call, OptoRuntime::stub_name(slow_call), oldcall->jvms()->bci(), TypeRawPtr::BOTTOM );
 152 
 153   // Slow path call has no side-effects, uses few values
 154   copy_predefined_input_for_runtime_call(slow_path, oldcall, call );
 155   if (parm0 != NULL)  call->init_req(TypeFunc::Parms+0, parm0);
 156   if (parm1 != NULL)  call->init_req(TypeFunc::Parms+1, parm1);
 157   copy_call_debug_info(oldcall, call);
 158   call->set_cnt(PROB_UNLIKELY_MAG(4));  // Same effect as RC_UNCOMMON.
 159   _igvn.replace_node(oldcall, call);
 160   transform_later(call);
 161 
 162   return call;
 163 }
 164 
 165 void PhaseMacroExpand::extract_call_projections(CallNode *call) {
 166   _fallthroughproj = NULL;
 167   _fallthroughcatchproj = NULL;
 168   _ioproj_fallthrough = NULL;
 169   _ioproj_catchall = NULL;
 170   _catchallcatchproj = NULL;
 171   _memproj_fallthrough = NULL;


 406     if (phi->is_Phi() && phi != mem &&
 407         phi->as_Phi()->is_same_inst_field(phi_type, instance_id, alias_idx, offset)) {
 408       return phi;
 409     }
 410   }
 411   // Check if an appropriate new value phi already exists.
 412   Node* new_phi = value_phis->find(mem->_idx);
 413   if (new_phi != NULL)
 414     return new_phi;
 415 
 416   if (level <= 0) {
 417     return NULL; // Give up: phi tree too deep
 418   }
 419   Node *start_mem = C->start()->proj_out(TypeFunc::Memory);
 420   Node *alloc_mem = alloc->in(TypeFunc::Memory);
 421 
 422   uint length = mem->req();
 423   GrowableArray <Node *> values(length, length, NULL, false);
 424 
 425   // create a new Phi for the value
 426   PhiNode *phi = new PhiNode(mem->in(0), phi_type, NULL, instance_id, alias_idx, offset);
 427   transform_later(phi);
 428   value_phis->push(phi, mem->_idx);
 429 
 430   for (uint j = 1; j < length; j++) {
 431     Node *in = mem->in(j);
 432     if (in == NULL || in->is_top()) {
 433       values.at_put(j, in);
 434     } else  {
 435       Node *val = scan_mem_chain(in, alias_idx, offset, start_mem, alloc, &_igvn);
 436       if (val == start_mem || val == alloc_mem) {
 437         // hit a sentinel, return appropriate 0 value
 438         values.at_put(j, _igvn.zerocon(ft));
 439         continue;
 440       }
 441       if (val->is_Initialize()) {
 442         val = val->as_Initialize()->find_captured_store(offset, type2aelembytes(ft), &_igvn);
 443       }
 444       if (val == NULL) {
 445         return NULL;  // can't find a value on this path
 446       }


 718       // find the array's elements which will be needed for safepoint debug information
 719       nfields = alloc->in(AllocateNode::ALength)->find_int_con(-1);
 720       assert(klass->is_array_klass() && nfields >= 0, "must be an array klass.");
 721       elem_type = klass->as_array_klass()->element_type();
 722       basic_elem_type = elem_type->basic_type();
 723       array_base = arrayOopDesc::base_offset_in_bytes(basic_elem_type);
 724       element_size = type2aelembytes(basic_elem_type);
 725     }
 726   }
 727   //
 728   // Process the safepoint uses
 729   //
 730   while (safepoints.length() > 0) {
 731     SafePointNode* sfpt = safepoints.pop();
 732     Node* mem = sfpt->memory();
 733     assert(sfpt->jvms() != NULL, "missed JVMS");
 734     // Fields of scalar objs are referenced only at the end
 735     // of regular debuginfo at the last (youngest) JVMS.
 736     // Record relative start index.
 737     uint first_ind = (sfpt->req() - sfpt->jvms()->scloff());
 738     SafePointScalarObjectNode* sobj = new SafePointScalarObjectNode(res_type,
 739 #ifdef ASSERT
 740                                                  alloc,
 741 #endif
 742                                                  first_ind, nfields);
 743     sobj->init_req(0, C->root());
 744     transform_later(sobj);
 745 
 746     // Scan object's fields adding an input to the safepoint for each field.
 747     for (int j = 0; j < nfields; j++) {
 748       intptr_t offset;
 749       ciField* field = NULL;
 750       if (iklass != NULL) {
 751         field = iklass->nonstatic_field_at(j);
 752         offset = field->offset();
 753         elem_type = field->type();
 754         basic_elem_type = field->layout_type();
 755       } else {
 756         offset = array_base + j * (intptr_t)element_size;
 757       }
 758 


 826             tty->print(" (alias_idx=%d)", field_idx);
 827           } else { // Array's element
 828             tty->print("=== At SafePoint node %d can't find value of array element [%d]",
 829                        sfpt->_idx, j);
 830           }
 831           tty->print(", which prevents elimination of: ");
 832           if (res == NULL)
 833             alloc->dump();
 834           else
 835             res->dump();
 836         }
 837 #endif
 838         return false;
 839       }
 840       if (UseCompressedOops && field_type->isa_narrowoop()) {
 841         // Enable "DecodeN(EncodeP(Allocate)) --> Allocate" transformation
 842         // to be able scalar replace the allocation.
 843         if (field_val->is_EncodeP()) {
 844           field_val = field_val->in(1);
 845         } else {
 846           field_val = transform_later(new DecodeNNode(field_val, field_val->get_ptr_type()));
 847         }
 848       }
 849       sfpt->add_req(field_val);
 850     }
 851     JVMState *jvms = sfpt->jvms();
 852     jvms->set_endoff(sfpt->req());
 853     // Now make a pass over the debug information replacing any references
 854     // to the allocated object with "sobj"
 855     int start = jvms->debug_start();
 856     int end   = jvms->debug_end();
 857     sfpt->replace_edges_in_range(res, sobj, start, end);
 858     safepoints_done.append_if_missing(sfpt); // keep it for rollback
 859   }
 860   return true;
 861 }
 862 
 863 // Process users of eliminated allocation.
 864 void PhaseMacroExpand::process_users_of_allocation(CallNode *alloc) {
 865   Node* res = alloc->result_cast();
 866   if (res != NULL) {


1052     }
1053     log->tail("eliminate_boxing");
1054   }
1055 
1056   process_users_of_allocation(boxing);
1057 
1058 #ifndef PRODUCT
1059   if (PrintEliminateAllocations) {
1060     tty->print("++++ Eliminated: %d ", boxing->_idx);
1061     boxing->method()->print_short_name(tty);
1062     tty->cr();
1063   }
1064 #endif
1065 
1066   return true;
1067 }
1068 
1069 //---------------------------set_eden_pointers-------------------------
1070 void PhaseMacroExpand::set_eden_pointers(Node* &eden_top_adr, Node* &eden_end_adr) {
1071   if (UseTLAB) {                // Private allocation: load from TLS
1072     Node* thread = transform_later(new ThreadLocalNode());
1073     int tlab_top_offset = in_bytes(JavaThread::tlab_top_offset());
1074     int tlab_end_offset = in_bytes(JavaThread::tlab_end_offset());
1075     eden_top_adr = basic_plus_adr(top()/*not oop*/, thread, tlab_top_offset);
1076     eden_end_adr = basic_plus_adr(top()/*not oop*/, thread, tlab_end_offset);
1077   } else {                      // Shared allocation: load from globals
1078     CollectedHeap* ch = Universe::heap();
1079     address top_adr = (address)ch->top_addr();
1080     address end_adr = (address)ch->end_addr();
1081     eden_top_adr = makecon(TypeRawPtr::make(top_adr));
1082     eden_end_adr = basic_plus_adr(eden_top_adr, end_adr - top_adr);
1083   }
1084 }
1085 
1086 
1087 Node* PhaseMacroExpand::make_load(Node* ctl, Node* mem, Node* base, int offset, const Type* value_type, BasicType bt) {
1088   Node* adr = basic_plus_adr(base, offset);
1089   const TypePtr* adr_type = adr->bottom_type()->is_ptr();
1090   Node* value = LoadNode::make(_igvn, ctl, mem, adr, adr_type, value_type, bt, MemNode::unordered);
1091   transform_later(value);
1092   return value;


1188   } else {
1189     initial_slow_test = BoolNode::make_predicate(initial_slow_test, &_igvn);
1190   }
1191 
1192   if (C->env()->dtrace_alloc_probes() ||
1193       !UseTLAB && (!Universe::heap()->supports_inline_contig_alloc() ||
1194                    (UseConcMarkSweepGC && CMSIncrementalMode))) {
1195     // Force slow-path allocation
1196     always_slow = true;
1197     initial_slow_test = NULL;
1198   }
1199 
1200 
1201   enum { too_big_or_final_path = 1, need_gc_path = 2 };
1202   Node *slow_region = NULL;
1203   Node *toobig_false = ctrl;
1204 
1205   assert (initial_slow_test == NULL || !always_slow, "arguments must be consistent");
1206   // generate the initial test if necessary
1207   if (initial_slow_test != NULL ) {
1208     slow_region = new RegionNode(3);
1209 
1210     // Now make the initial failure test.  Usually a too-big test but
1211     // might be a TRUE for finalizers or a fancy class check for
1212     // newInstance0.
1213     IfNode *toobig_iff = new IfNode(ctrl, initial_slow_test, PROB_MIN, COUNT_UNKNOWN);
1214     transform_later(toobig_iff);
1215     // Plug the failing-too-big test into the slow-path region
1216     Node *toobig_true = new IfTrueNode( toobig_iff );
1217     transform_later(toobig_true);
1218     slow_region    ->init_req( too_big_or_final_path, toobig_true );
1219     toobig_false = new IfFalseNode( toobig_iff );
1220     transform_later(toobig_false);
1221   } else {         // No initial test, just fall into next case
1222     toobig_false = ctrl;
1223     debug_only(slow_region = NodeSentinel);
1224   }
1225 
1226   Node *slow_mem = mem;  // save the current memory state for slow path
1227   // generate the fast allocation code unless we know that the initial test will always go slow
1228   if (!always_slow) {
1229     // Fast path modifies only raw memory.
1230     if (mem->is_MergeMem()) {
1231       mem = mem->as_MergeMem()->memory_at(Compile::AliasIdxRaw);
1232     }
1233 
1234     Node* eden_top_adr;
1235     Node* eden_end_adr;
1236 
1237     set_eden_pointers(eden_top_adr, eden_end_adr);
1238 
1239     // Load Eden::end.  Loop invariant and hoisted.
1240     //
1241     // Note: We set the control input on "eden_end" and "old_eden_top" when using
1242     //       a TLAB to work around a bug where these values were being moved across
1243     //       a safepoint.  These are not oops, so they cannot be include in the oop
1244     //       map, but they can be changed by a GC.   The proper way to fix this would
1245     //       be to set the raw memory state when generating a  SafepointNode.  However
1246     //       this will require extensive changes to the loop optimization in order to
1247     //       prevent a degradation of the optimization.
1248     //       See comment in memnode.hpp, around line 227 in class LoadPNode.
1249     Node *eden_end = make_load(ctrl, mem, eden_end_adr, 0, TypeRawPtr::BOTTOM, T_ADDRESS);
1250 
1251     // allocate the Region and Phi nodes for the result
1252     result_region = new RegionNode(3);
1253     result_phi_rawmem = new PhiNode(result_region, Type::MEMORY, TypeRawPtr::BOTTOM);
1254     result_phi_rawoop = new PhiNode(result_region, TypeRawPtr::BOTTOM);
1255     result_phi_i_o    = new PhiNode(result_region, Type::ABIO); // I/O is used for Prefetch
1256 
1257     // We need a Region for the loop-back contended case.
1258     enum { fall_in_path = 1, contended_loopback_path = 2 };
1259     Node *contended_region;
1260     Node *contended_phi_rawmem;
1261     if (UseTLAB) {
1262       contended_region = toobig_false;
1263       contended_phi_rawmem = mem;
1264     } else {
1265       contended_region = new RegionNode(3);
1266       contended_phi_rawmem = new PhiNode(contended_region, Type::MEMORY, TypeRawPtr::BOTTOM);
1267       // Now handle the passing-too-big test.  We fall into the contended
1268       // loop-back merge point.
1269       contended_region    ->init_req(fall_in_path, toobig_false);
1270       contended_phi_rawmem->init_req(fall_in_path, mem);
1271       transform_later(contended_region);
1272       transform_later(contended_phi_rawmem);
1273     }
1274 
1275     // Load(-locked) the heap top.
1276     // See note above concerning the control input when using a TLAB
1277     Node *old_eden_top = UseTLAB
1278       ? new LoadPNode      (ctrl, contended_phi_rawmem, eden_top_adr, TypeRawPtr::BOTTOM, TypeRawPtr::BOTTOM, MemNode::unordered)
1279       : new LoadPLockedNode(contended_region, contended_phi_rawmem, eden_top_adr, MemNode::acquire);
1280 
1281     transform_later(old_eden_top);
1282     // Add to heap top to get a new heap top
1283     Node *new_eden_top = new AddPNode(top(), old_eden_top, size_in_bytes);
1284     transform_later(new_eden_top);
1285     // Check for needing a GC; compare against heap end
1286     Node *needgc_cmp = new CmpPNode(new_eden_top, eden_end);
1287     transform_later(needgc_cmp);
1288     Node *needgc_bol = new BoolNode(needgc_cmp, BoolTest::ge);
1289     transform_later(needgc_bol);
1290     IfNode *needgc_iff = new IfNode(contended_region, needgc_bol, PROB_UNLIKELY_MAG(4), COUNT_UNKNOWN);
1291     transform_later(needgc_iff);
1292 
1293     // Plug the failing-heap-space-need-gc test into the slow-path region
1294     Node *needgc_true = new IfTrueNode(needgc_iff);
1295     transform_later(needgc_true);
1296     if (initial_slow_test) {
1297       slow_region->init_req(need_gc_path, needgc_true);
1298       // This completes all paths into the slow merge point
1299       transform_later(slow_region);
1300     } else {                      // No initial slow path needed!
1301       // Just fall from the need-GC path straight into the VM call.
1302       slow_region = needgc_true;
1303     }
1304     // No need for a GC.  Setup for the Store-Conditional
1305     Node *needgc_false = new IfFalseNode(needgc_iff);
1306     transform_later(needgc_false);
1307 
1308     // Grab regular I/O before optional prefetch may change it.
1309     // Slow-path does no I/O so just set it to the original I/O.
1310     result_phi_i_o->init_req(slow_result_path, i_o);
1311 
1312     i_o = prefetch_allocation(i_o, needgc_false, contended_phi_rawmem,
1313                               old_eden_top, new_eden_top, length);
1314 
1315     // Name successful fast-path variables
1316     Node* fast_oop = old_eden_top;
1317     Node* fast_oop_ctrl;
1318     Node* fast_oop_rawmem;
1319 
1320     // Store (-conditional) the modified eden top back down.
1321     // StorePConditional produces flags for a test PLUS a modified raw
1322     // memory state.
1323     if (UseTLAB) {
1324       Node* store_eden_top =
1325         new StorePNode(needgc_false, contended_phi_rawmem, eden_top_adr,
1326                               TypeRawPtr::BOTTOM, new_eden_top, MemNode::unordered);
1327       transform_later(store_eden_top);
1328       fast_oop_ctrl = needgc_false; // No contention, so this is the fast path
1329       fast_oop_rawmem = store_eden_top;
1330     } else {
1331       Node* store_eden_top =
1332         new StorePConditionalNode(needgc_false, contended_phi_rawmem, eden_top_adr,
1333                                          new_eden_top, fast_oop/*old_eden_top*/);
1334       transform_later(store_eden_top);
1335       Node *contention_check = new BoolNode(store_eden_top, BoolTest::ne);
1336       transform_later(contention_check);
1337       store_eden_top = new SCMemProjNode(store_eden_top);
1338       transform_later(store_eden_top);
1339 
1340       // If not using TLABs, check to see if there was contention.
1341       IfNode *contention_iff = new IfNode (needgc_false, contention_check, PROB_MIN, COUNT_UNKNOWN);
1342       transform_later(contention_iff);
1343       Node *contention_true = new IfTrueNode(contention_iff);
1344       transform_later(contention_true);
1345       // If contention, loopback and try again.
1346       contended_region->init_req(contended_loopback_path, contention_true);
1347       contended_phi_rawmem->init_req(contended_loopback_path, store_eden_top);
1348 
1349       // Fast-path succeeded with no contention!
1350       Node *contention_false = new IfFalseNode(contention_iff);
1351       transform_later(contention_false);
1352       fast_oop_ctrl = contention_false;
1353 
1354       // Bump total allocated bytes for this thread
1355       Node* thread = new ThreadLocalNode();
1356       transform_later(thread);
1357       Node* alloc_bytes_adr = basic_plus_adr(top()/*not oop*/, thread,
1358                                              in_bytes(JavaThread::allocated_bytes_offset()));
1359       Node* alloc_bytes = make_load(fast_oop_ctrl, store_eden_top, alloc_bytes_adr,
1360                                     0, TypeLong::LONG, T_LONG);
1361 #ifdef _LP64
1362       Node* alloc_size = size_in_bytes;
1363 #else
1364       Node* alloc_size = new ConvI2LNode(size_in_bytes);
1365       transform_later(alloc_size);
1366 #endif
1367       Node* new_alloc_bytes = new AddLNode(alloc_bytes, alloc_size);
1368       transform_later(new_alloc_bytes);
1369       fast_oop_rawmem = make_store(fast_oop_ctrl, store_eden_top, alloc_bytes_adr,
1370                                    0, new_alloc_bytes, T_LONG);
1371     }
1372 
1373     InitializeNode* init = alloc->initialization();
1374     fast_oop_rawmem = initialize_object(alloc,
1375                                         fast_oop_ctrl, fast_oop_rawmem, fast_oop,
1376                                         klass_node, length, size_in_bytes);
1377 
1378     // If initialization is performed by an array copy, any required
1379     // MemBarStoreStore was already added. If the object does not
1380     // escape no need for a MemBarStoreStore. Otherwise we need a
1381     // MemBarStoreStore so that stores that initialize this object
1382     // can't be reordered with a subsequent store that makes this
1383     // object accessible by other threads.
1384     if (init == NULL || (!init->is_complete_with_arraycopy() && !init->does_not_escape())) {
1385       if (init == NULL || init->req() < InitializeNode::RawStores) {
1386         // No InitializeNode or no stores captured by zeroing
1387         // elimination. Simply add the MemBarStoreStore after object
1388         // initialization.
1389         MemBarNode* mb = MemBarNode::make(C, Op_MemBarStoreStore, Compile::AliasIdxBot);
1390         transform_later(mb);
1391 
1392         mb->init_req(TypeFunc::Memory, fast_oop_rawmem);
1393         mb->init_req(TypeFunc::Control, fast_oop_ctrl);
1394         fast_oop_ctrl = new ProjNode(mb,TypeFunc::Control);
1395         transform_later(fast_oop_ctrl);
1396         fast_oop_rawmem = new ProjNode(mb,TypeFunc::Memory);
1397         transform_later(fast_oop_rawmem);
1398       } else {
1399         // Add the MemBarStoreStore after the InitializeNode so that
1400         // all stores performing the initialization that were moved
1401         // before the InitializeNode happen before the storestore
1402         // barrier.
1403 
1404         Node* init_ctrl = init->proj_out(TypeFunc::Control);
1405         Node* init_mem = init->proj_out(TypeFunc::Memory);
1406 
1407         MemBarNode* mb = MemBarNode::make(C, Op_MemBarStoreStore, Compile::AliasIdxBot);
1408         transform_later(mb);
1409 
1410         Node* ctrl = new ProjNode(init,TypeFunc::Control);
1411         transform_later(ctrl);
1412         Node* mem = new ProjNode(init,TypeFunc::Memory);
1413         transform_later(mem);
1414 
1415         // The MemBarStoreStore depends on control and memory coming
1416         // from the InitializeNode
1417         mb->init_req(TypeFunc::Memory, mem);
1418         mb->init_req(TypeFunc::Control, ctrl);
1419 
1420         ctrl = new ProjNode(mb,TypeFunc::Control);
1421         transform_later(ctrl);
1422         mem = new ProjNode(mb,TypeFunc::Memory);
1423         transform_later(mem);
1424 
1425         // All nodes that depended on the InitializeNode for control
1426         // and memory must now depend on the MemBarNode that itself
1427         // depends on the InitializeNode
1428         _igvn.replace_node(init_ctrl, ctrl);
1429         _igvn.replace_node(init_mem, mem);
1430       }
1431     }
1432 
1433     if (C->env()->dtrace_extended_probes()) {
1434       // Slow-path call
1435       int size = TypeFunc::Parms + 2;
1436       CallLeafNode *call = new CallLeafNode(OptoRuntime::dtrace_object_alloc_Type(),
1437                                             CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_object_alloc_base),
1438                                             "dtrace_object_alloc",
1439                                             TypeRawPtr::BOTTOM);
1440 
1441       // Get base of thread-local storage area
1442       Node* thread = new ThreadLocalNode();
1443       transform_later(thread);
1444 
1445       call->init_req(TypeFunc::Parms+0, thread);
1446       call->init_req(TypeFunc::Parms+1, fast_oop);
1447       call->init_req(TypeFunc::Control, fast_oop_ctrl);
1448       call->init_req(TypeFunc::I_O    , top()); // does no i/o
1449       call->init_req(TypeFunc::Memory , fast_oop_rawmem);
1450       call->init_req(TypeFunc::ReturnAdr, alloc->in(TypeFunc::ReturnAdr));
1451       call->init_req(TypeFunc::FramePtr, alloc->in(TypeFunc::FramePtr));
1452       transform_later(call);
1453       fast_oop_ctrl = new ProjNode(call,TypeFunc::Control);
1454       transform_later(fast_oop_ctrl);
1455       fast_oop_rawmem = new ProjNode(call,TypeFunc::Memory);
1456       transform_later(fast_oop_rawmem);
1457     }
1458 
1459     // Plug in the successful fast-path into the result merge point
1460     result_region    ->init_req(fast_result_path, fast_oop_ctrl);
1461     result_phi_rawoop->init_req(fast_result_path, fast_oop);
1462     result_phi_i_o   ->init_req(fast_result_path, i_o);
1463     result_phi_rawmem->init_req(fast_result_path, fast_oop_rawmem);
1464   } else {
1465     slow_region = ctrl;
1466     result_phi_i_o = i_o; // Rename it to use in the following code.
1467   }
1468 
1469   // Generate slow-path call
1470   CallNode *call = new CallStaticJavaNode(slow_call_type, slow_call_address,
1471                                OptoRuntime::stub_name(slow_call_address),
1472                                alloc->jvms()->bci(),
1473                                TypePtr::BOTTOM);
1474   call->init_req( TypeFunc::Control, slow_region );
1475   call->init_req( TypeFunc::I_O    , top() )     ;   // does no i/o
1476   call->init_req( TypeFunc::Memory , slow_mem ); // may gc ptrs
1477   call->init_req( TypeFunc::ReturnAdr, alloc->in(TypeFunc::ReturnAdr) );
1478   call->init_req( TypeFunc::FramePtr, alloc->in(TypeFunc::FramePtr) );
1479 
1480   call->init_req(TypeFunc::Parms+0, klass_node);
1481   if (length != NULL) {
1482     call->init_req(TypeFunc::Parms+1, length);
1483   }
1484 
1485   // Copy debug information and adjust JVMState information, then replace
1486   // allocate node with the call
1487   copy_call_debug_info((CallNode *) alloc,  call);
1488   if (!always_slow) {
1489     call->set_cnt(PROB_UNLIKELY_MAG(4));  // Same effect as RC_UNCOMMON.
1490   } else {


1507   //
1508   extract_call_projections(call);
1509 
1510   // An allocate node has separate memory projections for the uses on
1511   // the control and i_o paths. Replace the control memory projection with
1512   // result_phi_rawmem (unless we are only generating a slow call when
1513   // both memory projections are combined)
1514   if (!always_slow && _memproj_fallthrough != NULL) {
1515     for (DUIterator_Fast imax, i = _memproj_fallthrough->fast_outs(imax); i < imax; i++) {
1516       Node *use = _memproj_fallthrough->fast_out(i);
1517       _igvn.rehash_node_delayed(use);
1518       imax -= replace_input(use, _memproj_fallthrough, result_phi_rawmem);
1519       // back up iterator
1520       --i;
1521     }
1522   }
1523   // Now change uses of _memproj_catchall to use _memproj_fallthrough and delete
1524   // _memproj_catchall so we end up with a call that has only 1 memory projection.
1525   if (_memproj_catchall != NULL ) {
1526     if (_memproj_fallthrough == NULL) {
1527       _memproj_fallthrough = new ProjNode(call, TypeFunc::Memory);
1528       transform_later(_memproj_fallthrough);
1529     }
1530     for (DUIterator_Fast imax, i = _memproj_catchall->fast_outs(imax); i < imax; i++) {
1531       Node *use = _memproj_catchall->fast_out(i);
1532       _igvn.rehash_node_delayed(use);
1533       imax -= replace_input(use, _memproj_catchall, _memproj_fallthrough);
1534       // back up iterator
1535       --i;
1536     }
1537     assert(_memproj_catchall->outcnt() == 0, "all uses must be deleted");
1538     _igvn.remove_dead_node(_memproj_catchall);
1539   }
1540 
1541   // An allocate node has separate i_o projections for the uses on the control
1542   // and i_o paths. Always replace the control i_o projection with result i_o
1543   // otherwise incoming i_o become dead when only a slow call is generated
1544   // (it is different from memory projections where both projections are
1545   // combined in such case).
1546   if (_ioproj_fallthrough != NULL) {
1547     for (DUIterator_Fast imax, i = _ioproj_fallthrough->fast_outs(imax); i < imax; i++) {
1548       Node *use = _ioproj_fallthrough->fast_out(i);
1549       _igvn.rehash_node_delayed(use);
1550       imax -= replace_input(use, _ioproj_fallthrough, result_phi_i_o);
1551       // back up iterator
1552       --i;
1553     }
1554   }
1555   // Now change uses of _ioproj_catchall to use _ioproj_fallthrough and delete
1556   // _ioproj_catchall so we end up with a call that has only 1 i_o projection.
1557   if (_ioproj_catchall != NULL ) {
1558     if (_ioproj_fallthrough == NULL) {
1559       _ioproj_fallthrough = new ProjNode(call, TypeFunc::I_O);
1560       transform_later(_ioproj_fallthrough);
1561     }
1562     for (DUIterator_Fast imax, i = _ioproj_catchall->fast_outs(imax); i < imax; i++) {
1563       Node *use = _ioproj_catchall->fast_out(i);
1564       _igvn.rehash_node_delayed(use);
1565       imax -= replace_input(use, _ioproj_catchall, _ioproj_fallthrough);
1566       // back up iterator
1567       --i;
1568     }
1569     assert(_ioproj_catchall->outcnt() == 0, "all uses must be deleted");
1570     _igvn.remove_dead_node(_ioproj_catchall);
1571   }
1572 
1573   // if we generated only a slow call, we are done
1574   if (always_slow) {
1575     // Now we can unhook i_o.
1576     if (result_phi_i_o->outcnt() > 1) {
1577       call->set_req(TypeFunc::I_O, top());
1578     } else {
1579       assert(result_phi_i_o->unique_ctrl_out() == call, "");


1673     // We have no more use for this link, since the AllocateNode goes away:
1674     init->set_req(InitializeNode::RawAddress, top());
1675     // (If we keep the link, it just confuses the register allocator,
1676     // who thinks he sees a real use of the address by the membar.)
1677   }
1678 
1679   return rawmem;
1680 }
1681 
1682 // Generate prefetch instructions for next allocations.
1683 Node* PhaseMacroExpand::prefetch_allocation(Node* i_o, Node*& needgc_false,
1684                                         Node*& contended_phi_rawmem,
1685                                         Node* old_eden_top, Node* new_eden_top,
1686                                         Node* length) {
1687    enum { fall_in_path = 1, pf_path = 2 };
1688    if( UseTLAB && AllocatePrefetchStyle == 2 ) {
1689       // Generate prefetch allocation with watermark check.
1690       // As an allocation hits the watermark, we will prefetch starting
1691       // at a "distance" away from watermark.
1692 
1693       Node *pf_region = new RegionNode(3);
1694       Node *pf_phi_rawmem = new PhiNode( pf_region, Type::MEMORY,
1695                                                 TypeRawPtr::BOTTOM );
1696       // I/O is used for Prefetch
1697       Node *pf_phi_abio = new PhiNode( pf_region, Type::ABIO );
1698 
1699       Node *thread = new ThreadLocalNode();
1700       transform_later(thread);
1701 
1702       Node *eden_pf_adr = new AddPNode( top()/*not oop*/, thread,
1703                    _igvn.MakeConX(in_bytes(JavaThread::tlab_pf_top_offset())) );
1704       transform_later(eden_pf_adr);
1705 
1706       Node *old_pf_wm = new LoadPNode(needgc_false,
1707                                    contended_phi_rawmem, eden_pf_adr,
1708                                    TypeRawPtr::BOTTOM, TypeRawPtr::BOTTOM,
1709                                    MemNode::unordered);
1710       transform_later(old_pf_wm);
1711 
1712       // check against new_eden_top
1713       Node *need_pf_cmp = new CmpPNode( new_eden_top, old_pf_wm );
1714       transform_later(need_pf_cmp);
1715       Node *need_pf_bol = new BoolNode( need_pf_cmp, BoolTest::ge );
1716       transform_later(need_pf_bol);
1717       IfNode *need_pf_iff = new IfNode( needgc_false, need_pf_bol,
1718                                        PROB_UNLIKELY_MAG(4), COUNT_UNKNOWN );
1719       transform_later(need_pf_iff);
1720 
1721       // true node, add prefetchdistance
1722       Node *need_pf_true = new IfTrueNode( need_pf_iff );
1723       transform_later(need_pf_true);
1724 
1725       Node *need_pf_false = new IfFalseNode( need_pf_iff );
1726       transform_later(need_pf_false);
1727 
1728       Node *new_pf_wmt = new AddPNode( top(), old_pf_wm,
1729                                     _igvn.MakeConX(AllocatePrefetchDistance) );
1730       transform_later(new_pf_wmt );
1731       new_pf_wmt->set_req(0, need_pf_true);
1732 
1733       Node *store_new_wmt = new StorePNode(need_pf_true,
1734                                        contended_phi_rawmem, eden_pf_adr,
1735                                        TypeRawPtr::BOTTOM, new_pf_wmt,
1736                                        MemNode::unordered);
1737       transform_later(store_new_wmt);
1738 
1739       // adding prefetches
1740       pf_phi_abio->init_req( fall_in_path, i_o );
1741 
1742       Node *prefetch_adr;
1743       Node *prefetch;
1744       uint lines = AllocatePrefetchDistance / AllocatePrefetchStepSize;
1745       uint step_size = AllocatePrefetchStepSize;
1746       uint distance = 0;
1747 
1748       for ( uint i = 0; i < lines; i++ ) {
1749         prefetch_adr = new AddPNode( old_pf_wm, new_pf_wmt,
1750                                             _igvn.MakeConX(distance) );
1751         transform_later(prefetch_adr);
1752         prefetch = new PrefetchAllocationNode( i_o, prefetch_adr );
1753         transform_later(prefetch);
1754         distance += step_size;
1755         i_o = prefetch;
1756       }
1757       pf_phi_abio->set_req( pf_path, i_o );
1758 
1759       pf_region->init_req( fall_in_path, need_pf_false );
1760       pf_region->init_req( pf_path, need_pf_true );
1761 
1762       pf_phi_rawmem->init_req( fall_in_path, contended_phi_rawmem );
1763       pf_phi_rawmem->init_req( pf_path, store_new_wmt );
1764 
1765       transform_later(pf_region);
1766       transform_later(pf_phi_rawmem);
1767       transform_later(pf_phi_abio);
1768 
1769       needgc_false = pf_region;
1770       contended_phi_rawmem = pf_phi_rawmem;
1771       i_o = pf_phi_abio;
1772    } else if( UseTLAB && AllocatePrefetchStyle == 3 ) {
1773       // Insert a prefetch for each allocation.
1774       // This code is used for Sparc with BIS.
1775       Node *pf_region = new RegionNode(3);
1776       Node *pf_phi_rawmem = new PhiNode( pf_region, Type::MEMORY,
1777                                              TypeRawPtr::BOTTOM );
1778 
1779       // Generate several prefetch instructions.
1780       uint lines = (length != NULL) ? AllocatePrefetchLines : AllocateInstancePrefetchLines;
1781       uint step_size = AllocatePrefetchStepSize;
1782       uint distance = AllocatePrefetchDistance;
1783 
1784       // Next cache address.
1785       Node *cache_adr = new AddPNode(old_eden_top, old_eden_top,
1786                                             _igvn.MakeConX(distance));
1787       transform_later(cache_adr);
1788       cache_adr = new CastP2XNode(needgc_false, cache_adr);
1789       transform_later(cache_adr);
1790       Node* mask = _igvn.MakeConX(~(intptr_t)(step_size-1));
1791       cache_adr = new AndXNode(cache_adr, mask);
1792       transform_later(cache_adr);
1793       cache_adr = new CastX2PNode(cache_adr);
1794       transform_later(cache_adr);
1795 
1796       // Prefetch
1797       Node *prefetch = new PrefetchAllocationNode( contended_phi_rawmem, cache_adr );
1798       prefetch->set_req(0, needgc_false);
1799       transform_later(prefetch);
1800       contended_phi_rawmem = prefetch;
1801       Node *prefetch_adr;
1802       distance = step_size;
1803       for ( uint i = 1; i < lines; i++ ) {
1804         prefetch_adr = new AddPNode( cache_adr, cache_adr,
1805                                             _igvn.MakeConX(distance) );
1806         transform_later(prefetch_adr);
1807         prefetch = new PrefetchAllocationNode( contended_phi_rawmem, prefetch_adr );
1808         transform_later(prefetch);
1809         distance += step_size;
1810         contended_phi_rawmem = prefetch;
1811       }
1812    } else if( AllocatePrefetchStyle > 0 ) {
1813       // Insert a prefetch for each allocation only on the fast-path
1814       Node *prefetch_adr;
1815       Node *prefetch;
1816       // Generate several prefetch instructions.
1817       uint lines = (length != NULL) ? AllocatePrefetchLines : AllocateInstancePrefetchLines;
1818       uint step_size = AllocatePrefetchStepSize;
1819       uint distance = AllocatePrefetchDistance;
1820       for ( uint i = 0; i < lines; i++ ) {
1821         prefetch_adr = new AddPNode( old_eden_top, new_eden_top,
1822                                             _igvn.MakeConX(distance) );
1823         transform_later(prefetch_adr);
1824         prefetch = new PrefetchAllocationNode( i_o, prefetch_adr );
1825         // Do not let it float too high, since if eden_top == eden_end,
1826         // both might be null.
1827         if( i == 0 ) { // Set control for first prefetch, next follows it
1828           prefetch->init_req(0, needgc_false);
1829         }
1830         transform_later(prefetch);
1831         distance += step_size;
1832         i_o = prefetch;
1833       }
1834    }
1835    return i_o;
1836 }
1837 
1838 
1839 void PhaseMacroExpand::expand_allocate(AllocateNode *alloc) {
1840   expand_allocate_common(alloc, NULL,
1841                          OptoRuntime::new_instance_Type(),
1842                          OptoRuntime::new_instance_Java());
1843 }
1844 


2153      *        // Try to rebias.
2154      *        if( cas(&mark_word, old, new) == 0 ) {
2155      *          // Done.
2156      *        } else {
2157      *          goto slow_path; // Failed.
2158      *        }
2159      *      }
2160      *    }
2161      *  } else {
2162      *    // The object is not biased.
2163      *    cas_lock:
2164      *    if( FastLock(obj) == 0 ) {
2165      *      // Done.
2166      *    } else {
2167      *      slow_path:
2168      *      OptoRuntime::complete_monitor_locking_Java(obj);
2169      *    }
2170      *  }
2171      */
2172 
2173     region  = new RegionNode(5);
2174     // create a Phi for the memory state
2175     mem_phi = new PhiNode( region, Type::MEMORY, TypeRawPtr::BOTTOM);
2176 
2177     Node* fast_lock_region  = new RegionNode(3);
2178     Node* fast_lock_mem_phi = new PhiNode( fast_lock_region, Type::MEMORY, TypeRawPtr::BOTTOM);
2179 
2180     // First, check mark word for the biased lock pattern.
2181     Node* mark_node = make_load(ctrl, mem, obj, oopDesc::mark_offset_in_bytes(), TypeX_X, TypeX_X->basic_type());
2182 
2183     // Get fast path - mark word has the biased lock pattern.
2184     ctrl = opt_bits_test(ctrl, fast_lock_region, 1, mark_node,
2185                          markOopDesc::biased_lock_mask_in_place,
2186                          markOopDesc::biased_lock_pattern, true);
2187     // fast_lock_region->in(1) is set to slow path.
2188     fast_lock_mem_phi->init_req(1, mem);
2189 
2190     // Now check that the lock is biased to the current thread and has
2191     // the same epoch and bias as Klass::_prototype_header.
2192 
2193     // Special-case a fresh allocation to avoid building nodes:
2194     Node* klass_node = AllocateNode::Ideal_klass(obj, &_igvn);
2195     if (klass_node == NULL) {
2196       Node* k_adr = basic_plus_adr(obj, oopDesc::klass_offset_in_bytes());
2197       klass_node = transform_later( LoadKlassNode::make(_igvn, mem, k_adr, _igvn.type(k_adr)->is_ptr()) );
2198 #ifdef _LP64
2199       if (UseCompressedClassPointers && klass_node->is_DecodeNKlass()) {
2200         assert(klass_node->in(1)->Opcode() == Op_LoadNKlass, "sanity");
2201         klass_node->in(1)->init_req(0, ctrl);
2202       } else
2203 #endif
2204       klass_node->init_req(0, ctrl);
2205     }
2206     Node *proto_node = make_load(ctrl, mem, klass_node, in_bytes(Klass::prototype_header_offset()), TypeX_X, TypeX_X->basic_type());
2207 
2208     Node* thread = transform_later(new ThreadLocalNode());
2209     Node* cast_thread = transform_later(new CastP2XNode(ctrl, thread));
2210     Node* o_node = transform_later(new OrXNode(cast_thread, proto_node));
2211     Node* x_node = transform_later(new XorXNode(o_node, mark_node));
2212 
2213     // Get slow path - mark word does NOT match the value.
2214     Node* not_biased_ctrl =  opt_bits_test(ctrl, region, 3, x_node,
2215                                       (~markOopDesc::age_mask_in_place), 0);
2216     // region->in(3) is set to fast path - the object is biased to the current thread.
2217     mem_phi->init_req(3, mem);
2218 
2219 
2220     // Mark word does NOT match the value (thread | Klass::_prototype_header).
2221 
2222 
2223     // First, check biased pattern.
2224     // Get fast path - _prototype_header has the same biased lock pattern.
2225     ctrl =  opt_bits_test(not_biased_ctrl, fast_lock_region, 2, x_node,
2226                           markOopDesc::biased_lock_mask_in_place, 0, true);
2227 
2228     not_biased_ctrl = fast_lock_region->in(2); // Slow path
2229     // fast_lock_region->in(2) - the prototype header is no longer biased
2230     // and we have to revoke the bias on this object.
2231     // We are going to try to reset the mark of this object to the prototype
2232     // value and fall through to the CAS-based locking scheme.
2233     Node* adr = basic_plus_adr(obj, oopDesc::mark_offset_in_bytes());
2234     Node* cas = new StoreXConditionalNode(not_biased_ctrl, mem, adr,
2235                                           proto_node, mark_node);
2236     transform_later(cas);
2237     Node* proj = transform_later(new SCMemProjNode(cas));
2238     fast_lock_mem_phi->init_req(2, proj);
2239 
2240 
2241     // Second, check epoch bits.
2242     Node* rebiased_region  = new RegionNode(3);
2243     Node* old_phi = new PhiNode( rebiased_region, TypeX_X);
2244     Node* new_phi = new PhiNode( rebiased_region, TypeX_X);
2245 
2246     // Get slow path - mark word does NOT match epoch bits.
2247     Node* epoch_ctrl =  opt_bits_test(ctrl, rebiased_region, 1, x_node,
2248                                       markOopDesc::epoch_mask_in_place, 0);
2249     // The epoch of the current bias is not valid, attempt to rebias the object
2250     // toward the current thread.
2251     rebiased_region->init_req(2, epoch_ctrl);
2252     old_phi->init_req(2, mark_node);
2253     new_phi->init_req(2, o_node);
2254 
2255     // rebiased_region->in(1) is set to fast path.
2256     // The epoch of the current bias is still valid but we know
2257     // nothing about the owner; it might be set or it might be clear.
2258     Node* cmask   = MakeConX(markOopDesc::biased_lock_mask_in_place |
2259                              markOopDesc::age_mask_in_place |
2260                              markOopDesc::epoch_mask_in_place);
2261     Node* old = transform_later(new AndXNode(mark_node, cmask));
2262     cast_thread = transform_later(new CastP2XNode(ctrl, thread));
2263     Node* new_mark = transform_later(new OrXNode(cast_thread, old));
2264     old_phi->init_req(1, old);
2265     new_phi->init_req(1, new_mark);
2266 
2267     transform_later(rebiased_region);
2268     transform_later(old_phi);
2269     transform_later(new_phi);
2270 
2271     // Try to acquire the bias of the object using an atomic operation.
2272     // If this fails we will go in to the runtime to revoke the object's bias.
2273     cas = new StoreXConditionalNode(rebiased_region, mem, adr, new_phi, old_phi);

2274     transform_later(cas);
2275     proj = transform_later(new SCMemProjNode(cas));
2276 
2277     // Get slow path - Failed to CAS.
2278     not_biased_ctrl = opt_bits_test(rebiased_region, region, 4, cas, 0, 0);
2279     mem_phi->init_req(4, proj);
2280     // region->in(4) is set to fast path - the object is rebiased to the current thread.
2281 
2282     // Failed to CAS.
2283     slow_path  = new RegionNode(3);
2284     Node *slow_mem = new PhiNode( slow_path, Type::MEMORY, TypeRawPtr::BOTTOM);
2285 
2286     slow_path->init_req(1, not_biased_ctrl); // Capture slow-control
2287     slow_mem->init_req(1, proj);
2288 
2289     // Call CAS-based locking scheme (FastLock node).
2290 
2291     transform_later(fast_lock_region);
2292     transform_later(fast_lock_mem_phi);
2293 
2294     // Get slow path - FastLock failed to lock the object.
2295     ctrl = opt_bits_test(fast_lock_region, region, 2, flock, 0, 0);
2296     mem_phi->init_req(2, fast_lock_mem_phi);
2297     // region->in(2) is set to fast path - the object is locked to the current thread.
2298 
2299     slow_path->init_req(2, ctrl); // Capture slow-control
2300     slow_mem->init_req(2, fast_lock_mem_phi);
2301 
2302     transform_later(slow_path);
2303     transform_later(slow_mem);
2304     // Reset lock's memory edge.
2305     lock->set_req(TypeFunc::Memory, slow_mem);
2306 
2307   } else {
2308     region  = new RegionNode(3);
2309     // create a Phi for the memory state
2310     mem_phi = new PhiNode( region, Type::MEMORY, TypeRawPtr::BOTTOM);
2311 
2312     // Optimize test; set region slot 2
2313     slow_path = opt_bits_test(ctrl, region, 2, flock, 0, 0);
2314     mem_phi->init_req(2, mem);
2315   }
2316 
2317   // Make slow path call
2318   CallNode *call = make_slow_call( (CallNode *) lock, OptoRuntime::complete_monitor_enter_Type(), OptoRuntime::complete_monitor_locking_Java(), NULL, slow_path, obj, box );
2319 
2320   extract_call_projections(call);
2321 
2322   // Slow path can only throw asynchronous exceptions, which are always
2323   // de-opted.  So the compiler thinks the slow-call can never throw an
2324   // exception.  If it DOES throw an exception we would need the debug
2325   // info removed first (since if it throws there is no monitor).
2326   assert ( _ioproj_fallthrough == NULL && _ioproj_catchall == NULL &&
2327            _memproj_catchall == NULL && _catchallcatchproj == NULL, "Unexpected projection from Lock");
2328 
2329   // Capture slow path
2330   // disconnect fall-through projection from call and create a new one
2331   // hook up users of fall-through projection to region
2332   Node *slow_ctrl = _fallthroughproj->clone();
2333   transform_later(slow_ctrl);
2334   _igvn.hash_delete(_fallthroughproj);
2335   _fallthroughproj->disconnect_inputs(NULL, C);
2336   region->init_req(1, slow_ctrl);
2337   // region inputs are now complete
2338   transform_later(region);
2339   _igvn.replace_node(_fallthroughproj, region);
2340 
2341   Node *memproj = transform_later(new ProjNode(call, TypeFunc::Memory));
2342   mem_phi->init_req(1, memproj );
2343   transform_later(mem_phi);
2344   _igvn.replace_node(_memproj_fallthrough, mem_phi);
2345 }
2346 
2347 //------------------------------expand_unlock_node----------------------
2348 void PhaseMacroExpand::expand_unlock_node(UnlockNode *unlock) {
2349 
2350   Node* ctrl = unlock->in(TypeFunc::Control);
2351   Node* mem = unlock->in(TypeFunc::Memory);
2352   Node* obj = unlock->obj_node();
2353   Node* box = unlock->box_node();
2354 
2355   assert(!box->as_BoxLock()->is_eliminated(), "sanity");
2356 
2357   // No need for a null check on unlock
2358 
2359   // Make the merge point
2360   Node *region;
2361   Node *mem_phi;
2362 
2363   if (UseOptoBiasInlining) {
2364     // Check for biased locking unlock case, which is a no-op.
2365     // See the full description in MacroAssembler::biased_locking_exit().
2366     region  = new RegionNode(4);
2367     // create a Phi for the memory state
2368     mem_phi = new PhiNode( region, Type::MEMORY, TypeRawPtr::BOTTOM);
2369     mem_phi->init_req(3, mem);
2370 
2371     Node* mark_node = make_load(ctrl, mem, obj, oopDesc::mark_offset_in_bytes(), TypeX_X, TypeX_X->basic_type());
2372     ctrl = opt_bits_test(ctrl, region, 3, mark_node,
2373                          markOopDesc::biased_lock_mask_in_place,
2374                          markOopDesc::biased_lock_pattern);
2375   } else {
2376     region  = new RegionNode(3);
2377     // create a Phi for the memory state
2378     mem_phi = new PhiNode( region, Type::MEMORY, TypeRawPtr::BOTTOM);
2379   }
2380 
2381   FastUnlockNode *funlock = new FastUnlockNode( ctrl, obj, box );
2382   funlock = transform_later( funlock )->as_FastUnlock();
2383   // Optimize test; set region slot 2
2384   Node *slow_path = opt_bits_test(ctrl, region, 2, funlock, 0, 0);
2385 
2386   CallNode *call = make_slow_call( (CallNode *) unlock, OptoRuntime::complete_monitor_exit_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::complete_monitor_unlocking_C), "complete_monitor_unlocking_C", slow_path, obj, box );
2387 
2388   extract_call_projections(call);
2389 
2390   assert ( _ioproj_fallthrough == NULL && _ioproj_catchall == NULL &&
2391            _memproj_catchall == NULL && _catchallcatchproj == NULL, "Unexpected projection from Lock");
2392 
2393   // No exceptions for unlocking
2394   // Capture slow path
2395   // disconnect fall-through projection from call and create a new one
2396   // hook up users of fall-through projection to region
2397   Node *slow_ctrl = _fallthroughproj->clone();
2398   transform_later(slow_ctrl);
2399   _igvn.hash_delete(_fallthroughproj);
2400   _fallthroughproj->disconnect_inputs(NULL, C);
2401   region->init_req(1, slow_ctrl);
2402   // region inputs are now complete
2403   transform_later(region);
2404   _igvn.replace_node(_fallthroughproj, region);
2405 
2406   Node *memproj = transform_later(new ProjNode(call, TypeFunc::Memory) );
2407   mem_phi->init_req(1, memproj );
2408   mem_phi->init_req(2, mem);
2409   transform_later(mem_phi);
2410   _igvn.replace_node(_memproj_fallthrough, mem_phi);
2411 }
2412 
2413 //---------------------------eliminate_macro_nodes----------------------
2414 // Eliminate scalar replaced allocations and associated locks.
2415 void PhaseMacroExpand::eliminate_macro_nodes() {
2416   if (C->macro_count() == 0)
2417     return;
2418 
2419   // First, attempt to eliminate locks
2420   int cnt = C->macro_count();
2421   for (int i=0; i < cnt; i++) {
2422     Node *n = C->macro_node(i);
2423     if (n->is_AbstractLock()) { // Lock and Unlock nodes
2424       // Before elimination mark all associated (same box and obj)
2425       // lock and unlock nodes.
2426       mark_eliminated_locking_nodes(n->as_AbstractLock());


src/share/vm/opto/macro.cpp
Index Unified diffs Context diffs Sdiffs Patch New Old Previous File Next File