91 }
92 old_in = new_in;
93 }
94 newcall->add_req(old_in);
95 }
96
97 newcall->set_jvms(oldcall->jvms());
98 for (JVMState *jvms = newcall->jvms(); jvms != NULL; jvms = jvms->caller()) {
99 jvms->set_map(newcall);
100 jvms->set_locoff(jvms->locoff()+jvms_adj);
101 jvms->set_stkoff(jvms->stkoff()+jvms_adj);
102 jvms->set_monoff(jvms->monoff()+jvms_adj);
103 jvms->set_scloff(jvms->scloff()+jvms_adj);
104 jvms->set_endoff(jvms->endoff()+jvms_adj);
105 }
106 }
107
108 Node* PhaseMacroExpand::opt_bits_test(Node* ctrl, Node* region, int edge, Node* word, int mask, int bits, bool return_fast_path) {
109 Node* cmp;
110 if (mask != 0) {
111 Node* and_node = transform_later(new (C) AndXNode(word, MakeConX(mask)));
112 cmp = transform_later(new (C) CmpXNode(and_node, MakeConX(bits)));
113 } else {
114 cmp = word;
115 }
116 Node* bol = transform_later(new (C) BoolNode(cmp, BoolTest::ne));
117 IfNode* iff = new (C) IfNode( ctrl, bol, PROB_MIN, COUNT_UNKNOWN );
118 transform_later(iff);
119
120 // Fast path taken.
121 Node *fast_taken = transform_later( new (C) IfFalseNode(iff) );
122
123 // Fast path not-taken, i.e. slow path
124 Node *slow_taken = transform_later( new (C) IfTrueNode(iff) );
125
126 if (return_fast_path) {
127 region->init_req(edge, slow_taken); // Capture slow-control
128 return fast_taken;
129 } else {
130 region->init_req(edge, fast_taken); // Capture fast-control
131 return slow_taken;
132 }
133 }
134
135 //--------------------copy_predefined_input_for_runtime_call--------------------
136 void PhaseMacroExpand::copy_predefined_input_for_runtime_call(Node * ctrl, CallNode* oldcall, CallNode* call) {
137 // Set fixed predefined input arguments
138 call->init_req( TypeFunc::Control, ctrl );
139 call->init_req( TypeFunc::I_O , oldcall->in( TypeFunc::I_O) );
140 call->init_req( TypeFunc::Memory , oldcall->in( TypeFunc::Memory ) ); // ?????
141 call->init_req( TypeFunc::ReturnAdr, oldcall->in( TypeFunc::ReturnAdr ) );
142 call->init_req( TypeFunc::FramePtr, oldcall->in( TypeFunc::FramePtr ) );
143 }
144
145 //------------------------------make_slow_call---------------------------------
146 CallNode* PhaseMacroExpand::make_slow_call(CallNode *oldcall, const TypeFunc* slow_call_type, address slow_call, const char* leaf_name, Node* slow_path, Node* parm0, Node* parm1) {
147
148 // Slow-path call
149 CallNode *call = leaf_name
150 ? (CallNode*)new (C) CallLeafNode ( slow_call_type, slow_call, leaf_name, TypeRawPtr::BOTTOM )
151 : (CallNode*)new (C) CallStaticJavaNode( slow_call_type, slow_call, OptoRuntime::stub_name(slow_call), oldcall->jvms()->bci(), TypeRawPtr::BOTTOM );
152
153 // Slow path call has no side-effects, uses few values
154 copy_predefined_input_for_runtime_call(slow_path, oldcall, call );
155 if (parm0 != NULL) call->init_req(TypeFunc::Parms+0, parm0);
156 if (parm1 != NULL) call->init_req(TypeFunc::Parms+1, parm1);
157 copy_call_debug_info(oldcall, call);
158 call->set_cnt(PROB_UNLIKELY_MAG(4)); // Same effect as RC_UNCOMMON.
159 _igvn.replace_node(oldcall, call);
160 transform_later(call);
161
162 return call;
163 }
164
165 void PhaseMacroExpand::extract_call_projections(CallNode *call) {
166 _fallthroughproj = NULL;
167 _fallthroughcatchproj = NULL;
168 _ioproj_fallthrough = NULL;
169 _ioproj_catchall = NULL;
170 _catchallcatchproj = NULL;
171 _memproj_fallthrough = NULL;
406 if (phi->is_Phi() && phi != mem &&
407 phi->as_Phi()->is_same_inst_field(phi_type, instance_id, alias_idx, offset)) {
408 return phi;
409 }
410 }
411 // Check if an appropriate new value phi already exists.
412 Node* new_phi = value_phis->find(mem->_idx);
413 if (new_phi != NULL)
414 return new_phi;
415
416 if (level <= 0) {
417 return NULL; // Give up: phi tree too deep
418 }
419 Node *start_mem = C->start()->proj_out(TypeFunc::Memory);
420 Node *alloc_mem = alloc->in(TypeFunc::Memory);
421
422 uint length = mem->req();
423 GrowableArray <Node *> values(length, length, NULL, false);
424
425 // create a new Phi for the value
426 PhiNode *phi = new (C) PhiNode(mem->in(0), phi_type, NULL, instance_id, alias_idx, offset);
427 transform_later(phi);
428 value_phis->push(phi, mem->_idx);
429
430 for (uint j = 1; j < length; j++) {
431 Node *in = mem->in(j);
432 if (in == NULL || in->is_top()) {
433 values.at_put(j, in);
434 } else {
435 Node *val = scan_mem_chain(in, alias_idx, offset, start_mem, alloc, &_igvn);
436 if (val == start_mem || val == alloc_mem) {
437 // hit a sentinel, return appropriate 0 value
438 values.at_put(j, _igvn.zerocon(ft));
439 continue;
440 }
441 if (val->is_Initialize()) {
442 val = val->as_Initialize()->find_captured_store(offset, type2aelembytes(ft), &_igvn);
443 }
444 if (val == NULL) {
445 return NULL; // can't find a value on this path
446 }
718 // find the array's elements which will be needed for safepoint debug information
719 nfields = alloc->in(AllocateNode::ALength)->find_int_con(-1);
720 assert(klass->is_array_klass() && nfields >= 0, "must be an array klass.");
721 elem_type = klass->as_array_klass()->element_type();
722 basic_elem_type = elem_type->basic_type();
723 array_base = arrayOopDesc::base_offset_in_bytes(basic_elem_type);
724 element_size = type2aelembytes(basic_elem_type);
725 }
726 }
727 //
728 // Process the safepoint uses
729 //
730 while (safepoints.length() > 0) {
731 SafePointNode* sfpt = safepoints.pop();
732 Node* mem = sfpt->memory();
733 assert(sfpt->jvms() != NULL, "missed JVMS");
734 // Fields of scalar objs are referenced only at the end
735 // of regular debuginfo at the last (youngest) JVMS.
736 // Record relative start index.
737 uint first_ind = (sfpt->req() - sfpt->jvms()->scloff());
738 SafePointScalarObjectNode* sobj = new (C) SafePointScalarObjectNode(res_type,
739 #ifdef ASSERT
740 alloc,
741 #endif
742 first_ind, nfields);
743 sobj->init_req(0, C->root());
744 transform_later(sobj);
745
746 // Scan object's fields adding an input to the safepoint for each field.
747 for (int j = 0; j < nfields; j++) {
748 intptr_t offset;
749 ciField* field = NULL;
750 if (iklass != NULL) {
751 field = iklass->nonstatic_field_at(j);
752 offset = field->offset();
753 elem_type = field->type();
754 basic_elem_type = field->layout_type();
755 } else {
756 offset = array_base + j * (intptr_t)element_size;
757 }
758
826 tty->print(" (alias_idx=%d)", field_idx);
827 } else { // Array's element
828 tty->print("=== At SafePoint node %d can't find value of array element [%d]",
829 sfpt->_idx, j);
830 }
831 tty->print(", which prevents elimination of: ");
832 if (res == NULL)
833 alloc->dump();
834 else
835 res->dump();
836 }
837 #endif
838 return false;
839 }
840 if (UseCompressedOops && field_type->isa_narrowoop()) {
841 // Enable "DecodeN(EncodeP(Allocate)) --> Allocate" transformation
842 // to be able scalar replace the allocation.
843 if (field_val->is_EncodeP()) {
844 field_val = field_val->in(1);
845 } else {
846 field_val = transform_later(new (C) DecodeNNode(field_val, field_val->get_ptr_type()));
847 }
848 }
849 sfpt->add_req(field_val);
850 }
851 JVMState *jvms = sfpt->jvms();
852 jvms->set_endoff(sfpt->req());
853 // Now make a pass over the debug information replacing any references
854 // to the allocated object with "sobj"
855 int start = jvms->debug_start();
856 int end = jvms->debug_end();
857 sfpt->replace_edges_in_range(res, sobj, start, end);
858 safepoints_done.append_if_missing(sfpt); // keep it for rollback
859 }
860 return true;
861 }
862
863 // Process users of eliminated allocation.
864 void PhaseMacroExpand::process_users_of_allocation(CallNode *alloc) {
865 Node* res = alloc->result_cast();
866 if (res != NULL) {
1052 }
1053 log->tail("eliminate_boxing");
1054 }
1055
1056 process_users_of_allocation(boxing);
1057
1058 #ifndef PRODUCT
1059 if (PrintEliminateAllocations) {
1060 tty->print("++++ Eliminated: %d ", boxing->_idx);
1061 boxing->method()->print_short_name(tty);
1062 tty->cr();
1063 }
1064 #endif
1065
1066 return true;
1067 }
1068
1069 //---------------------------set_eden_pointers-------------------------
1070 void PhaseMacroExpand::set_eden_pointers(Node* &eden_top_adr, Node* &eden_end_adr) {
1071 if (UseTLAB) { // Private allocation: load from TLS
1072 Node* thread = transform_later(new (C) ThreadLocalNode());
1073 int tlab_top_offset = in_bytes(JavaThread::tlab_top_offset());
1074 int tlab_end_offset = in_bytes(JavaThread::tlab_end_offset());
1075 eden_top_adr = basic_plus_adr(top()/*not oop*/, thread, tlab_top_offset);
1076 eden_end_adr = basic_plus_adr(top()/*not oop*/, thread, tlab_end_offset);
1077 } else { // Shared allocation: load from globals
1078 CollectedHeap* ch = Universe::heap();
1079 address top_adr = (address)ch->top_addr();
1080 address end_adr = (address)ch->end_addr();
1081 eden_top_adr = makecon(TypeRawPtr::make(top_adr));
1082 eden_end_adr = basic_plus_adr(eden_top_adr, end_adr - top_adr);
1083 }
1084 }
1085
1086
1087 Node* PhaseMacroExpand::make_load(Node* ctl, Node* mem, Node* base, int offset, const Type* value_type, BasicType bt) {
1088 Node* adr = basic_plus_adr(base, offset);
1089 const TypePtr* adr_type = adr->bottom_type()->is_ptr();
1090 Node* value = LoadNode::make(_igvn, ctl, mem, adr, adr_type, value_type, bt, MemNode::unordered);
1091 transform_later(value);
1092 return value;
1188 } else {
1189 initial_slow_test = BoolNode::make_predicate(initial_slow_test, &_igvn);
1190 }
1191
1192 if (C->env()->dtrace_alloc_probes() ||
1193 !UseTLAB && (!Universe::heap()->supports_inline_contig_alloc() ||
1194 (UseConcMarkSweepGC && CMSIncrementalMode))) {
1195 // Force slow-path allocation
1196 always_slow = true;
1197 initial_slow_test = NULL;
1198 }
1199
1200
1201 enum { too_big_or_final_path = 1, need_gc_path = 2 };
1202 Node *slow_region = NULL;
1203 Node *toobig_false = ctrl;
1204
1205 assert (initial_slow_test == NULL || !always_slow, "arguments must be consistent");
1206 // generate the initial test if necessary
1207 if (initial_slow_test != NULL ) {
1208 slow_region = new (C) RegionNode(3);
1209
1210 // Now make the initial failure test. Usually a too-big test but
1211 // might be a TRUE for finalizers or a fancy class check for
1212 // newInstance0.
1213 IfNode *toobig_iff = new (C) IfNode(ctrl, initial_slow_test, PROB_MIN, COUNT_UNKNOWN);
1214 transform_later(toobig_iff);
1215 // Plug the failing-too-big test into the slow-path region
1216 Node *toobig_true = new (C) IfTrueNode( toobig_iff );
1217 transform_later(toobig_true);
1218 slow_region ->init_req( too_big_or_final_path, toobig_true );
1219 toobig_false = new (C) IfFalseNode( toobig_iff );
1220 transform_later(toobig_false);
1221 } else { // No initial test, just fall into next case
1222 toobig_false = ctrl;
1223 debug_only(slow_region = NodeSentinel);
1224 }
1225
1226 Node *slow_mem = mem; // save the current memory state for slow path
1227 // generate the fast allocation code unless we know that the initial test will always go slow
1228 if (!always_slow) {
1229 // Fast path modifies only raw memory.
1230 if (mem->is_MergeMem()) {
1231 mem = mem->as_MergeMem()->memory_at(Compile::AliasIdxRaw);
1232 }
1233
1234 Node* eden_top_adr;
1235 Node* eden_end_adr;
1236
1237 set_eden_pointers(eden_top_adr, eden_end_adr);
1238
1239 // Load Eden::end. Loop invariant and hoisted.
1240 //
1241 // Note: We set the control input on "eden_end" and "old_eden_top" when using
1242 // a TLAB to work around a bug where these values were being moved across
1243 // a safepoint. These are not oops, so they cannot be include in the oop
1244 // map, but they can be changed by a GC. The proper way to fix this would
1245 // be to set the raw memory state when generating a SafepointNode. However
1246 // this will require extensive changes to the loop optimization in order to
1247 // prevent a degradation of the optimization.
1248 // See comment in memnode.hpp, around line 227 in class LoadPNode.
1249 Node *eden_end = make_load(ctrl, mem, eden_end_adr, 0, TypeRawPtr::BOTTOM, T_ADDRESS);
1250
1251 // allocate the Region and Phi nodes for the result
1252 result_region = new (C) RegionNode(3);
1253 result_phi_rawmem = new (C) PhiNode(result_region, Type::MEMORY, TypeRawPtr::BOTTOM);
1254 result_phi_rawoop = new (C) PhiNode(result_region, TypeRawPtr::BOTTOM);
1255 result_phi_i_o = new (C) PhiNode(result_region, Type::ABIO); // I/O is used for Prefetch
1256
1257 // We need a Region for the loop-back contended case.
1258 enum { fall_in_path = 1, contended_loopback_path = 2 };
1259 Node *contended_region;
1260 Node *contended_phi_rawmem;
1261 if (UseTLAB) {
1262 contended_region = toobig_false;
1263 contended_phi_rawmem = mem;
1264 } else {
1265 contended_region = new (C) RegionNode(3);
1266 contended_phi_rawmem = new (C) PhiNode(contended_region, Type::MEMORY, TypeRawPtr::BOTTOM);
1267 // Now handle the passing-too-big test. We fall into the contended
1268 // loop-back merge point.
1269 contended_region ->init_req(fall_in_path, toobig_false);
1270 contended_phi_rawmem->init_req(fall_in_path, mem);
1271 transform_later(contended_region);
1272 transform_later(contended_phi_rawmem);
1273 }
1274
1275 // Load(-locked) the heap top.
1276 // See note above concerning the control input when using a TLAB
1277 Node *old_eden_top = UseTLAB
1278 ? new (C) LoadPNode (ctrl, contended_phi_rawmem, eden_top_adr, TypeRawPtr::BOTTOM, TypeRawPtr::BOTTOM, MemNode::unordered)
1279 : new (C) LoadPLockedNode(contended_region, contended_phi_rawmem, eden_top_adr, MemNode::acquire);
1280
1281 transform_later(old_eden_top);
1282 // Add to heap top to get a new heap top
1283 Node *new_eden_top = new (C) AddPNode(top(), old_eden_top, size_in_bytes);
1284 transform_later(new_eden_top);
1285 // Check for needing a GC; compare against heap end
1286 Node *needgc_cmp = new (C) CmpPNode(new_eden_top, eden_end);
1287 transform_later(needgc_cmp);
1288 Node *needgc_bol = new (C) BoolNode(needgc_cmp, BoolTest::ge);
1289 transform_later(needgc_bol);
1290 IfNode *needgc_iff = new (C) IfNode(contended_region, needgc_bol, PROB_UNLIKELY_MAG(4), COUNT_UNKNOWN);
1291 transform_later(needgc_iff);
1292
1293 // Plug the failing-heap-space-need-gc test into the slow-path region
1294 Node *needgc_true = new (C) IfTrueNode(needgc_iff);
1295 transform_later(needgc_true);
1296 if (initial_slow_test) {
1297 slow_region->init_req(need_gc_path, needgc_true);
1298 // This completes all paths into the slow merge point
1299 transform_later(slow_region);
1300 } else { // No initial slow path needed!
1301 // Just fall from the need-GC path straight into the VM call.
1302 slow_region = needgc_true;
1303 }
1304 // No need for a GC. Setup for the Store-Conditional
1305 Node *needgc_false = new (C) IfFalseNode(needgc_iff);
1306 transform_later(needgc_false);
1307
1308 // Grab regular I/O before optional prefetch may change it.
1309 // Slow-path does no I/O so just set it to the original I/O.
1310 result_phi_i_o->init_req(slow_result_path, i_o);
1311
1312 i_o = prefetch_allocation(i_o, needgc_false, contended_phi_rawmem,
1313 old_eden_top, new_eden_top, length);
1314
1315 // Name successful fast-path variables
1316 Node* fast_oop = old_eden_top;
1317 Node* fast_oop_ctrl;
1318 Node* fast_oop_rawmem;
1319
1320 // Store (-conditional) the modified eden top back down.
1321 // StorePConditional produces flags for a test PLUS a modified raw
1322 // memory state.
1323 if (UseTLAB) {
1324 Node* store_eden_top =
1325 new (C) StorePNode(needgc_false, contended_phi_rawmem, eden_top_adr,
1326 TypeRawPtr::BOTTOM, new_eden_top, MemNode::unordered);
1327 transform_later(store_eden_top);
1328 fast_oop_ctrl = needgc_false; // No contention, so this is the fast path
1329 fast_oop_rawmem = store_eden_top;
1330 } else {
1331 Node* store_eden_top =
1332 new (C) StorePConditionalNode(needgc_false, contended_phi_rawmem, eden_top_adr,
1333 new_eden_top, fast_oop/*old_eden_top*/);
1334 transform_later(store_eden_top);
1335 Node *contention_check = new (C) BoolNode(store_eden_top, BoolTest::ne);
1336 transform_later(contention_check);
1337 store_eden_top = new (C) SCMemProjNode(store_eden_top);
1338 transform_later(store_eden_top);
1339
1340 // If not using TLABs, check to see if there was contention.
1341 IfNode *contention_iff = new (C) IfNode (needgc_false, contention_check, PROB_MIN, COUNT_UNKNOWN);
1342 transform_later(contention_iff);
1343 Node *contention_true = new (C) IfTrueNode(contention_iff);
1344 transform_later(contention_true);
1345 // If contention, loopback and try again.
1346 contended_region->init_req(contended_loopback_path, contention_true);
1347 contended_phi_rawmem->init_req(contended_loopback_path, store_eden_top);
1348
1349 // Fast-path succeeded with no contention!
1350 Node *contention_false = new (C) IfFalseNode(contention_iff);
1351 transform_later(contention_false);
1352 fast_oop_ctrl = contention_false;
1353
1354 // Bump total allocated bytes for this thread
1355 Node* thread = new (C) ThreadLocalNode();
1356 transform_later(thread);
1357 Node* alloc_bytes_adr = basic_plus_adr(top()/*not oop*/, thread,
1358 in_bytes(JavaThread::allocated_bytes_offset()));
1359 Node* alloc_bytes = make_load(fast_oop_ctrl, store_eden_top, alloc_bytes_adr,
1360 0, TypeLong::LONG, T_LONG);
1361 #ifdef _LP64
1362 Node* alloc_size = size_in_bytes;
1363 #else
1364 Node* alloc_size = new (C) ConvI2LNode(size_in_bytes);
1365 transform_later(alloc_size);
1366 #endif
1367 Node* new_alloc_bytes = new (C) AddLNode(alloc_bytes, alloc_size);
1368 transform_later(new_alloc_bytes);
1369 fast_oop_rawmem = make_store(fast_oop_ctrl, store_eden_top, alloc_bytes_adr,
1370 0, new_alloc_bytes, T_LONG);
1371 }
1372
1373 InitializeNode* init = alloc->initialization();
1374 fast_oop_rawmem = initialize_object(alloc,
1375 fast_oop_ctrl, fast_oop_rawmem, fast_oop,
1376 klass_node, length, size_in_bytes);
1377
1378 // If initialization is performed by an array copy, any required
1379 // MemBarStoreStore was already added. If the object does not
1380 // escape no need for a MemBarStoreStore. Otherwise we need a
1381 // MemBarStoreStore so that stores that initialize this object
1382 // can't be reordered with a subsequent store that makes this
1383 // object accessible by other threads.
1384 if (init == NULL || (!init->is_complete_with_arraycopy() && !init->does_not_escape())) {
1385 if (init == NULL || init->req() < InitializeNode::RawStores) {
1386 // No InitializeNode or no stores captured by zeroing
1387 // elimination. Simply add the MemBarStoreStore after object
1388 // initialization.
1389 MemBarNode* mb = MemBarNode::make(C, Op_MemBarStoreStore, Compile::AliasIdxBot);
1390 transform_later(mb);
1391
1392 mb->init_req(TypeFunc::Memory, fast_oop_rawmem);
1393 mb->init_req(TypeFunc::Control, fast_oop_ctrl);
1394 fast_oop_ctrl = new (C) ProjNode(mb,TypeFunc::Control);
1395 transform_later(fast_oop_ctrl);
1396 fast_oop_rawmem = new (C) ProjNode(mb,TypeFunc::Memory);
1397 transform_later(fast_oop_rawmem);
1398 } else {
1399 // Add the MemBarStoreStore after the InitializeNode so that
1400 // all stores performing the initialization that were moved
1401 // before the InitializeNode happen before the storestore
1402 // barrier.
1403
1404 Node* init_ctrl = init->proj_out(TypeFunc::Control);
1405 Node* init_mem = init->proj_out(TypeFunc::Memory);
1406
1407 MemBarNode* mb = MemBarNode::make(C, Op_MemBarStoreStore, Compile::AliasIdxBot);
1408 transform_later(mb);
1409
1410 Node* ctrl = new (C) ProjNode(init,TypeFunc::Control);
1411 transform_later(ctrl);
1412 Node* mem = new (C) ProjNode(init,TypeFunc::Memory);
1413 transform_later(mem);
1414
1415 // The MemBarStoreStore depends on control and memory coming
1416 // from the InitializeNode
1417 mb->init_req(TypeFunc::Memory, mem);
1418 mb->init_req(TypeFunc::Control, ctrl);
1419
1420 ctrl = new (C) ProjNode(mb,TypeFunc::Control);
1421 transform_later(ctrl);
1422 mem = new (C) ProjNode(mb,TypeFunc::Memory);
1423 transform_later(mem);
1424
1425 // All nodes that depended on the InitializeNode for control
1426 // and memory must now depend on the MemBarNode that itself
1427 // depends on the InitializeNode
1428 _igvn.replace_node(init_ctrl, ctrl);
1429 _igvn.replace_node(init_mem, mem);
1430 }
1431 }
1432
1433 if (C->env()->dtrace_extended_probes()) {
1434 // Slow-path call
1435 int size = TypeFunc::Parms + 2;
1436 CallLeafNode *call = new (C) CallLeafNode(OptoRuntime::dtrace_object_alloc_Type(),
1437 CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_object_alloc_base),
1438 "dtrace_object_alloc",
1439 TypeRawPtr::BOTTOM);
1440
1441 // Get base of thread-local storage area
1442 Node* thread = new (C) ThreadLocalNode();
1443 transform_later(thread);
1444
1445 call->init_req(TypeFunc::Parms+0, thread);
1446 call->init_req(TypeFunc::Parms+1, fast_oop);
1447 call->init_req(TypeFunc::Control, fast_oop_ctrl);
1448 call->init_req(TypeFunc::I_O , top()); // does no i/o
1449 call->init_req(TypeFunc::Memory , fast_oop_rawmem);
1450 call->init_req(TypeFunc::ReturnAdr, alloc->in(TypeFunc::ReturnAdr));
1451 call->init_req(TypeFunc::FramePtr, alloc->in(TypeFunc::FramePtr));
1452 transform_later(call);
1453 fast_oop_ctrl = new (C) ProjNode(call,TypeFunc::Control);
1454 transform_later(fast_oop_ctrl);
1455 fast_oop_rawmem = new (C) ProjNode(call,TypeFunc::Memory);
1456 transform_later(fast_oop_rawmem);
1457 }
1458
1459 // Plug in the successful fast-path into the result merge point
1460 result_region ->init_req(fast_result_path, fast_oop_ctrl);
1461 result_phi_rawoop->init_req(fast_result_path, fast_oop);
1462 result_phi_i_o ->init_req(fast_result_path, i_o);
1463 result_phi_rawmem->init_req(fast_result_path, fast_oop_rawmem);
1464 } else {
1465 slow_region = ctrl;
1466 result_phi_i_o = i_o; // Rename it to use in the following code.
1467 }
1468
1469 // Generate slow-path call
1470 CallNode *call = new (C) CallStaticJavaNode(slow_call_type, slow_call_address,
1471 OptoRuntime::stub_name(slow_call_address),
1472 alloc->jvms()->bci(),
1473 TypePtr::BOTTOM);
1474 call->init_req( TypeFunc::Control, slow_region );
1475 call->init_req( TypeFunc::I_O , top() ) ; // does no i/o
1476 call->init_req( TypeFunc::Memory , slow_mem ); // may gc ptrs
1477 call->init_req( TypeFunc::ReturnAdr, alloc->in(TypeFunc::ReturnAdr) );
1478 call->init_req( TypeFunc::FramePtr, alloc->in(TypeFunc::FramePtr) );
1479
1480 call->init_req(TypeFunc::Parms+0, klass_node);
1481 if (length != NULL) {
1482 call->init_req(TypeFunc::Parms+1, length);
1483 }
1484
1485 // Copy debug information and adjust JVMState information, then replace
1486 // allocate node with the call
1487 copy_call_debug_info((CallNode *) alloc, call);
1488 if (!always_slow) {
1489 call->set_cnt(PROB_UNLIKELY_MAG(4)); // Same effect as RC_UNCOMMON.
1490 } else {
1507 //
1508 extract_call_projections(call);
1509
1510 // An allocate node has separate memory projections for the uses on
1511 // the control and i_o paths. Replace the control memory projection with
1512 // result_phi_rawmem (unless we are only generating a slow call when
1513 // both memory projections are combined)
1514 if (!always_slow && _memproj_fallthrough != NULL) {
1515 for (DUIterator_Fast imax, i = _memproj_fallthrough->fast_outs(imax); i < imax; i++) {
1516 Node *use = _memproj_fallthrough->fast_out(i);
1517 _igvn.rehash_node_delayed(use);
1518 imax -= replace_input(use, _memproj_fallthrough, result_phi_rawmem);
1519 // back up iterator
1520 --i;
1521 }
1522 }
1523 // Now change uses of _memproj_catchall to use _memproj_fallthrough and delete
1524 // _memproj_catchall so we end up with a call that has only 1 memory projection.
1525 if (_memproj_catchall != NULL ) {
1526 if (_memproj_fallthrough == NULL) {
1527 _memproj_fallthrough = new (C) ProjNode(call, TypeFunc::Memory);
1528 transform_later(_memproj_fallthrough);
1529 }
1530 for (DUIterator_Fast imax, i = _memproj_catchall->fast_outs(imax); i < imax; i++) {
1531 Node *use = _memproj_catchall->fast_out(i);
1532 _igvn.rehash_node_delayed(use);
1533 imax -= replace_input(use, _memproj_catchall, _memproj_fallthrough);
1534 // back up iterator
1535 --i;
1536 }
1537 assert(_memproj_catchall->outcnt() == 0, "all uses must be deleted");
1538 _igvn.remove_dead_node(_memproj_catchall);
1539 }
1540
1541 // An allocate node has separate i_o projections for the uses on the control
1542 // and i_o paths. Always replace the control i_o projection with result i_o
1543 // otherwise incoming i_o become dead when only a slow call is generated
1544 // (it is different from memory projections where both projections are
1545 // combined in such case).
1546 if (_ioproj_fallthrough != NULL) {
1547 for (DUIterator_Fast imax, i = _ioproj_fallthrough->fast_outs(imax); i < imax; i++) {
1548 Node *use = _ioproj_fallthrough->fast_out(i);
1549 _igvn.rehash_node_delayed(use);
1550 imax -= replace_input(use, _ioproj_fallthrough, result_phi_i_o);
1551 // back up iterator
1552 --i;
1553 }
1554 }
1555 // Now change uses of _ioproj_catchall to use _ioproj_fallthrough and delete
1556 // _ioproj_catchall so we end up with a call that has only 1 i_o projection.
1557 if (_ioproj_catchall != NULL ) {
1558 if (_ioproj_fallthrough == NULL) {
1559 _ioproj_fallthrough = new (C) ProjNode(call, TypeFunc::I_O);
1560 transform_later(_ioproj_fallthrough);
1561 }
1562 for (DUIterator_Fast imax, i = _ioproj_catchall->fast_outs(imax); i < imax; i++) {
1563 Node *use = _ioproj_catchall->fast_out(i);
1564 _igvn.rehash_node_delayed(use);
1565 imax -= replace_input(use, _ioproj_catchall, _ioproj_fallthrough);
1566 // back up iterator
1567 --i;
1568 }
1569 assert(_ioproj_catchall->outcnt() == 0, "all uses must be deleted");
1570 _igvn.remove_dead_node(_ioproj_catchall);
1571 }
1572
1573 // if we generated only a slow call, we are done
1574 if (always_slow) {
1575 // Now we can unhook i_o.
1576 if (result_phi_i_o->outcnt() > 1) {
1577 call->set_req(TypeFunc::I_O, top());
1578 } else {
1579 assert(result_phi_i_o->unique_ctrl_out() == call, "");
1673 // We have no more use for this link, since the AllocateNode goes away:
1674 init->set_req(InitializeNode::RawAddress, top());
1675 // (If we keep the link, it just confuses the register allocator,
1676 // who thinks he sees a real use of the address by the membar.)
1677 }
1678
1679 return rawmem;
1680 }
1681
1682 // Generate prefetch instructions for next allocations.
1683 Node* PhaseMacroExpand::prefetch_allocation(Node* i_o, Node*& needgc_false,
1684 Node*& contended_phi_rawmem,
1685 Node* old_eden_top, Node* new_eden_top,
1686 Node* length) {
1687 enum { fall_in_path = 1, pf_path = 2 };
1688 if( UseTLAB && AllocatePrefetchStyle == 2 ) {
1689 // Generate prefetch allocation with watermark check.
1690 // As an allocation hits the watermark, we will prefetch starting
1691 // at a "distance" away from watermark.
1692
1693 Node *pf_region = new (C) RegionNode(3);
1694 Node *pf_phi_rawmem = new (C) PhiNode( pf_region, Type::MEMORY,
1695 TypeRawPtr::BOTTOM );
1696 // I/O is used for Prefetch
1697 Node *pf_phi_abio = new (C) PhiNode( pf_region, Type::ABIO );
1698
1699 Node *thread = new (C) ThreadLocalNode();
1700 transform_later(thread);
1701
1702 Node *eden_pf_adr = new (C) AddPNode( top()/*not oop*/, thread,
1703 _igvn.MakeConX(in_bytes(JavaThread::tlab_pf_top_offset())) );
1704 transform_later(eden_pf_adr);
1705
1706 Node *old_pf_wm = new (C) LoadPNode(needgc_false,
1707 contended_phi_rawmem, eden_pf_adr,
1708 TypeRawPtr::BOTTOM, TypeRawPtr::BOTTOM,
1709 MemNode::unordered);
1710 transform_later(old_pf_wm);
1711
1712 // check against new_eden_top
1713 Node *need_pf_cmp = new (C) CmpPNode( new_eden_top, old_pf_wm );
1714 transform_later(need_pf_cmp);
1715 Node *need_pf_bol = new (C) BoolNode( need_pf_cmp, BoolTest::ge );
1716 transform_later(need_pf_bol);
1717 IfNode *need_pf_iff = new (C) IfNode( needgc_false, need_pf_bol,
1718 PROB_UNLIKELY_MAG(4), COUNT_UNKNOWN );
1719 transform_later(need_pf_iff);
1720
1721 // true node, add prefetchdistance
1722 Node *need_pf_true = new (C) IfTrueNode( need_pf_iff );
1723 transform_later(need_pf_true);
1724
1725 Node *need_pf_false = new (C) IfFalseNode( need_pf_iff );
1726 transform_later(need_pf_false);
1727
1728 Node *new_pf_wmt = new (C) AddPNode( top(), old_pf_wm,
1729 _igvn.MakeConX(AllocatePrefetchDistance) );
1730 transform_later(new_pf_wmt );
1731 new_pf_wmt->set_req(0, need_pf_true);
1732
1733 Node *store_new_wmt = new (C) StorePNode(need_pf_true,
1734 contended_phi_rawmem, eden_pf_adr,
1735 TypeRawPtr::BOTTOM, new_pf_wmt,
1736 MemNode::unordered);
1737 transform_later(store_new_wmt);
1738
1739 // adding prefetches
1740 pf_phi_abio->init_req( fall_in_path, i_o );
1741
1742 Node *prefetch_adr;
1743 Node *prefetch;
1744 uint lines = AllocatePrefetchDistance / AllocatePrefetchStepSize;
1745 uint step_size = AllocatePrefetchStepSize;
1746 uint distance = 0;
1747
1748 for ( uint i = 0; i < lines; i++ ) {
1749 prefetch_adr = new (C) AddPNode( old_pf_wm, new_pf_wmt,
1750 _igvn.MakeConX(distance) );
1751 transform_later(prefetch_adr);
1752 prefetch = new (C) PrefetchAllocationNode( i_o, prefetch_adr );
1753 transform_later(prefetch);
1754 distance += step_size;
1755 i_o = prefetch;
1756 }
1757 pf_phi_abio->set_req( pf_path, i_o );
1758
1759 pf_region->init_req( fall_in_path, need_pf_false );
1760 pf_region->init_req( pf_path, need_pf_true );
1761
1762 pf_phi_rawmem->init_req( fall_in_path, contended_phi_rawmem );
1763 pf_phi_rawmem->init_req( pf_path, store_new_wmt );
1764
1765 transform_later(pf_region);
1766 transform_later(pf_phi_rawmem);
1767 transform_later(pf_phi_abio);
1768
1769 needgc_false = pf_region;
1770 contended_phi_rawmem = pf_phi_rawmem;
1771 i_o = pf_phi_abio;
1772 } else if( UseTLAB && AllocatePrefetchStyle == 3 ) {
1773 // Insert a prefetch for each allocation.
1774 // This code is used for Sparc with BIS.
1775 Node *pf_region = new (C) RegionNode(3);
1776 Node *pf_phi_rawmem = new (C) PhiNode( pf_region, Type::MEMORY,
1777 TypeRawPtr::BOTTOM );
1778
1779 // Generate several prefetch instructions.
1780 uint lines = (length != NULL) ? AllocatePrefetchLines : AllocateInstancePrefetchLines;
1781 uint step_size = AllocatePrefetchStepSize;
1782 uint distance = AllocatePrefetchDistance;
1783
1784 // Next cache address.
1785 Node *cache_adr = new (C) AddPNode(old_eden_top, old_eden_top,
1786 _igvn.MakeConX(distance));
1787 transform_later(cache_adr);
1788 cache_adr = new (C) CastP2XNode(needgc_false, cache_adr);
1789 transform_later(cache_adr);
1790 Node* mask = _igvn.MakeConX(~(intptr_t)(step_size-1));
1791 cache_adr = new (C) AndXNode(cache_adr, mask);
1792 transform_later(cache_adr);
1793 cache_adr = new (C) CastX2PNode(cache_adr);
1794 transform_later(cache_adr);
1795
1796 // Prefetch
1797 Node *prefetch = new (C) PrefetchAllocationNode( contended_phi_rawmem, cache_adr );
1798 prefetch->set_req(0, needgc_false);
1799 transform_later(prefetch);
1800 contended_phi_rawmem = prefetch;
1801 Node *prefetch_adr;
1802 distance = step_size;
1803 for ( uint i = 1; i < lines; i++ ) {
1804 prefetch_adr = new (C) AddPNode( cache_adr, cache_adr,
1805 _igvn.MakeConX(distance) );
1806 transform_later(prefetch_adr);
1807 prefetch = new (C) PrefetchAllocationNode( contended_phi_rawmem, prefetch_adr );
1808 transform_later(prefetch);
1809 distance += step_size;
1810 contended_phi_rawmem = prefetch;
1811 }
1812 } else if( AllocatePrefetchStyle > 0 ) {
1813 // Insert a prefetch for each allocation only on the fast-path
1814 Node *prefetch_adr;
1815 Node *prefetch;
1816 // Generate several prefetch instructions.
1817 uint lines = (length != NULL) ? AllocatePrefetchLines : AllocateInstancePrefetchLines;
1818 uint step_size = AllocatePrefetchStepSize;
1819 uint distance = AllocatePrefetchDistance;
1820 for ( uint i = 0; i < lines; i++ ) {
1821 prefetch_adr = new (C) AddPNode( old_eden_top, new_eden_top,
1822 _igvn.MakeConX(distance) );
1823 transform_later(prefetch_adr);
1824 prefetch = new (C) PrefetchAllocationNode( i_o, prefetch_adr );
1825 // Do not let it float too high, since if eden_top == eden_end,
1826 // both might be null.
1827 if( i == 0 ) { // Set control for first prefetch, next follows it
1828 prefetch->init_req(0, needgc_false);
1829 }
1830 transform_later(prefetch);
1831 distance += step_size;
1832 i_o = prefetch;
1833 }
1834 }
1835 return i_o;
1836 }
1837
1838
1839 void PhaseMacroExpand::expand_allocate(AllocateNode *alloc) {
1840 expand_allocate_common(alloc, NULL,
1841 OptoRuntime::new_instance_Type(),
1842 OptoRuntime::new_instance_Java());
1843 }
1844
2153 * // Try to rebias.
2154 * if( cas(&mark_word, old, new) == 0 ) {
2155 * // Done.
2156 * } else {
2157 * goto slow_path; // Failed.
2158 * }
2159 * }
2160 * }
2161 * } else {
2162 * // The object is not biased.
2163 * cas_lock:
2164 * if( FastLock(obj) == 0 ) {
2165 * // Done.
2166 * } else {
2167 * slow_path:
2168 * OptoRuntime::complete_monitor_locking_Java(obj);
2169 * }
2170 * }
2171 */
2172
2173 region = new (C) RegionNode(5);
2174 // create a Phi for the memory state
2175 mem_phi = new (C) PhiNode( region, Type::MEMORY, TypeRawPtr::BOTTOM);
2176
2177 Node* fast_lock_region = new (C) RegionNode(3);
2178 Node* fast_lock_mem_phi = new (C) PhiNode( fast_lock_region, Type::MEMORY, TypeRawPtr::BOTTOM);
2179
2180 // First, check mark word for the biased lock pattern.
2181 Node* mark_node = make_load(ctrl, mem, obj, oopDesc::mark_offset_in_bytes(), TypeX_X, TypeX_X->basic_type());
2182
2183 // Get fast path - mark word has the biased lock pattern.
2184 ctrl = opt_bits_test(ctrl, fast_lock_region, 1, mark_node,
2185 markOopDesc::biased_lock_mask_in_place,
2186 markOopDesc::biased_lock_pattern, true);
2187 // fast_lock_region->in(1) is set to slow path.
2188 fast_lock_mem_phi->init_req(1, mem);
2189
2190 // Now check that the lock is biased to the current thread and has
2191 // the same epoch and bias as Klass::_prototype_header.
2192
2193 // Special-case a fresh allocation to avoid building nodes:
2194 Node* klass_node = AllocateNode::Ideal_klass(obj, &_igvn);
2195 if (klass_node == NULL) {
2196 Node* k_adr = basic_plus_adr(obj, oopDesc::klass_offset_in_bytes());
2197 klass_node = transform_later( LoadKlassNode::make(_igvn, mem, k_adr, _igvn.type(k_adr)->is_ptr()) );
2198 #ifdef _LP64
2199 if (UseCompressedClassPointers && klass_node->is_DecodeNKlass()) {
2200 assert(klass_node->in(1)->Opcode() == Op_LoadNKlass, "sanity");
2201 klass_node->in(1)->init_req(0, ctrl);
2202 } else
2203 #endif
2204 klass_node->init_req(0, ctrl);
2205 }
2206 Node *proto_node = make_load(ctrl, mem, klass_node, in_bytes(Klass::prototype_header_offset()), TypeX_X, TypeX_X->basic_type());
2207
2208 Node* thread = transform_later(new (C) ThreadLocalNode());
2209 Node* cast_thread = transform_later(new (C) CastP2XNode(ctrl, thread));
2210 Node* o_node = transform_later(new (C) OrXNode(cast_thread, proto_node));
2211 Node* x_node = transform_later(new (C) XorXNode(o_node, mark_node));
2212
2213 // Get slow path - mark word does NOT match the value.
2214 Node* not_biased_ctrl = opt_bits_test(ctrl, region, 3, x_node,
2215 (~markOopDesc::age_mask_in_place), 0);
2216 // region->in(3) is set to fast path - the object is biased to the current thread.
2217 mem_phi->init_req(3, mem);
2218
2219
2220 // Mark word does NOT match the value (thread | Klass::_prototype_header).
2221
2222
2223 // First, check biased pattern.
2224 // Get fast path - _prototype_header has the same biased lock pattern.
2225 ctrl = opt_bits_test(not_biased_ctrl, fast_lock_region, 2, x_node,
2226 markOopDesc::biased_lock_mask_in_place, 0, true);
2227
2228 not_biased_ctrl = fast_lock_region->in(2); // Slow path
2229 // fast_lock_region->in(2) - the prototype header is no longer biased
2230 // and we have to revoke the bias on this object.
2231 // We are going to try to reset the mark of this object to the prototype
2232 // value and fall through to the CAS-based locking scheme.
2233 Node* adr = basic_plus_adr(obj, oopDesc::mark_offset_in_bytes());
2234 Node* cas = new (C) StoreXConditionalNode(not_biased_ctrl, mem, adr,
2235 proto_node, mark_node);
2236 transform_later(cas);
2237 Node* proj = transform_later( new (C) SCMemProjNode(cas));
2238 fast_lock_mem_phi->init_req(2, proj);
2239
2240
2241 // Second, check epoch bits.
2242 Node* rebiased_region = new (C) RegionNode(3);
2243 Node* old_phi = new (C) PhiNode( rebiased_region, TypeX_X);
2244 Node* new_phi = new (C) PhiNode( rebiased_region, TypeX_X);
2245
2246 // Get slow path - mark word does NOT match epoch bits.
2247 Node* epoch_ctrl = opt_bits_test(ctrl, rebiased_region, 1, x_node,
2248 markOopDesc::epoch_mask_in_place, 0);
2249 // The epoch of the current bias is not valid, attempt to rebias the object
2250 // toward the current thread.
2251 rebiased_region->init_req(2, epoch_ctrl);
2252 old_phi->init_req(2, mark_node);
2253 new_phi->init_req(2, o_node);
2254
2255 // rebiased_region->in(1) is set to fast path.
2256 // The epoch of the current bias is still valid but we know
2257 // nothing about the owner; it might be set or it might be clear.
2258 Node* cmask = MakeConX(markOopDesc::biased_lock_mask_in_place |
2259 markOopDesc::age_mask_in_place |
2260 markOopDesc::epoch_mask_in_place);
2261 Node* old = transform_later(new (C) AndXNode(mark_node, cmask));
2262 cast_thread = transform_later(new (C) CastP2XNode(ctrl, thread));
2263 Node* new_mark = transform_later(new (C) OrXNode(cast_thread, old));
2264 old_phi->init_req(1, old);
2265 new_phi->init_req(1, new_mark);
2266
2267 transform_later(rebiased_region);
2268 transform_later(old_phi);
2269 transform_later(new_phi);
2270
2271 // Try to acquire the bias of the object using an atomic operation.
2272 // If this fails we will go in to the runtime to revoke the object's bias.
2273 cas = new (C) StoreXConditionalNode(rebiased_region, mem, adr,
2274 new_phi, old_phi);
2275 transform_later(cas);
2276 proj = transform_later( new (C) SCMemProjNode(cas));
2277
2278 // Get slow path - Failed to CAS.
2279 not_biased_ctrl = opt_bits_test(rebiased_region, region, 4, cas, 0, 0);
2280 mem_phi->init_req(4, proj);
2281 // region->in(4) is set to fast path - the object is rebiased to the current thread.
2282
2283 // Failed to CAS.
2284 slow_path = new (C) RegionNode(3);
2285 Node *slow_mem = new (C) PhiNode( slow_path, Type::MEMORY, TypeRawPtr::BOTTOM);
2286
2287 slow_path->init_req(1, not_biased_ctrl); // Capture slow-control
2288 slow_mem->init_req(1, proj);
2289
2290 // Call CAS-based locking scheme (FastLock node).
2291
2292 transform_later(fast_lock_region);
2293 transform_later(fast_lock_mem_phi);
2294
2295 // Get slow path - FastLock failed to lock the object.
2296 ctrl = opt_bits_test(fast_lock_region, region, 2, flock, 0, 0);
2297 mem_phi->init_req(2, fast_lock_mem_phi);
2298 // region->in(2) is set to fast path - the object is locked to the current thread.
2299
2300 slow_path->init_req(2, ctrl); // Capture slow-control
2301 slow_mem->init_req(2, fast_lock_mem_phi);
2302
2303 transform_later(slow_path);
2304 transform_later(slow_mem);
2305 // Reset lock's memory edge.
2306 lock->set_req(TypeFunc::Memory, slow_mem);
2307
2308 } else {
2309 region = new (C) RegionNode(3);
2310 // create a Phi for the memory state
2311 mem_phi = new (C) PhiNode( region, Type::MEMORY, TypeRawPtr::BOTTOM);
2312
2313 // Optimize test; set region slot 2
2314 slow_path = opt_bits_test(ctrl, region, 2, flock, 0, 0);
2315 mem_phi->init_req(2, mem);
2316 }
2317
2318 // Make slow path call
2319 CallNode *call = make_slow_call( (CallNode *) lock, OptoRuntime::complete_monitor_enter_Type(), OptoRuntime::complete_monitor_locking_Java(), NULL, slow_path, obj, box );
2320
2321 extract_call_projections(call);
2322
2323 // Slow path can only throw asynchronous exceptions, which are always
2324 // de-opted. So the compiler thinks the slow-call can never throw an
2325 // exception. If it DOES throw an exception we would need the debug
2326 // info removed first (since if it throws there is no monitor).
2327 assert ( _ioproj_fallthrough == NULL && _ioproj_catchall == NULL &&
2328 _memproj_catchall == NULL && _catchallcatchproj == NULL, "Unexpected projection from Lock");
2329
2330 // Capture slow path
2331 // disconnect fall-through projection from call and create a new one
2332 // hook up users of fall-through projection to region
2333 Node *slow_ctrl = _fallthroughproj->clone();
2334 transform_later(slow_ctrl);
2335 _igvn.hash_delete(_fallthroughproj);
2336 _fallthroughproj->disconnect_inputs(NULL, C);
2337 region->init_req(1, slow_ctrl);
2338 // region inputs are now complete
2339 transform_later(region);
2340 _igvn.replace_node(_fallthroughproj, region);
2341
2342 Node *memproj = transform_later( new(C) ProjNode(call, TypeFunc::Memory) );
2343 mem_phi->init_req(1, memproj );
2344 transform_later(mem_phi);
2345 _igvn.replace_node(_memproj_fallthrough, mem_phi);
2346 }
2347
2348 //------------------------------expand_unlock_node----------------------
2349 void PhaseMacroExpand::expand_unlock_node(UnlockNode *unlock) {
2350
2351 Node* ctrl = unlock->in(TypeFunc::Control);
2352 Node* mem = unlock->in(TypeFunc::Memory);
2353 Node* obj = unlock->obj_node();
2354 Node* box = unlock->box_node();
2355
2356 assert(!box->as_BoxLock()->is_eliminated(), "sanity");
2357
2358 // No need for a null check on unlock
2359
2360 // Make the merge point
2361 Node *region;
2362 Node *mem_phi;
2363
2364 if (UseOptoBiasInlining) {
2365 // Check for biased locking unlock case, which is a no-op.
2366 // See the full description in MacroAssembler::biased_locking_exit().
2367 region = new (C) RegionNode(4);
2368 // create a Phi for the memory state
2369 mem_phi = new (C) PhiNode( region, Type::MEMORY, TypeRawPtr::BOTTOM);
2370 mem_phi->init_req(3, mem);
2371
2372 Node* mark_node = make_load(ctrl, mem, obj, oopDesc::mark_offset_in_bytes(), TypeX_X, TypeX_X->basic_type());
2373 ctrl = opt_bits_test(ctrl, region, 3, mark_node,
2374 markOopDesc::biased_lock_mask_in_place,
2375 markOopDesc::biased_lock_pattern);
2376 } else {
2377 region = new (C) RegionNode(3);
2378 // create a Phi for the memory state
2379 mem_phi = new (C) PhiNode( region, Type::MEMORY, TypeRawPtr::BOTTOM);
2380 }
2381
2382 FastUnlockNode *funlock = new (C) FastUnlockNode( ctrl, obj, box );
2383 funlock = transform_later( funlock )->as_FastUnlock();
2384 // Optimize test; set region slot 2
2385 Node *slow_path = opt_bits_test(ctrl, region, 2, funlock, 0, 0);
2386
2387 CallNode *call = make_slow_call( (CallNode *) unlock, OptoRuntime::complete_monitor_exit_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::complete_monitor_unlocking_C), "complete_monitor_unlocking_C", slow_path, obj, box );
2388
2389 extract_call_projections(call);
2390
2391 assert ( _ioproj_fallthrough == NULL && _ioproj_catchall == NULL &&
2392 _memproj_catchall == NULL && _catchallcatchproj == NULL, "Unexpected projection from Lock");
2393
2394 // No exceptions for unlocking
2395 // Capture slow path
2396 // disconnect fall-through projection from call and create a new one
2397 // hook up users of fall-through projection to region
2398 Node *slow_ctrl = _fallthroughproj->clone();
2399 transform_later(slow_ctrl);
2400 _igvn.hash_delete(_fallthroughproj);
2401 _fallthroughproj->disconnect_inputs(NULL, C);
2402 region->init_req(1, slow_ctrl);
2403 // region inputs are now complete
2404 transform_later(region);
2405 _igvn.replace_node(_fallthroughproj, region);
2406
2407 Node *memproj = transform_later( new(C) ProjNode(call, TypeFunc::Memory) );
2408 mem_phi->init_req(1, memproj );
2409 mem_phi->init_req(2, mem);
2410 transform_later(mem_phi);
2411 _igvn.replace_node(_memproj_fallthrough, mem_phi);
2412 }
2413
2414 //---------------------------eliminate_macro_nodes----------------------
2415 // Eliminate scalar replaced allocations and associated locks.
2416 void PhaseMacroExpand::eliminate_macro_nodes() {
2417 if (C->macro_count() == 0)
2418 return;
2419
2420 // First, attempt to eliminate locks
2421 int cnt = C->macro_count();
2422 for (int i=0; i < cnt; i++) {
2423 Node *n = C->macro_node(i);
2424 if (n->is_AbstractLock()) { // Lock and Unlock nodes
2425 // Before elimination mark all associated (same box and obj)
2426 // lock and unlock nodes.
2427 mark_eliminated_locking_nodes(n->as_AbstractLock());
|
91 }
92 old_in = new_in;
93 }
94 newcall->add_req(old_in);
95 }
96
97 newcall->set_jvms(oldcall->jvms());
98 for (JVMState *jvms = newcall->jvms(); jvms != NULL; jvms = jvms->caller()) {
99 jvms->set_map(newcall);
100 jvms->set_locoff(jvms->locoff()+jvms_adj);
101 jvms->set_stkoff(jvms->stkoff()+jvms_adj);
102 jvms->set_monoff(jvms->monoff()+jvms_adj);
103 jvms->set_scloff(jvms->scloff()+jvms_adj);
104 jvms->set_endoff(jvms->endoff()+jvms_adj);
105 }
106 }
107
108 Node* PhaseMacroExpand::opt_bits_test(Node* ctrl, Node* region, int edge, Node* word, int mask, int bits, bool return_fast_path) {
109 Node* cmp;
110 if (mask != 0) {
111 Node* and_node = transform_later(new AndXNode(word, MakeConX(mask)));
112 cmp = transform_later(new CmpXNode(and_node, MakeConX(bits)));
113 } else {
114 cmp = word;
115 }
116 Node* bol = transform_later(new BoolNode(cmp, BoolTest::ne));
117 IfNode* iff = new IfNode( ctrl, bol, PROB_MIN, COUNT_UNKNOWN );
118 transform_later(iff);
119
120 // Fast path taken.
121 Node *fast_taken = transform_later(new IfFalseNode(iff));
122
123 // Fast path not-taken, i.e. slow path
124 Node *slow_taken = transform_later(new IfTrueNode(iff));
125
126 if (return_fast_path) {
127 region->init_req(edge, slow_taken); // Capture slow-control
128 return fast_taken;
129 } else {
130 region->init_req(edge, fast_taken); // Capture fast-control
131 return slow_taken;
132 }
133 }
134
135 //--------------------copy_predefined_input_for_runtime_call--------------------
136 void PhaseMacroExpand::copy_predefined_input_for_runtime_call(Node * ctrl, CallNode* oldcall, CallNode* call) {
137 // Set fixed predefined input arguments
138 call->init_req( TypeFunc::Control, ctrl );
139 call->init_req( TypeFunc::I_O , oldcall->in( TypeFunc::I_O) );
140 call->init_req( TypeFunc::Memory , oldcall->in( TypeFunc::Memory ) ); // ?????
141 call->init_req( TypeFunc::ReturnAdr, oldcall->in( TypeFunc::ReturnAdr ) );
142 call->init_req( TypeFunc::FramePtr, oldcall->in( TypeFunc::FramePtr ) );
143 }
144
145 //------------------------------make_slow_call---------------------------------
146 CallNode* PhaseMacroExpand::make_slow_call(CallNode *oldcall, const TypeFunc* slow_call_type, address slow_call, const char* leaf_name, Node* slow_path, Node* parm0, Node* parm1) {
147
148 // Slow-path call
149 CallNode *call = leaf_name
150 ? (CallNode*)new CallLeafNode ( slow_call_type, slow_call, leaf_name, TypeRawPtr::BOTTOM )
151 : (CallNode*)new CallStaticJavaNode( slow_call_type, slow_call, OptoRuntime::stub_name(slow_call), oldcall->jvms()->bci(), TypeRawPtr::BOTTOM );
152
153 // Slow path call has no side-effects, uses few values
154 copy_predefined_input_for_runtime_call(slow_path, oldcall, call );
155 if (parm0 != NULL) call->init_req(TypeFunc::Parms+0, parm0);
156 if (parm1 != NULL) call->init_req(TypeFunc::Parms+1, parm1);
157 copy_call_debug_info(oldcall, call);
158 call->set_cnt(PROB_UNLIKELY_MAG(4)); // Same effect as RC_UNCOMMON.
159 _igvn.replace_node(oldcall, call);
160 transform_later(call);
161
162 return call;
163 }
164
165 void PhaseMacroExpand::extract_call_projections(CallNode *call) {
166 _fallthroughproj = NULL;
167 _fallthroughcatchproj = NULL;
168 _ioproj_fallthrough = NULL;
169 _ioproj_catchall = NULL;
170 _catchallcatchproj = NULL;
171 _memproj_fallthrough = NULL;
406 if (phi->is_Phi() && phi != mem &&
407 phi->as_Phi()->is_same_inst_field(phi_type, instance_id, alias_idx, offset)) {
408 return phi;
409 }
410 }
411 // Check if an appropriate new value phi already exists.
412 Node* new_phi = value_phis->find(mem->_idx);
413 if (new_phi != NULL)
414 return new_phi;
415
416 if (level <= 0) {
417 return NULL; // Give up: phi tree too deep
418 }
419 Node *start_mem = C->start()->proj_out(TypeFunc::Memory);
420 Node *alloc_mem = alloc->in(TypeFunc::Memory);
421
422 uint length = mem->req();
423 GrowableArray <Node *> values(length, length, NULL, false);
424
425 // create a new Phi for the value
426 PhiNode *phi = new PhiNode(mem->in(0), phi_type, NULL, instance_id, alias_idx, offset);
427 transform_later(phi);
428 value_phis->push(phi, mem->_idx);
429
430 for (uint j = 1; j < length; j++) {
431 Node *in = mem->in(j);
432 if (in == NULL || in->is_top()) {
433 values.at_put(j, in);
434 } else {
435 Node *val = scan_mem_chain(in, alias_idx, offset, start_mem, alloc, &_igvn);
436 if (val == start_mem || val == alloc_mem) {
437 // hit a sentinel, return appropriate 0 value
438 values.at_put(j, _igvn.zerocon(ft));
439 continue;
440 }
441 if (val->is_Initialize()) {
442 val = val->as_Initialize()->find_captured_store(offset, type2aelembytes(ft), &_igvn);
443 }
444 if (val == NULL) {
445 return NULL; // can't find a value on this path
446 }
718 // find the array's elements which will be needed for safepoint debug information
719 nfields = alloc->in(AllocateNode::ALength)->find_int_con(-1);
720 assert(klass->is_array_klass() && nfields >= 0, "must be an array klass.");
721 elem_type = klass->as_array_klass()->element_type();
722 basic_elem_type = elem_type->basic_type();
723 array_base = arrayOopDesc::base_offset_in_bytes(basic_elem_type);
724 element_size = type2aelembytes(basic_elem_type);
725 }
726 }
727 //
728 // Process the safepoint uses
729 //
730 while (safepoints.length() > 0) {
731 SafePointNode* sfpt = safepoints.pop();
732 Node* mem = sfpt->memory();
733 assert(sfpt->jvms() != NULL, "missed JVMS");
734 // Fields of scalar objs are referenced only at the end
735 // of regular debuginfo at the last (youngest) JVMS.
736 // Record relative start index.
737 uint first_ind = (sfpt->req() - sfpt->jvms()->scloff());
738 SafePointScalarObjectNode* sobj = new SafePointScalarObjectNode(res_type,
739 #ifdef ASSERT
740 alloc,
741 #endif
742 first_ind, nfields);
743 sobj->init_req(0, C->root());
744 transform_later(sobj);
745
746 // Scan object's fields adding an input to the safepoint for each field.
747 for (int j = 0; j < nfields; j++) {
748 intptr_t offset;
749 ciField* field = NULL;
750 if (iklass != NULL) {
751 field = iklass->nonstatic_field_at(j);
752 offset = field->offset();
753 elem_type = field->type();
754 basic_elem_type = field->layout_type();
755 } else {
756 offset = array_base + j * (intptr_t)element_size;
757 }
758
826 tty->print(" (alias_idx=%d)", field_idx);
827 } else { // Array's element
828 tty->print("=== At SafePoint node %d can't find value of array element [%d]",
829 sfpt->_idx, j);
830 }
831 tty->print(", which prevents elimination of: ");
832 if (res == NULL)
833 alloc->dump();
834 else
835 res->dump();
836 }
837 #endif
838 return false;
839 }
840 if (UseCompressedOops && field_type->isa_narrowoop()) {
841 // Enable "DecodeN(EncodeP(Allocate)) --> Allocate" transformation
842 // to be able scalar replace the allocation.
843 if (field_val->is_EncodeP()) {
844 field_val = field_val->in(1);
845 } else {
846 field_val = transform_later(new DecodeNNode(field_val, field_val->get_ptr_type()));
847 }
848 }
849 sfpt->add_req(field_val);
850 }
851 JVMState *jvms = sfpt->jvms();
852 jvms->set_endoff(sfpt->req());
853 // Now make a pass over the debug information replacing any references
854 // to the allocated object with "sobj"
855 int start = jvms->debug_start();
856 int end = jvms->debug_end();
857 sfpt->replace_edges_in_range(res, sobj, start, end);
858 safepoints_done.append_if_missing(sfpt); // keep it for rollback
859 }
860 return true;
861 }
862
863 // Process users of eliminated allocation.
864 void PhaseMacroExpand::process_users_of_allocation(CallNode *alloc) {
865 Node* res = alloc->result_cast();
866 if (res != NULL) {
1052 }
1053 log->tail("eliminate_boxing");
1054 }
1055
1056 process_users_of_allocation(boxing);
1057
1058 #ifndef PRODUCT
1059 if (PrintEliminateAllocations) {
1060 tty->print("++++ Eliminated: %d ", boxing->_idx);
1061 boxing->method()->print_short_name(tty);
1062 tty->cr();
1063 }
1064 #endif
1065
1066 return true;
1067 }
1068
1069 //---------------------------set_eden_pointers-------------------------
1070 void PhaseMacroExpand::set_eden_pointers(Node* &eden_top_adr, Node* &eden_end_adr) {
1071 if (UseTLAB) { // Private allocation: load from TLS
1072 Node* thread = transform_later(new ThreadLocalNode());
1073 int tlab_top_offset = in_bytes(JavaThread::tlab_top_offset());
1074 int tlab_end_offset = in_bytes(JavaThread::tlab_end_offset());
1075 eden_top_adr = basic_plus_adr(top()/*not oop*/, thread, tlab_top_offset);
1076 eden_end_adr = basic_plus_adr(top()/*not oop*/, thread, tlab_end_offset);
1077 } else { // Shared allocation: load from globals
1078 CollectedHeap* ch = Universe::heap();
1079 address top_adr = (address)ch->top_addr();
1080 address end_adr = (address)ch->end_addr();
1081 eden_top_adr = makecon(TypeRawPtr::make(top_adr));
1082 eden_end_adr = basic_plus_adr(eden_top_adr, end_adr - top_adr);
1083 }
1084 }
1085
1086
1087 Node* PhaseMacroExpand::make_load(Node* ctl, Node* mem, Node* base, int offset, const Type* value_type, BasicType bt) {
1088 Node* adr = basic_plus_adr(base, offset);
1089 const TypePtr* adr_type = adr->bottom_type()->is_ptr();
1090 Node* value = LoadNode::make(_igvn, ctl, mem, adr, adr_type, value_type, bt, MemNode::unordered);
1091 transform_later(value);
1092 return value;
1188 } else {
1189 initial_slow_test = BoolNode::make_predicate(initial_slow_test, &_igvn);
1190 }
1191
1192 if (C->env()->dtrace_alloc_probes() ||
1193 !UseTLAB && (!Universe::heap()->supports_inline_contig_alloc() ||
1194 (UseConcMarkSweepGC && CMSIncrementalMode))) {
1195 // Force slow-path allocation
1196 always_slow = true;
1197 initial_slow_test = NULL;
1198 }
1199
1200
1201 enum { too_big_or_final_path = 1, need_gc_path = 2 };
1202 Node *slow_region = NULL;
1203 Node *toobig_false = ctrl;
1204
1205 assert (initial_slow_test == NULL || !always_slow, "arguments must be consistent");
1206 // generate the initial test if necessary
1207 if (initial_slow_test != NULL ) {
1208 slow_region = new RegionNode(3);
1209
1210 // Now make the initial failure test. Usually a too-big test but
1211 // might be a TRUE for finalizers or a fancy class check for
1212 // newInstance0.
1213 IfNode *toobig_iff = new IfNode(ctrl, initial_slow_test, PROB_MIN, COUNT_UNKNOWN);
1214 transform_later(toobig_iff);
1215 // Plug the failing-too-big test into the slow-path region
1216 Node *toobig_true = new IfTrueNode( toobig_iff );
1217 transform_later(toobig_true);
1218 slow_region ->init_req( too_big_or_final_path, toobig_true );
1219 toobig_false = new IfFalseNode( toobig_iff );
1220 transform_later(toobig_false);
1221 } else { // No initial test, just fall into next case
1222 toobig_false = ctrl;
1223 debug_only(slow_region = NodeSentinel);
1224 }
1225
1226 Node *slow_mem = mem; // save the current memory state for slow path
1227 // generate the fast allocation code unless we know that the initial test will always go slow
1228 if (!always_slow) {
1229 // Fast path modifies only raw memory.
1230 if (mem->is_MergeMem()) {
1231 mem = mem->as_MergeMem()->memory_at(Compile::AliasIdxRaw);
1232 }
1233
1234 Node* eden_top_adr;
1235 Node* eden_end_adr;
1236
1237 set_eden_pointers(eden_top_adr, eden_end_adr);
1238
1239 // Load Eden::end. Loop invariant and hoisted.
1240 //
1241 // Note: We set the control input on "eden_end" and "old_eden_top" when using
1242 // a TLAB to work around a bug where these values were being moved across
1243 // a safepoint. These are not oops, so they cannot be include in the oop
1244 // map, but they can be changed by a GC. The proper way to fix this would
1245 // be to set the raw memory state when generating a SafepointNode. However
1246 // this will require extensive changes to the loop optimization in order to
1247 // prevent a degradation of the optimization.
1248 // See comment in memnode.hpp, around line 227 in class LoadPNode.
1249 Node *eden_end = make_load(ctrl, mem, eden_end_adr, 0, TypeRawPtr::BOTTOM, T_ADDRESS);
1250
1251 // allocate the Region and Phi nodes for the result
1252 result_region = new RegionNode(3);
1253 result_phi_rawmem = new PhiNode(result_region, Type::MEMORY, TypeRawPtr::BOTTOM);
1254 result_phi_rawoop = new PhiNode(result_region, TypeRawPtr::BOTTOM);
1255 result_phi_i_o = new PhiNode(result_region, Type::ABIO); // I/O is used for Prefetch
1256
1257 // We need a Region for the loop-back contended case.
1258 enum { fall_in_path = 1, contended_loopback_path = 2 };
1259 Node *contended_region;
1260 Node *contended_phi_rawmem;
1261 if (UseTLAB) {
1262 contended_region = toobig_false;
1263 contended_phi_rawmem = mem;
1264 } else {
1265 contended_region = new RegionNode(3);
1266 contended_phi_rawmem = new PhiNode(contended_region, Type::MEMORY, TypeRawPtr::BOTTOM);
1267 // Now handle the passing-too-big test. We fall into the contended
1268 // loop-back merge point.
1269 contended_region ->init_req(fall_in_path, toobig_false);
1270 contended_phi_rawmem->init_req(fall_in_path, mem);
1271 transform_later(contended_region);
1272 transform_later(contended_phi_rawmem);
1273 }
1274
1275 // Load(-locked) the heap top.
1276 // See note above concerning the control input when using a TLAB
1277 Node *old_eden_top = UseTLAB
1278 ? new LoadPNode (ctrl, contended_phi_rawmem, eden_top_adr, TypeRawPtr::BOTTOM, TypeRawPtr::BOTTOM, MemNode::unordered)
1279 : new LoadPLockedNode(contended_region, contended_phi_rawmem, eden_top_adr, MemNode::acquire);
1280
1281 transform_later(old_eden_top);
1282 // Add to heap top to get a new heap top
1283 Node *new_eden_top = new AddPNode(top(), old_eden_top, size_in_bytes);
1284 transform_later(new_eden_top);
1285 // Check for needing a GC; compare against heap end
1286 Node *needgc_cmp = new CmpPNode(new_eden_top, eden_end);
1287 transform_later(needgc_cmp);
1288 Node *needgc_bol = new BoolNode(needgc_cmp, BoolTest::ge);
1289 transform_later(needgc_bol);
1290 IfNode *needgc_iff = new IfNode(contended_region, needgc_bol, PROB_UNLIKELY_MAG(4), COUNT_UNKNOWN);
1291 transform_later(needgc_iff);
1292
1293 // Plug the failing-heap-space-need-gc test into the slow-path region
1294 Node *needgc_true = new IfTrueNode(needgc_iff);
1295 transform_later(needgc_true);
1296 if (initial_slow_test) {
1297 slow_region->init_req(need_gc_path, needgc_true);
1298 // This completes all paths into the slow merge point
1299 transform_later(slow_region);
1300 } else { // No initial slow path needed!
1301 // Just fall from the need-GC path straight into the VM call.
1302 slow_region = needgc_true;
1303 }
1304 // No need for a GC. Setup for the Store-Conditional
1305 Node *needgc_false = new IfFalseNode(needgc_iff);
1306 transform_later(needgc_false);
1307
1308 // Grab regular I/O before optional prefetch may change it.
1309 // Slow-path does no I/O so just set it to the original I/O.
1310 result_phi_i_o->init_req(slow_result_path, i_o);
1311
1312 i_o = prefetch_allocation(i_o, needgc_false, contended_phi_rawmem,
1313 old_eden_top, new_eden_top, length);
1314
1315 // Name successful fast-path variables
1316 Node* fast_oop = old_eden_top;
1317 Node* fast_oop_ctrl;
1318 Node* fast_oop_rawmem;
1319
1320 // Store (-conditional) the modified eden top back down.
1321 // StorePConditional produces flags for a test PLUS a modified raw
1322 // memory state.
1323 if (UseTLAB) {
1324 Node* store_eden_top =
1325 new StorePNode(needgc_false, contended_phi_rawmem, eden_top_adr,
1326 TypeRawPtr::BOTTOM, new_eden_top, MemNode::unordered);
1327 transform_later(store_eden_top);
1328 fast_oop_ctrl = needgc_false; // No contention, so this is the fast path
1329 fast_oop_rawmem = store_eden_top;
1330 } else {
1331 Node* store_eden_top =
1332 new StorePConditionalNode(needgc_false, contended_phi_rawmem, eden_top_adr,
1333 new_eden_top, fast_oop/*old_eden_top*/);
1334 transform_later(store_eden_top);
1335 Node *contention_check = new BoolNode(store_eden_top, BoolTest::ne);
1336 transform_later(contention_check);
1337 store_eden_top = new SCMemProjNode(store_eden_top);
1338 transform_later(store_eden_top);
1339
1340 // If not using TLABs, check to see if there was contention.
1341 IfNode *contention_iff = new IfNode (needgc_false, contention_check, PROB_MIN, COUNT_UNKNOWN);
1342 transform_later(contention_iff);
1343 Node *contention_true = new IfTrueNode(contention_iff);
1344 transform_later(contention_true);
1345 // If contention, loopback and try again.
1346 contended_region->init_req(contended_loopback_path, contention_true);
1347 contended_phi_rawmem->init_req(contended_loopback_path, store_eden_top);
1348
1349 // Fast-path succeeded with no contention!
1350 Node *contention_false = new IfFalseNode(contention_iff);
1351 transform_later(contention_false);
1352 fast_oop_ctrl = contention_false;
1353
1354 // Bump total allocated bytes for this thread
1355 Node* thread = new ThreadLocalNode();
1356 transform_later(thread);
1357 Node* alloc_bytes_adr = basic_plus_adr(top()/*not oop*/, thread,
1358 in_bytes(JavaThread::allocated_bytes_offset()));
1359 Node* alloc_bytes = make_load(fast_oop_ctrl, store_eden_top, alloc_bytes_adr,
1360 0, TypeLong::LONG, T_LONG);
1361 #ifdef _LP64
1362 Node* alloc_size = size_in_bytes;
1363 #else
1364 Node* alloc_size = new ConvI2LNode(size_in_bytes);
1365 transform_later(alloc_size);
1366 #endif
1367 Node* new_alloc_bytes = new AddLNode(alloc_bytes, alloc_size);
1368 transform_later(new_alloc_bytes);
1369 fast_oop_rawmem = make_store(fast_oop_ctrl, store_eden_top, alloc_bytes_adr,
1370 0, new_alloc_bytes, T_LONG);
1371 }
1372
1373 InitializeNode* init = alloc->initialization();
1374 fast_oop_rawmem = initialize_object(alloc,
1375 fast_oop_ctrl, fast_oop_rawmem, fast_oop,
1376 klass_node, length, size_in_bytes);
1377
1378 // If initialization is performed by an array copy, any required
1379 // MemBarStoreStore was already added. If the object does not
1380 // escape no need for a MemBarStoreStore. Otherwise we need a
1381 // MemBarStoreStore so that stores that initialize this object
1382 // can't be reordered with a subsequent store that makes this
1383 // object accessible by other threads.
1384 if (init == NULL || (!init->is_complete_with_arraycopy() && !init->does_not_escape())) {
1385 if (init == NULL || init->req() < InitializeNode::RawStores) {
1386 // No InitializeNode or no stores captured by zeroing
1387 // elimination. Simply add the MemBarStoreStore after object
1388 // initialization.
1389 MemBarNode* mb = MemBarNode::make(C, Op_MemBarStoreStore, Compile::AliasIdxBot);
1390 transform_later(mb);
1391
1392 mb->init_req(TypeFunc::Memory, fast_oop_rawmem);
1393 mb->init_req(TypeFunc::Control, fast_oop_ctrl);
1394 fast_oop_ctrl = new ProjNode(mb,TypeFunc::Control);
1395 transform_later(fast_oop_ctrl);
1396 fast_oop_rawmem = new ProjNode(mb,TypeFunc::Memory);
1397 transform_later(fast_oop_rawmem);
1398 } else {
1399 // Add the MemBarStoreStore after the InitializeNode so that
1400 // all stores performing the initialization that were moved
1401 // before the InitializeNode happen before the storestore
1402 // barrier.
1403
1404 Node* init_ctrl = init->proj_out(TypeFunc::Control);
1405 Node* init_mem = init->proj_out(TypeFunc::Memory);
1406
1407 MemBarNode* mb = MemBarNode::make(C, Op_MemBarStoreStore, Compile::AliasIdxBot);
1408 transform_later(mb);
1409
1410 Node* ctrl = new ProjNode(init,TypeFunc::Control);
1411 transform_later(ctrl);
1412 Node* mem = new ProjNode(init,TypeFunc::Memory);
1413 transform_later(mem);
1414
1415 // The MemBarStoreStore depends on control and memory coming
1416 // from the InitializeNode
1417 mb->init_req(TypeFunc::Memory, mem);
1418 mb->init_req(TypeFunc::Control, ctrl);
1419
1420 ctrl = new ProjNode(mb,TypeFunc::Control);
1421 transform_later(ctrl);
1422 mem = new ProjNode(mb,TypeFunc::Memory);
1423 transform_later(mem);
1424
1425 // All nodes that depended on the InitializeNode for control
1426 // and memory must now depend on the MemBarNode that itself
1427 // depends on the InitializeNode
1428 _igvn.replace_node(init_ctrl, ctrl);
1429 _igvn.replace_node(init_mem, mem);
1430 }
1431 }
1432
1433 if (C->env()->dtrace_extended_probes()) {
1434 // Slow-path call
1435 int size = TypeFunc::Parms + 2;
1436 CallLeafNode *call = new CallLeafNode(OptoRuntime::dtrace_object_alloc_Type(),
1437 CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_object_alloc_base),
1438 "dtrace_object_alloc",
1439 TypeRawPtr::BOTTOM);
1440
1441 // Get base of thread-local storage area
1442 Node* thread = new ThreadLocalNode();
1443 transform_later(thread);
1444
1445 call->init_req(TypeFunc::Parms+0, thread);
1446 call->init_req(TypeFunc::Parms+1, fast_oop);
1447 call->init_req(TypeFunc::Control, fast_oop_ctrl);
1448 call->init_req(TypeFunc::I_O , top()); // does no i/o
1449 call->init_req(TypeFunc::Memory , fast_oop_rawmem);
1450 call->init_req(TypeFunc::ReturnAdr, alloc->in(TypeFunc::ReturnAdr));
1451 call->init_req(TypeFunc::FramePtr, alloc->in(TypeFunc::FramePtr));
1452 transform_later(call);
1453 fast_oop_ctrl = new ProjNode(call,TypeFunc::Control);
1454 transform_later(fast_oop_ctrl);
1455 fast_oop_rawmem = new ProjNode(call,TypeFunc::Memory);
1456 transform_later(fast_oop_rawmem);
1457 }
1458
1459 // Plug in the successful fast-path into the result merge point
1460 result_region ->init_req(fast_result_path, fast_oop_ctrl);
1461 result_phi_rawoop->init_req(fast_result_path, fast_oop);
1462 result_phi_i_o ->init_req(fast_result_path, i_o);
1463 result_phi_rawmem->init_req(fast_result_path, fast_oop_rawmem);
1464 } else {
1465 slow_region = ctrl;
1466 result_phi_i_o = i_o; // Rename it to use in the following code.
1467 }
1468
1469 // Generate slow-path call
1470 CallNode *call = new CallStaticJavaNode(slow_call_type, slow_call_address,
1471 OptoRuntime::stub_name(slow_call_address),
1472 alloc->jvms()->bci(),
1473 TypePtr::BOTTOM);
1474 call->init_req( TypeFunc::Control, slow_region );
1475 call->init_req( TypeFunc::I_O , top() ) ; // does no i/o
1476 call->init_req( TypeFunc::Memory , slow_mem ); // may gc ptrs
1477 call->init_req( TypeFunc::ReturnAdr, alloc->in(TypeFunc::ReturnAdr) );
1478 call->init_req( TypeFunc::FramePtr, alloc->in(TypeFunc::FramePtr) );
1479
1480 call->init_req(TypeFunc::Parms+0, klass_node);
1481 if (length != NULL) {
1482 call->init_req(TypeFunc::Parms+1, length);
1483 }
1484
1485 // Copy debug information and adjust JVMState information, then replace
1486 // allocate node with the call
1487 copy_call_debug_info((CallNode *) alloc, call);
1488 if (!always_slow) {
1489 call->set_cnt(PROB_UNLIKELY_MAG(4)); // Same effect as RC_UNCOMMON.
1490 } else {
1507 //
1508 extract_call_projections(call);
1509
1510 // An allocate node has separate memory projections for the uses on
1511 // the control and i_o paths. Replace the control memory projection with
1512 // result_phi_rawmem (unless we are only generating a slow call when
1513 // both memory projections are combined)
1514 if (!always_slow && _memproj_fallthrough != NULL) {
1515 for (DUIterator_Fast imax, i = _memproj_fallthrough->fast_outs(imax); i < imax; i++) {
1516 Node *use = _memproj_fallthrough->fast_out(i);
1517 _igvn.rehash_node_delayed(use);
1518 imax -= replace_input(use, _memproj_fallthrough, result_phi_rawmem);
1519 // back up iterator
1520 --i;
1521 }
1522 }
1523 // Now change uses of _memproj_catchall to use _memproj_fallthrough and delete
1524 // _memproj_catchall so we end up with a call that has only 1 memory projection.
1525 if (_memproj_catchall != NULL ) {
1526 if (_memproj_fallthrough == NULL) {
1527 _memproj_fallthrough = new ProjNode(call, TypeFunc::Memory);
1528 transform_later(_memproj_fallthrough);
1529 }
1530 for (DUIterator_Fast imax, i = _memproj_catchall->fast_outs(imax); i < imax; i++) {
1531 Node *use = _memproj_catchall->fast_out(i);
1532 _igvn.rehash_node_delayed(use);
1533 imax -= replace_input(use, _memproj_catchall, _memproj_fallthrough);
1534 // back up iterator
1535 --i;
1536 }
1537 assert(_memproj_catchall->outcnt() == 0, "all uses must be deleted");
1538 _igvn.remove_dead_node(_memproj_catchall);
1539 }
1540
1541 // An allocate node has separate i_o projections for the uses on the control
1542 // and i_o paths. Always replace the control i_o projection with result i_o
1543 // otherwise incoming i_o become dead when only a slow call is generated
1544 // (it is different from memory projections where both projections are
1545 // combined in such case).
1546 if (_ioproj_fallthrough != NULL) {
1547 for (DUIterator_Fast imax, i = _ioproj_fallthrough->fast_outs(imax); i < imax; i++) {
1548 Node *use = _ioproj_fallthrough->fast_out(i);
1549 _igvn.rehash_node_delayed(use);
1550 imax -= replace_input(use, _ioproj_fallthrough, result_phi_i_o);
1551 // back up iterator
1552 --i;
1553 }
1554 }
1555 // Now change uses of _ioproj_catchall to use _ioproj_fallthrough and delete
1556 // _ioproj_catchall so we end up with a call that has only 1 i_o projection.
1557 if (_ioproj_catchall != NULL ) {
1558 if (_ioproj_fallthrough == NULL) {
1559 _ioproj_fallthrough = new ProjNode(call, TypeFunc::I_O);
1560 transform_later(_ioproj_fallthrough);
1561 }
1562 for (DUIterator_Fast imax, i = _ioproj_catchall->fast_outs(imax); i < imax; i++) {
1563 Node *use = _ioproj_catchall->fast_out(i);
1564 _igvn.rehash_node_delayed(use);
1565 imax -= replace_input(use, _ioproj_catchall, _ioproj_fallthrough);
1566 // back up iterator
1567 --i;
1568 }
1569 assert(_ioproj_catchall->outcnt() == 0, "all uses must be deleted");
1570 _igvn.remove_dead_node(_ioproj_catchall);
1571 }
1572
1573 // if we generated only a slow call, we are done
1574 if (always_slow) {
1575 // Now we can unhook i_o.
1576 if (result_phi_i_o->outcnt() > 1) {
1577 call->set_req(TypeFunc::I_O, top());
1578 } else {
1579 assert(result_phi_i_o->unique_ctrl_out() == call, "");
1673 // We have no more use for this link, since the AllocateNode goes away:
1674 init->set_req(InitializeNode::RawAddress, top());
1675 // (If we keep the link, it just confuses the register allocator,
1676 // who thinks he sees a real use of the address by the membar.)
1677 }
1678
1679 return rawmem;
1680 }
1681
1682 // Generate prefetch instructions for next allocations.
1683 Node* PhaseMacroExpand::prefetch_allocation(Node* i_o, Node*& needgc_false,
1684 Node*& contended_phi_rawmem,
1685 Node* old_eden_top, Node* new_eden_top,
1686 Node* length) {
1687 enum { fall_in_path = 1, pf_path = 2 };
1688 if( UseTLAB && AllocatePrefetchStyle == 2 ) {
1689 // Generate prefetch allocation with watermark check.
1690 // As an allocation hits the watermark, we will prefetch starting
1691 // at a "distance" away from watermark.
1692
1693 Node *pf_region = new RegionNode(3);
1694 Node *pf_phi_rawmem = new PhiNode( pf_region, Type::MEMORY,
1695 TypeRawPtr::BOTTOM );
1696 // I/O is used for Prefetch
1697 Node *pf_phi_abio = new PhiNode( pf_region, Type::ABIO );
1698
1699 Node *thread = new ThreadLocalNode();
1700 transform_later(thread);
1701
1702 Node *eden_pf_adr = new AddPNode( top()/*not oop*/, thread,
1703 _igvn.MakeConX(in_bytes(JavaThread::tlab_pf_top_offset())) );
1704 transform_later(eden_pf_adr);
1705
1706 Node *old_pf_wm = new LoadPNode(needgc_false,
1707 contended_phi_rawmem, eden_pf_adr,
1708 TypeRawPtr::BOTTOM, TypeRawPtr::BOTTOM,
1709 MemNode::unordered);
1710 transform_later(old_pf_wm);
1711
1712 // check against new_eden_top
1713 Node *need_pf_cmp = new CmpPNode( new_eden_top, old_pf_wm );
1714 transform_later(need_pf_cmp);
1715 Node *need_pf_bol = new BoolNode( need_pf_cmp, BoolTest::ge );
1716 transform_later(need_pf_bol);
1717 IfNode *need_pf_iff = new IfNode( needgc_false, need_pf_bol,
1718 PROB_UNLIKELY_MAG(4), COUNT_UNKNOWN );
1719 transform_later(need_pf_iff);
1720
1721 // true node, add prefetchdistance
1722 Node *need_pf_true = new IfTrueNode( need_pf_iff );
1723 transform_later(need_pf_true);
1724
1725 Node *need_pf_false = new IfFalseNode( need_pf_iff );
1726 transform_later(need_pf_false);
1727
1728 Node *new_pf_wmt = new AddPNode( top(), old_pf_wm,
1729 _igvn.MakeConX(AllocatePrefetchDistance) );
1730 transform_later(new_pf_wmt );
1731 new_pf_wmt->set_req(0, need_pf_true);
1732
1733 Node *store_new_wmt = new StorePNode(need_pf_true,
1734 contended_phi_rawmem, eden_pf_adr,
1735 TypeRawPtr::BOTTOM, new_pf_wmt,
1736 MemNode::unordered);
1737 transform_later(store_new_wmt);
1738
1739 // adding prefetches
1740 pf_phi_abio->init_req( fall_in_path, i_o );
1741
1742 Node *prefetch_adr;
1743 Node *prefetch;
1744 uint lines = AllocatePrefetchDistance / AllocatePrefetchStepSize;
1745 uint step_size = AllocatePrefetchStepSize;
1746 uint distance = 0;
1747
1748 for ( uint i = 0; i < lines; i++ ) {
1749 prefetch_adr = new AddPNode( old_pf_wm, new_pf_wmt,
1750 _igvn.MakeConX(distance) );
1751 transform_later(prefetch_adr);
1752 prefetch = new PrefetchAllocationNode( i_o, prefetch_adr );
1753 transform_later(prefetch);
1754 distance += step_size;
1755 i_o = prefetch;
1756 }
1757 pf_phi_abio->set_req( pf_path, i_o );
1758
1759 pf_region->init_req( fall_in_path, need_pf_false );
1760 pf_region->init_req( pf_path, need_pf_true );
1761
1762 pf_phi_rawmem->init_req( fall_in_path, contended_phi_rawmem );
1763 pf_phi_rawmem->init_req( pf_path, store_new_wmt );
1764
1765 transform_later(pf_region);
1766 transform_later(pf_phi_rawmem);
1767 transform_later(pf_phi_abio);
1768
1769 needgc_false = pf_region;
1770 contended_phi_rawmem = pf_phi_rawmem;
1771 i_o = pf_phi_abio;
1772 } else if( UseTLAB && AllocatePrefetchStyle == 3 ) {
1773 // Insert a prefetch for each allocation.
1774 // This code is used for Sparc with BIS.
1775 Node *pf_region = new RegionNode(3);
1776 Node *pf_phi_rawmem = new PhiNode( pf_region, Type::MEMORY,
1777 TypeRawPtr::BOTTOM );
1778
1779 // Generate several prefetch instructions.
1780 uint lines = (length != NULL) ? AllocatePrefetchLines : AllocateInstancePrefetchLines;
1781 uint step_size = AllocatePrefetchStepSize;
1782 uint distance = AllocatePrefetchDistance;
1783
1784 // Next cache address.
1785 Node *cache_adr = new AddPNode(old_eden_top, old_eden_top,
1786 _igvn.MakeConX(distance));
1787 transform_later(cache_adr);
1788 cache_adr = new CastP2XNode(needgc_false, cache_adr);
1789 transform_later(cache_adr);
1790 Node* mask = _igvn.MakeConX(~(intptr_t)(step_size-1));
1791 cache_adr = new AndXNode(cache_adr, mask);
1792 transform_later(cache_adr);
1793 cache_adr = new CastX2PNode(cache_adr);
1794 transform_later(cache_adr);
1795
1796 // Prefetch
1797 Node *prefetch = new PrefetchAllocationNode( contended_phi_rawmem, cache_adr );
1798 prefetch->set_req(0, needgc_false);
1799 transform_later(prefetch);
1800 contended_phi_rawmem = prefetch;
1801 Node *prefetch_adr;
1802 distance = step_size;
1803 for ( uint i = 1; i < lines; i++ ) {
1804 prefetch_adr = new AddPNode( cache_adr, cache_adr,
1805 _igvn.MakeConX(distance) );
1806 transform_later(prefetch_adr);
1807 prefetch = new PrefetchAllocationNode( contended_phi_rawmem, prefetch_adr );
1808 transform_later(prefetch);
1809 distance += step_size;
1810 contended_phi_rawmem = prefetch;
1811 }
1812 } else if( AllocatePrefetchStyle > 0 ) {
1813 // Insert a prefetch for each allocation only on the fast-path
1814 Node *prefetch_adr;
1815 Node *prefetch;
1816 // Generate several prefetch instructions.
1817 uint lines = (length != NULL) ? AllocatePrefetchLines : AllocateInstancePrefetchLines;
1818 uint step_size = AllocatePrefetchStepSize;
1819 uint distance = AllocatePrefetchDistance;
1820 for ( uint i = 0; i < lines; i++ ) {
1821 prefetch_adr = new AddPNode( old_eden_top, new_eden_top,
1822 _igvn.MakeConX(distance) );
1823 transform_later(prefetch_adr);
1824 prefetch = new PrefetchAllocationNode( i_o, prefetch_adr );
1825 // Do not let it float too high, since if eden_top == eden_end,
1826 // both might be null.
1827 if( i == 0 ) { // Set control for first prefetch, next follows it
1828 prefetch->init_req(0, needgc_false);
1829 }
1830 transform_later(prefetch);
1831 distance += step_size;
1832 i_o = prefetch;
1833 }
1834 }
1835 return i_o;
1836 }
1837
1838
1839 void PhaseMacroExpand::expand_allocate(AllocateNode *alloc) {
1840 expand_allocate_common(alloc, NULL,
1841 OptoRuntime::new_instance_Type(),
1842 OptoRuntime::new_instance_Java());
1843 }
1844
2153 * // Try to rebias.
2154 * if( cas(&mark_word, old, new) == 0 ) {
2155 * // Done.
2156 * } else {
2157 * goto slow_path; // Failed.
2158 * }
2159 * }
2160 * }
2161 * } else {
2162 * // The object is not biased.
2163 * cas_lock:
2164 * if( FastLock(obj) == 0 ) {
2165 * // Done.
2166 * } else {
2167 * slow_path:
2168 * OptoRuntime::complete_monitor_locking_Java(obj);
2169 * }
2170 * }
2171 */
2172
2173 region = new RegionNode(5);
2174 // create a Phi for the memory state
2175 mem_phi = new PhiNode( region, Type::MEMORY, TypeRawPtr::BOTTOM);
2176
2177 Node* fast_lock_region = new RegionNode(3);
2178 Node* fast_lock_mem_phi = new PhiNode( fast_lock_region, Type::MEMORY, TypeRawPtr::BOTTOM);
2179
2180 // First, check mark word for the biased lock pattern.
2181 Node* mark_node = make_load(ctrl, mem, obj, oopDesc::mark_offset_in_bytes(), TypeX_X, TypeX_X->basic_type());
2182
2183 // Get fast path - mark word has the biased lock pattern.
2184 ctrl = opt_bits_test(ctrl, fast_lock_region, 1, mark_node,
2185 markOopDesc::biased_lock_mask_in_place,
2186 markOopDesc::biased_lock_pattern, true);
2187 // fast_lock_region->in(1) is set to slow path.
2188 fast_lock_mem_phi->init_req(1, mem);
2189
2190 // Now check that the lock is biased to the current thread and has
2191 // the same epoch and bias as Klass::_prototype_header.
2192
2193 // Special-case a fresh allocation to avoid building nodes:
2194 Node* klass_node = AllocateNode::Ideal_klass(obj, &_igvn);
2195 if (klass_node == NULL) {
2196 Node* k_adr = basic_plus_adr(obj, oopDesc::klass_offset_in_bytes());
2197 klass_node = transform_later( LoadKlassNode::make(_igvn, mem, k_adr, _igvn.type(k_adr)->is_ptr()) );
2198 #ifdef _LP64
2199 if (UseCompressedClassPointers && klass_node->is_DecodeNKlass()) {
2200 assert(klass_node->in(1)->Opcode() == Op_LoadNKlass, "sanity");
2201 klass_node->in(1)->init_req(0, ctrl);
2202 } else
2203 #endif
2204 klass_node->init_req(0, ctrl);
2205 }
2206 Node *proto_node = make_load(ctrl, mem, klass_node, in_bytes(Klass::prototype_header_offset()), TypeX_X, TypeX_X->basic_type());
2207
2208 Node* thread = transform_later(new ThreadLocalNode());
2209 Node* cast_thread = transform_later(new CastP2XNode(ctrl, thread));
2210 Node* o_node = transform_later(new OrXNode(cast_thread, proto_node));
2211 Node* x_node = transform_later(new XorXNode(o_node, mark_node));
2212
2213 // Get slow path - mark word does NOT match the value.
2214 Node* not_biased_ctrl = opt_bits_test(ctrl, region, 3, x_node,
2215 (~markOopDesc::age_mask_in_place), 0);
2216 // region->in(3) is set to fast path - the object is biased to the current thread.
2217 mem_phi->init_req(3, mem);
2218
2219
2220 // Mark word does NOT match the value (thread | Klass::_prototype_header).
2221
2222
2223 // First, check biased pattern.
2224 // Get fast path - _prototype_header has the same biased lock pattern.
2225 ctrl = opt_bits_test(not_biased_ctrl, fast_lock_region, 2, x_node,
2226 markOopDesc::biased_lock_mask_in_place, 0, true);
2227
2228 not_biased_ctrl = fast_lock_region->in(2); // Slow path
2229 // fast_lock_region->in(2) - the prototype header is no longer biased
2230 // and we have to revoke the bias on this object.
2231 // We are going to try to reset the mark of this object to the prototype
2232 // value and fall through to the CAS-based locking scheme.
2233 Node* adr = basic_plus_adr(obj, oopDesc::mark_offset_in_bytes());
2234 Node* cas = new StoreXConditionalNode(not_biased_ctrl, mem, adr,
2235 proto_node, mark_node);
2236 transform_later(cas);
2237 Node* proj = transform_later(new SCMemProjNode(cas));
2238 fast_lock_mem_phi->init_req(2, proj);
2239
2240
2241 // Second, check epoch bits.
2242 Node* rebiased_region = new RegionNode(3);
2243 Node* old_phi = new PhiNode( rebiased_region, TypeX_X);
2244 Node* new_phi = new PhiNode( rebiased_region, TypeX_X);
2245
2246 // Get slow path - mark word does NOT match epoch bits.
2247 Node* epoch_ctrl = opt_bits_test(ctrl, rebiased_region, 1, x_node,
2248 markOopDesc::epoch_mask_in_place, 0);
2249 // The epoch of the current bias is not valid, attempt to rebias the object
2250 // toward the current thread.
2251 rebiased_region->init_req(2, epoch_ctrl);
2252 old_phi->init_req(2, mark_node);
2253 new_phi->init_req(2, o_node);
2254
2255 // rebiased_region->in(1) is set to fast path.
2256 // The epoch of the current bias is still valid but we know
2257 // nothing about the owner; it might be set or it might be clear.
2258 Node* cmask = MakeConX(markOopDesc::biased_lock_mask_in_place |
2259 markOopDesc::age_mask_in_place |
2260 markOopDesc::epoch_mask_in_place);
2261 Node* old = transform_later(new AndXNode(mark_node, cmask));
2262 cast_thread = transform_later(new CastP2XNode(ctrl, thread));
2263 Node* new_mark = transform_later(new OrXNode(cast_thread, old));
2264 old_phi->init_req(1, old);
2265 new_phi->init_req(1, new_mark);
2266
2267 transform_later(rebiased_region);
2268 transform_later(old_phi);
2269 transform_later(new_phi);
2270
2271 // Try to acquire the bias of the object using an atomic operation.
2272 // If this fails we will go in to the runtime to revoke the object's bias.
2273 cas = new StoreXConditionalNode(rebiased_region, mem, adr, new_phi, old_phi);
2274 transform_later(cas);
2275 proj = transform_later(new SCMemProjNode(cas));
2276
2277 // Get slow path - Failed to CAS.
2278 not_biased_ctrl = opt_bits_test(rebiased_region, region, 4, cas, 0, 0);
2279 mem_phi->init_req(4, proj);
2280 // region->in(4) is set to fast path - the object is rebiased to the current thread.
2281
2282 // Failed to CAS.
2283 slow_path = new RegionNode(3);
2284 Node *slow_mem = new PhiNode( slow_path, Type::MEMORY, TypeRawPtr::BOTTOM);
2285
2286 slow_path->init_req(1, not_biased_ctrl); // Capture slow-control
2287 slow_mem->init_req(1, proj);
2288
2289 // Call CAS-based locking scheme (FastLock node).
2290
2291 transform_later(fast_lock_region);
2292 transform_later(fast_lock_mem_phi);
2293
2294 // Get slow path - FastLock failed to lock the object.
2295 ctrl = opt_bits_test(fast_lock_region, region, 2, flock, 0, 0);
2296 mem_phi->init_req(2, fast_lock_mem_phi);
2297 // region->in(2) is set to fast path - the object is locked to the current thread.
2298
2299 slow_path->init_req(2, ctrl); // Capture slow-control
2300 slow_mem->init_req(2, fast_lock_mem_phi);
2301
2302 transform_later(slow_path);
2303 transform_later(slow_mem);
2304 // Reset lock's memory edge.
2305 lock->set_req(TypeFunc::Memory, slow_mem);
2306
2307 } else {
2308 region = new RegionNode(3);
2309 // create a Phi for the memory state
2310 mem_phi = new PhiNode( region, Type::MEMORY, TypeRawPtr::BOTTOM);
2311
2312 // Optimize test; set region slot 2
2313 slow_path = opt_bits_test(ctrl, region, 2, flock, 0, 0);
2314 mem_phi->init_req(2, mem);
2315 }
2316
2317 // Make slow path call
2318 CallNode *call = make_slow_call( (CallNode *) lock, OptoRuntime::complete_monitor_enter_Type(), OptoRuntime::complete_monitor_locking_Java(), NULL, slow_path, obj, box );
2319
2320 extract_call_projections(call);
2321
2322 // Slow path can only throw asynchronous exceptions, which are always
2323 // de-opted. So the compiler thinks the slow-call can never throw an
2324 // exception. If it DOES throw an exception we would need the debug
2325 // info removed first (since if it throws there is no monitor).
2326 assert ( _ioproj_fallthrough == NULL && _ioproj_catchall == NULL &&
2327 _memproj_catchall == NULL && _catchallcatchproj == NULL, "Unexpected projection from Lock");
2328
2329 // Capture slow path
2330 // disconnect fall-through projection from call and create a new one
2331 // hook up users of fall-through projection to region
2332 Node *slow_ctrl = _fallthroughproj->clone();
2333 transform_later(slow_ctrl);
2334 _igvn.hash_delete(_fallthroughproj);
2335 _fallthroughproj->disconnect_inputs(NULL, C);
2336 region->init_req(1, slow_ctrl);
2337 // region inputs are now complete
2338 transform_later(region);
2339 _igvn.replace_node(_fallthroughproj, region);
2340
2341 Node *memproj = transform_later(new ProjNode(call, TypeFunc::Memory));
2342 mem_phi->init_req(1, memproj );
2343 transform_later(mem_phi);
2344 _igvn.replace_node(_memproj_fallthrough, mem_phi);
2345 }
2346
2347 //------------------------------expand_unlock_node----------------------
2348 void PhaseMacroExpand::expand_unlock_node(UnlockNode *unlock) {
2349
2350 Node* ctrl = unlock->in(TypeFunc::Control);
2351 Node* mem = unlock->in(TypeFunc::Memory);
2352 Node* obj = unlock->obj_node();
2353 Node* box = unlock->box_node();
2354
2355 assert(!box->as_BoxLock()->is_eliminated(), "sanity");
2356
2357 // No need for a null check on unlock
2358
2359 // Make the merge point
2360 Node *region;
2361 Node *mem_phi;
2362
2363 if (UseOptoBiasInlining) {
2364 // Check for biased locking unlock case, which is a no-op.
2365 // See the full description in MacroAssembler::biased_locking_exit().
2366 region = new RegionNode(4);
2367 // create a Phi for the memory state
2368 mem_phi = new PhiNode( region, Type::MEMORY, TypeRawPtr::BOTTOM);
2369 mem_phi->init_req(3, mem);
2370
2371 Node* mark_node = make_load(ctrl, mem, obj, oopDesc::mark_offset_in_bytes(), TypeX_X, TypeX_X->basic_type());
2372 ctrl = opt_bits_test(ctrl, region, 3, mark_node,
2373 markOopDesc::biased_lock_mask_in_place,
2374 markOopDesc::biased_lock_pattern);
2375 } else {
2376 region = new RegionNode(3);
2377 // create a Phi for the memory state
2378 mem_phi = new PhiNode( region, Type::MEMORY, TypeRawPtr::BOTTOM);
2379 }
2380
2381 FastUnlockNode *funlock = new FastUnlockNode( ctrl, obj, box );
2382 funlock = transform_later( funlock )->as_FastUnlock();
2383 // Optimize test; set region slot 2
2384 Node *slow_path = opt_bits_test(ctrl, region, 2, funlock, 0, 0);
2385
2386 CallNode *call = make_slow_call( (CallNode *) unlock, OptoRuntime::complete_monitor_exit_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::complete_monitor_unlocking_C), "complete_monitor_unlocking_C", slow_path, obj, box );
2387
2388 extract_call_projections(call);
2389
2390 assert ( _ioproj_fallthrough == NULL && _ioproj_catchall == NULL &&
2391 _memproj_catchall == NULL && _catchallcatchproj == NULL, "Unexpected projection from Lock");
2392
2393 // No exceptions for unlocking
2394 // Capture slow path
2395 // disconnect fall-through projection from call and create a new one
2396 // hook up users of fall-through projection to region
2397 Node *slow_ctrl = _fallthroughproj->clone();
2398 transform_later(slow_ctrl);
2399 _igvn.hash_delete(_fallthroughproj);
2400 _fallthroughproj->disconnect_inputs(NULL, C);
2401 region->init_req(1, slow_ctrl);
2402 // region inputs are now complete
2403 transform_later(region);
2404 _igvn.replace_node(_fallthroughproj, region);
2405
2406 Node *memproj = transform_later(new ProjNode(call, TypeFunc::Memory) );
2407 mem_phi->init_req(1, memproj );
2408 mem_phi->init_req(2, mem);
2409 transform_later(mem_phi);
2410 _igvn.replace_node(_memproj_fallthrough, mem_phi);
2411 }
2412
2413 //---------------------------eliminate_macro_nodes----------------------
2414 // Eliminate scalar replaced allocations and associated locks.
2415 void PhaseMacroExpand::eliminate_macro_nodes() {
2416 if (C->macro_count() == 0)
2417 return;
2418
2419 // First, attempt to eliminate locks
2420 int cnt = C->macro_count();
2421 for (int i=0; i < cnt; i++) {
2422 Node *n = C->macro_node(i);
2423 if (n->is_AbstractLock()) { // Lock and Unlock nodes
2424 // Before elimination mark all associated (same box and obj)
2425 // lock and unlock nodes.
2426 mark_eliminated_locking_nodes(n->as_AbstractLock());
|