1067 void PhaseMacroExpand::set_eden_pointers(Node* &eden_top_adr, Node* &eden_end_adr) {
1068 if (UseTLAB) { // Private allocation: load from TLS
1069 Node* thread = transform_later(new (C) ThreadLocalNode());
1070 int tlab_top_offset = in_bytes(JavaThread::tlab_top_offset());
1071 int tlab_end_offset = in_bytes(JavaThread::tlab_end_offset());
1072 eden_top_adr = basic_plus_adr(top()/*not oop*/, thread, tlab_top_offset);
1073 eden_end_adr = basic_plus_adr(top()/*not oop*/, thread, tlab_end_offset);
1074 } else { // Shared allocation: load from globals
1075 CollectedHeap* ch = Universe::heap();
1076 address top_adr = (address)ch->top_addr();
1077 address end_adr = (address)ch->end_addr();
1078 eden_top_adr = makecon(TypeRawPtr::make(top_adr));
1079 eden_end_adr = basic_plus_adr(eden_top_adr, end_adr - top_adr);
1080 }
1081 }
1082
1083
1084 Node* PhaseMacroExpand::make_load(Node* ctl, Node* mem, Node* base, int offset, const Type* value_type, BasicType bt) {
1085 Node* adr = basic_plus_adr(base, offset);
1086 const TypePtr* adr_type = adr->bottom_type()->is_ptr();
1087 Node* value = LoadNode::make(_igvn, ctl, mem, adr, adr_type, value_type, bt);
1088 transform_later(value);
1089 return value;
1090 }
1091
1092
1093 Node* PhaseMacroExpand::make_store(Node* ctl, Node* mem, Node* base, int offset, Node* value, BasicType bt) {
1094 Node* adr = basic_plus_adr(base, offset);
1095 mem = StoreNode::make(_igvn, ctl, mem, adr, NULL, value, bt);
1096 transform_later(mem);
1097 return mem;
1098 }
1099
1100 //=============================================================================
1101 //
1102 // A L L O C A T I O N
1103 //
1104 // Allocation attempts to be fast in the case of frequent small objects.
1105 // It breaks down like this:
1106 //
1107 // 1) Size in doublewords is computed. This is a constant for objects and
1108 // variable for most arrays. Doubleword units are used to avoid size
1109 // overflow of huge doubleword arrays. We need doublewords in the end for
1110 // rounding.
1111 //
1112 // 2) Size is checked for being 'too large'. Too-large allocations will go
1113 // the slow path into the VM. The slow path can throw any required
1114 // exceptions, and does all the special checks for very large arrays. The
1115 // size test can constant-fold away for objects. For objects with
1255 enum { fall_in_path = 1, contended_loopback_path = 2 };
1256 Node *contended_region;
1257 Node *contended_phi_rawmem;
1258 if (UseTLAB) {
1259 contended_region = toobig_false;
1260 contended_phi_rawmem = mem;
1261 } else {
1262 contended_region = new (C) RegionNode(3);
1263 contended_phi_rawmem = new (C) PhiNode(contended_region, Type::MEMORY, TypeRawPtr::BOTTOM);
1264 // Now handle the passing-too-big test. We fall into the contended
1265 // loop-back merge point.
1266 contended_region ->init_req(fall_in_path, toobig_false);
1267 contended_phi_rawmem->init_req(fall_in_path, mem);
1268 transform_later(contended_region);
1269 transform_later(contended_phi_rawmem);
1270 }
1271
1272 // Load(-locked) the heap top.
1273 // See note above concerning the control input when using a TLAB
1274 Node *old_eden_top = UseTLAB
1275 ? new (C) LoadPNode (ctrl, contended_phi_rawmem, eden_top_adr, TypeRawPtr::BOTTOM, TypeRawPtr::BOTTOM)
1276 : new (C) LoadPLockedNode(contended_region, contended_phi_rawmem, eden_top_adr);
1277
1278 transform_later(old_eden_top);
1279 // Add to heap top to get a new heap top
1280 Node *new_eden_top = new (C) AddPNode(top(), old_eden_top, size_in_bytes);
1281 transform_later(new_eden_top);
1282 // Check for needing a GC; compare against heap end
1283 Node *needgc_cmp = new (C) CmpPNode(new_eden_top, eden_end);
1284 transform_later(needgc_cmp);
1285 Node *needgc_bol = new (C) BoolNode(needgc_cmp, BoolTest::ge);
1286 transform_later(needgc_bol);
1287 IfNode *needgc_iff = new (C) IfNode(contended_region, needgc_bol, PROB_UNLIKELY_MAG(4), COUNT_UNKNOWN);
1288 transform_later(needgc_iff);
1289
1290 // Plug the failing-heap-space-need-gc test into the slow-path region
1291 Node *needgc_true = new (C) IfTrueNode(needgc_iff);
1292 transform_later(needgc_true);
1293 if (initial_slow_test) {
1294 slow_region->init_req(need_gc_path, needgc_true);
1295 // This completes all paths into the slow merge point
1296 transform_later(slow_region);
1303 transform_later(needgc_false);
1304
1305 // Grab regular I/O before optional prefetch may change it.
1306 // Slow-path does no I/O so just set it to the original I/O.
1307 result_phi_i_o->init_req(slow_result_path, i_o);
1308
1309 i_o = prefetch_allocation(i_o, needgc_false, contended_phi_rawmem,
1310 old_eden_top, new_eden_top, length);
1311
1312 // Name successful fast-path variables
1313 Node* fast_oop = old_eden_top;
1314 Node* fast_oop_ctrl;
1315 Node* fast_oop_rawmem;
1316
1317 // Store (-conditional) the modified eden top back down.
1318 // StorePConditional produces flags for a test PLUS a modified raw
1319 // memory state.
1320 if (UseTLAB) {
1321 Node* store_eden_top =
1322 new (C) StorePNode(needgc_false, contended_phi_rawmem, eden_top_adr,
1323 TypeRawPtr::BOTTOM, new_eden_top);
1324 transform_later(store_eden_top);
1325 fast_oop_ctrl = needgc_false; // No contention, so this is the fast path
1326 fast_oop_rawmem = store_eden_top;
1327 } else {
1328 Node* store_eden_top =
1329 new (C) StorePConditionalNode(needgc_false, contended_phi_rawmem, eden_top_adr,
1330 new_eden_top, fast_oop/*old_eden_top*/);
1331 transform_later(store_eden_top);
1332 Node *contention_check = new (C) BoolNode(store_eden_top, BoolTest::ne);
1333 transform_later(contention_check);
1334 store_eden_top = new (C) SCMemProjNode(store_eden_top);
1335 transform_later(store_eden_top);
1336
1337 // If not using TLABs, check to see if there was contention.
1338 IfNode *contention_iff = new (C) IfNode (needgc_false, contention_check, PROB_MIN, COUNT_UNKNOWN);
1339 transform_later(contention_iff);
1340 Node *contention_true = new (C) IfTrueNode(contention_iff);
1341 transform_later(contention_true);
1342 // If contention, loopback and try again.
1343 contended_region->init_req(contended_loopback_path, contention_true);
1683 Node* length) {
1684 enum { fall_in_path = 1, pf_path = 2 };
1685 if( UseTLAB && AllocatePrefetchStyle == 2 ) {
1686 // Generate prefetch allocation with watermark check.
1687 // As an allocation hits the watermark, we will prefetch starting
1688 // at a "distance" away from watermark.
1689
1690 Node *pf_region = new (C) RegionNode(3);
1691 Node *pf_phi_rawmem = new (C) PhiNode( pf_region, Type::MEMORY,
1692 TypeRawPtr::BOTTOM );
1693 // I/O is used for Prefetch
1694 Node *pf_phi_abio = new (C) PhiNode( pf_region, Type::ABIO );
1695
1696 Node *thread = new (C) ThreadLocalNode();
1697 transform_later(thread);
1698
1699 Node *eden_pf_adr = new (C) AddPNode( top()/*not oop*/, thread,
1700 _igvn.MakeConX(in_bytes(JavaThread::tlab_pf_top_offset())) );
1701 transform_later(eden_pf_adr);
1702
1703 Node *old_pf_wm = new (C) LoadPNode( needgc_false,
1704 contended_phi_rawmem, eden_pf_adr,
1705 TypeRawPtr::BOTTOM, TypeRawPtr::BOTTOM );
1706 transform_later(old_pf_wm);
1707
1708 // check against new_eden_top
1709 Node *need_pf_cmp = new (C) CmpPNode( new_eden_top, old_pf_wm );
1710 transform_later(need_pf_cmp);
1711 Node *need_pf_bol = new (C) BoolNode( need_pf_cmp, BoolTest::ge );
1712 transform_later(need_pf_bol);
1713 IfNode *need_pf_iff = new (C) IfNode( needgc_false, need_pf_bol,
1714 PROB_UNLIKELY_MAG(4), COUNT_UNKNOWN );
1715 transform_later(need_pf_iff);
1716
1717 // true node, add prefetchdistance
1718 Node *need_pf_true = new (C) IfTrueNode( need_pf_iff );
1719 transform_later(need_pf_true);
1720
1721 Node *need_pf_false = new (C) IfFalseNode( need_pf_iff );
1722 transform_later(need_pf_false);
1723
1724 Node *new_pf_wmt = new (C) AddPNode( top(), old_pf_wm,
1725 _igvn.MakeConX(AllocatePrefetchDistance) );
1726 transform_later(new_pf_wmt );
1727 new_pf_wmt->set_req(0, need_pf_true);
1728
1729 Node *store_new_wmt = new (C) StorePNode( need_pf_true,
1730 contended_phi_rawmem, eden_pf_adr,
1731 TypeRawPtr::BOTTOM, new_pf_wmt );
1732 transform_later(store_new_wmt);
1733
1734 // adding prefetches
1735 pf_phi_abio->init_req( fall_in_path, i_o );
1736
1737 Node *prefetch_adr;
1738 Node *prefetch;
1739 uint lines = AllocatePrefetchDistance / AllocatePrefetchStepSize;
1740 uint step_size = AllocatePrefetchStepSize;
1741 uint distance = 0;
1742
1743 for ( uint i = 0; i < lines; i++ ) {
1744 prefetch_adr = new (C) AddPNode( old_pf_wm, new_pf_wmt,
1745 _igvn.MakeConX(distance) );
1746 transform_later(prefetch_adr);
1747 prefetch = new (C) PrefetchAllocationNode( i_o, prefetch_adr );
1748 transform_later(prefetch);
1749 distance += step_size;
1750 i_o = prefetch;
1751 }
|
1067 void PhaseMacroExpand::set_eden_pointers(Node* &eden_top_adr, Node* &eden_end_adr) {
1068 if (UseTLAB) { // Private allocation: load from TLS
1069 Node* thread = transform_later(new (C) ThreadLocalNode());
1070 int tlab_top_offset = in_bytes(JavaThread::tlab_top_offset());
1071 int tlab_end_offset = in_bytes(JavaThread::tlab_end_offset());
1072 eden_top_adr = basic_plus_adr(top()/*not oop*/, thread, tlab_top_offset);
1073 eden_end_adr = basic_plus_adr(top()/*not oop*/, thread, tlab_end_offset);
1074 } else { // Shared allocation: load from globals
1075 CollectedHeap* ch = Universe::heap();
1076 address top_adr = (address)ch->top_addr();
1077 address end_adr = (address)ch->end_addr();
1078 eden_top_adr = makecon(TypeRawPtr::make(top_adr));
1079 eden_end_adr = basic_plus_adr(eden_top_adr, end_adr - top_adr);
1080 }
1081 }
1082
1083
1084 Node* PhaseMacroExpand::make_load(Node* ctl, Node* mem, Node* base, int offset, const Type* value_type, BasicType bt) {
1085 Node* adr = basic_plus_adr(base, offset);
1086 const TypePtr* adr_type = adr->bottom_type()->is_ptr();
1087 Node* value = LoadNode::make(_igvn, ctl, mem, adr, adr_type, value_type, bt, LoadNode::unordered);
1088 transform_later(value);
1089 return value;
1090 }
1091
1092
1093 Node* PhaseMacroExpand::make_store(Node* ctl, Node* mem, Node* base, int offset, Node* value, BasicType bt) {
1094 Node* adr = basic_plus_adr(base, offset);
1095 mem = StoreNode::make(_igvn, ctl, mem, adr, NULL, value, bt, StoreNode::unordered);
1096 transform_later(mem);
1097 return mem;
1098 }
1099
1100 //=============================================================================
1101 //
1102 // A L L O C A T I O N
1103 //
1104 // Allocation attempts to be fast in the case of frequent small objects.
1105 // It breaks down like this:
1106 //
1107 // 1) Size in doublewords is computed. This is a constant for objects and
1108 // variable for most arrays. Doubleword units are used to avoid size
1109 // overflow of huge doubleword arrays. We need doublewords in the end for
1110 // rounding.
1111 //
1112 // 2) Size is checked for being 'too large'. Too-large allocations will go
1113 // the slow path into the VM. The slow path can throw any required
1114 // exceptions, and does all the special checks for very large arrays. The
1115 // size test can constant-fold away for objects. For objects with
1255 enum { fall_in_path = 1, contended_loopback_path = 2 };
1256 Node *contended_region;
1257 Node *contended_phi_rawmem;
1258 if (UseTLAB) {
1259 contended_region = toobig_false;
1260 contended_phi_rawmem = mem;
1261 } else {
1262 contended_region = new (C) RegionNode(3);
1263 contended_phi_rawmem = new (C) PhiNode(contended_region, Type::MEMORY, TypeRawPtr::BOTTOM);
1264 // Now handle the passing-too-big test. We fall into the contended
1265 // loop-back merge point.
1266 contended_region ->init_req(fall_in_path, toobig_false);
1267 contended_phi_rawmem->init_req(fall_in_path, mem);
1268 transform_later(contended_region);
1269 transform_later(contended_phi_rawmem);
1270 }
1271
1272 // Load(-locked) the heap top.
1273 // See note above concerning the control input when using a TLAB
1274 Node *old_eden_top = UseTLAB
1275 ? new (C) LoadPNode (ctrl, contended_phi_rawmem, eden_top_adr, TypeRawPtr::BOTTOM, TypeRawPtr::BOTTOM, LoadNode::unordered)
1276 : new (C) LoadPLockedNode(contended_region, contended_phi_rawmem, eden_top_adr, LoadNode::acquire);
1277
1278 transform_later(old_eden_top);
1279 // Add to heap top to get a new heap top
1280 Node *new_eden_top = new (C) AddPNode(top(), old_eden_top, size_in_bytes);
1281 transform_later(new_eden_top);
1282 // Check for needing a GC; compare against heap end
1283 Node *needgc_cmp = new (C) CmpPNode(new_eden_top, eden_end);
1284 transform_later(needgc_cmp);
1285 Node *needgc_bol = new (C) BoolNode(needgc_cmp, BoolTest::ge);
1286 transform_later(needgc_bol);
1287 IfNode *needgc_iff = new (C) IfNode(contended_region, needgc_bol, PROB_UNLIKELY_MAG(4), COUNT_UNKNOWN);
1288 transform_later(needgc_iff);
1289
1290 // Plug the failing-heap-space-need-gc test into the slow-path region
1291 Node *needgc_true = new (C) IfTrueNode(needgc_iff);
1292 transform_later(needgc_true);
1293 if (initial_slow_test) {
1294 slow_region->init_req(need_gc_path, needgc_true);
1295 // This completes all paths into the slow merge point
1296 transform_later(slow_region);
1303 transform_later(needgc_false);
1304
1305 // Grab regular I/O before optional prefetch may change it.
1306 // Slow-path does no I/O so just set it to the original I/O.
1307 result_phi_i_o->init_req(slow_result_path, i_o);
1308
1309 i_o = prefetch_allocation(i_o, needgc_false, contended_phi_rawmem,
1310 old_eden_top, new_eden_top, length);
1311
1312 // Name successful fast-path variables
1313 Node* fast_oop = old_eden_top;
1314 Node* fast_oop_ctrl;
1315 Node* fast_oop_rawmem;
1316
1317 // Store (-conditional) the modified eden top back down.
1318 // StorePConditional produces flags for a test PLUS a modified raw
1319 // memory state.
1320 if (UseTLAB) {
1321 Node* store_eden_top =
1322 new (C) StorePNode(needgc_false, contended_phi_rawmem, eden_top_adr,
1323 TypeRawPtr::BOTTOM, new_eden_top, StoreNode::unordered);
1324 transform_later(store_eden_top);
1325 fast_oop_ctrl = needgc_false; // No contention, so this is the fast path
1326 fast_oop_rawmem = store_eden_top;
1327 } else {
1328 Node* store_eden_top =
1329 new (C) StorePConditionalNode(needgc_false, contended_phi_rawmem, eden_top_adr,
1330 new_eden_top, fast_oop/*old_eden_top*/);
1331 transform_later(store_eden_top);
1332 Node *contention_check = new (C) BoolNode(store_eden_top, BoolTest::ne);
1333 transform_later(contention_check);
1334 store_eden_top = new (C) SCMemProjNode(store_eden_top);
1335 transform_later(store_eden_top);
1336
1337 // If not using TLABs, check to see if there was contention.
1338 IfNode *contention_iff = new (C) IfNode (needgc_false, contention_check, PROB_MIN, COUNT_UNKNOWN);
1339 transform_later(contention_iff);
1340 Node *contention_true = new (C) IfTrueNode(contention_iff);
1341 transform_later(contention_true);
1342 // If contention, loopback and try again.
1343 contended_region->init_req(contended_loopback_path, contention_true);
1683 Node* length) {
1684 enum { fall_in_path = 1, pf_path = 2 };
1685 if( UseTLAB && AllocatePrefetchStyle == 2 ) {
1686 // Generate prefetch allocation with watermark check.
1687 // As an allocation hits the watermark, we will prefetch starting
1688 // at a "distance" away from watermark.
1689
1690 Node *pf_region = new (C) RegionNode(3);
1691 Node *pf_phi_rawmem = new (C) PhiNode( pf_region, Type::MEMORY,
1692 TypeRawPtr::BOTTOM );
1693 // I/O is used for Prefetch
1694 Node *pf_phi_abio = new (C) PhiNode( pf_region, Type::ABIO );
1695
1696 Node *thread = new (C) ThreadLocalNode();
1697 transform_later(thread);
1698
1699 Node *eden_pf_adr = new (C) AddPNode( top()/*not oop*/, thread,
1700 _igvn.MakeConX(in_bytes(JavaThread::tlab_pf_top_offset())) );
1701 transform_later(eden_pf_adr);
1702
1703 Node *old_pf_wm = new (C) LoadPNode(needgc_false,
1704 contended_phi_rawmem, eden_pf_adr,
1705 TypeRawPtr::BOTTOM, TypeRawPtr::BOTTOM,
1706 LoadNode::unordered);
1707 transform_later(old_pf_wm);
1708
1709 // check against new_eden_top
1710 Node *need_pf_cmp = new (C) CmpPNode( new_eden_top, old_pf_wm );
1711 transform_later(need_pf_cmp);
1712 Node *need_pf_bol = new (C) BoolNode( need_pf_cmp, BoolTest::ge );
1713 transform_later(need_pf_bol);
1714 IfNode *need_pf_iff = new (C) IfNode( needgc_false, need_pf_bol,
1715 PROB_UNLIKELY_MAG(4), COUNT_UNKNOWN );
1716 transform_later(need_pf_iff);
1717
1718 // true node, add prefetchdistance
1719 Node *need_pf_true = new (C) IfTrueNode( need_pf_iff );
1720 transform_later(need_pf_true);
1721
1722 Node *need_pf_false = new (C) IfFalseNode( need_pf_iff );
1723 transform_later(need_pf_false);
1724
1725 Node *new_pf_wmt = new (C) AddPNode( top(), old_pf_wm,
1726 _igvn.MakeConX(AllocatePrefetchDistance) );
1727 transform_later(new_pf_wmt );
1728 new_pf_wmt->set_req(0, need_pf_true);
1729
1730 Node *store_new_wmt = new (C) StorePNode(need_pf_true,
1731 contended_phi_rawmem, eden_pf_adr,
1732 TypeRawPtr::BOTTOM, new_pf_wmt,
1733 StoreNode::unordered);
1734 transform_later(store_new_wmt);
1735
1736 // adding prefetches
1737 pf_phi_abio->init_req( fall_in_path, i_o );
1738
1739 Node *prefetch_adr;
1740 Node *prefetch;
1741 uint lines = AllocatePrefetchDistance / AllocatePrefetchStepSize;
1742 uint step_size = AllocatePrefetchStepSize;
1743 uint distance = 0;
1744
1745 for ( uint i = 0; i < lines; i++ ) {
1746 prefetch_adr = new (C) AddPNode( old_pf_wm, new_pf_wmt,
1747 _igvn.MakeConX(distance) );
1748 transform_later(prefetch_adr);
1749 prefetch = new (C) PrefetchAllocationNode( i_o, prefetch_adr );
1750 transform_later(prefetch);
1751 distance += step_size;
1752 i_o = prefetch;
1753 }
|