src/share/vm/opto/macro.cpp

Print this page
rev 2892 : 7121140: Allocation paths require explicit memory synchronization operations for RMO systems
Summary: adds store store barrier after initialization of header and body of objects.
Reviewed-by:


1071 // code shape produced here, so if you are changing this code shape
1072 // make sure the GC info for the heap-top is correct in and around the
1073 // slow-path call.
1074 //
1075 
1076 void PhaseMacroExpand::expand_allocate_common(
1077             AllocateNode* alloc, // allocation node to be expanded
1078             Node* length,  // array length for an array allocation
1079             const TypeFunc* slow_call_type, // Type of slow call
1080             address slow_call_address  // Address of slow call
1081     )
1082 {
1083 
1084   Node* ctrl = alloc->in(TypeFunc::Control);
1085   Node* mem  = alloc->in(TypeFunc::Memory);
1086   Node* i_o  = alloc->in(TypeFunc::I_O);
1087   Node* size_in_bytes     = alloc->in(AllocateNode::AllocSize);
1088   Node* klass_node        = alloc->in(AllocateNode::KlassNode);
1089   Node* initial_slow_test = alloc->in(AllocateNode::InitialTest);
1090 






1091   assert(ctrl != NULL, "must have control");
1092   // We need a Region and corresponding Phi's to merge the slow-path and fast-path results.
1093   // they will not be used if "always_slow" is set
1094   enum { slow_result_path = 1, fast_result_path = 2 };
1095   Node *result_region;
1096   Node *result_phi_rawmem;
1097   Node *result_phi_rawoop;
1098   Node *result_phi_i_o;
1099 
1100   // The initial slow comparison is a size check, the comparison
1101   // we want to do is a BoolTest::gt
1102   bool always_slow = false;
1103   int tv = _igvn.find_int_con(initial_slow_test, -1);
1104   if (tv >= 0) {
1105     always_slow = (tv == 1);
1106     initial_slow_test = NULL;
1107   } else {
1108     initial_slow_test = BoolNode::make_predicate(initial_slow_test, &_igvn);
1109   }
1110 


1272 
1273       // Bump total allocated bytes for this thread
1274       Node* thread = new (C, 1) ThreadLocalNode();
1275       transform_later(thread);
1276       Node* alloc_bytes_adr = basic_plus_adr(top()/*not oop*/, thread,
1277                                              in_bytes(JavaThread::allocated_bytes_offset()));
1278       Node* alloc_bytes = make_load(fast_oop_ctrl, store_eden_top, alloc_bytes_adr,
1279                                     0, TypeLong::LONG, T_LONG);
1280 #ifdef _LP64
1281       Node* alloc_size = size_in_bytes;
1282 #else
1283       Node* alloc_size = new (C, 2) ConvI2LNode(size_in_bytes);
1284       transform_later(alloc_size);
1285 #endif
1286       Node* new_alloc_bytes = new (C, 3) AddLNode(alloc_bytes, alloc_size);
1287       transform_later(new_alloc_bytes);
1288       fast_oop_rawmem = make_store(fast_oop_ctrl, store_eden_top, alloc_bytes_adr,
1289                                    0, new_alloc_bytes, T_LONG);
1290     }
1291 

1292     fast_oop_rawmem = initialize_object(alloc,
1293                                         fast_oop_ctrl, fast_oop_rawmem, fast_oop,
1294                                         klass_node, length, size_in_bytes);























































1295 
1296     if (C->env()->dtrace_extended_probes()) {
1297       // Slow-path call
1298       int size = TypeFunc::Parms + 2;
1299       CallLeafNode *call = new (C, size) CallLeafNode(OptoRuntime::dtrace_object_alloc_Type(),
1300                                                       CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_object_alloc_base),
1301                                                       "dtrace_object_alloc",
1302                                                       TypeRawPtr::BOTTOM);
1303 
1304       // Get base of thread-local storage area
1305       Node* thread = new (C, 1) ThreadLocalNode();
1306       transform_later(thread);
1307 
1308       call->init_req(TypeFunc::Parms+0, thread);
1309       call->init_req(TypeFunc::Parms+1, fast_oop);
1310       call->init_req(TypeFunc::Control, fast_oop_ctrl);
1311       call->init_req(TypeFunc::I_O    , top()); // does no i/o
1312       call->init_req(TypeFunc::Memory , fast_oop_rawmem);
1313       call->init_req(TypeFunc::ReturnAdr, alloc->in(TypeFunc::ReturnAdr));
1314       call->init_req(TypeFunc::FramePtr, alloc->in(TypeFunc::FramePtr));




1071 // code shape produced here, so if you are changing this code shape
1072 // make sure the GC info for the heap-top is correct in and around the
1073 // slow-path call.
1074 //
1075 
1076 void PhaseMacroExpand::expand_allocate_common(
1077             AllocateNode* alloc, // allocation node to be expanded
1078             Node* length,  // array length for an array allocation
1079             const TypeFunc* slow_call_type, // Type of slow call
1080             address slow_call_address  // Address of slow call
1081     )
1082 {
1083 
1084   Node* ctrl = alloc->in(TypeFunc::Control);
1085   Node* mem  = alloc->in(TypeFunc::Memory);
1086   Node* i_o  = alloc->in(TypeFunc::I_O);
1087   Node* size_in_bytes     = alloc->in(AllocateNode::AllocSize);
1088   Node* klass_node        = alloc->in(AllocateNode::KlassNode);
1089   Node* initial_slow_test = alloc->in(AllocateNode::InitialTest);
1090 
1091   Node* storestore = alloc->storestore();
1092   if (storestore != NULL) {
1093     // Break this link that is no longer useful and confuses register allocation
1094     storestore->set_req(MemBarNode::Precedent, top());
1095   }
1096 
1097   assert(ctrl != NULL, "must have control");
1098   // We need a Region and corresponding Phi's to merge the slow-path and fast-path results.
1099   // they will not be used if "always_slow" is set
1100   enum { slow_result_path = 1, fast_result_path = 2 };
1101   Node *result_region;
1102   Node *result_phi_rawmem;
1103   Node *result_phi_rawoop;
1104   Node *result_phi_i_o;
1105 
1106   // The initial slow comparison is a size check, the comparison
1107   // we want to do is a BoolTest::gt
1108   bool always_slow = false;
1109   int tv = _igvn.find_int_con(initial_slow_test, -1);
1110   if (tv >= 0) {
1111     always_slow = (tv == 1);
1112     initial_slow_test = NULL;
1113   } else {
1114     initial_slow_test = BoolNode::make_predicate(initial_slow_test, &_igvn);
1115   }
1116 


1278 
1279       // Bump total allocated bytes for this thread
1280       Node* thread = new (C, 1) ThreadLocalNode();
1281       transform_later(thread);
1282       Node* alloc_bytes_adr = basic_plus_adr(top()/*not oop*/, thread,
1283                                              in_bytes(JavaThread::allocated_bytes_offset()));
1284       Node* alloc_bytes = make_load(fast_oop_ctrl, store_eden_top, alloc_bytes_adr,
1285                                     0, TypeLong::LONG, T_LONG);
1286 #ifdef _LP64
1287       Node* alloc_size = size_in_bytes;
1288 #else
1289       Node* alloc_size = new (C, 2) ConvI2LNode(size_in_bytes);
1290       transform_later(alloc_size);
1291 #endif
1292       Node* new_alloc_bytes = new (C, 3) AddLNode(alloc_bytes, alloc_size);
1293       transform_later(new_alloc_bytes);
1294       fast_oop_rawmem = make_store(fast_oop_ctrl, store_eden_top, alloc_bytes_adr,
1295                                    0, new_alloc_bytes, T_LONG);
1296     }
1297 
1298     InitializeNode* init = alloc->initialization();
1299     fast_oop_rawmem = initialize_object(alloc,
1300                                         fast_oop_ctrl, fast_oop_rawmem, fast_oop,
1301                                         klass_node, length, size_in_bytes);
1302 
1303     // If initialization is performed by an array copy, any required
1304     // MemBarStoreStore was already added. If the object does not
1305     // escape no need for a MemBarStoreStore. Otherwise we need a
1306     // MemBarStoreStore so that stores that initialize this object
1307     // can't be reordered with a subsequent store that makes this
1308     // object accessible by other threads.
1309     if (init == NULL || (!init->is_complete_with_arraycopy() && !init->does_not_escape())) {
1310       if (init == NULL || init->req() < InitializeNode::RawStores) {
1311         // No InitializeNode or no stores captured by zeroing
1312         // elimination. Simply add the MemBarStoreStore after object
1313         // initialization.
1314         MemBarNode* mb = MemBarNode::make(C, Op_MemBarStoreStore, Compile::AliasIdxBot, fast_oop_rawmem);
1315         transform_later(mb);
1316  
1317         mb->init_req(TypeFunc::Memory, fast_oop_rawmem);
1318         mb->init_req(TypeFunc::Control, fast_oop_ctrl);
1319         fast_oop_ctrl = new (C, 1) ProjNode(mb,TypeFunc::Control);
1320         transform_later(fast_oop_ctrl);
1321         fast_oop_rawmem = new (C, 1) ProjNode(mb,TypeFunc::Memory);
1322         transform_later(fast_oop_rawmem);
1323       } else {
1324         // Add the MemBarStoreStore after the InitializeNode so that
1325         // all stores performing the initialization that were moved
1326         // before the InitializeNode happen before the storestore
1327         // barrier.
1328 
1329         Node* init_ctrl = init->proj_out(TypeFunc::Control);
1330         Node* init_mem = init->proj_out(TypeFunc::Memory);
1331 
1332         MemBarNode* mb = MemBarNode::make(C, Op_MemBarStoreStore, Compile::AliasIdxBot);
1333         transform_later(mb);
1334 
1335         Node* ctrl = new (C, 1) ProjNode(init,TypeFunc::Control);
1336         transform_later(ctrl);
1337         Node* mem = new (C, 1) ProjNode(init,TypeFunc::Memory);
1338         transform_later(mem);
1339 
1340         // The MemBarStoreStore depends on control and memory coming
1341         // from the InitializeNode
1342         mb->init_req(TypeFunc::Memory, mem);
1343         mb->init_req(TypeFunc::Control, ctrl);
1344      
1345         ctrl = new (C, 1) ProjNode(mb,TypeFunc::Control);
1346         transform_later(ctrl);
1347         mem = new (C, 1) ProjNode(mb,TypeFunc::Memory);
1348         transform_later(mem);
1349 
1350         // All nodes that depended on the InitializeNode for control
1351         // and memory must now depend on the MemBarNode that itself
1352         // depends on the InitializeNode
1353         _igvn.replace_node(init_ctrl, ctrl);
1354         _igvn.replace_node(init_mem, mem);
1355       }
1356     }
1357 
1358     if (C->env()->dtrace_extended_probes()) {
1359       // Slow-path call
1360       int size = TypeFunc::Parms + 2;
1361       CallLeafNode *call = new (C, size) CallLeafNode(OptoRuntime::dtrace_object_alloc_Type(),
1362                                                       CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_object_alloc_base),
1363                                                       "dtrace_object_alloc",
1364                                                       TypeRawPtr::BOTTOM);
1365 
1366       // Get base of thread-local storage area
1367       Node* thread = new (C, 1) ThreadLocalNode();
1368       transform_later(thread);
1369 
1370       call->init_req(TypeFunc::Parms+0, thread);
1371       call->init_req(TypeFunc::Parms+1, fast_oop);
1372       call->init_req(TypeFunc::Control, fast_oop_ctrl);
1373       call->init_req(TypeFunc::I_O    , top()); // does no i/o
1374       call->init_req(TypeFunc::Memory , fast_oop_rawmem);
1375       call->init_req(TypeFunc::ReturnAdr, alloc->in(TypeFunc::ReturnAdr));
1376       call->init_req(TypeFunc::FramePtr, alloc->in(TypeFunc::FramePtr));