< prev index next >

src/share/vm/opto/library_call.cpp

Print this page




 190   Node* generate_objArray_guard(Node* kls, RegionNode* region) {
 191     return generate_array_guard_common(kls, region, true, false);
 192   }
 193   Node* generate_non_objArray_guard(Node* kls, RegionNode* region) {
 194     return generate_array_guard_common(kls, region, true, true);
 195   }
 196   Node* generate_array_guard_common(Node* kls, RegionNode* region,
 197                                     bool obj_array, bool not_array);
 198   Node* generate_virtual_guard(Node* obj_klass, RegionNode* slow_region);
 199   CallJavaNode* generate_method_call(vmIntrinsics::ID method_id,
 200                                      bool is_virtual = false, bool is_static = false);
 201   CallJavaNode* generate_method_call_static(vmIntrinsics::ID method_id) {
 202     return generate_method_call(method_id, false, true);
 203   }
 204   CallJavaNode* generate_method_call_virtual(vmIntrinsics::ID method_id) {
 205     return generate_method_call(method_id, true, false);
 206   }
 207   Node * load_field_from_object(Node * fromObj, const char * fieldName, const char * fieldTypeString, bool is_exact, bool is_static, ciInstanceKlass * fromKls);
 208   Node * field_address_from_object(Node * fromObj, const char * fieldName, const char * fieldTypeString, bool is_exact, bool is_static, ciInstanceKlass * fromKls);
 209 
 210   Node* make_string_method_node(int opcode, Node* str1_start, Node* cnt1, Node* str2_start, Node* cnt2, StrIntrinsicNode::ArgEnc ae);
 211   bool inline_string_compareTo(StrIntrinsicNode::ArgEnc ae);
 212   bool inline_string_indexOf(StrIntrinsicNode::ArgEnc ae);
 213   bool inline_string_indexOfI(StrIntrinsicNode::ArgEnc ae);
 214   Node* make_indexOf_node(Node* src_start, Node* src_count, Node* tgt_start, Node* tgt_count,
 215                           RegionNode* region, Node* phi, StrIntrinsicNode::ArgEnc ae);
 216   bool inline_string_indexOfChar();
 217   bool inline_string_equals(StrIntrinsicNode::ArgEnc ae);
 218   bool inline_string_toBytesU();
 219   bool inline_string_getCharsU();
 220   bool inline_string_copy(bool compress);
 221   bool inline_string_char_access(bool is_store);
 222   Node* round_double_node(Node* n);
 223   bool runtime_math(const TypeFunc* call_type, address funcAddr, const char* funcName);
 224   bool inline_math_native(vmIntrinsics::ID id);
 225   bool inline_trig(vmIntrinsics::ID id);
 226   bool inline_math(vmIntrinsics::ID id);
 227   template <typename OverflowOp>
 228   bool inline_math_overflow(Node* arg1, Node* arg2);
 229   void inline_math_mathExact(Node* math, Node* test);
 230   bool inline_math_addExactI(bool is_increment);


 463     } else {
 464       // Root compile
 465       C->print_inlining_stream()->print("Did not generate predicate for intrinsic %s%s at bci:%d in",
 466                                         vmIntrinsics::name_at(intrinsic_id()),
 467                                         (is_virtual() ? " (virtual)" : ""), bci);
 468     }
 469   }
 470   C->gather_intrinsic_statistics(intrinsic_id(), is_virtual(), Compile::_intrinsic_failed);
 471   return NULL;
 472 }
 473 
 474 bool LibraryCallKit::try_to_inline(int predicate) {
 475   // Handle symbolic names for otherwise undistinguished boolean switches:
 476   const bool is_store       = true;
 477   const bool is_compress    = true;
 478   const bool is_static      = true;
 479   const bool is_volatile    = true;
 480 
 481   if (!jvms()->has_method()) {
 482     // Root JVMState has a null method.
 483     assert(map()->memory()->Opcode() == Op_Parm, "");
 484     // Insert the memory aliasing node
 485     set_all_memory(reset_memory());
 486   }
 487   assert(merged_memory(), "");
 488 
 489 
 490   switch (intrinsic_id()) {
 491   case vmIntrinsics::_hashCode:                 return inline_native_hashcode(intrinsic()->is_virtual(), !is_static);
 492   case vmIntrinsics::_identityHashCode:         return inline_native_hashcode(/*!virtual*/ false,         is_static);
 493   case vmIntrinsics::_getClass:                 return inline_native_getClass();
 494 
 495   case vmIntrinsics::_dsin:
 496   case vmIntrinsics::_dcos:
 497   case vmIntrinsics::_dtan:
 498   case vmIntrinsics::_dabs:
 499   case vmIntrinsics::_datan2:
 500   case vmIntrinsics::_dsqrt:
 501   case vmIntrinsics::_dexp:
 502   case vmIntrinsics::_dlog:
 503   case vmIntrinsics::_dlog10:


 825 
 826   case vmIntrinsics::_hasNegatives:
 827     return inline_hasNegatives();
 828 
 829   default:
 830     // If you get here, it may be that someone has added a new intrinsic
 831     // to the list in vmSymbols.hpp without implementing it here.
 832 #ifndef PRODUCT
 833     if ((PrintMiscellaneous && (Verbose || WizardMode)) || PrintOpto) {
 834       tty->print_cr("*** Warning: Unimplemented intrinsic %s(%d)",
 835                     vmIntrinsics::name_at(intrinsic_id()), intrinsic_id());
 836     }
 837 #endif
 838     return false;
 839   }
 840 }
 841 
 842 Node* LibraryCallKit::try_to_predicate(int predicate) {
 843   if (!jvms()->has_method()) {
 844     // Root JVMState has a null method.
 845     assert(map()->memory()->Opcode() == Op_Parm, "");
 846     // Insert the memory aliasing node
 847     set_all_memory(reset_memory());
 848   }
 849   assert(merged_memory(), "");
 850 
 851   switch (intrinsic_id()) {
 852   case vmIntrinsics::_cipherBlockChaining_encryptAESCrypt:
 853     return inline_cipherBlockChaining_AESCrypt_predicate(false);
 854   case vmIntrinsics::_cipherBlockChaining_decryptAESCrypt:
 855     return inline_cipherBlockChaining_AESCrypt_predicate(true);
 856   case vmIntrinsics::_counterMode_AESCrypt:
 857     return inline_counterMode_AESCrypt_predicate();
 858   case vmIntrinsics::_digestBase_implCompressMB:
 859     return inline_digestBase_implCompressMB_predicate(predicate);
 860 
 861   default:
 862     // If you get here, it may be that someone has added a new intrinsic
 863     // to the list in vmSymbols.hpp without implementing it here.
 864 #ifndef PRODUCT
 865     if ((PrintMiscellaneous && (Verbose || WizardMode)) || PrintOpto) {


1005   }
1006 }
1007 
1008 //--------------------------generate_current_thread--------------------
1009 Node* LibraryCallKit::generate_current_thread(Node* &tls_output) {
1010   ciKlass*    thread_klass = env()->Thread_klass();
1011   const Type* thread_type  = TypeOopPtr::make_from_klass(thread_klass)->cast_to_ptr_type(TypePtr::NotNull);
1012   Node* thread = _gvn.transform(new ThreadLocalNode());
1013   Node* p = basic_plus_adr(top()/*!oop*/, thread, in_bytes(JavaThread::threadObj_offset()));
1014   Node* threadObj = make_load(NULL, p, thread_type, T_OBJECT, MemNode::unordered);
1015   tls_output = thread;
1016   return threadObj;
1017 }
1018 
1019 
1020 //------------------------------make_string_method_node------------------------
1021 // Helper method for String intrinsic functions. This version is called with
1022 // str1 and str2 pointing to byte[] nodes containing Latin1 or UTF16 encoded
1023 // characters (depending on 'is_byte'). cnt1 and cnt2 are pointing to Int nodes
1024 // containing the lengths of str1 and str2.
1025 Node* LibraryCallKit::make_string_method_node(int opcode, Node* str1_start, Node* cnt1, Node* str2_start, Node* cnt2, StrIntrinsicNode::ArgEnc ae) {
1026   Node* result = NULL;
1027   switch (opcode) {
1028   case Op_StrIndexOf:
1029     result = new StrIndexOfNode(control(), memory(TypeAryPtr::BYTES),
1030                                 str1_start, cnt1, str2_start, cnt2, ae);
1031     break;
1032   case Op_StrComp:
1033     result = new StrCompNode(control(), memory(TypeAryPtr::BYTES),
1034                              str1_start, cnt1, str2_start, cnt2, ae);
1035     break;
1036   case Op_StrEquals:
1037     // We already know that cnt1 == cnt2 here (checked in 'inline_string_equals').
1038     // Use the constant length if there is one because optimized match rule may exist.
1039     result = new StrEqualsNode(control(), memory(TypeAryPtr::BYTES),
1040                                str1_start, str2_start, cnt2->is_Con() ? cnt2 : cnt1, ae);
1041     break;
1042   default:
1043     ShouldNotReachHere();
1044     return NULL;
1045   }
1046 
1047   // All these intrinsics have checks.
1048   C->set_has_split_ifs(true); // Has chance for split-if optimization
1049 
1050   return _gvn.transform(result);
1051 }
1052 
1053 //------------------------------inline_string_compareTo------------------------
1054 bool LibraryCallKit::inline_string_compareTo(StrIntrinsicNode::ArgEnc ae) {
1055   Node* arg1 = argument(0);
1056   Node* arg2 = argument(1);
1057 
1058   // Get start addr and length of first argument
1059   Node* arg1_start  = array_element_address(arg1, intcon(0), T_BYTE);
1060   Node* arg1_cnt    = load_array_length(arg1);
1061 
1062   // Get start addr and length of second argument
1063   Node* arg2_start  = array_element_address(arg2, intcon(0), T_BYTE);
1064   Node* arg2_cnt    = load_array_length(arg2);
1065 
1066   Node* result = make_string_method_node(Op_StrComp, arg1_start, arg1_cnt, arg2_start, arg2_cnt, ae);
1067   set_result(result);
1068   return true;
1069 }
1070 
1071 //------------------------------inline_string_equals------------------------
1072 bool LibraryCallKit::inline_string_equals(StrIntrinsicNode::ArgEnc ae) {
1073   Node* arg1 = argument(0);
1074   Node* arg2 = argument(1);
1075 
1076   // paths (plus control) merge
1077   RegionNode* region = new RegionNode(3);
1078   Node* phi = new PhiNode(region, TypeInt::BOOL);
1079 
1080   if (!stopped()) {
1081     // Get start addr and length of first argument
1082     Node* arg1_start  = array_element_address(arg1, intcon(0), T_BYTE);
1083     Node* arg1_cnt    = load_array_length(arg1);
1084 
1085     // Get start addr and length of second argument
1086     Node* arg2_start  = array_element_address(arg2, intcon(0), T_BYTE);
1087     Node* arg2_cnt    = load_array_length(arg2);
1088 
1089     // Check for arg1_cnt != arg2_cnt
1090     Node* cmp = _gvn.transform(new CmpINode(arg1_cnt, arg2_cnt));
1091     Node* bol = _gvn.transform(new BoolNode(cmp, BoolTest::ne));
1092     Node* if_ne = generate_slow_guard(bol, NULL);
1093     if (if_ne != NULL) {
1094       phi->init_req(2, intcon(0));
1095       region->init_req(2, if_ne);
1096     }
1097 
1098     // Check for count == 0 is done by assembler code for StrEquals.
1099 
1100     if (!stopped()) {
1101       Node* equals = make_string_method_node(Op_StrEquals, arg1_start, arg1_cnt, arg2_start, arg2_cnt, ae);
1102       phi->init_req(1, equals);
1103       region->init_req(1, control());
1104     }
1105   }
1106 
1107   // post merge
1108   set_control(_gvn.transform(region));
1109   record_for_igvn(region);
1110 
1111   set_result(_gvn.transform(phi));
1112   return true;
1113 }
1114 
1115 //------------------------------inline_array_equals----------------------------
1116 bool LibraryCallKit::inline_array_equals(StrIntrinsicNode::ArgEnc ae) {
1117   assert(ae == StrIntrinsicNode::UU || ae == StrIntrinsicNode::LL, "unsupported array types");
1118   Node* arg1 = argument(0);
1119   Node* arg2 = argument(1);
1120 
1121   const TypeAryPtr* mtype = (ae == StrIntrinsicNode::UU) ? TypeAryPtr::CHARS : TypeAryPtr::BYTES;


1179     PreserveJVMState pjvms(this);
1180     set_control(_gvn.transform(new IfFalseNode(rc)));
1181     uncommon_trap(Deoptimization::Reason_range_check,
1182                   Deoptimization::Action_make_not_entrant);
1183   }
1184 
1185   if (stopped()) {
1186     return false;
1187   }
1188 
1189   Node* result = new CastIINode(index, TypeInt::make(0, _gvn.type(length)->is_int()->_hi, Type::WidenMax));
1190   result->set_req(0, control());
1191   result = _gvn.transform(result);
1192   set_result(result);
1193   replace_in_map(index, result);
1194   return true;
1195 }
1196 
1197 //------------------------------inline_string_indexOf------------------------
1198 bool LibraryCallKit::inline_string_indexOf(StrIntrinsicNode::ArgEnc ae) {
1199   if (!Matcher::match_rule_supported(Op_StrIndexOf)) {
1200     return false;
1201   }
1202   Node* src = argument(0);
1203   Node* tgt = argument(1);
1204 
1205   // Make the merge point
1206   RegionNode* result_rgn = new RegionNode(4);
1207   Node*       result_phi = new PhiNode(result_rgn, TypeInt::INT);
1208 
1209   // Get start addr and length of source string
1210   Node* src_start = array_element_address(src, intcon(0), T_BYTE);
1211   Node* src_count = load_array_length(src);
1212 
1213   // Get start addr and length of substring
1214   Node* tgt_start = array_element_address(tgt, intcon(0), T_BYTE);
1215   Node* tgt_count = load_array_length(tgt);
1216 
1217   if (ae == StrIntrinsicNode::UU || ae == StrIntrinsicNode::UL) {
1218     // Divide src size by 2 if String is UTF16 encoded
1219     src_count = _gvn.transform(new RShiftINode(src_count, intcon(1)));


1223     tgt_count = _gvn.transform(new RShiftINode(tgt_count, intcon(1)));
1224   }
1225 
1226   Node* result = make_indexOf_node(src_start, src_count, tgt_start, tgt_count, result_rgn, result_phi, ae);
1227   if (result != NULL) {
1228     result_phi->init_req(3, result);
1229     result_rgn->init_req(3, control());
1230   }
1231   set_control(_gvn.transform(result_rgn));
1232   record_for_igvn(result_rgn);
1233   set_result(_gvn.transform(result_phi));
1234 
1235   return true;
1236 }
1237 
1238 //-----------------------------inline_string_indexOf-----------------------
1239 bool LibraryCallKit::inline_string_indexOfI(StrIntrinsicNode::ArgEnc ae) {
1240   if (too_many_traps(Deoptimization::Reason_intrinsic)) {
1241     return false;
1242   }
1243   if (!Matcher::match_rule_supported(Op_StrIndexOf)) {
1244     return false;
1245   }
1246   assert(callee()->signature()->size() == 5, "String.indexOf() has 5 arguments");
1247   Node* src         = argument(0); // byte[]
1248   Node* src_count   = argument(1); // char count
1249   Node* tgt         = argument(2); // byte[]
1250   Node* tgt_count   = argument(3); // char count
1251   Node* from_index  = argument(4); // char index
1252 
1253   // Multiply byte array index by 2 if String is UTF16 encoded
1254   Node* src_offset = (ae == StrIntrinsicNode::LL) ? from_index : _gvn.transform(new LShiftINode(from_index, intcon(1)));
1255   src_count = _gvn.transform(new SubINode(src_count, from_index));
1256   Node* src_start = array_element_address(src, src_offset, T_BYTE);
1257   Node* tgt_start = array_element_address(tgt, intcon(0), T_BYTE);
1258 
1259   // Range checks
1260   generate_string_range_check(src, src_offset, src_count, ae != StrIntrinsicNode::LL);
1261   generate_string_range_check(tgt, intcon(0), tgt_count, ae == StrIntrinsicNode::UU);
1262   if (stopped()) {
1263     return true;


1298                                         RegionNode* region, Node* phi, StrIntrinsicNode::ArgEnc ae) {
1299   // Check for substr count > string count
1300   Node* cmp = _gvn.transform(new CmpINode(tgt_count, src_count));
1301   Node* bol = _gvn.transform(new BoolNode(cmp, BoolTest::gt));
1302   Node* if_gt = generate_slow_guard(bol, NULL);
1303   if (if_gt != NULL) {
1304     phi->init_req(1, intcon(-1));
1305     region->init_req(1, if_gt);
1306   }
1307   if (!stopped()) {
1308     // Check for substr count == 0
1309     cmp = _gvn.transform(new CmpINode(tgt_count, intcon(0)));
1310     bol = _gvn.transform(new BoolNode(cmp, BoolTest::eq));
1311     Node* if_zero = generate_slow_guard(bol, NULL);
1312     if (if_zero != NULL) {
1313       phi->init_req(2, intcon(0));
1314       region->init_req(2, if_zero);
1315     }
1316   }
1317   if (!stopped()) {
1318     return make_string_method_node(Op_StrIndexOf, src_start, src_count, tgt_start, tgt_count, ae);
1319   }
1320   return NULL;
1321 }
1322 
1323 //-----------------------------inline_string_indexOfChar-----------------------
1324 bool LibraryCallKit::inline_string_indexOfChar() {
1325   if (too_many_traps(Deoptimization::Reason_intrinsic)) {
1326     return false;
1327   }
1328   if (!Matcher::match_rule_supported(Op_StrIndexOfChar)) {
1329     return false;
1330   }
1331   assert(callee()->signature()->size() == 4, "String.indexOfChar() has 4 arguments");
1332   Node* src         = argument(0); // byte[]
1333   Node* tgt         = argument(1); // tgt is int ch
1334   Node* from_index  = argument(2);
1335   Node* max         = argument(3);
1336 
1337   Node* src_offset = _gvn.transform(new LShiftINode(from_index, intcon(1)));
1338   Node* src_start = array_element_address(src, src_offset, T_BYTE);
1339   Node* src_count = _gvn.transform(new SubINode(max, from_index));
1340 
1341   // Range checks
1342   generate_string_range_check(src, src_offset, src_count, true);
1343   if (stopped()) {
1344     return true;
1345   }
1346 
1347   RegionNode* region = new RegionNode(3);
1348   Node* phi = new PhiNode(region, TypeInt::INT);


1428   } else {
1429     inflate_string(src_start, dst_start, TypeAryPtr::get_array_body_type(dst_elem), length);
1430   }
1431 
1432   if (alloc != NULL) {
1433     if (alloc->maybe_set_complete(&_gvn)) {
1434       // "You break it, you buy it."
1435       InitializeNode* init = alloc->initialization();
1436       assert(init->is_complete(), "we just did this");
1437       init->set_complete_with_arraycopy();
1438       assert(dst->is_CheckCastPP(), "sanity");
1439       assert(dst->in(0)->in(0) == init, "dest pinned");
1440     }
1441     // Do not let stores that initialize this object be reordered with
1442     // a subsequent store that would make this object accessible by
1443     // other threads.
1444     // Record what AllocateNode this StoreStore protects so that
1445     // escape analysis can go from the MemBarStoreStoreNode to the
1446     // AllocateNode and eliminate the MemBarStoreStoreNode if possible
1447     // based on the escape status of the AllocateNode.
1448     insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out(AllocateNode::RawAddress));
1449   }
1450   if (compress) {
1451     set_result(_gvn.transform(count));
1452   }
1453   return true;
1454 }
1455 
1456 #ifdef _LP64
1457 #define XTOP ,top() /*additional argument*/
1458 #else  //_LP64
1459 #define XTOP        /*no additional argument*/
1460 #endif //_LP64
1461 
1462 //------------------------inline_string_toBytesU--------------------------
1463 // public static byte[] StringUTF16.toBytes(char[] value, int off, int len)
1464 bool LibraryCallKit::inline_string_toBytesU() {
1465   if (too_many_traps(Deoptimization::Reason_intrinsic)) {
1466     return false;
1467   }
1468   // Get the arguments.


1520                       OptoRuntime::fast_arraycopy_Type(),
1521                       copyfunc_addr, copyfunc_name, TypeRawPtr::BOTTOM,
1522                       src_start, dst_start, ConvI2X(length) XTOP);
1523     // Do not let reads from the cloned object float above the arraycopy.
1524     if (alloc != NULL) {
1525       if (alloc->maybe_set_complete(&_gvn)) {
1526         // "You break it, you buy it."
1527         InitializeNode* init = alloc->initialization();
1528         assert(init->is_complete(), "we just did this");
1529         init->set_complete_with_arraycopy();
1530         assert(newcopy->is_CheckCastPP(), "sanity");
1531         assert(newcopy->in(0)->in(0) == init, "dest pinned");
1532       }
1533       // Do not let stores that initialize this object be reordered with
1534       // a subsequent store that would make this object accessible by
1535       // other threads.
1536       // Record what AllocateNode this StoreStore protects so that
1537       // escape analysis can go from the MemBarStoreStoreNode to the
1538       // AllocateNode and eliminate the MemBarStoreStoreNode if possible
1539       // based on the escape status of the AllocateNode.
1540       insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out(AllocateNode::RawAddress));
1541     } else {
1542       insert_mem_bar(Op_MemBarCPUOrder);
1543     }
1544   } // original reexecute is set back here
1545 
1546   C->set_has_split_ifs(true); // Has chance for split-if optimization
1547   if (!stopped()) {
1548     set_result(newcopy);
1549   }
1550   return true;
1551 }
1552 
1553 //------------------------inline_string_getCharsU--------------------------
1554 // public void StringUTF16.getChars(byte[] src, int srcBegin, int srcEnd, char dst[], int dstBegin)
1555 bool LibraryCallKit::inline_string_getCharsU() {
1556   if (too_many_traps(Deoptimization::Reason_intrinsic)) {
1557     return false;
1558   }
1559 
1560   // Get the arguments.
1561   Node* src       = argument(0);
1562   Node* src_begin = argument(1);


1604                       OptoRuntime::fast_arraycopy_Type(),
1605                       copyfunc_addr, copyfunc_name, TypeRawPtr::BOTTOM,
1606                       src_start, dst_start, ConvI2X(length) XTOP);
1607     // Do not let reads from the cloned object float above the arraycopy.
1608     if (alloc != NULL) {
1609       if (alloc->maybe_set_complete(&_gvn)) {
1610         // "You break it, you buy it."
1611         InitializeNode* init = alloc->initialization();
1612         assert(init->is_complete(), "we just did this");
1613         init->set_complete_with_arraycopy();
1614         assert(dst->is_CheckCastPP(), "sanity");
1615         assert(dst->in(0)->in(0) == init, "dest pinned");
1616       }
1617       // Do not let stores that initialize this object be reordered with
1618       // a subsequent store that would make this object accessible by
1619       // other threads.
1620       // Record what AllocateNode this StoreStore protects so that
1621       // escape analysis can go from the MemBarStoreStoreNode to the
1622       // AllocateNode and eliminate the MemBarStoreStoreNode if possible
1623       // based on the escape status of the AllocateNode.
1624       insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out(AllocateNode::RawAddress));
1625     } else {
1626       insert_mem_bar(Op_MemBarCPUOrder);
1627     }
1628   }
1629 
1630   C->set_has_split_ifs(true); // Has chance for split-if optimization
1631   return true;
1632 }
1633 
1634 //----------------------inline_string_char_access----------------------------
1635 // Store/Load char to/from byte[] array.
1636 // static void StringUTF16.putChar(byte[] val, int index, int c)
1637 // static char StringUTF16.getChar(byte[] val, int index)
1638 bool LibraryCallKit::inline_string_char_access(bool is_store) {
1639   Node* value  = argument(0);
1640   Node* index  = argument(1);
1641   Node* ch = is_store ? argument(2) : NULL;
1642 
1643   // This intrinsic accesses byte[] array as char[] array. Computing the offsets
1644   // correctly requires matched array shapes.
1645   assert (arrayOopDesc::base_offset_in_bytes(T_CHAR) == arrayOopDesc::base_offset_in_bytes(T_BYTE),
1646           "sanity: byte[] and char[] bases agree");


1812       runtime_math(OptoRuntime::Math_D_D_Type(), StubRoutines::dsin(), "dsin") :
1813       runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dsin),   "SIN");
1814   case vmIntrinsics::_dcos:
1815     return StubRoutines::dcos() != NULL ?
1816       runtime_math(OptoRuntime::Math_D_D_Type(), StubRoutines::dcos(), "dcos") :
1817       runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dcos),   "COS");
1818   case vmIntrinsics::_dtan:
1819     return StubRoutines::dtan() != NULL ?
1820       runtime_math(OptoRuntime::Math_D_D_Type(), StubRoutines::dtan(), "dtan") :
1821       runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dtan), "TAN");
1822   case vmIntrinsics::_dlog:
1823     return StubRoutines::dlog() != NULL ?
1824       runtime_math(OptoRuntime::Math_D_D_Type(), StubRoutines::dlog(), "dlog") :
1825       runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dlog),   "LOG");
1826   case vmIntrinsics::_dlog10:
1827     return StubRoutines::dlog10() != NULL ?
1828       runtime_math(OptoRuntime::Math_D_D_Type(), StubRoutines::dlog10(), "dlog10") :
1829       runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dlog10), "LOG10");
1830 
1831     // These intrinsics are supported on all hardware
1832   case vmIntrinsics::_dsqrt:  return Matcher::match_rule_supported(Op_SqrtD) ? inline_math(id) : false;
1833   case vmIntrinsics::_dabs:   return Matcher::has_match_rule(Op_AbsD)   ? inline_math(id) : false;
1834 
1835   case vmIntrinsics::_dexp:
1836     return StubRoutines::dexp() != NULL ?
1837       runtime_math(OptoRuntime::Math_D_D_Type(), StubRoutines::dexp(),  "dexp") :
1838       runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dexp),  "EXP");
1839   case vmIntrinsics::_dpow:
1840     return StubRoutines::dpow() != NULL ?
1841       runtime_math(OptoRuntime::Math_DD_D_Type(), StubRoutines::dpow(), "dpow") :
1842       runtime_math(OptoRuntime::Math_DD_D_Type(), FN_PTR(SharedRuntime::dpow),  "POW");
1843 #undef FN_PTR
1844 
1845    // These intrinsics are not yet correctly implemented
1846   case vmIntrinsics::_datan2:
1847     return false;
1848 
1849   default:
1850     fatal_unexpected_iid(id);
1851     return false;
1852   }
1853 }


1957   }
1958 
1959   bool want_max = (id == vmIntrinsics::_max);
1960 
1961   const TypeInt* txvalue = _gvn.type(xvalue)->isa_int();
1962   const TypeInt* tyvalue = _gvn.type(yvalue)->isa_int();
1963   if (txvalue == NULL || tyvalue == NULL)  return top();
1964   // This is not really necessary, but it is consistent with a
1965   // hypothetical MaxINode::Value method:
1966   int widen = MAX2(txvalue->_widen, tyvalue->_widen);
1967 
1968   // %%% This folding logic should (ideally) be in a different place.
1969   // Some should be inside IfNode, and there to be a more reliable
1970   // transformation of ?: style patterns into cmoves.  We also want
1971   // more powerful optimizations around cmove and min/max.
1972 
1973   // Try to find a dominating comparison of these guys.
1974   // It can simplify the index computation for Arrays.copyOf
1975   // and similar uses of System.arraycopy.
1976   // First, compute the normalized version of CmpI(x, y).
1977   int   cmp_op = Op_CmpI;
1978   Node* xkey = xvalue;
1979   Node* ykey = yvalue;
1980   Node* ideal_cmpxy = _gvn.transform(new CmpINode(xkey, ykey));
1981   if (ideal_cmpxy->is_Cmp()) {
1982     // E.g., if we have CmpI(length - offset, count),
1983     // it might idealize to CmpI(length, count + offset)
1984     cmp_op = ideal_cmpxy->Opcode();
1985     xkey = ideal_cmpxy->in(1);
1986     ykey = ideal_cmpxy->in(2);
1987   }
1988 
1989   // Start by locating any relevant comparisons.
1990   Node* start_from = (xkey->outcnt() < ykey->outcnt()) ? xkey : ykey;
1991   Node* cmpxy = NULL;
1992   Node* cmpyx = NULL;
1993   for (DUIterator_Fast kmax, k = start_from->fast_outs(kmax); k < kmax; k++) {
1994     Node* cmp = start_from->fast_out(k);
1995     if (cmp->outcnt() > 0 &&            // must have prior uses
1996         cmp->in(0) == NULL &&           // must be context-independent
1997         cmp->Opcode() == cmp_op) {      // right kind of compare


2270 
2271       // Update IdealKit memory and control from graphKit.
2272       __ sync_kit(this);
2273 
2274       Node* one = __ ConI(1);
2275       // is_instof == 0 if base_oop == NULL
2276       __ if_then(is_instof, BoolTest::eq, one, unlikely); {
2277 
2278         // Update graphKit from IdeakKit.
2279         sync_kit(ideal);
2280 
2281         // Use the pre-barrier to record the value in the referent field
2282         pre_barrier(false /* do_load */,
2283                     __ ctrl(),
2284                     NULL /* obj */, NULL /* adr */, max_juint /* alias_idx */, NULL /* val */, NULL /* val_type */,
2285                     pre_val /* pre_val */,
2286                     T_OBJECT);
2287         if (need_mem_bar) {
2288           // Add memory barrier to prevent commoning reads from this field
2289           // across safepoint since GC can change its value.
2290           insert_mem_bar(Op_MemBarCPUOrder);
2291         }
2292         // Update IdealKit from graphKit.
2293         __ sync_kit(this);
2294 
2295       } __ end_if(); // _ref_type != ref_none
2296   } __ end_if(); // offset == referent_offset
2297 
2298   // Final sync IdealKit and GraphKit.
2299   final_sync(ideal);
2300 #undef __
2301 }
2302 
2303 
2304 const TypeOopPtr* LibraryCallKit::sharpen_unsafe_type(Compile::AliasType* alias_type, const TypePtr *adr_type) {
2305   // Attempt to infer a sharper value type from the offset and base type.
2306   ciKlass* sharpened_klass = NULL;
2307 
2308   // See if it is an instance field, with an object type.
2309   if (alias_type->field() != NULL) {
2310     if (alias_type->field()->type()->is_klass()) {


2495   // Heap pointers get a null-check from the interpreter,
2496   // as a courtesy.  However, this is not guaranteed by Unsafe,
2497   // and it is not possible to fully distinguish unintended nulls
2498   // from intended ones in this API.
2499 
2500   // We need to emit leading and trailing CPU membars (see below) in
2501   // addition to memory membars for special access modes. This is a little
2502   // too strong, but avoids the need to insert per-alias-type
2503   // volatile membars (for stores; compare Parse::do_put_xxx), which
2504   // we cannot do effectively here because we probably only have a
2505   // rough approximation of type.
2506 
2507   switch(kind) {
2508     case Relaxed:
2509     case Opaque:
2510     case Acquire:
2511       break;
2512     case Release:
2513     case Volatile:
2514       if (is_store) {
2515         insert_mem_bar(Op_MemBarRelease);
2516       } else {
2517         if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
2518           insert_mem_bar(Op_MemBarVolatile);
2519         }
2520       }
2521       break;
2522     default:
2523       ShouldNotReachHere();
2524   }
2525 
2526   // Memory barrier to prevent normal and 'unsafe' accesses from
2527   // bypassing each other.  Happens after null checks, so the
2528   // exception paths do not take memory state from the memory barrier,
2529   // so there's no problems making a strong assert about mixing users
2530   // of safe & unsafe memory.
2531   if (need_mem_bar) insert_mem_bar(Op_MemBarCPUOrder);
2532 
2533   if (!is_store) {
2534     Node* p = NULL;
2535     // Try to constant fold a load from a constant field
2536     ciField* field = alias_type->field();
2537     if (heap_base_oop != top() && field != NULL && field->is_constant() && !mismatched) {
2538       // final or stable field
2539       p = make_constant_from_field(field, heap_base_oop);
2540     }
2541     if (p == NULL) {
2542       // To be valid, unsafe loads may depend on other conditions than
2543       // the one that guards them: pin the Load node
2544       p = make_load(control(), adr, value_type, type, adr_type, mo, LoadNode::Pinned, requires_atomic_access, unaligned, mismatched);
2545       // load value
2546       switch (type) {
2547       case T_BOOLEAN:
2548       case T_CHAR:
2549       case T_BYTE:
2550       case T_SHORT:
2551       case T_INT:


2610           // Update IdealKit memory.
2611           __ sync_kit(this);
2612         } __ else_(); {
2613           __ store(__ ctrl(), adr, val, type, alias_type->index(), mo, requires_atomic_access, mismatched);
2614         } __ end_if();
2615         // Final sync IdealKit and GraphKit.
2616         final_sync(ideal);
2617 #undef __
2618       }
2619     }
2620   }
2621 
2622   switch(kind) {
2623     case Relaxed:
2624     case Opaque:
2625     case Release:
2626       break;
2627     case Acquire:
2628     case Volatile:
2629       if (!is_store) {
2630         insert_mem_bar(Op_MemBarAcquire);
2631       } else {
2632         if (!support_IRIW_for_not_multiple_copy_atomic_cpu) {
2633           insert_mem_bar(Op_MemBarVolatile);
2634         }
2635       }
2636       break;
2637     default:
2638       ShouldNotReachHere();
2639   }
2640 
2641   if (need_mem_bar) insert_mem_bar(Op_MemBarCPUOrder);
2642 
2643   return true;
2644 }
2645 
2646 //----------------------------inline_unsafe_load_store----------------------------
2647 // This method serves a couple of different customers (depending on LoadStoreKind):
2648 //
2649 // LS_cmp_swap:
2650 //
2651 //   boolean compareAndSwapObject(Object o, long offset, Object expected, Object x);
2652 //   boolean compareAndSwapInt(   Object o, long offset, int    expected, int    x);
2653 //   boolean compareAndSwapLong(  Object o, long offset, long   expected, long   x);
2654 //
2655 // LS_cmp_swap_weak:
2656 //
2657 //   boolean weakCompareAndSwapObject(       Object o, long offset, Object expected, Object x);
2658 //   boolean weakCompareAndSwapObjectAcquire(Object o, long offset, Object expected, Object x);
2659 //   boolean weakCompareAndSwapObjectRelease(Object o, long offset, Object expected, Object x);
2660 //
2661 //   boolean weakCompareAndSwapInt(          Object o, long offset, int    expected, int    x);


2827     case LS_cmp_swap:
2828     case LS_cmp_swap_weak:
2829     case LS_get_add:
2830       break;
2831     default:
2832       ShouldNotReachHere();
2833   }
2834 
2835   int alias_idx = C->get_alias_index(adr_type);
2836 
2837   // Memory-model-wise, a LoadStore acts like a little synchronized
2838   // block, so needs barriers on each side.  These don't translate
2839   // into actual barriers on most machines, but we still need rest of
2840   // compiler to respect ordering.
2841 
2842   switch (access_kind) {
2843     case Relaxed:
2844     case Acquire:
2845       break;
2846     case Release:
2847       insert_mem_bar(Op_MemBarRelease);
2848       break;
2849     case Volatile:
2850       if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
2851         insert_mem_bar(Op_MemBarVolatile);
2852       } else {
2853         insert_mem_bar(Op_MemBarRelease);
2854       }
2855       break;
2856     default:
2857       ShouldNotReachHere();
2858   }
2859   insert_mem_bar(Op_MemBarCPUOrder);
2860 
2861   // Figure out the memory ordering.
2862   MemNode::MemOrd mo = access_kind_to_memord(access_kind);
2863 
2864   // 4984716: MemBars must be inserted before this
2865   //          memory node in order to avoid a false
2866   //          dependency which will confuse the scheduler.
2867   Node *mem = memory(alias_idx);
2868 
2869   // For now, we handle only those cases that actually exist: ints,
2870   // longs, and Object. Adding others should be straightforward.
2871   Node* load_store = NULL;
2872   switch(type) {
2873   case T_BYTE:
2874     switch(kind) {
2875       case LS_get_add:
2876         load_store = _gvn.transform(new GetAndAddBNode(control(), mem, adr, newval, adr_type));
2877         break;
2878       case LS_get_set:
2879         load_store = _gvn.transform(new GetAndSetBNode(control(), mem, adr, newval, adr_type));


3079   set_memory(proj, alias_idx);
3080 
3081   if (type == T_OBJECT && (kind == LS_get_set || kind == LS_cmp_exchange)) {
3082 #ifdef _LP64
3083     if (adr->bottom_type()->is_ptr_to_narrowoop()) {
3084       load_store = _gvn.transform(new DecodeNNode(load_store, load_store->get_ptr_type()));
3085     }
3086 #endif
3087     if (can_move_pre_barrier()) {
3088       // Don't need to load pre_val. The old value is returned by load_store.
3089       // The pre_barrier can execute after the xchg as long as no safepoint
3090       // gets inserted between them.
3091       pre_barrier(false /* do_load */,
3092                   control(), NULL, NULL, max_juint, NULL, NULL,
3093                   load_store /* pre_val */,
3094                   T_OBJECT);
3095     }
3096   }
3097 
3098   // Add the trailing membar surrounding the access
3099   insert_mem_bar(Op_MemBarCPUOrder);
3100 
3101   switch (access_kind) {
3102     case Relaxed:
3103     case Release:
3104       break; // do nothing
3105     case Acquire:
3106     case Volatile:
3107       insert_mem_bar(Op_MemBarAcquire);
3108       // !support_IRIW_for_not_multiple_copy_atomic_cpu handled in platform code
3109       break;
3110     default:
3111       ShouldNotReachHere();
3112   }
3113 
3114   assert(type2size[load_store->bottom_type()->basic_type()] == type2size[rtype], "result type should match");
3115   set_result(load_store);
3116   return true;
3117 }
3118 
3119 MemNode::MemOrd LibraryCallKit::access_kind_to_memord_LS(AccessKind kind, bool is_store) {
3120   MemNode::MemOrd mo = MemNode::unset;
3121   switch(kind) {
3122     case Opaque:
3123     case Relaxed:  mo = MemNode::unordered; break;
3124     case Acquire:  mo = MemNode::acquire;   break;
3125     case Release:  mo = MemNode::release;   break;
3126     case Volatile: mo = is_store ? MemNode::release : MemNode::acquire; break;
3127     default:


3132 }
3133 
3134 MemNode::MemOrd LibraryCallKit::access_kind_to_memord(AccessKind kind) {
3135   MemNode::MemOrd mo = MemNode::unset;
3136   switch(kind) {
3137     case Opaque:
3138     case Relaxed:  mo = MemNode::unordered; break;
3139     case Acquire:  mo = MemNode::acquire;   break;
3140     case Release:  mo = MemNode::release;   break;
3141     case Volatile: mo = MemNode::seqcst;    break;
3142     default:
3143       ShouldNotReachHere();
3144   }
3145   guarantee(mo != MemNode::unset, "Should select memory ordering");
3146   return mo;
3147 }
3148 
3149 bool LibraryCallKit::inline_unsafe_fence(vmIntrinsics::ID id) {
3150   // Regardless of form, don't allow previous ld/st to move down,
3151   // then issue acquire, release, or volatile mem_bar.
3152   insert_mem_bar(Op_MemBarCPUOrder);
3153   switch(id) {
3154     case vmIntrinsics::_loadFence:
3155       insert_mem_bar(Op_LoadFence);
3156       return true;
3157     case vmIntrinsics::_storeFence:
3158       insert_mem_bar(Op_StoreFence);
3159       return true;
3160     case vmIntrinsics::_fullFence:
3161       insert_mem_bar(Op_MemBarVolatile);
3162       return true;
3163     default:
3164       fatal_unexpected_iid(id);
3165       return false;
3166   }
3167 }
3168 
3169 bool LibraryCallKit::inline_onspinwait() {
3170   insert_mem_bar(Op_OnSpinWait);
3171   return true;
3172 }
3173 
3174 bool LibraryCallKit::klass_needs_init_guard(Node* kls) {
3175   if (!kls->is_Con()) {
3176     return true;
3177   }
3178   const TypeKlassPtr* klsptr = kls->bottom_type()->isa_klassptr();
3179   if (klsptr == NULL) {
3180     return true;
3181   }
3182   ciInstanceKlass* ik = klsptr->klass()->as_instance_klass();
3183   // don't need a guard for a klass that is already initialized
3184   return !ik->is_initialized();
3185 }
3186 
3187 //----------------------------inline_unsafe_allocate---------------------------
3188 // public native Object Unsafe.allocateInstance(Class<?> cls);
3189 bool LibraryCallKit::inline_unsafe_allocate() {
3190   if (callee()->is_static())  return false;  // caller must have the capability!


3247   //    (!TLS._osthread._interrupted || WINDOWS_ONLY(false) NOT_WINDOWS(!clear_int)))
3248   //   ? TLS._osthread._interrupted : /*slow path:*/ t.isInterrupted(clear_int)
3249   // So, in the common case that the interrupt bit is false,
3250   // we avoid making a call into the VM.  Even if the interrupt bit
3251   // is true, if the clear_int argument is false, we avoid the VM call.
3252   // However, if the receiver is not currentThread, we must call the VM,
3253   // because there must be some locking done around the operation.
3254 
3255   // We only go to the fast case code if we pass two guards.
3256   // Paths which do not pass are accumulated in the slow_region.
3257 
3258   enum {
3259     no_int_result_path   = 1, // t == Thread.current() && !TLS._osthread._interrupted
3260     no_clear_result_path = 2, // t == Thread.current() &&  TLS._osthread._interrupted && !clear_int
3261     slow_result_path     = 3, // slow path: t.isInterrupted(clear_int)
3262     PATH_LIMIT
3263   };
3264 
3265   // Ensure that it's not possible to move the load of TLS._osthread._interrupted flag
3266   // out of the function.
3267   insert_mem_bar(Op_MemBarCPUOrder);
3268 
3269   RegionNode* result_rgn = new RegionNode(PATH_LIMIT);
3270   PhiNode*    result_val = new PhiNode(result_rgn, TypeInt::BOOL);
3271 
3272   RegionNode* slow_region = new RegionNode(1);
3273   record_for_igvn(slow_region);
3274 
3275   // (a) Receiving thread must be the current thread.
3276   Node* rec_thr = argument(0);
3277   Node* tls_ptr = NULL;
3278   Node* cur_thr = generate_current_thread(tls_ptr);
3279   Node* cmp_thr = _gvn.transform(new CmpPNode(cur_thr, rec_thr));
3280   Node* bol_thr = _gvn.transform(new BoolNode(cmp_thr, BoolTest::ne));
3281 
3282   generate_slow_guard(bol_thr, slow_region);
3283 
3284   // (b) Interrupt bit on TLS must be false.
3285   Node* p = basic_plus_adr(top()/*!oop*/, tls_ptr, in_bytes(JavaThread::osthread_offset()));
3286   Node* osthread = make_load(NULL, p, TypeRawPtr::NOTNULL, T_ADDRESS, MemNode::unordered);
3287   p = basic_plus_adr(top()/*!oop*/, osthread, in_bytes(OSThread::interrupted_offset()));


4471   if (callee()->is_static())  return false;  // caller must have the capability!
4472   null_check_receiver();  // null-check receiver
4473   if (stopped())  return true;
4474 
4475   C->set_has_unsafe_access(true);  // Mark eventual nmethod as "unsafe".
4476 
4477   Node* src_ptr =         argument(1);   // type: oop
4478   Node* src_off = ConvL2X(argument(2));  // type: long
4479   Node* dst_ptr =         argument(4);   // type: oop
4480   Node* dst_off = ConvL2X(argument(5));  // type: long
4481   Node* size    = ConvL2X(argument(7));  // type: long
4482 
4483   assert(Unsafe_field_offset_to_byte_offset(11) == 11,
4484          "fieldOffset must be byte-scaled");
4485 
4486   Node* src = make_unsafe_address(src_ptr, src_off);
4487   Node* dst = make_unsafe_address(dst_ptr, dst_off);
4488 
4489   // Conservatively insert a memory barrier on all memory slices.
4490   // Do not let writes of the copy source or destination float below the copy.
4491   insert_mem_bar(Op_MemBarCPUOrder);
4492 
4493   // Call it.  Note that the length argument is not scaled.
4494   make_runtime_call(RC_LEAF|RC_NO_FP,
4495                     OptoRuntime::fast_arraycopy_Type(),
4496                     StubRoutines::unsafe_arraycopy(),
4497                     "unsafe_arraycopy",
4498                     TypeRawPtr::BOTTOM,
4499                     src, dst, size XTOP);
4500 
4501   // Do not let reads of the copy destination float above the copy.
4502   insert_mem_bar(Op_MemBarCPUOrder);
4503 
4504   return true;
4505 }
4506 
4507 //------------------------clone_coping-----------------------------------
4508 // Helper function for inline_native_clone.
4509 void LibraryCallKit::copy_to_clone(Node* obj, Node* alloc_obj, Node* obj_size, bool is_array, bool card_mark) {
4510   assert(obj_size != NULL, "");
4511   Node* raw_obj = alloc_obj->in(1);
4512   assert(alloc_obj->is_CheckCastPP() && raw_obj->is_Proj() && raw_obj->in(0)->is_Allocate(), "");
4513 
4514   AllocateNode* alloc = NULL;
4515   if (ReduceBulkZeroing) {
4516     // We will be completely responsible for initializing this object -
4517     // mark Initialize node as complete.
4518     alloc = AllocateNode::Ideal_allocation(alloc_obj, &_gvn);
4519     // The object was just allocated - there should be no any stores!
4520     guarantee(alloc != NULL && alloc->maybe_set_complete(&_gvn), "");
4521     // Mark as complete_with_arraycopy so that on AllocateNode
4522     // expansion, we know this AllocateNode is initialized by an array


4579     int raw_adr_idx = Compile::AliasIdxRaw;
4580     post_barrier(control(),
4581                  memory(raw_adr_type),
4582                  alloc_obj,
4583                  no_particular_field,
4584                  raw_adr_idx,
4585                  no_particular_value,
4586                  T_OBJECT,
4587                  false);
4588   }
4589 
4590   // Do not let reads from the cloned object float above the arraycopy.
4591   if (alloc != NULL) {
4592     // Do not let stores that initialize this object be reordered with
4593     // a subsequent store that would make this object accessible by
4594     // other threads.
4595     // Record what AllocateNode this StoreStore protects so that
4596     // escape analysis can go from the MemBarStoreStoreNode to the
4597     // AllocateNode and eliminate the MemBarStoreStoreNode if possible
4598     // based on the escape status of the AllocateNode.
4599     insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out(AllocateNode::RawAddress));
4600   } else {
4601     insert_mem_bar(Op_MemBarCPUOrder);
4602   }
4603 }
4604 
4605 //------------------------inline_native_clone----------------------------
4606 // protected native Object java.lang.Object.clone();
4607 //
4608 // Here are the simple edge cases:
4609 //  null receiver => normal trap
4610 //  virtual and clone was overridden => slow path to out-of-line clone
4611 //  not cloneable or finalizer => slow path to out-of-line Object.clone
4612 //
4613 // The general case has two steps, allocation and copying.
4614 // Allocation has two cases, and uses GraphKit::new_instance or new_array.
4615 //
4616 // Copying also has two cases, oop arrays and everything else.
4617 // Oop arrays use arrayof_oop_arraycopy (same as System.arraycopy).
4618 // Everything else uses the tight inline loop supplied by CopyArrayNode.
4619 //
4620 // These steps fold up nicely if and when the cloned object's klass
4621 // can be sharply typed as an object array, a type array, or an instance.


4642       ciInstanceKlass* spec_ik = obj_type->speculative_type()->as_instance_klass();
4643       if (spec_ik->nof_nonstatic_fields() <= ArrayCopyLoadStoreMaxElem &&
4644           !spec_ik->has_injected_fields()) {
4645         ciKlass* k = obj_type->klass();
4646         if (!k->is_instance_klass() ||
4647             k->as_instance_klass()->is_interface() ||
4648             k->as_instance_klass()->has_subklass()) {
4649           obj = maybe_cast_profiled_obj(obj, obj_type->speculative_type(), false);
4650         }
4651       }
4652     }
4653 
4654     Node* obj_klass = load_object_klass(obj);
4655     const TypeKlassPtr* tklass = _gvn.type(obj_klass)->isa_klassptr();
4656     const TypeOopPtr*   toop   = ((tklass != NULL)
4657                                 ? tklass->as_instance_type()
4658                                 : TypeInstPtr::NOTNULL);
4659 
4660     // Conservatively insert a memory barrier on all memory slices.
4661     // Do not let writes into the original float below the clone.
4662     insert_mem_bar(Op_MemBarCPUOrder);
4663 
4664     // paths into result_reg:
4665     enum {
4666       _slow_path = 1,     // out-of-line call to clone method (virtual or not)
4667       _objArray_path,     // plain array allocation, plus arrayof_oop_arraycopy
4668       _array_path,        // plain array allocation, plus arrayof_long_arraycopy
4669       _instance_path,     // plain instance allocation, plus arrayof_long_arraycopy
4670       PATH_LIMIT
4671     };
4672     RegionNode* result_reg = new RegionNode(PATH_LIMIT);
4673     result_val             = new PhiNode(result_reg, TypeInstPtr::NOTNULL);
4674     PhiNode*    result_i_o = new PhiNode(result_reg, Type::ABIO);
4675     PhiNode*    result_mem = new PhiNode(result_reg, Type::MEMORY, TypePtr::BOTTOM);
4676     record_for_igvn(result_reg);
4677 
4678     const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
4679     int raw_adr_idx = Compile::AliasIdxRaw;
4680 
4681     Node* array_ctl = generate_array_guard(obj_klass, (RegionNode*)NULL);
4682     if (array_ctl != NULL) {


5949   Node* reference_obj = null_check_receiver();
5950   if (stopped()) return true;
5951 
5952   Node* adr = basic_plus_adr(reference_obj, reference_obj, referent_offset);
5953 
5954   ciInstanceKlass* klass = env()->Object_klass();
5955   const TypeOopPtr* object_type = TypeOopPtr::make_from_klass(klass);
5956 
5957   Node* no_ctrl = NULL;
5958   Node* result = make_load(no_ctrl, adr, object_type, T_OBJECT, MemNode::unordered);
5959 
5960   // Use the pre-barrier to record the value in the referent field
5961   pre_barrier(false /* do_load */,
5962               control(),
5963               NULL /* obj */, NULL /* adr */, max_juint /* alias_idx */, NULL /* val */, NULL /* val_type */,
5964               result /* pre_val */,
5965               T_OBJECT);
5966 
5967   // Add memory barrier to prevent commoning reads from this field
5968   // across safepoint since GC can change its value.
5969   insert_mem_bar(Op_MemBarCPUOrder);
5970 
5971   set_result(result);
5972   return true;
5973 }
5974 
5975 
5976 Node * LibraryCallKit::load_field_from_object(Node * fromObj, const char * fieldName, const char * fieldTypeString,
5977                                               bool is_exact=true, bool is_static=false,
5978                                               ciInstanceKlass * fromKls=NULL) {
5979   if (fromKls == NULL) {
5980     const TypeInstPtr* tinst = _gvn.type(fromObj)->isa_instptr();
5981     assert(tinst != NULL, "obj is null");
5982     assert(tinst->klass()->is_loaded(), "obj is not loaded");
5983     assert(!is_exact || tinst->klass_is_exact(), "klass not exact");
5984     fromKls = tinst->klass()->as_instance_klass();
5985   } else {
5986     assert(is_static, "only for static field access");
5987   }
5988   ciField* field = fromKls->get_field_by_name(ciSymbol::make(fieldName),
5989                                               ciSymbol::make(fieldTypeString),


6000   // Next code  copied from Parse::do_get_xxx():
6001 
6002   // Compute address and memory type.
6003   int offset  = field->offset_in_bytes();
6004   bool is_vol = field->is_volatile();
6005   ciType* field_klass = field->type();
6006   assert(field_klass->is_loaded(), "should be loaded");
6007   const TypePtr* adr_type = C->alias_type(field)->adr_type();
6008   Node *adr = basic_plus_adr(fromObj, fromObj, offset);
6009   BasicType bt = field->layout_type();
6010 
6011   // Build the resultant type of the load
6012   const Type *type;
6013   if (bt == T_OBJECT) {
6014     type = TypeOopPtr::make_from_klass(field_klass->as_klass());
6015   } else {
6016     type = Type::get_const_basic_type(bt);
6017   }
6018 
6019   if (support_IRIW_for_not_multiple_copy_atomic_cpu && is_vol) {
6020     insert_mem_bar(Op_MemBarVolatile);   // StoreLoad barrier
6021   }
6022   // Build the load.
6023   MemNode::MemOrd mo = is_vol ? MemNode::acquire : MemNode::unordered;
6024   Node* loadedField = make_load(NULL, adr, type, bt, adr_type, mo, LoadNode::DependsOnlyOnTest, is_vol);
6025   // If reference is volatile, prevent following memory ops from
6026   // floating up past the volatile read.  Also prevents commoning
6027   // another volatile read.
6028   if (is_vol) {
6029     // Memory barrier includes bogus read of value to force load BEFORE membar
6030     insert_mem_bar(Op_MemBarAcquire, loadedField);
6031   }
6032   return loadedField;
6033 }
6034 
6035 Node * LibraryCallKit::field_address_from_object(Node * fromObj, const char * fieldName, const char * fieldTypeString,
6036                                                  bool is_exact = true, bool is_static = false,
6037                                                  ciInstanceKlass * fromKls = NULL) {
6038   if (fromKls == NULL) {
6039     const TypeInstPtr* tinst = _gvn.type(fromObj)->isa_instptr();
6040     assert(tinst != NULL, "obj is null");
6041     assert(tinst->klass()->is_loaded(), "obj is not loaded");
6042     assert(!is_exact || tinst->klass_is_exact(), "klass not exact");
6043     fromKls = tinst->klass()->as_instance_klass();
6044   }
6045   else {
6046     assert(is_static, "only for static field access");
6047   }
6048   ciField* field = fromKls->get_field_by_name(ciSymbol::make(fieldName),
6049     ciSymbol::make(fieldTypeString),
6050     is_static);




 190   Node* generate_objArray_guard(Node* kls, RegionNode* region) {
 191     return generate_array_guard_common(kls, region, true, false);
 192   }
 193   Node* generate_non_objArray_guard(Node* kls, RegionNode* region) {
 194     return generate_array_guard_common(kls, region, true, true);
 195   }
 196   Node* generate_array_guard_common(Node* kls, RegionNode* region,
 197                                     bool obj_array, bool not_array);
 198   Node* generate_virtual_guard(Node* obj_klass, RegionNode* slow_region);
 199   CallJavaNode* generate_method_call(vmIntrinsics::ID method_id,
 200                                      bool is_virtual = false, bool is_static = false);
 201   CallJavaNode* generate_method_call_static(vmIntrinsics::ID method_id) {
 202     return generate_method_call(method_id, false, true);
 203   }
 204   CallJavaNode* generate_method_call_virtual(vmIntrinsics::ID method_id) {
 205     return generate_method_call(method_id, true, false);
 206   }
 207   Node * load_field_from_object(Node * fromObj, const char * fieldName, const char * fieldTypeString, bool is_exact, bool is_static, ciInstanceKlass * fromKls);
 208   Node * field_address_from_object(Node * fromObj, const char * fieldName, const char * fieldTypeString, bool is_exact, bool is_static, ciInstanceKlass * fromKls);
 209 
 210   Node* make_string_method_node(Opcodes opcode, Node* str1_start, Node* cnt1, Node* str2_start, Node* cnt2, StrIntrinsicNode::ArgEnc ae);
 211   bool inline_string_compareTo(StrIntrinsicNode::ArgEnc ae);
 212   bool inline_string_indexOf(StrIntrinsicNode::ArgEnc ae);
 213   bool inline_string_indexOfI(StrIntrinsicNode::ArgEnc ae);
 214   Node* make_indexOf_node(Node* src_start, Node* src_count, Node* tgt_start, Node* tgt_count,
 215                           RegionNode* region, Node* phi, StrIntrinsicNode::ArgEnc ae);
 216   bool inline_string_indexOfChar();
 217   bool inline_string_equals(StrIntrinsicNode::ArgEnc ae);
 218   bool inline_string_toBytesU();
 219   bool inline_string_getCharsU();
 220   bool inline_string_copy(bool compress);
 221   bool inline_string_char_access(bool is_store);
 222   Node* round_double_node(Node* n);
 223   bool runtime_math(const TypeFunc* call_type, address funcAddr, const char* funcName);
 224   bool inline_math_native(vmIntrinsics::ID id);
 225   bool inline_trig(vmIntrinsics::ID id);
 226   bool inline_math(vmIntrinsics::ID id);
 227   template <typename OverflowOp>
 228   bool inline_math_overflow(Node* arg1, Node* arg2);
 229   void inline_math_mathExact(Node* math, Node* test);
 230   bool inline_math_addExactI(bool is_increment);


 463     } else {
 464       // Root compile
 465       C->print_inlining_stream()->print("Did not generate predicate for intrinsic %s%s at bci:%d in",
 466                                         vmIntrinsics::name_at(intrinsic_id()),
 467                                         (is_virtual() ? " (virtual)" : ""), bci);
 468     }
 469   }
 470   C->gather_intrinsic_statistics(intrinsic_id(), is_virtual(), Compile::_intrinsic_failed);
 471   return NULL;
 472 }
 473 
 474 bool LibraryCallKit::try_to_inline(int predicate) {
 475   // Handle symbolic names for otherwise undistinguished boolean switches:
 476   const bool is_store       = true;
 477   const bool is_compress    = true;
 478   const bool is_static      = true;
 479   const bool is_volatile    = true;
 480 
 481   if (!jvms()->has_method()) {
 482     // Root JVMState has a null method.
 483     assert(map()->memory()->Opcode() == Opcodes::Op_Parm, "");
 484     // Insert the memory aliasing node
 485     set_all_memory(reset_memory());
 486   }
 487   assert(merged_memory(), "");
 488 
 489 
 490   switch (intrinsic_id()) {
 491   case vmIntrinsics::_hashCode:                 return inline_native_hashcode(intrinsic()->is_virtual(), !is_static);
 492   case vmIntrinsics::_identityHashCode:         return inline_native_hashcode(/*!virtual*/ false,         is_static);
 493   case vmIntrinsics::_getClass:                 return inline_native_getClass();
 494 
 495   case vmIntrinsics::_dsin:
 496   case vmIntrinsics::_dcos:
 497   case vmIntrinsics::_dtan:
 498   case vmIntrinsics::_dabs:
 499   case vmIntrinsics::_datan2:
 500   case vmIntrinsics::_dsqrt:
 501   case vmIntrinsics::_dexp:
 502   case vmIntrinsics::_dlog:
 503   case vmIntrinsics::_dlog10:


 825 
 826   case vmIntrinsics::_hasNegatives:
 827     return inline_hasNegatives();
 828 
 829   default:
 830     // If you get here, it may be that someone has added a new intrinsic
 831     // to the list in vmSymbols.hpp without implementing it here.
 832 #ifndef PRODUCT
 833     if ((PrintMiscellaneous && (Verbose || WizardMode)) || PrintOpto) {
 834       tty->print_cr("*** Warning: Unimplemented intrinsic %s(%d)",
 835                     vmIntrinsics::name_at(intrinsic_id()), intrinsic_id());
 836     }
 837 #endif
 838     return false;
 839   }
 840 }
 841 
 842 Node* LibraryCallKit::try_to_predicate(int predicate) {
 843   if (!jvms()->has_method()) {
 844     // Root JVMState has a null method.
 845     assert(map()->memory()->Opcode() == Opcodes::Op_Parm, "");
 846     // Insert the memory aliasing node
 847     set_all_memory(reset_memory());
 848   }
 849   assert(merged_memory(), "");
 850 
 851   switch (intrinsic_id()) {
 852   case vmIntrinsics::_cipherBlockChaining_encryptAESCrypt:
 853     return inline_cipherBlockChaining_AESCrypt_predicate(false);
 854   case vmIntrinsics::_cipherBlockChaining_decryptAESCrypt:
 855     return inline_cipherBlockChaining_AESCrypt_predicate(true);
 856   case vmIntrinsics::_counterMode_AESCrypt:
 857     return inline_counterMode_AESCrypt_predicate();
 858   case vmIntrinsics::_digestBase_implCompressMB:
 859     return inline_digestBase_implCompressMB_predicate(predicate);
 860 
 861   default:
 862     // If you get here, it may be that someone has added a new intrinsic
 863     // to the list in vmSymbols.hpp without implementing it here.
 864 #ifndef PRODUCT
 865     if ((PrintMiscellaneous && (Verbose || WizardMode)) || PrintOpto) {


1005   }
1006 }
1007 
1008 //--------------------------generate_current_thread--------------------
1009 Node* LibraryCallKit::generate_current_thread(Node* &tls_output) {
1010   ciKlass*    thread_klass = env()->Thread_klass();
1011   const Type* thread_type  = TypeOopPtr::make_from_klass(thread_klass)->cast_to_ptr_type(TypePtr::NotNull);
1012   Node* thread = _gvn.transform(new ThreadLocalNode());
1013   Node* p = basic_plus_adr(top()/*!oop*/, thread, in_bytes(JavaThread::threadObj_offset()));
1014   Node* threadObj = make_load(NULL, p, thread_type, T_OBJECT, MemNode::unordered);
1015   tls_output = thread;
1016   return threadObj;
1017 }
1018 
1019 
1020 //------------------------------make_string_method_node------------------------
1021 // Helper method for String intrinsic functions. This version is called with
1022 // str1 and str2 pointing to byte[] nodes containing Latin1 or UTF16 encoded
1023 // characters (depending on 'is_byte'). cnt1 and cnt2 are pointing to Int nodes
1024 // containing the lengths of str1 and str2.
1025 Node* LibraryCallKit::make_string_method_node(Opcodes opcode, Node* str1_start, Node* cnt1, Node* str2_start, Node* cnt2, StrIntrinsicNode::ArgEnc ae) {
1026   Node* result = NULL;
1027   switch (opcode) {
1028   case Opcodes::Op_StrIndexOf:
1029     result = new StrIndexOfNode(control(), memory(TypeAryPtr::BYTES),
1030                                 str1_start, cnt1, str2_start, cnt2, ae);
1031     break;
1032   case Opcodes::Op_StrComp:
1033     result = new StrCompNode(control(), memory(TypeAryPtr::BYTES),
1034                              str1_start, cnt1, str2_start, cnt2, ae);
1035     break;
1036   case Opcodes::Op_StrEquals:
1037     // We already know that cnt1 == cnt2 here (checked in 'inline_string_equals').
1038     // Use the constant length if there is one because optimized match rule may exist.
1039     result = new StrEqualsNode(control(), memory(TypeAryPtr::BYTES),
1040                                str1_start, str2_start, cnt2->is_Con() ? cnt2 : cnt1, ae);
1041     break;
1042   default:
1043     ShouldNotReachHere();
1044     return NULL;
1045   }
1046 
1047   // All these intrinsics have checks.
1048   C->set_has_split_ifs(true); // Has chance for split-if optimization
1049 
1050   return _gvn.transform(result);
1051 }
1052 
1053 //------------------------------inline_string_compareTo------------------------
1054 bool LibraryCallKit::inline_string_compareTo(StrIntrinsicNode::ArgEnc ae) {
1055   Node* arg1 = argument(0);
1056   Node* arg2 = argument(1);
1057 
1058   // Get start addr and length of first argument
1059   Node* arg1_start  = array_element_address(arg1, intcon(0), T_BYTE);
1060   Node* arg1_cnt    = load_array_length(arg1);
1061 
1062   // Get start addr and length of second argument
1063   Node* arg2_start  = array_element_address(arg2, intcon(0), T_BYTE);
1064   Node* arg2_cnt    = load_array_length(arg2);
1065 
1066   Node* result = make_string_method_node(Opcodes::Op_StrComp, arg1_start, arg1_cnt, arg2_start, arg2_cnt, ae);
1067   set_result(result);
1068   return true;
1069 }
1070 
1071 //------------------------------inline_string_equals------------------------
1072 bool LibraryCallKit::inline_string_equals(StrIntrinsicNode::ArgEnc ae) {
1073   Node* arg1 = argument(0);
1074   Node* arg2 = argument(1);
1075 
1076   // paths (plus control) merge
1077   RegionNode* region = new RegionNode(3);
1078   Node* phi = new PhiNode(region, TypeInt::BOOL);
1079 
1080   if (!stopped()) {
1081     // Get start addr and length of first argument
1082     Node* arg1_start  = array_element_address(arg1, intcon(0), T_BYTE);
1083     Node* arg1_cnt    = load_array_length(arg1);
1084 
1085     // Get start addr and length of second argument
1086     Node* arg2_start  = array_element_address(arg2, intcon(0), T_BYTE);
1087     Node* arg2_cnt    = load_array_length(arg2);
1088 
1089     // Check for arg1_cnt != arg2_cnt
1090     Node* cmp = _gvn.transform(new CmpINode(arg1_cnt, arg2_cnt));
1091     Node* bol = _gvn.transform(new BoolNode(cmp, BoolTest::ne));
1092     Node* if_ne = generate_slow_guard(bol, NULL);
1093     if (if_ne != NULL) {
1094       phi->init_req(2, intcon(0));
1095       region->init_req(2, if_ne);
1096     }
1097 
1098     // Check for count == 0 is done by assembler code for StrEquals.
1099 
1100     if (!stopped()) {
1101       Node* equals = make_string_method_node(Opcodes::Op_StrEquals, arg1_start, arg1_cnt, arg2_start, arg2_cnt, ae);
1102       phi->init_req(1, equals);
1103       region->init_req(1, control());
1104     }
1105   }
1106 
1107   // post merge
1108   set_control(_gvn.transform(region));
1109   record_for_igvn(region);
1110 
1111   set_result(_gvn.transform(phi));
1112   return true;
1113 }
1114 
1115 //------------------------------inline_array_equals----------------------------
1116 bool LibraryCallKit::inline_array_equals(StrIntrinsicNode::ArgEnc ae) {
1117   assert(ae == StrIntrinsicNode::UU || ae == StrIntrinsicNode::LL, "unsupported array types");
1118   Node* arg1 = argument(0);
1119   Node* arg2 = argument(1);
1120 
1121   const TypeAryPtr* mtype = (ae == StrIntrinsicNode::UU) ? TypeAryPtr::CHARS : TypeAryPtr::BYTES;


1179     PreserveJVMState pjvms(this);
1180     set_control(_gvn.transform(new IfFalseNode(rc)));
1181     uncommon_trap(Deoptimization::Reason_range_check,
1182                   Deoptimization::Action_make_not_entrant);
1183   }
1184 
1185   if (stopped()) {
1186     return false;
1187   }
1188 
1189   Node* result = new CastIINode(index, TypeInt::make(0, _gvn.type(length)->is_int()->_hi, Type::WidenMax));
1190   result->set_req(0, control());
1191   result = _gvn.transform(result);
1192   set_result(result);
1193   replace_in_map(index, result);
1194   return true;
1195 }
1196 
1197 //------------------------------inline_string_indexOf------------------------
1198 bool LibraryCallKit::inline_string_indexOf(StrIntrinsicNode::ArgEnc ae) {
1199   if (!Matcher::match_rule_supported(Opcodes::Op_StrIndexOf)) {
1200     return false;
1201   }
1202   Node* src = argument(0);
1203   Node* tgt = argument(1);
1204 
1205   // Make the merge point
1206   RegionNode* result_rgn = new RegionNode(4);
1207   Node*       result_phi = new PhiNode(result_rgn, TypeInt::INT);
1208 
1209   // Get start addr and length of source string
1210   Node* src_start = array_element_address(src, intcon(0), T_BYTE);
1211   Node* src_count = load_array_length(src);
1212 
1213   // Get start addr and length of substring
1214   Node* tgt_start = array_element_address(tgt, intcon(0), T_BYTE);
1215   Node* tgt_count = load_array_length(tgt);
1216 
1217   if (ae == StrIntrinsicNode::UU || ae == StrIntrinsicNode::UL) {
1218     // Divide src size by 2 if String is UTF16 encoded
1219     src_count = _gvn.transform(new RShiftINode(src_count, intcon(1)));


1223     tgt_count = _gvn.transform(new RShiftINode(tgt_count, intcon(1)));
1224   }
1225 
1226   Node* result = make_indexOf_node(src_start, src_count, tgt_start, tgt_count, result_rgn, result_phi, ae);
1227   if (result != NULL) {
1228     result_phi->init_req(3, result);
1229     result_rgn->init_req(3, control());
1230   }
1231   set_control(_gvn.transform(result_rgn));
1232   record_for_igvn(result_rgn);
1233   set_result(_gvn.transform(result_phi));
1234 
1235   return true;
1236 }
1237 
1238 //-----------------------------inline_string_indexOf-----------------------
1239 bool LibraryCallKit::inline_string_indexOfI(StrIntrinsicNode::ArgEnc ae) {
1240   if (too_many_traps(Deoptimization::Reason_intrinsic)) {
1241     return false;
1242   }
1243   if (!Matcher::match_rule_supported(Opcodes::Op_StrIndexOf)) {
1244     return false;
1245   }
1246   assert(callee()->signature()->size() == 5, "String.indexOf() has 5 arguments");
1247   Node* src         = argument(0); // byte[]
1248   Node* src_count   = argument(1); // char count
1249   Node* tgt         = argument(2); // byte[]
1250   Node* tgt_count   = argument(3); // char count
1251   Node* from_index  = argument(4); // char index
1252 
1253   // Multiply byte array index by 2 if String is UTF16 encoded
1254   Node* src_offset = (ae == StrIntrinsicNode::LL) ? from_index : _gvn.transform(new LShiftINode(from_index, intcon(1)));
1255   src_count = _gvn.transform(new SubINode(src_count, from_index));
1256   Node* src_start = array_element_address(src, src_offset, T_BYTE);
1257   Node* tgt_start = array_element_address(tgt, intcon(0), T_BYTE);
1258 
1259   // Range checks
1260   generate_string_range_check(src, src_offset, src_count, ae != StrIntrinsicNode::LL);
1261   generate_string_range_check(tgt, intcon(0), tgt_count, ae == StrIntrinsicNode::UU);
1262   if (stopped()) {
1263     return true;


1298                                         RegionNode* region, Node* phi, StrIntrinsicNode::ArgEnc ae) {
1299   // Check for substr count > string count
1300   Node* cmp = _gvn.transform(new CmpINode(tgt_count, src_count));
1301   Node* bol = _gvn.transform(new BoolNode(cmp, BoolTest::gt));
1302   Node* if_gt = generate_slow_guard(bol, NULL);
1303   if (if_gt != NULL) {
1304     phi->init_req(1, intcon(-1));
1305     region->init_req(1, if_gt);
1306   }
1307   if (!stopped()) {
1308     // Check for substr count == 0
1309     cmp = _gvn.transform(new CmpINode(tgt_count, intcon(0)));
1310     bol = _gvn.transform(new BoolNode(cmp, BoolTest::eq));
1311     Node* if_zero = generate_slow_guard(bol, NULL);
1312     if (if_zero != NULL) {
1313       phi->init_req(2, intcon(0));
1314       region->init_req(2, if_zero);
1315     }
1316   }
1317   if (!stopped()) {
1318     return make_string_method_node(Opcodes::Op_StrIndexOf, src_start, src_count, tgt_start, tgt_count, ae);
1319   }
1320   return NULL;
1321 }
1322 
1323 //-----------------------------inline_string_indexOfChar-----------------------
1324 bool LibraryCallKit::inline_string_indexOfChar() {
1325   if (too_many_traps(Deoptimization::Reason_intrinsic)) {
1326     return false;
1327   }
1328   if (!Matcher::match_rule_supported(Opcodes::Op_StrIndexOfChar)) {
1329     return false;
1330   }
1331   assert(callee()->signature()->size() == 4, "String.indexOfChar() has 4 arguments");
1332   Node* src         = argument(0); // byte[]
1333   Node* tgt         = argument(1); // tgt is int ch
1334   Node* from_index  = argument(2);
1335   Node* max         = argument(3);
1336 
1337   Node* src_offset = _gvn.transform(new LShiftINode(from_index, intcon(1)));
1338   Node* src_start = array_element_address(src, src_offset, T_BYTE);
1339   Node* src_count = _gvn.transform(new SubINode(max, from_index));
1340 
1341   // Range checks
1342   generate_string_range_check(src, src_offset, src_count, true);
1343   if (stopped()) {
1344     return true;
1345   }
1346 
1347   RegionNode* region = new RegionNode(3);
1348   Node* phi = new PhiNode(region, TypeInt::INT);


1428   } else {
1429     inflate_string(src_start, dst_start, TypeAryPtr::get_array_body_type(dst_elem), length);
1430   }
1431 
1432   if (alloc != NULL) {
1433     if (alloc->maybe_set_complete(&_gvn)) {
1434       // "You break it, you buy it."
1435       InitializeNode* init = alloc->initialization();
1436       assert(init->is_complete(), "we just did this");
1437       init->set_complete_with_arraycopy();
1438       assert(dst->is_CheckCastPP(), "sanity");
1439       assert(dst->in(0)->in(0) == init, "dest pinned");
1440     }
1441     // Do not let stores that initialize this object be reordered with
1442     // a subsequent store that would make this object accessible by
1443     // other threads.
1444     // Record what AllocateNode this StoreStore protects so that
1445     // escape analysis can go from the MemBarStoreStoreNode to the
1446     // AllocateNode and eliminate the MemBarStoreStoreNode if possible
1447     // based on the escape status of the AllocateNode.
1448     insert_mem_bar(Opcodes::Op_MemBarStoreStore, alloc->proj_out(AllocateNode::RawAddress));
1449   }
1450   if (compress) {
1451     set_result(_gvn.transform(count));
1452   }
1453   return true;
1454 }
1455 
1456 #ifdef _LP64
1457 #define XTOP ,top() /*additional argument*/
1458 #else  //_LP64
1459 #define XTOP        /*no additional argument*/
1460 #endif //_LP64
1461 
1462 //------------------------inline_string_toBytesU--------------------------
1463 // public static byte[] StringUTF16.toBytes(char[] value, int off, int len)
1464 bool LibraryCallKit::inline_string_toBytesU() {
1465   if (too_many_traps(Deoptimization::Reason_intrinsic)) {
1466     return false;
1467   }
1468   // Get the arguments.


1520                       OptoRuntime::fast_arraycopy_Type(),
1521                       copyfunc_addr, copyfunc_name, TypeRawPtr::BOTTOM,
1522                       src_start, dst_start, ConvI2X(length) XTOP);
1523     // Do not let reads from the cloned object float above the arraycopy.
1524     if (alloc != NULL) {
1525       if (alloc->maybe_set_complete(&_gvn)) {
1526         // "You break it, you buy it."
1527         InitializeNode* init = alloc->initialization();
1528         assert(init->is_complete(), "we just did this");
1529         init->set_complete_with_arraycopy();
1530         assert(newcopy->is_CheckCastPP(), "sanity");
1531         assert(newcopy->in(0)->in(0) == init, "dest pinned");
1532       }
1533       // Do not let stores that initialize this object be reordered with
1534       // a subsequent store that would make this object accessible by
1535       // other threads.
1536       // Record what AllocateNode this StoreStore protects so that
1537       // escape analysis can go from the MemBarStoreStoreNode to the
1538       // AllocateNode and eliminate the MemBarStoreStoreNode if possible
1539       // based on the escape status of the AllocateNode.
1540       insert_mem_bar(Opcodes::Op_MemBarStoreStore, alloc->proj_out(AllocateNode::RawAddress));
1541     } else {
1542       insert_mem_bar(Opcodes::Op_MemBarCPUOrder);
1543     }
1544   } // original reexecute is set back here
1545 
1546   C->set_has_split_ifs(true); // Has chance for split-if optimization
1547   if (!stopped()) {
1548     set_result(newcopy);
1549   }
1550   return true;
1551 }
1552 
1553 //------------------------inline_string_getCharsU--------------------------
1554 // public void StringUTF16.getChars(byte[] src, int srcBegin, int srcEnd, char dst[], int dstBegin)
1555 bool LibraryCallKit::inline_string_getCharsU() {
1556   if (too_many_traps(Deoptimization::Reason_intrinsic)) {
1557     return false;
1558   }
1559 
1560   // Get the arguments.
1561   Node* src       = argument(0);
1562   Node* src_begin = argument(1);


1604                       OptoRuntime::fast_arraycopy_Type(),
1605                       copyfunc_addr, copyfunc_name, TypeRawPtr::BOTTOM,
1606                       src_start, dst_start, ConvI2X(length) XTOP);
1607     // Do not let reads from the cloned object float above the arraycopy.
1608     if (alloc != NULL) {
1609       if (alloc->maybe_set_complete(&_gvn)) {
1610         // "You break it, you buy it."
1611         InitializeNode* init = alloc->initialization();
1612         assert(init->is_complete(), "we just did this");
1613         init->set_complete_with_arraycopy();
1614         assert(dst->is_CheckCastPP(), "sanity");
1615         assert(dst->in(0)->in(0) == init, "dest pinned");
1616       }
1617       // Do not let stores that initialize this object be reordered with
1618       // a subsequent store that would make this object accessible by
1619       // other threads.
1620       // Record what AllocateNode this StoreStore protects so that
1621       // escape analysis can go from the MemBarStoreStoreNode to the
1622       // AllocateNode and eliminate the MemBarStoreStoreNode if possible
1623       // based on the escape status of the AllocateNode.
1624       insert_mem_bar(Opcodes::Op_MemBarStoreStore, alloc->proj_out(AllocateNode::RawAddress));
1625     } else {
1626       insert_mem_bar(Opcodes::Op_MemBarCPUOrder);
1627     }
1628   }
1629 
1630   C->set_has_split_ifs(true); // Has chance for split-if optimization
1631   return true;
1632 }
1633 
1634 //----------------------inline_string_char_access----------------------------
1635 // Store/Load char to/from byte[] array.
1636 // static void StringUTF16.putChar(byte[] val, int index, int c)
1637 // static char StringUTF16.getChar(byte[] val, int index)
1638 bool LibraryCallKit::inline_string_char_access(bool is_store) {
1639   Node* value  = argument(0);
1640   Node* index  = argument(1);
1641   Node* ch = is_store ? argument(2) : NULL;
1642 
1643   // This intrinsic accesses byte[] array as char[] array. Computing the offsets
1644   // correctly requires matched array shapes.
1645   assert (arrayOopDesc::base_offset_in_bytes(T_CHAR) == arrayOopDesc::base_offset_in_bytes(T_BYTE),
1646           "sanity: byte[] and char[] bases agree");


1812       runtime_math(OptoRuntime::Math_D_D_Type(), StubRoutines::dsin(), "dsin") :
1813       runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dsin),   "SIN");
1814   case vmIntrinsics::_dcos:
1815     return StubRoutines::dcos() != NULL ?
1816       runtime_math(OptoRuntime::Math_D_D_Type(), StubRoutines::dcos(), "dcos") :
1817       runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dcos),   "COS");
1818   case vmIntrinsics::_dtan:
1819     return StubRoutines::dtan() != NULL ?
1820       runtime_math(OptoRuntime::Math_D_D_Type(), StubRoutines::dtan(), "dtan") :
1821       runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dtan), "TAN");
1822   case vmIntrinsics::_dlog:
1823     return StubRoutines::dlog() != NULL ?
1824       runtime_math(OptoRuntime::Math_D_D_Type(), StubRoutines::dlog(), "dlog") :
1825       runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dlog),   "LOG");
1826   case vmIntrinsics::_dlog10:
1827     return StubRoutines::dlog10() != NULL ?
1828       runtime_math(OptoRuntime::Math_D_D_Type(), StubRoutines::dlog10(), "dlog10") :
1829       runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dlog10), "LOG10");
1830 
1831     // These intrinsics are supported on all hardware
1832   case vmIntrinsics::_dsqrt:  return Matcher::match_rule_supported(Opcodes::Op_SqrtD) ? inline_math(id) : false;
1833   case vmIntrinsics::_dabs:   return Matcher::has_match_rule(Opcodes::Op_AbsD)   ? inline_math(id) : false;
1834 
1835   case vmIntrinsics::_dexp:
1836     return StubRoutines::dexp() != NULL ?
1837       runtime_math(OptoRuntime::Math_D_D_Type(), StubRoutines::dexp(),  "dexp") :
1838       runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dexp),  "EXP");
1839   case vmIntrinsics::_dpow:
1840     return StubRoutines::dpow() != NULL ?
1841       runtime_math(OptoRuntime::Math_DD_D_Type(), StubRoutines::dpow(), "dpow") :
1842       runtime_math(OptoRuntime::Math_DD_D_Type(), FN_PTR(SharedRuntime::dpow),  "POW");
1843 #undef FN_PTR
1844 
1845    // These intrinsics are not yet correctly implemented
1846   case vmIntrinsics::_datan2:
1847     return false;
1848 
1849   default:
1850     fatal_unexpected_iid(id);
1851     return false;
1852   }
1853 }


1957   }
1958 
1959   bool want_max = (id == vmIntrinsics::_max);
1960 
1961   const TypeInt* txvalue = _gvn.type(xvalue)->isa_int();
1962   const TypeInt* tyvalue = _gvn.type(yvalue)->isa_int();
1963   if (txvalue == NULL || tyvalue == NULL)  return top();
1964   // This is not really necessary, but it is consistent with a
1965   // hypothetical MaxINode::Value method:
1966   int widen = MAX2(txvalue->_widen, tyvalue->_widen);
1967 
1968   // %%% This folding logic should (ideally) be in a different place.
1969   // Some should be inside IfNode, and there to be a more reliable
1970   // transformation of ?: style patterns into cmoves.  We also want
1971   // more powerful optimizations around cmove and min/max.
1972 
1973   // Try to find a dominating comparison of these guys.
1974   // It can simplify the index computation for Arrays.copyOf
1975   // and similar uses of System.arraycopy.
1976   // First, compute the normalized version of CmpI(x, y).
1977   Opcodes   cmp_op = Opcodes::Op_CmpI;
1978   Node* xkey = xvalue;
1979   Node* ykey = yvalue;
1980   Node* ideal_cmpxy = _gvn.transform(new CmpINode(xkey, ykey));
1981   if (ideal_cmpxy->is_Cmp()) {
1982     // E.g., if we have CmpI(length - offset, count),
1983     // it might idealize to CmpI(length, count + offset)
1984     cmp_op = ideal_cmpxy->Opcode();
1985     xkey = ideal_cmpxy->in(1);
1986     ykey = ideal_cmpxy->in(2);
1987   }
1988 
1989   // Start by locating any relevant comparisons.
1990   Node* start_from = (xkey->outcnt() < ykey->outcnt()) ? xkey : ykey;
1991   Node* cmpxy = NULL;
1992   Node* cmpyx = NULL;
1993   for (DUIterator_Fast kmax, k = start_from->fast_outs(kmax); k < kmax; k++) {
1994     Node* cmp = start_from->fast_out(k);
1995     if (cmp->outcnt() > 0 &&            // must have prior uses
1996         cmp->in(0) == NULL &&           // must be context-independent
1997         cmp->Opcode() == cmp_op) {      // right kind of compare


2270 
2271       // Update IdealKit memory and control from graphKit.
2272       __ sync_kit(this);
2273 
2274       Node* one = __ ConI(1);
2275       // is_instof == 0 if base_oop == NULL
2276       __ if_then(is_instof, BoolTest::eq, one, unlikely); {
2277 
2278         // Update graphKit from IdeakKit.
2279         sync_kit(ideal);
2280 
2281         // Use the pre-barrier to record the value in the referent field
2282         pre_barrier(false /* do_load */,
2283                     __ ctrl(),
2284                     NULL /* obj */, NULL /* adr */, max_juint /* alias_idx */, NULL /* val */, NULL /* val_type */,
2285                     pre_val /* pre_val */,
2286                     T_OBJECT);
2287         if (need_mem_bar) {
2288           // Add memory barrier to prevent commoning reads from this field
2289           // across safepoint since GC can change its value.
2290           insert_mem_bar(Opcodes::Op_MemBarCPUOrder);
2291         }
2292         // Update IdealKit from graphKit.
2293         __ sync_kit(this);
2294 
2295       } __ end_if(); // _ref_type != ref_none
2296   } __ end_if(); // offset == referent_offset
2297 
2298   // Final sync IdealKit and GraphKit.
2299   final_sync(ideal);
2300 #undef __
2301 }
2302 
2303 
2304 const TypeOopPtr* LibraryCallKit::sharpen_unsafe_type(Compile::AliasType* alias_type, const TypePtr *adr_type) {
2305   // Attempt to infer a sharper value type from the offset and base type.
2306   ciKlass* sharpened_klass = NULL;
2307 
2308   // See if it is an instance field, with an object type.
2309   if (alias_type->field() != NULL) {
2310     if (alias_type->field()->type()->is_klass()) {


2495   // Heap pointers get a null-check from the interpreter,
2496   // as a courtesy.  However, this is not guaranteed by Unsafe,
2497   // and it is not possible to fully distinguish unintended nulls
2498   // from intended ones in this API.
2499 
2500   // We need to emit leading and trailing CPU membars (see below) in
2501   // addition to memory membars for special access modes. This is a little
2502   // too strong, but avoids the need to insert per-alias-type
2503   // volatile membars (for stores; compare Parse::do_put_xxx), which
2504   // we cannot do effectively here because we probably only have a
2505   // rough approximation of type.
2506 
2507   switch(kind) {
2508     case Relaxed:
2509     case Opaque:
2510     case Acquire:
2511       break;
2512     case Release:
2513     case Volatile:
2514       if (is_store) {
2515         insert_mem_bar(Opcodes::Op_MemBarRelease);
2516       } else {
2517         if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
2518           insert_mem_bar(Opcodes::Op_MemBarVolatile);
2519         }
2520       }
2521       break;
2522     default:
2523       ShouldNotReachHere();
2524   }
2525 
2526   // Memory barrier to prevent normal and 'unsafe' accesses from
2527   // bypassing each other.  Happens after null checks, so the
2528   // exception paths do not take memory state from the memory barrier,
2529   // so there's no problems making a strong assert about mixing users
2530   // of safe & unsafe memory.
2531   if (need_mem_bar) insert_mem_bar(Opcodes::Op_MemBarCPUOrder);
2532 
2533   if (!is_store) {
2534     Node* p = NULL;
2535     // Try to constant fold a load from a constant field
2536     ciField* field = alias_type->field();
2537     if (heap_base_oop != top() && field != NULL && field->is_constant() && !mismatched) {
2538       // final or stable field
2539       p = make_constant_from_field(field, heap_base_oop);
2540     }
2541     if (p == NULL) {
2542       // To be valid, unsafe loads may depend on other conditions than
2543       // the one that guards them: pin the Load node
2544       p = make_load(control(), adr, value_type, type, adr_type, mo, LoadNode::Pinned, requires_atomic_access, unaligned, mismatched);
2545       // load value
2546       switch (type) {
2547       case T_BOOLEAN:
2548       case T_CHAR:
2549       case T_BYTE:
2550       case T_SHORT:
2551       case T_INT:


2610           // Update IdealKit memory.
2611           __ sync_kit(this);
2612         } __ else_(); {
2613           __ store(__ ctrl(), adr, val, type, alias_type->index(), mo, requires_atomic_access, mismatched);
2614         } __ end_if();
2615         // Final sync IdealKit and GraphKit.
2616         final_sync(ideal);
2617 #undef __
2618       }
2619     }
2620   }
2621 
2622   switch(kind) {
2623     case Relaxed:
2624     case Opaque:
2625     case Release:
2626       break;
2627     case Acquire:
2628     case Volatile:
2629       if (!is_store) {
2630         insert_mem_bar(Opcodes::Op_MemBarAcquire);
2631       } else {
2632         if (!support_IRIW_for_not_multiple_copy_atomic_cpu) {
2633           insert_mem_bar(Opcodes::Op_MemBarVolatile);
2634         }
2635       }
2636       break;
2637     default:
2638       ShouldNotReachHere();
2639   }
2640 
2641   if (need_mem_bar) insert_mem_bar(Opcodes::Op_MemBarCPUOrder);
2642 
2643   return true;
2644 }
2645 
2646 //----------------------------inline_unsafe_load_store----------------------------
2647 // This method serves a couple of different customers (depending on LoadStoreKind):
2648 //
2649 // LS_cmp_swap:
2650 //
2651 //   boolean compareAndSwapObject(Object o, long offset, Object expected, Object x);
2652 //   boolean compareAndSwapInt(   Object o, long offset, int    expected, int    x);
2653 //   boolean compareAndSwapLong(  Object o, long offset, long   expected, long   x);
2654 //
2655 // LS_cmp_swap_weak:
2656 //
2657 //   boolean weakCompareAndSwapObject(       Object o, long offset, Object expected, Object x);
2658 //   boolean weakCompareAndSwapObjectAcquire(Object o, long offset, Object expected, Object x);
2659 //   boolean weakCompareAndSwapObjectRelease(Object o, long offset, Object expected, Object x);
2660 //
2661 //   boolean weakCompareAndSwapInt(          Object o, long offset, int    expected, int    x);


2827     case LS_cmp_swap:
2828     case LS_cmp_swap_weak:
2829     case LS_get_add:
2830       break;
2831     default:
2832       ShouldNotReachHere();
2833   }
2834 
2835   int alias_idx = C->get_alias_index(adr_type);
2836 
2837   // Memory-model-wise, a LoadStore acts like a little synchronized
2838   // block, so needs barriers on each side.  These don't translate
2839   // into actual barriers on most machines, but we still need rest of
2840   // compiler to respect ordering.
2841 
2842   switch (access_kind) {
2843     case Relaxed:
2844     case Acquire:
2845       break;
2846     case Release:
2847       insert_mem_bar(Opcodes::Op_MemBarRelease);
2848       break;
2849     case Volatile:
2850       if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
2851         insert_mem_bar(Opcodes::Op_MemBarVolatile);
2852       } else {
2853         insert_mem_bar(Opcodes::Op_MemBarRelease);
2854       }
2855       break;
2856     default:
2857       ShouldNotReachHere();
2858   }
2859   insert_mem_bar(Opcodes::Op_MemBarCPUOrder);
2860 
2861   // Figure out the memory ordering.
2862   MemNode::MemOrd mo = access_kind_to_memord(access_kind);
2863 
2864   // 4984716: MemBars must be inserted before this
2865   //          memory node in order to avoid a false
2866   //          dependency which will confuse the scheduler.
2867   Node *mem = memory(alias_idx);
2868 
2869   // For now, we handle only those cases that actually exist: ints,
2870   // longs, and Object. Adding others should be straightforward.
2871   Node* load_store = NULL;
2872   switch(type) {
2873   case T_BYTE:
2874     switch(kind) {
2875       case LS_get_add:
2876         load_store = _gvn.transform(new GetAndAddBNode(control(), mem, adr, newval, adr_type));
2877         break;
2878       case LS_get_set:
2879         load_store = _gvn.transform(new GetAndSetBNode(control(), mem, adr, newval, adr_type));


3079   set_memory(proj, alias_idx);
3080 
3081   if (type == T_OBJECT && (kind == LS_get_set || kind == LS_cmp_exchange)) {
3082 #ifdef _LP64
3083     if (adr->bottom_type()->is_ptr_to_narrowoop()) {
3084       load_store = _gvn.transform(new DecodeNNode(load_store, load_store->get_ptr_type()));
3085     }
3086 #endif
3087     if (can_move_pre_barrier()) {
3088       // Don't need to load pre_val. The old value is returned by load_store.
3089       // The pre_barrier can execute after the xchg as long as no safepoint
3090       // gets inserted between them.
3091       pre_barrier(false /* do_load */,
3092                   control(), NULL, NULL, max_juint, NULL, NULL,
3093                   load_store /* pre_val */,
3094                   T_OBJECT);
3095     }
3096   }
3097 
3098   // Add the trailing membar surrounding the access
3099   insert_mem_bar(Opcodes::Op_MemBarCPUOrder);
3100 
3101   switch (access_kind) {
3102     case Relaxed:
3103     case Release:
3104       break; // do nothing
3105     case Acquire:
3106     case Volatile:
3107       insert_mem_bar(Opcodes::Op_MemBarAcquire);
3108       // !support_IRIW_for_not_multiple_copy_atomic_cpu handled in platform code
3109       break;
3110     default:
3111       ShouldNotReachHere();
3112   }
3113 
3114   assert(type2size[load_store->bottom_type()->basic_type()] == type2size[rtype], "result type should match");
3115   set_result(load_store);
3116   return true;
3117 }
3118 
3119 MemNode::MemOrd LibraryCallKit::access_kind_to_memord_LS(AccessKind kind, bool is_store) {
3120   MemNode::MemOrd mo = MemNode::unset;
3121   switch(kind) {
3122     case Opaque:
3123     case Relaxed:  mo = MemNode::unordered; break;
3124     case Acquire:  mo = MemNode::acquire;   break;
3125     case Release:  mo = MemNode::release;   break;
3126     case Volatile: mo = is_store ? MemNode::release : MemNode::acquire; break;
3127     default:


3132 }
3133 
3134 MemNode::MemOrd LibraryCallKit::access_kind_to_memord(AccessKind kind) {
3135   MemNode::MemOrd mo = MemNode::unset;
3136   switch(kind) {
3137     case Opaque:
3138     case Relaxed:  mo = MemNode::unordered; break;
3139     case Acquire:  mo = MemNode::acquire;   break;
3140     case Release:  mo = MemNode::release;   break;
3141     case Volatile: mo = MemNode::seqcst;    break;
3142     default:
3143       ShouldNotReachHere();
3144   }
3145   guarantee(mo != MemNode::unset, "Should select memory ordering");
3146   return mo;
3147 }
3148 
3149 bool LibraryCallKit::inline_unsafe_fence(vmIntrinsics::ID id) {
3150   // Regardless of form, don't allow previous ld/st to move down,
3151   // then issue acquire, release, or volatile mem_bar.
3152   insert_mem_bar(Opcodes::Op_MemBarCPUOrder);
3153   switch(id) {
3154     case vmIntrinsics::_loadFence:
3155       insert_mem_bar(Opcodes::Op_LoadFence);
3156       return true;
3157     case vmIntrinsics::_storeFence:
3158       insert_mem_bar(Opcodes::Op_StoreFence);
3159       return true;
3160     case vmIntrinsics::_fullFence:
3161       insert_mem_bar(Opcodes::Op_MemBarVolatile);
3162       return true;
3163     default:
3164       fatal_unexpected_iid(id);
3165       return false;
3166   }
3167 }
3168 
3169 bool LibraryCallKit::inline_onspinwait() {
3170   insert_mem_bar(Opcodes::Op_OnSpinWait);
3171   return true;
3172 }
3173 
3174 bool LibraryCallKit::klass_needs_init_guard(Node* kls) {
3175   if (!kls->is_Con()) {
3176     return true;
3177   }
3178   const TypeKlassPtr* klsptr = kls->bottom_type()->isa_klassptr();
3179   if (klsptr == NULL) {
3180     return true;
3181   }
3182   ciInstanceKlass* ik = klsptr->klass()->as_instance_klass();
3183   // don't need a guard for a klass that is already initialized
3184   return !ik->is_initialized();
3185 }
3186 
3187 //----------------------------inline_unsafe_allocate---------------------------
3188 // public native Object Unsafe.allocateInstance(Class<?> cls);
3189 bool LibraryCallKit::inline_unsafe_allocate() {
3190   if (callee()->is_static())  return false;  // caller must have the capability!


3247   //    (!TLS._osthread._interrupted || WINDOWS_ONLY(false) NOT_WINDOWS(!clear_int)))
3248   //   ? TLS._osthread._interrupted : /*slow path:*/ t.isInterrupted(clear_int)
3249   // So, in the common case that the interrupt bit is false,
3250   // we avoid making a call into the VM.  Even if the interrupt bit
3251   // is true, if the clear_int argument is false, we avoid the VM call.
3252   // However, if the receiver is not currentThread, we must call the VM,
3253   // because there must be some locking done around the operation.
3254 
3255   // We only go to the fast case code if we pass two guards.
3256   // Paths which do not pass are accumulated in the slow_region.
3257 
3258   enum {
3259     no_int_result_path   = 1, // t == Thread.current() && !TLS._osthread._interrupted
3260     no_clear_result_path = 2, // t == Thread.current() &&  TLS._osthread._interrupted && !clear_int
3261     slow_result_path     = 3, // slow path: t.isInterrupted(clear_int)
3262     PATH_LIMIT
3263   };
3264 
3265   // Ensure that it's not possible to move the load of TLS._osthread._interrupted flag
3266   // out of the function.
3267   insert_mem_bar(Opcodes::Op_MemBarCPUOrder);
3268 
3269   RegionNode* result_rgn = new RegionNode(PATH_LIMIT);
3270   PhiNode*    result_val = new PhiNode(result_rgn, TypeInt::BOOL);
3271 
3272   RegionNode* slow_region = new RegionNode(1);
3273   record_for_igvn(slow_region);
3274 
3275   // (a) Receiving thread must be the current thread.
3276   Node* rec_thr = argument(0);
3277   Node* tls_ptr = NULL;
3278   Node* cur_thr = generate_current_thread(tls_ptr);
3279   Node* cmp_thr = _gvn.transform(new CmpPNode(cur_thr, rec_thr));
3280   Node* bol_thr = _gvn.transform(new BoolNode(cmp_thr, BoolTest::ne));
3281 
3282   generate_slow_guard(bol_thr, slow_region);
3283 
3284   // (b) Interrupt bit on TLS must be false.
3285   Node* p = basic_plus_adr(top()/*!oop*/, tls_ptr, in_bytes(JavaThread::osthread_offset()));
3286   Node* osthread = make_load(NULL, p, TypeRawPtr::NOTNULL, T_ADDRESS, MemNode::unordered);
3287   p = basic_plus_adr(top()/*!oop*/, osthread, in_bytes(OSThread::interrupted_offset()));


4471   if (callee()->is_static())  return false;  // caller must have the capability!
4472   null_check_receiver();  // null-check receiver
4473   if (stopped())  return true;
4474 
4475   C->set_has_unsafe_access(true);  // Mark eventual nmethod as "unsafe".
4476 
4477   Node* src_ptr =         argument(1);   // type: oop
4478   Node* src_off = ConvL2X(argument(2));  // type: long
4479   Node* dst_ptr =         argument(4);   // type: oop
4480   Node* dst_off = ConvL2X(argument(5));  // type: long
4481   Node* size    = ConvL2X(argument(7));  // type: long
4482 
4483   assert(Unsafe_field_offset_to_byte_offset(11) == 11,
4484          "fieldOffset must be byte-scaled");
4485 
4486   Node* src = make_unsafe_address(src_ptr, src_off);
4487   Node* dst = make_unsafe_address(dst_ptr, dst_off);
4488 
4489   // Conservatively insert a memory barrier on all memory slices.
4490   // Do not let writes of the copy source or destination float below the copy.
4491   insert_mem_bar(Opcodes::Op_MemBarCPUOrder);
4492 
4493   // Call it.  Note that the length argument is not scaled.
4494   make_runtime_call(RC_LEAF|RC_NO_FP,
4495                     OptoRuntime::fast_arraycopy_Type(),
4496                     StubRoutines::unsafe_arraycopy(),
4497                     "unsafe_arraycopy",
4498                     TypeRawPtr::BOTTOM,
4499                     src, dst, size XTOP);
4500 
4501   // Do not let reads of the copy destination float above the copy.
4502   insert_mem_bar(Opcodes::Op_MemBarCPUOrder);
4503 
4504   return true;
4505 }
4506 
4507 //------------------------clone_coping-----------------------------------
4508 // Helper function for inline_native_clone.
4509 void LibraryCallKit::copy_to_clone(Node* obj, Node* alloc_obj, Node* obj_size, bool is_array, bool card_mark) {
4510   assert(obj_size != NULL, "");
4511   Node* raw_obj = alloc_obj->in(1);
4512   assert(alloc_obj->is_CheckCastPP() && raw_obj->is_Proj() && raw_obj->in(0)->is_Allocate(), "");
4513 
4514   AllocateNode* alloc = NULL;
4515   if (ReduceBulkZeroing) {
4516     // We will be completely responsible for initializing this object -
4517     // mark Initialize node as complete.
4518     alloc = AllocateNode::Ideal_allocation(alloc_obj, &_gvn);
4519     // The object was just allocated - there should be no any stores!
4520     guarantee(alloc != NULL && alloc->maybe_set_complete(&_gvn), "");
4521     // Mark as complete_with_arraycopy so that on AllocateNode
4522     // expansion, we know this AllocateNode is initialized by an array


4579     int raw_adr_idx = Compile::AliasIdxRaw;
4580     post_barrier(control(),
4581                  memory(raw_adr_type),
4582                  alloc_obj,
4583                  no_particular_field,
4584                  raw_adr_idx,
4585                  no_particular_value,
4586                  T_OBJECT,
4587                  false);
4588   }
4589 
4590   // Do not let reads from the cloned object float above the arraycopy.
4591   if (alloc != NULL) {
4592     // Do not let stores that initialize this object be reordered with
4593     // a subsequent store that would make this object accessible by
4594     // other threads.
4595     // Record what AllocateNode this StoreStore protects so that
4596     // escape analysis can go from the MemBarStoreStoreNode to the
4597     // AllocateNode and eliminate the MemBarStoreStoreNode if possible
4598     // based on the escape status of the AllocateNode.
4599     insert_mem_bar(Opcodes::Op_MemBarStoreStore, alloc->proj_out(AllocateNode::RawAddress));
4600   } else {
4601     insert_mem_bar(Opcodes::Op_MemBarCPUOrder);
4602   }
4603 }
4604 
4605 //------------------------inline_native_clone----------------------------
4606 // protected native Object java.lang.Object.clone();
4607 //
4608 // Here are the simple edge cases:
4609 //  null receiver => normal trap
4610 //  virtual and clone was overridden => slow path to out-of-line clone
4611 //  not cloneable or finalizer => slow path to out-of-line Object.clone
4612 //
4613 // The general case has two steps, allocation and copying.
4614 // Allocation has two cases, and uses GraphKit::new_instance or new_array.
4615 //
4616 // Copying also has two cases, oop arrays and everything else.
4617 // Oop arrays use arrayof_oop_arraycopy (same as System.arraycopy).
4618 // Everything else uses the tight inline loop supplied by CopyArrayNode.
4619 //
4620 // These steps fold up nicely if and when the cloned object's klass
4621 // can be sharply typed as an object array, a type array, or an instance.


4642       ciInstanceKlass* spec_ik = obj_type->speculative_type()->as_instance_klass();
4643       if (spec_ik->nof_nonstatic_fields() <= ArrayCopyLoadStoreMaxElem &&
4644           !spec_ik->has_injected_fields()) {
4645         ciKlass* k = obj_type->klass();
4646         if (!k->is_instance_klass() ||
4647             k->as_instance_klass()->is_interface() ||
4648             k->as_instance_klass()->has_subklass()) {
4649           obj = maybe_cast_profiled_obj(obj, obj_type->speculative_type(), false);
4650         }
4651       }
4652     }
4653 
4654     Node* obj_klass = load_object_klass(obj);
4655     const TypeKlassPtr* tklass = _gvn.type(obj_klass)->isa_klassptr();
4656     const TypeOopPtr*   toop   = ((tklass != NULL)
4657                                 ? tklass->as_instance_type()
4658                                 : TypeInstPtr::NOTNULL);
4659 
4660     // Conservatively insert a memory barrier on all memory slices.
4661     // Do not let writes into the original float below the clone.
4662     insert_mem_bar(Opcodes::Op_MemBarCPUOrder);
4663 
4664     // paths into result_reg:
4665     enum {
4666       _slow_path = 1,     // out-of-line call to clone method (virtual or not)
4667       _objArray_path,     // plain array allocation, plus arrayof_oop_arraycopy
4668       _array_path,        // plain array allocation, plus arrayof_long_arraycopy
4669       _instance_path,     // plain instance allocation, plus arrayof_long_arraycopy
4670       PATH_LIMIT
4671     };
4672     RegionNode* result_reg = new RegionNode(PATH_LIMIT);
4673     result_val             = new PhiNode(result_reg, TypeInstPtr::NOTNULL);
4674     PhiNode*    result_i_o = new PhiNode(result_reg, Type::ABIO);
4675     PhiNode*    result_mem = new PhiNode(result_reg, Type::MEMORY, TypePtr::BOTTOM);
4676     record_for_igvn(result_reg);
4677 
4678     const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
4679     int raw_adr_idx = Compile::AliasIdxRaw;
4680 
4681     Node* array_ctl = generate_array_guard(obj_klass, (RegionNode*)NULL);
4682     if (array_ctl != NULL) {


5949   Node* reference_obj = null_check_receiver();
5950   if (stopped()) return true;
5951 
5952   Node* adr = basic_plus_adr(reference_obj, reference_obj, referent_offset);
5953 
5954   ciInstanceKlass* klass = env()->Object_klass();
5955   const TypeOopPtr* object_type = TypeOopPtr::make_from_klass(klass);
5956 
5957   Node* no_ctrl = NULL;
5958   Node* result = make_load(no_ctrl, adr, object_type, T_OBJECT, MemNode::unordered);
5959 
5960   // Use the pre-barrier to record the value in the referent field
5961   pre_barrier(false /* do_load */,
5962               control(),
5963               NULL /* obj */, NULL /* adr */, max_juint /* alias_idx */, NULL /* val */, NULL /* val_type */,
5964               result /* pre_val */,
5965               T_OBJECT);
5966 
5967   // Add memory barrier to prevent commoning reads from this field
5968   // across safepoint since GC can change its value.
5969   insert_mem_bar(Opcodes::Op_MemBarCPUOrder);
5970 
5971   set_result(result);
5972   return true;
5973 }
5974 
5975 
5976 Node * LibraryCallKit::load_field_from_object(Node * fromObj, const char * fieldName, const char * fieldTypeString,
5977                                               bool is_exact=true, bool is_static=false,
5978                                               ciInstanceKlass * fromKls=NULL) {
5979   if (fromKls == NULL) {
5980     const TypeInstPtr* tinst = _gvn.type(fromObj)->isa_instptr();
5981     assert(tinst != NULL, "obj is null");
5982     assert(tinst->klass()->is_loaded(), "obj is not loaded");
5983     assert(!is_exact || tinst->klass_is_exact(), "klass not exact");
5984     fromKls = tinst->klass()->as_instance_klass();
5985   } else {
5986     assert(is_static, "only for static field access");
5987   }
5988   ciField* field = fromKls->get_field_by_name(ciSymbol::make(fieldName),
5989                                               ciSymbol::make(fieldTypeString),


6000   // Next code  copied from Parse::do_get_xxx():
6001 
6002   // Compute address and memory type.
6003   int offset  = field->offset_in_bytes();
6004   bool is_vol = field->is_volatile();
6005   ciType* field_klass = field->type();
6006   assert(field_klass->is_loaded(), "should be loaded");
6007   const TypePtr* adr_type = C->alias_type(field)->adr_type();
6008   Node *adr = basic_plus_adr(fromObj, fromObj, offset);
6009   BasicType bt = field->layout_type();
6010 
6011   // Build the resultant type of the load
6012   const Type *type;
6013   if (bt == T_OBJECT) {
6014     type = TypeOopPtr::make_from_klass(field_klass->as_klass());
6015   } else {
6016     type = Type::get_const_basic_type(bt);
6017   }
6018 
6019   if (support_IRIW_for_not_multiple_copy_atomic_cpu && is_vol) {
6020     insert_mem_bar(Opcodes::Op_MemBarVolatile);   // StoreLoad barrier
6021   }
6022   // Build the load.
6023   MemNode::MemOrd mo = is_vol ? MemNode::acquire : MemNode::unordered;
6024   Node* loadedField = make_load(NULL, adr, type, bt, adr_type, mo, LoadNode::DependsOnlyOnTest, is_vol);
6025   // If reference is volatile, prevent following memory ops from
6026   // floating up past the volatile read.  Also prevents commoning
6027   // another volatile read.
6028   if (is_vol) {
6029     // Memory barrier includes bogus read of value to force load BEFORE membar
6030     insert_mem_bar(Opcodes::Op_MemBarAcquire, loadedField);
6031   }
6032   return loadedField;
6033 }
6034 
6035 Node * LibraryCallKit::field_address_from_object(Node * fromObj, const char * fieldName, const char * fieldTypeString,
6036                                                  bool is_exact = true, bool is_static = false,
6037                                                  ciInstanceKlass * fromKls = NULL) {
6038   if (fromKls == NULL) {
6039     const TypeInstPtr* tinst = _gvn.type(fromObj)->isa_instptr();
6040     assert(tinst != NULL, "obj is null");
6041     assert(tinst->klass()->is_loaded(), "obj is not loaded");
6042     assert(!is_exact || tinst->klass_is_exact(), "klass not exact");
6043     fromKls = tinst->klass()->as_instance_klass();
6044   }
6045   else {
6046     assert(is_static, "only for static field access");
6047   }
6048   ciField* field = fromKls->get_field_by_name(ciSymbol::make(fieldName),
6049     ciSymbol::make(fieldTypeString),
6050     is_static);


< prev index next >