190 Node* generate_objArray_guard(Node* kls, RegionNode* region) {
191 return generate_array_guard_common(kls, region, true, false);
192 }
193 Node* generate_non_objArray_guard(Node* kls, RegionNode* region) {
194 return generate_array_guard_common(kls, region, true, true);
195 }
196 Node* generate_array_guard_common(Node* kls, RegionNode* region,
197 bool obj_array, bool not_array);
198 Node* generate_virtual_guard(Node* obj_klass, RegionNode* slow_region);
199 CallJavaNode* generate_method_call(vmIntrinsics::ID method_id,
200 bool is_virtual = false, bool is_static = false);
201 CallJavaNode* generate_method_call_static(vmIntrinsics::ID method_id) {
202 return generate_method_call(method_id, false, true);
203 }
204 CallJavaNode* generate_method_call_virtual(vmIntrinsics::ID method_id) {
205 return generate_method_call(method_id, true, false);
206 }
207 Node * load_field_from_object(Node * fromObj, const char * fieldName, const char * fieldTypeString, bool is_exact, bool is_static, ciInstanceKlass * fromKls);
208 Node * field_address_from_object(Node * fromObj, const char * fieldName, const char * fieldTypeString, bool is_exact, bool is_static, ciInstanceKlass * fromKls);
209
210 Node* make_string_method_node(int opcode, Node* str1_start, Node* cnt1, Node* str2_start, Node* cnt2, StrIntrinsicNode::ArgEnc ae);
211 bool inline_string_compareTo(StrIntrinsicNode::ArgEnc ae);
212 bool inline_string_indexOf(StrIntrinsicNode::ArgEnc ae);
213 bool inline_string_indexOfI(StrIntrinsicNode::ArgEnc ae);
214 Node* make_indexOf_node(Node* src_start, Node* src_count, Node* tgt_start, Node* tgt_count,
215 RegionNode* region, Node* phi, StrIntrinsicNode::ArgEnc ae);
216 bool inline_string_indexOfChar();
217 bool inline_string_equals(StrIntrinsicNode::ArgEnc ae);
218 bool inline_string_toBytesU();
219 bool inline_string_getCharsU();
220 bool inline_string_copy(bool compress);
221 bool inline_string_char_access(bool is_store);
222 Node* round_double_node(Node* n);
223 bool runtime_math(const TypeFunc* call_type, address funcAddr, const char* funcName);
224 bool inline_math_native(vmIntrinsics::ID id);
225 bool inline_trig(vmIntrinsics::ID id);
226 bool inline_math(vmIntrinsics::ID id);
227 template <typename OverflowOp>
228 bool inline_math_overflow(Node* arg1, Node* arg2);
229 void inline_math_mathExact(Node* math, Node* test);
230 bool inline_math_addExactI(bool is_increment);
463 } else {
464 // Root compile
465 C->print_inlining_stream()->print("Did not generate predicate for intrinsic %s%s at bci:%d in",
466 vmIntrinsics::name_at(intrinsic_id()),
467 (is_virtual() ? " (virtual)" : ""), bci);
468 }
469 }
470 C->gather_intrinsic_statistics(intrinsic_id(), is_virtual(), Compile::_intrinsic_failed);
471 return NULL;
472 }
473
474 bool LibraryCallKit::try_to_inline(int predicate) {
475 // Handle symbolic names for otherwise undistinguished boolean switches:
476 const bool is_store = true;
477 const bool is_compress = true;
478 const bool is_static = true;
479 const bool is_volatile = true;
480
481 if (!jvms()->has_method()) {
482 // Root JVMState has a null method.
483 assert(map()->memory()->Opcode() == Op_Parm, "");
484 // Insert the memory aliasing node
485 set_all_memory(reset_memory());
486 }
487 assert(merged_memory(), "");
488
489
490 switch (intrinsic_id()) {
491 case vmIntrinsics::_hashCode: return inline_native_hashcode(intrinsic()->is_virtual(), !is_static);
492 case vmIntrinsics::_identityHashCode: return inline_native_hashcode(/*!virtual*/ false, is_static);
493 case vmIntrinsics::_getClass: return inline_native_getClass();
494
495 case vmIntrinsics::_dsin:
496 case vmIntrinsics::_dcos:
497 case vmIntrinsics::_dtan:
498 case vmIntrinsics::_dabs:
499 case vmIntrinsics::_datan2:
500 case vmIntrinsics::_dsqrt:
501 case vmIntrinsics::_dexp:
502 case vmIntrinsics::_dlog:
503 case vmIntrinsics::_dlog10:
825
826 case vmIntrinsics::_hasNegatives:
827 return inline_hasNegatives();
828
829 default:
830 // If you get here, it may be that someone has added a new intrinsic
831 // to the list in vmSymbols.hpp without implementing it here.
832 #ifndef PRODUCT
833 if ((PrintMiscellaneous && (Verbose || WizardMode)) || PrintOpto) {
834 tty->print_cr("*** Warning: Unimplemented intrinsic %s(%d)",
835 vmIntrinsics::name_at(intrinsic_id()), intrinsic_id());
836 }
837 #endif
838 return false;
839 }
840 }
841
842 Node* LibraryCallKit::try_to_predicate(int predicate) {
843 if (!jvms()->has_method()) {
844 // Root JVMState has a null method.
845 assert(map()->memory()->Opcode() == Op_Parm, "");
846 // Insert the memory aliasing node
847 set_all_memory(reset_memory());
848 }
849 assert(merged_memory(), "");
850
851 switch (intrinsic_id()) {
852 case vmIntrinsics::_cipherBlockChaining_encryptAESCrypt:
853 return inline_cipherBlockChaining_AESCrypt_predicate(false);
854 case vmIntrinsics::_cipherBlockChaining_decryptAESCrypt:
855 return inline_cipherBlockChaining_AESCrypt_predicate(true);
856 case vmIntrinsics::_counterMode_AESCrypt:
857 return inline_counterMode_AESCrypt_predicate();
858 case vmIntrinsics::_digestBase_implCompressMB:
859 return inline_digestBase_implCompressMB_predicate(predicate);
860
861 default:
862 // If you get here, it may be that someone has added a new intrinsic
863 // to the list in vmSymbols.hpp without implementing it here.
864 #ifndef PRODUCT
865 if ((PrintMiscellaneous && (Verbose || WizardMode)) || PrintOpto) {
1005 }
1006 }
1007
1008 //--------------------------generate_current_thread--------------------
1009 Node* LibraryCallKit::generate_current_thread(Node* &tls_output) {
1010 ciKlass* thread_klass = env()->Thread_klass();
1011 const Type* thread_type = TypeOopPtr::make_from_klass(thread_klass)->cast_to_ptr_type(TypePtr::NotNull);
1012 Node* thread = _gvn.transform(new ThreadLocalNode());
1013 Node* p = basic_plus_adr(top()/*!oop*/, thread, in_bytes(JavaThread::threadObj_offset()));
1014 Node* threadObj = make_load(NULL, p, thread_type, T_OBJECT, MemNode::unordered);
1015 tls_output = thread;
1016 return threadObj;
1017 }
1018
1019
1020 //------------------------------make_string_method_node------------------------
1021 // Helper method for String intrinsic functions. This version is called with
1022 // str1 and str2 pointing to byte[] nodes containing Latin1 or UTF16 encoded
1023 // characters (depending on 'is_byte'). cnt1 and cnt2 are pointing to Int nodes
1024 // containing the lengths of str1 and str2.
1025 Node* LibraryCallKit::make_string_method_node(int opcode, Node* str1_start, Node* cnt1, Node* str2_start, Node* cnt2, StrIntrinsicNode::ArgEnc ae) {
1026 Node* result = NULL;
1027 switch (opcode) {
1028 case Op_StrIndexOf:
1029 result = new StrIndexOfNode(control(), memory(TypeAryPtr::BYTES),
1030 str1_start, cnt1, str2_start, cnt2, ae);
1031 break;
1032 case Op_StrComp:
1033 result = new StrCompNode(control(), memory(TypeAryPtr::BYTES),
1034 str1_start, cnt1, str2_start, cnt2, ae);
1035 break;
1036 case Op_StrEquals:
1037 // We already know that cnt1 == cnt2 here (checked in 'inline_string_equals').
1038 // Use the constant length if there is one because optimized match rule may exist.
1039 result = new StrEqualsNode(control(), memory(TypeAryPtr::BYTES),
1040 str1_start, str2_start, cnt2->is_Con() ? cnt2 : cnt1, ae);
1041 break;
1042 default:
1043 ShouldNotReachHere();
1044 return NULL;
1045 }
1046
1047 // All these intrinsics have checks.
1048 C->set_has_split_ifs(true); // Has chance for split-if optimization
1049
1050 return _gvn.transform(result);
1051 }
1052
1053 //------------------------------inline_string_compareTo------------------------
1054 bool LibraryCallKit::inline_string_compareTo(StrIntrinsicNode::ArgEnc ae) {
1055 Node* arg1 = argument(0);
1056 Node* arg2 = argument(1);
1057
1058 // Get start addr and length of first argument
1059 Node* arg1_start = array_element_address(arg1, intcon(0), T_BYTE);
1060 Node* arg1_cnt = load_array_length(arg1);
1061
1062 // Get start addr and length of second argument
1063 Node* arg2_start = array_element_address(arg2, intcon(0), T_BYTE);
1064 Node* arg2_cnt = load_array_length(arg2);
1065
1066 Node* result = make_string_method_node(Op_StrComp, arg1_start, arg1_cnt, arg2_start, arg2_cnt, ae);
1067 set_result(result);
1068 return true;
1069 }
1070
1071 //------------------------------inline_string_equals------------------------
1072 bool LibraryCallKit::inline_string_equals(StrIntrinsicNode::ArgEnc ae) {
1073 Node* arg1 = argument(0);
1074 Node* arg2 = argument(1);
1075
1076 // paths (plus control) merge
1077 RegionNode* region = new RegionNode(3);
1078 Node* phi = new PhiNode(region, TypeInt::BOOL);
1079
1080 if (!stopped()) {
1081 // Get start addr and length of first argument
1082 Node* arg1_start = array_element_address(arg1, intcon(0), T_BYTE);
1083 Node* arg1_cnt = load_array_length(arg1);
1084
1085 // Get start addr and length of second argument
1086 Node* arg2_start = array_element_address(arg2, intcon(0), T_BYTE);
1087 Node* arg2_cnt = load_array_length(arg2);
1088
1089 // Check for arg1_cnt != arg2_cnt
1090 Node* cmp = _gvn.transform(new CmpINode(arg1_cnt, arg2_cnt));
1091 Node* bol = _gvn.transform(new BoolNode(cmp, BoolTest::ne));
1092 Node* if_ne = generate_slow_guard(bol, NULL);
1093 if (if_ne != NULL) {
1094 phi->init_req(2, intcon(0));
1095 region->init_req(2, if_ne);
1096 }
1097
1098 // Check for count == 0 is done by assembler code for StrEquals.
1099
1100 if (!stopped()) {
1101 Node* equals = make_string_method_node(Op_StrEquals, arg1_start, arg1_cnt, arg2_start, arg2_cnt, ae);
1102 phi->init_req(1, equals);
1103 region->init_req(1, control());
1104 }
1105 }
1106
1107 // post merge
1108 set_control(_gvn.transform(region));
1109 record_for_igvn(region);
1110
1111 set_result(_gvn.transform(phi));
1112 return true;
1113 }
1114
1115 //------------------------------inline_array_equals----------------------------
1116 bool LibraryCallKit::inline_array_equals(StrIntrinsicNode::ArgEnc ae) {
1117 assert(ae == StrIntrinsicNode::UU || ae == StrIntrinsicNode::LL, "unsupported array types");
1118 Node* arg1 = argument(0);
1119 Node* arg2 = argument(1);
1120
1121 const TypeAryPtr* mtype = (ae == StrIntrinsicNode::UU) ? TypeAryPtr::CHARS : TypeAryPtr::BYTES;
1179 PreserveJVMState pjvms(this);
1180 set_control(_gvn.transform(new IfFalseNode(rc)));
1181 uncommon_trap(Deoptimization::Reason_range_check,
1182 Deoptimization::Action_make_not_entrant);
1183 }
1184
1185 if (stopped()) {
1186 return false;
1187 }
1188
1189 Node* result = new CastIINode(index, TypeInt::make(0, _gvn.type(length)->is_int()->_hi, Type::WidenMax));
1190 result->set_req(0, control());
1191 result = _gvn.transform(result);
1192 set_result(result);
1193 replace_in_map(index, result);
1194 return true;
1195 }
1196
1197 //------------------------------inline_string_indexOf------------------------
1198 bool LibraryCallKit::inline_string_indexOf(StrIntrinsicNode::ArgEnc ae) {
1199 if (!Matcher::match_rule_supported(Op_StrIndexOf)) {
1200 return false;
1201 }
1202 Node* src = argument(0);
1203 Node* tgt = argument(1);
1204
1205 // Make the merge point
1206 RegionNode* result_rgn = new RegionNode(4);
1207 Node* result_phi = new PhiNode(result_rgn, TypeInt::INT);
1208
1209 // Get start addr and length of source string
1210 Node* src_start = array_element_address(src, intcon(0), T_BYTE);
1211 Node* src_count = load_array_length(src);
1212
1213 // Get start addr and length of substring
1214 Node* tgt_start = array_element_address(tgt, intcon(0), T_BYTE);
1215 Node* tgt_count = load_array_length(tgt);
1216
1217 if (ae == StrIntrinsicNode::UU || ae == StrIntrinsicNode::UL) {
1218 // Divide src size by 2 if String is UTF16 encoded
1219 src_count = _gvn.transform(new RShiftINode(src_count, intcon(1)));
1223 tgt_count = _gvn.transform(new RShiftINode(tgt_count, intcon(1)));
1224 }
1225
1226 Node* result = make_indexOf_node(src_start, src_count, tgt_start, tgt_count, result_rgn, result_phi, ae);
1227 if (result != NULL) {
1228 result_phi->init_req(3, result);
1229 result_rgn->init_req(3, control());
1230 }
1231 set_control(_gvn.transform(result_rgn));
1232 record_for_igvn(result_rgn);
1233 set_result(_gvn.transform(result_phi));
1234
1235 return true;
1236 }
1237
1238 //-----------------------------inline_string_indexOf-----------------------
1239 bool LibraryCallKit::inline_string_indexOfI(StrIntrinsicNode::ArgEnc ae) {
1240 if (too_many_traps(Deoptimization::Reason_intrinsic)) {
1241 return false;
1242 }
1243 if (!Matcher::match_rule_supported(Op_StrIndexOf)) {
1244 return false;
1245 }
1246 assert(callee()->signature()->size() == 5, "String.indexOf() has 5 arguments");
1247 Node* src = argument(0); // byte[]
1248 Node* src_count = argument(1); // char count
1249 Node* tgt = argument(2); // byte[]
1250 Node* tgt_count = argument(3); // char count
1251 Node* from_index = argument(4); // char index
1252
1253 // Multiply byte array index by 2 if String is UTF16 encoded
1254 Node* src_offset = (ae == StrIntrinsicNode::LL) ? from_index : _gvn.transform(new LShiftINode(from_index, intcon(1)));
1255 src_count = _gvn.transform(new SubINode(src_count, from_index));
1256 Node* src_start = array_element_address(src, src_offset, T_BYTE);
1257 Node* tgt_start = array_element_address(tgt, intcon(0), T_BYTE);
1258
1259 // Range checks
1260 generate_string_range_check(src, src_offset, src_count, ae != StrIntrinsicNode::LL);
1261 generate_string_range_check(tgt, intcon(0), tgt_count, ae == StrIntrinsicNode::UU);
1262 if (stopped()) {
1263 return true;
1298 RegionNode* region, Node* phi, StrIntrinsicNode::ArgEnc ae) {
1299 // Check for substr count > string count
1300 Node* cmp = _gvn.transform(new CmpINode(tgt_count, src_count));
1301 Node* bol = _gvn.transform(new BoolNode(cmp, BoolTest::gt));
1302 Node* if_gt = generate_slow_guard(bol, NULL);
1303 if (if_gt != NULL) {
1304 phi->init_req(1, intcon(-1));
1305 region->init_req(1, if_gt);
1306 }
1307 if (!stopped()) {
1308 // Check for substr count == 0
1309 cmp = _gvn.transform(new CmpINode(tgt_count, intcon(0)));
1310 bol = _gvn.transform(new BoolNode(cmp, BoolTest::eq));
1311 Node* if_zero = generate_slow_guard(bol, NULL);
1312 if (if_zero != NULL) {
1313 phi->init_req(2, intcon(0));
1314 region->init_req(2, if_zero);
1315 }
1316 }
1317 if (!stopped()) {
1318 return make_string_method_node(Op_StrIndexOf, src_start, src_count, tgt_start, tgt_count, ae);
1319 }
1320 return NULL;
1321 }
1322
1323 //-----------------------------inline_string_indexOfChar-----------------------
1324 bool LibraryCallKit::inline_string_indexOfChar() {
1325 if (too_many_traps(Deoptimization::Reason_intrinsic)) {
1326 return false;
1327 }
1328 if (!Matcher::match_rule_supported(Op_StrIndexOfChar)) {
1329 return false;
1330 }
1331 assert(callee()->signature()->size() == 4, "String.indexOfChar() has 4 arguments");
1332 Node* src = argument(0); // byte[]
1333 Node* tgt = argument(1); // tgt is int ch
1334 Node* from_index = argument(2);
1335 Node* max = argument(3);
1336
1337 Node* src_offset = _gvn.transform(new LShiftINode(from_index, intcon(1)));
1338 Node* src_start = array_element_address(src, src_offset, T_BYTE);
1339 Node* src_count = _gvn.transform(new SubINode(max, from_index));
1340
1341 // Range checks
1342 generate_string_range_check(src, src_offset, src_count, true);
1343 if (stopped()) {
1344 return true;
1345 }
1346
1347 RegionNode* region = new RegionNode(3);
1348 Node* phi = new PhiNode(region, TypeInt::INT);
1428 } else {
1429 inflate_string(src_start, dst_start, TypeAryPtr::get_array_body_type(dst_elem), length);
1430 }
1431
1432 if (alloc != NULL) {
1433 if (alloc->maybe_set_complete(&_gvn)) {
1434 // "You break it, you buy it."
1435 InitializeNode* init = alloc->initialization();
1436 assert(init->is_complete(), "we just did this");
1437 init->set_complete_with_arraycopy();
1438 assert(dst->is_CheckCastPP(), "sanity");
1439 assert(dst->in(0)->in(0) == init, "dest pinned");
1440 }
1441 // Do not let stores that initialize this object be reordered with
1442 // a subsequent store that would make this object accessible by
1443 // other threads.
1444 // Record what AllocateNode this StoreStore protects so that
1445 // escape analysis can go from the MemBarStoreStoreNode to the
1446 // AllocateNode and eliminate the MemBarStoreStoreNode if possible
1447 // based on the escape status of the AllocateNode.
1448 insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out(AllocateNode::RawAddress));
1449 }
1450 if (compress) {
1451 set_result(_gvn.transform(count));
1452 }
1453 return true;
1454 }
1455
1456 #ifdef _LP64
1457 #define XTOP ,top() /*additional argument*/
1458 #else //_LP64
1459 #define XTOP /*no additional argument*/
1460 #endif //_LP64
1461
1462 //------------------------inline_string_toBytesU--------------------------
1463 // public static byte[] StringUTF16.toBytes(char[] value, int off, int len)
1464 bool LibraryCallKit::inline_string_toBytesU() {
1465 if (too_many_traps(Deoptimization::Reason_intrinsic)) {
1466 return false;
1467 }
1468 // Get the arguments.
1520 OptoRuntime::fast_arraycopy_Type(),
1521 copyfunc_addr, copyfunc_name, TypeRawPtr::BOTTOM,
1522 src_start, dst_start, ConvI2X(length) XTOP);
1523 // Do not let reads from the cloned object float above the arraycopy.
1524 if (alloc != NULL) {
1525 if (alloc->maybe_set_complete(&_gvn)) {
1526 // "You break it, you buy it."
1527 InitializeNode* init = alloc->initialization();
1528 assert(init->is_complete(), "we just did this");
1529 init->set_complete_with_arraycopy();
1530 assert(newcopy->is_CheckCastPP(), "sanity");
1531 assert(newcopy->in(0)->in(0) == init, "dest pinned");
1532 }
1533 // Do not let stores that initialize this object be reordered with
1534 // a subsequent store that would make this object accessible by
1535 // other threads.
1536 // Record what AllocateNode this StoreStore protects so that
1537 // escape analysis can go from the MemBarStoreStoreNode to the
1538 // AllocateNode and eliminate the MemBarStoreStoreNode if possible
1539 // based on the escape status of the AllocateNode.
1540 insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out(AllocateNode::RawAddress));
1541 } else {
1542 insert_mem_bar(Op_MemBarCPUOrder);
1543 }
1544 } // original reexecute is set back here
1545
1546 C->set_has_split_ifs(true); // Has chance for split-if optimization
1547 if (!stopped()) {
1548 set_result(newcopy);
1549 }
1550 return true;
1551 }
1552
1553 //------------------------inline_string_getCharsU--------------------------
1554 // public void StringUTF16.getChars(byte[] src, int srcBegin, int srcEnd, char dst[], int dstBegin)
1555 bool LibraryCallKit::inline_string_getCharsU() {
1556 if (too_many_traps(Deoptimization::Reason_intrinsic)) {
1557 return false;
1558 }
1559
1560 // Get the arguments.
1561 Node* src = argument(0);
1562 Node* src_begin = argument(1);
1604 OptoRuntime::fast_arraycopy_Type(),
1605 copyfunc_addr, copyfunc_name, TypeRawPtr::BOTTOM,
1606 src_start, dst_start, ConvI2X(length) XTOP);
1607 // Do not let reads from the cloned object float above the arraycopy.
1608 if (alloc != NULL) {
1609 if (alloc->maybe_set_complete(&_gvn)) {
1610 // "You break it, you buy it."
1611 InitializeNode* init = alloc->initialization();
1612 assert(init->is_complete(), "we just did this");
1613 init->set_complete_with_arraycopy();
1614 assert(dst->is_CheckCastPP(), "sanity");
1615 assert(dst->in(0)->in(0) == init, "dest pinned");
1616 }
1617 // Do not let stores that initialize this object be reordered with
1618 // a subsequent store that would make this object accessible by
1619 // other threads.
1620 // Record what AllocateNode this StoreStore protects so that
1621 // escape analysis can go from the MemBarStoreStoreNode to the
1622 // AllocateNode and eliminate the MemBarStoreStoreNode if possible
1623 // based on the escape status of the AllocateNode.
1624 insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out(AllocateNode::RawAddress));
1625 } else {
1626 insert_mem_bar(Op_MemBarCPUOrder);
1627 }
1628 }
1629
1630 C->set_has_split_ifs(true); // Has chance for split-if optimization
1631 return true;
1632 }
1633
1634 //----------------------inline_string_char_access----------------------------
1635 // Store/Load char to/from byte[] array.
1636 // static void StringUTF16.putChar(byte[] val, int index, int c)
1637 // static char StringUTF16.getChar(byte[] val, int index)
1638 bool LibraryCallKit::inline_string_char_access(bool is_store) {
1639 Node* value = argument(0);
1640 Node* index = argument(1);
1641 Node* ch = is_store ? argument(2) : NULL;
1642
1643 // This intrinsic accesses byte[] array as char[] array. Computing the offsets
1644 // correctly requires matched array shapes.
1645 assert (arrayOopDesc::base_offset_in_bytes(T_CHAR) == arrayOopDesc::base_offset_in_bytes(T_BYTE),
1646 "sanity: byte[] and char[] bases agree");
1812 runtime_math(OptoRuntime::Math_D_D_Type(), StubRoutines::dsin(), "dsin") :
1813 runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dsin), "SIN");
1814 case vmIntrinsics::_dcos:
1815 return StubRoutines::dcos() != NULL ?
1816 runtime_math(OptoRuntime::Math_D_D_Type(), StubRoutines::dcos(), "dcos") :
1817 runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dcos), "COS");
1818 case vmIntrinsics::_dtan:
1819 return StubRoutines::dtan() != NULL ?
1820 runtime_math(OptoRuntime::Math_D_D_Type(), StubRoutines::dtan(), "dtan") :
1821 runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dtan), "TAN");
1822 case vmIntrinsics::_dlog:
1823 return StubRoutines::dlog() != NULL ?
1824 runtime_math(OptoRuntime::Math_D_D_Type(), StubRoutines::dlog(), "dlog") :
1825 runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dlog), "LOG");
1826 case vmIntrinsics::_dlog10:
1827 return StubRoutines::dlog10() != NULL ?
1828 runtime_math(OptoRuntime::Math_D_D_Type(), StubRoutines::dlog10(), "dlog10") :
1829 runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dlog10), "LOG10");
1830
1831 // These intrinsics are supported on all hardware
1832 case vmIntrinsics::_dsqrt: return Matcher::match_rule_supported(Op_SqrtD) ? inline_math(id) : false;
1833 case vmIntrinsics::_dabs: return Matcher::has_match_rule(Op_AbsD) ? inline_math(id) : false;
1834
1835 case vmIntrinsics::_dexp:
1836 return StubRoutines::dexp() != NULL ?
1837 runtime_math(OptoRuntime::Math_D_D_Type(), StubRoutines::dexp(), "dexp") :
1838 runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dexp), "EXP");
1839 case vmIntrinsics::_dpow:
1840 return StubRoutines::dpow() != NULL ?
1841 runtime_math(OptoRuntime::Math_DD_D_Type(), StubRoutines::dpow(), "dpow") :
1842 runtime_math(OptoRuntime::Math_DD_D_Type(), FN_PTR(SharedRuntime::dpow), "POW");
1843 #undef FN_PTR
1844
1845 // These intrinsics are not yet correctly implemented
1846 case vmIntrinsics::_datan2:
1847 return false;
1848
1849 default:
1850 fatal_unexpected_iid(id);
1851 return false;
1852 }
1853 }
1957 }
1958
1959 bool want_max = (id == vmIntrinsics::_max);
1960
1961 const TypeInt* txvalue = _gvn.type(xvalue)->isa_int();
1962 const TypeInt* tyvalue = _gvn.type(yvalue)->isa_int();
1963 if (txvalue == NULL || tyvalue == NULL) return top();
1964 // This is not really necessary, but it is consistent with a
1965 // hypothetical MaxINode::Value method:
1966 int widen = MAX2(txvalue->_widen, tyvalue->_widen);
1967
1968 // %%% This folding logic should (ideally) be in a different place.
1969 // Some should be inside IfNode, and there to be a more reliable
1970 // transformation of ?: style patterns into cmoves. We also want
1971 // more powerful optimizations around cmove and min/max.
1972
1973 // Try to find a dominating comparison of these guys.
1974 // It can simplify the index computation for Arrays.copyOf
1975 // and similar uses of System.arraycopy.
1976 // First, compute the normalized version of CmpI(x, y).
1977 int cmp_op = Op_CmpI;
1978 Node* xkey = xvalue;
1979 Node* ykey = yvalue;
1980 Node* ideal_cmpxy = _gvn.transform(new CmpINode(xkey, ykey));
1981 if (ideal_cmpxy->is_Cmp()) {
1982 // E.g., if we have CmpI(length - offset, count),
1983 // it might idealize to CmpI(length, count + offset)
1984 cmp_op = ideal_cmpxy->Opcode();
1985 xkey = ideal_cmpxy->in(1);
1986 ykey = ideal_cmpxy->in(2);
1987 }
1988
1989 // Start by locating any relevant comparisons.
1990 Node* start_from = (xkey->outcnt() < ykey->outcnt()) ? xkey : ykey;
1991 Node* cmpxy = NULL;
1992 Node* cmpyx = NULL;
1993 for (DUIterator_Fast kmax, k = start_from->fast_outs(kmax); k < kmax; k++) {
1994 Node* cmp = start_from->fast_out(k);
1995 if (cmp->outcnt() > 0 && // must have prior uses
1996 cmp->in(0) == NULL && // must be context-independent
1997 cmp->Opcode() == cmp_op) { // right kind of compare
2270
2271 // Update IdealKit memory and control from graphKit.
2272 __ sync_kit(this);
2273
2274 Node* one = __ ConI(1);
2275 // is_instof == 0 if base_oop == NULL
2276 __ if_then(is_instof, BoolTest::eq, one, unlikely); {
2277
2278 // Update graphKit from IdeakKit.
2279 sync_kit(ideal);
2280
2281 // Use the pre-barrier to record the value in the referent field
2282 pre_barrier(false /* do_load */,
2283 __ ctrl(),
2284 NULL /* obj */, NULL /* adr */, max_juint /* alias_idx */, NULL /* val */, NULL /* val_type */,
2285 pre_val /* pre_val */,
2286 T_OBJECT);
2287 if (need_mem_bar) {
2288 // Add memory barrier to prevent commoning reads from this field
2289 // across safepoint since GC can change its value.
2290 insert_mem_bar(Op_MemBarCPUOrder);
2291 }
2292 // Update IdealKit from graphKit.
2293 __ sync_kit(this);
2294
2295 } __ end_if(); // _ref_type != ref_none
2296 } __ end_if(); // offset == referent_offset
2297
2298 // Final sync IdealKit and GraphKit.
2299 final_sync(ideal);
2300 #undef __
2301 }
2302
2303
2304 const TypeOopPtr* LibraryCallKit::sharpen_unsafe_type(Compile::AliasType* alias_type, const TypePtr *adr_type) {
2305 // Attempt to infer a sharper value type from the offset and base type.
2306 ciKlass* sharpened_klass = NULL;
2307
2308 // See if it is an instance field, with an object type.
2309 if (alias_type->field() != NULL) {
2310 if (alias_type->field()->type()->is_klass()) {
2495 // Heap pointers get a null-check from the interpreter,
2496 // as a courtesy. However, this is not guaranteed by Unsafe,
2497 // and it is not possible to fully distinguish unintended nulls
2498 // from intended ones in this API.
2499
2500 // We need to emit leading and trailing CPU membars (see below) in
2501 // addition to memory membars for special access modes. This is a little
2502 // too strong, but avoids the need to insert per-alias-type
2503 // volatile membars (for stores; compare Parse::do_put_xxx), which
2504 // we cannot do effectively here because we probably only have a
2505 // rough approximation of type.
2506
2507 switch(kind) {
2508 case Relaxed:
2509 case Opaque:
2510 case Acquire:
2511 break;
2512 case Release:
2513 case Volatile:
2514 if (is_store) {
2515 insert_mem_bar(Op_MemBarRelease);
2516 } else {
2517 if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
2518 insert_mem_bar(Op_MemBarVolatile);
2519 }
2520 }
2521 break;
2522 default:
2523 ShouldNotReachHere();
2524 }
2525
2526 // Memory barrier to prevent normal and 'unsafe' accesses from
2527 // bypassing each other. Happens after null checks, so the
2528 // exception paths do not take memory state from the memory barrier,
2529 // so there's no problems making a strong assert about mixing users
2530 // of safe & unsafe memory.
2531 if (need_mem_bar) insert_mem_bar(Op_MemBarCPUOrder);
2532
2533 if (!is_store) {
2534 Node* p = NULL;
2535 // Try to constant fold a load from a constant field
2536 ciField* field = alias_type->field();
2537 if (heap_base_oop != top() && field != NULL && field->is_constant() && !mismatched) {
2538 // final or stable field
2539 p = make_constant_from_field(field, heap_base_oop);
2540 }
2541 if (p == NULL) {
2542 // To be valid, unsafe loads may depend on other conditions than
2543 // the one that guards them: pin the Load node
2544 p = make_load(control(), adr, value_type, type, adr_type, mo, LoadNode::Pinned, requires_atomic_access, unaligned, mismatched);
2545 // load value
2546 switch (type) {
2547 case T_BOOLEAN:
2548 case T_CHAR:
2549 case T_BYTE:
2550 case T_SHORT:
2551 case T_INT:
2610 // Update IdealKit memory.
2611 __ sync_kit(this);
2612 } __ else_(); {
2613 __ store(__ ctrl(), adr, val, type, alias_type->index(), mo, requires_atomic_access, mismatched);
2614 } __ end_if();
2615 // Final sync IdealKit and GraphKit.
2616 final_sync(ideal);
2617 #undef __
2618 }
2619 }
2620 }
2621
2622 switch(kind) {
2623 case Relaxed:
2624 case Opaque:
2625 case Release:
2626 break;
2627 case Acquire:
2628 case Volatile:
2629 if (!is_store) {
2630 insert_mem_bar(Op_MemBarAcquire);
2631 } else {
2632 if (!support_IRIW_for_not_multiple_copy_atomic_cpu) {
2633 insert_mem_bar(Op_MemBarVolatile);
2634 }
2635 }
2636 break;
2637 default:
2638 ShouldNotReachHere();
2639 }
2640
2641 if (need_mem_bar) insert_mem_bar(Op_MemBarCPUOrder);
2642
2643 return true;
2644 }
2645
2646 //----------------------------inline_unsafe_load_store----------------------------
2647 // This method serves a couple of different customers (depending on LoadStoreKind):
2648 //
2649 // LS_cmp_swap:
2650 //
2651 // boolean compareAndSwapObject(Object o, long offset, Object expected, Object x);
2652 // boolean compareAndSwapInt( Object o, long offset, int expected, int x);
2653 // boolean compareAndSwapLong( Object o, long offset, long expected, long x);
2654 //
2655 // LS_cmp_swap_weak:
2656 //
2657 // boolean weakCompareAndSwapObject( Object o, long offset, Object expected, Object x);
2658 // boolean weakCompareAndSwapObjectAcquire(Object o, long offset, Object expected, Object x);
2659 // boolean weakCompareAndSwapObjectRelease(Object o, long offset, Object expected, Object x);
2660 //
2661 // boolean weakCompareAndSwapInt( Object o, long offset, int expected, int x);
2827 case LS_cmp_swap:
2828 case LS_cmp_swap_weak:
2829 case LS_get_add:
2830 break;
2831 default:
2832 ShouldNotReachHere();
2833 }
2834
2835 int alias_idx = C->get_alias_index(adr_type);
2836
2837 // Memory-model-wise, a LoadStore acts like a little synchronized
2838 // block, so needs barriers on each side. These don't translate
2839 // into actual barriers on most machines, but we still need rest of
2840 // compiler to respect ordering.
2841
2842 switch (access_kind) {
2843 case Relaxed:
2844 case Acquire:
2845 break;
2846 case Release:
2847 insert_mem_bar(Op_MemBarRelease);
2848 break;
2849 case Volatile:
2850 if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
2851 insert_mem_bar(Op_MemBarVolatile);
2852 } else {
2853 insert_mem_bar(Op_MemBarRelease);
2854 }
2855 break;
2856 default:
2857 ShouldNotReachHere();
2858 }
2859 insert_mem_bar(Op_MemBarCPUOrder);
2860
2861 // Figure out the memory ordering.
2862 MemNode::MemOrd mo = access_kind_to_memord(access_kind);
2863
2864 // 4984716: MemBars must be inserted before this
2865 // memory node in order to avoid a false
2866 // dependency which will confuse the scheduler.
2867 Node *mem = memory(alias_idx);
2868
2869 // For now, we handle only those cases that actually exist: ints,
2870 // longs, and Object. Adding others should be straightforward.
2871 Node* load_store = NULL;
2872 switch(type) {
2873 case T_BYTE:
2874 switch(kind) {
2875 case LS_get_add:
2876 load_store = _gvn.transform(new GetAndAddBNode(control(), mem, adr, newval, adr_type));
2877 break;
2878 case LS_get_set:
2879 load_store = _gvn.transform(new GetAndSetBNode(control(), mem, adr, newval, adr_type));
3079 set_memory(proj, alias_idx);
3080
3081 if (type == T_OBJECT && (kind == LS_get_set || kind == LS_cmp_exchange)) {
3082 #ifdef _LP64
3083 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
3084 load_store = _gvn.transform(new DecodeNNode(load_store, load_store->get_ptr_type()));
3085 }
3086 #endif
3087 if (can_move_pre_barrier()) {
3088 // Don't need to load pre_val. The old value is returned by load_store.
3089 // The pre_barrier can execute after the xchg as long as no safepoint
3090 // gets inserted between them.
3091 pre_barrier(false /* do_load */,
3092 control(), NULL, NULL, max_juint, NULL, NULL,
3093 load_store /* pre_val */,
3094 T_OBJECT);
3095 }
3096 }
3097
3098 // Add the trailing membar surrounding the access
3099 insert_mem_bar(Op_MemBarCPUOrder);
3100
3101 switch (access_kind) {
3102 case Relaxed:
3103 case Release:
3104 break; // do nothing
3105 case Acquire:
3106 case Volatile:
3107 insert_mem_bar(Op_MemBarAcquire);
3108 // !support_IRIW_for_not_multiple_copy_atomic_cpu handled in platform code
3109 break;
3110 default:
3111 ShouldNotReachHere();
3112 }
3113
3114 assert(type2size[load_store->bottom_type()->basic_type()] == type2size[rtype], "result type should match");
3115 set_result(load_store);
3116 return true;
3117 }
3118
3119 MemNode::MemOrd LibraryCallKit::access_kind_to_memord_LS(AccessKind kind, bool is_store) {
3120 MemNode::MemOrd mo = MemNode::unset;
3121 switch(kind) {
3122 case Opaque:
3123 case Relaxed: mo = MemNode::unordered; break;
3124 case Acquire: mo = MemNode::acquire; break;
3125 case Release: mo = MemNode::release; break;
3126 case Volatile: mo = is_store ? MemNode::release : MemNode::acquire; break;
3127 default:
3132 }
3133
3134 MemNode::MemOrd LibraryCallKit::access_kind_to_memord(AccessKind kind) {
3135 MemNode::MemOrd mo = MemNode::unset;
3136 switch(kind) {
3137 case Opaque:
3138 case Relaxed: mo = MemNode::unordered; break;
3139 case Acquire: mo = MemNode::acquire; break;
3140 case Release: mo = MemNode::release; break;
3141 case Volatile: mo = MemNode::seqcst; break;
3142 default:
3143 ShouldNotReachHere();
3144 }
3145 guarantee(mo != MemNode::unset, "Should select memory ordering");
3146 return mo;
3147 }
3148
3149 bool LibraryCallKit::inline_unsafe_fence(vmIntrinsics::ID id) {
3150 // Regardless of form, don't allow previous ld/st to move down,
3151 // then issue acquire, release, or volatile mem_bar.
3152 insert_mem_bar(Op_MemBarCPUOrder);
3153 switch(id) {
3154 case vmIntrinsics::_loadFence:
3155 insert_mem_bar(Op_LoadFence);
3156 return true;
3157 case vmIntrinsics::_storeFence:
3158 insert_mem_bar(Op_StoreFence);
3159 return true;
3160 case vmIntrinsics::_fullFence:
3161 insert_mem_bar(Op_MemBarVolatile);
3162 return true;
3163 default:
3164 fatal_unexpected_iid(id);
3165 return false;
3166 }
3167 }
3168
3169 bool LibraryCallKit::inline_onspinwait() {
3170 insert_mem_bar(Op_OnSpinWait);
3171 return true;
3172 }
3173
3174 bool LibraryCallKit::klass_needs_init_guard(Node* kls) {
3175 if (!kls->is_Con()) {
3176 return true;
3177 }
3178 const TypeKlassPtr* klsptr = kls->bottom_type()->isa_klassptr();
3179 if (klsptr == NULL) {
3180 return true;
3181 }
3182 ciInstanceKlass* ik = klsptr->klass()->as_instance_klass();
3183 // don't need a guard for a klass that is already initialized
3184 return !ik->is_initialized();
3185 }
3186
3187 //----------------------------inline_unsafe_allocate---------------------------
3188 // public native Object Unsafe.allocateInstance(Class<?> cls);
3189 bool LibraryCallKit::inline_unsafe_allocate() {
3190 if (callee()->is_static()) return false; // caller must have the capability!
3247 // (!TLS._osthread._interrupted || WINDOWS_ONLY(false) NOT_WINDOWS(!clear_int)))
3248 // ? TLS._osthread._interrupted : /*slow path:*/ t.isInterrupted(clear_int)
3249 // So, in the common case that the interrupt bit is false,
3250 // we avoid making a call into the VM. Even if the interrupt bit
3251 // is true, if the clear_int argument is false, we avoid the VM call.
3252 // However, if the receiver is not currentThread, we must call the VM,
3253 // because there must be some locking done around the operation.
3254
3255 // We only go to the fast case code if we pass two guards.
3256 // Paths which do not pass are accumulated in the slow_region.
3257
3258 enum {
3259 no_int_result_path = 1, // t == Thread.current() && !TLS._osthread._interrupted
3260 no_clear_result_path = 2, // t == Thread.current() && TLS._osthread._interrupted && !clear_int
3261 slow_result_path = 3, // slow path: t.isInterrupted(clear_int)
3262 PATH_LIMIT
3263 };
3264
3265 // Ensure that it's not possible to move the load of TLS._osthread._interrupted flag
3266 // out of the function.
3267 insert_mem_bar(Op_MemBarCPUOrder);
3268
3269 RegionNode* result_rgn = new RegionNode(PATH_LIMIT);
3270 PhiNode* result_val = new PhiNode(result_rgn, TypeInt::BOOL);
3271
3272 RegionNode* slow_region = new RegionNode(1);
3273 record_for_igvn(slow_region);
3274
3275 // (a) Receiving thread must be the current thread.
3276 Node* rec_thr = argument(0);
3277 Node* tls_ptr = NULL;
3278 Node* cur_thr = generate_current_thread(tls_ptr);
3279 Node* cmp_thr = _gvn.transform(new CmpPNode(cur_thr, rec_thr));
3280 Node* bol_thr = _gvn.transform(new BoolNode(cmp_thr, BoolTest::ne));
3281
3282 generate_slow_guard(bol_thr, slow_region);
3283
3284 // (b) Interrupt bit on TLS must be false.
3285 Node* p = basic_plus_adr(top()/*!oop*/, tls_ptr, in_bytes(JavaThread::osthread_offset()));
3286 Node* osthread = make_load(NULL, p, TypeRawPtr::NOTNULL, T_ADDRESS, MemNode::unordered);
3287 p = basic_plus_adr(top()/*!oop*/, osthread, in_bytes(OSThread::interrupted_offset()));
4471 if (callee()->is_static()) return false; // caller must have the capability!
4472 null_check_receiver(); // null-check receiver
4473 if (stopped()) return true;
4474
4475 C->set_has_unsafe_access(true); // Mark eventual nmethod as "unsafe".
4476
4477 Node* src_ptr = argument(1); // type: oop
4478 Node* src_off = ConvL2X(argument(2)); // type: long
4479 Node* dst_ptr = argument(4); // type: oop
4480 Node* dst_off = ConvL2X(argument(5)); // type: long
4481 Node* size = ConvL2X(argument(7)); // type: long
4482
4483 assert(Unsafe_field_offset_to_byte_offset(11) == 11,
4484 "fieldOffset must be byte-scaled");
4485
4486 Node* src = make_unsafe_address(src_ptr, src_off);
4487 Node* dst = make_unsafe_address(dst_ptr, dst_off);
4488
4489 // Conservatively insert a memory barrier on all memory slices.
4490 // Do not let writes of the copy source or destination float below the copy.
4491 insert_mem_bar(Op_MemBarCPUOrder);
4492
4493 // Call it. Note that the length argument is not scaled.
4494 make_runtime_call(RC_LEAF|RC_NO_FP,
4495 OptoRuntime::fast_arraycopy_Type(),
4496 StubRoutines::unsafe_arraycopy(),
4497 "unsafe_arraycopy",
4498 TypeRawPtr::BOTTOM,
4499 src, dst, size XTOP);
4500
4501 // Do not let reads of the copy destination float above the copy.
4502 insert_mem_bar(Op_MemBarCPUOrder);
4503
4504 return true;
4505 }
4506
4507 //------------------------clone_coping-----------------------------------
4508 // Helper function for inline_native_clone.
4509 void LibraryCallKit::copy_to_clone(Node* obj, Node* alloc_obj, Node* obj_size, bool is_array, bool card_mark) {
4510 assert(obj_size != NULL, "");
4511 Node* raw_obj = alloc_obj->in(1);
4512 assert(alloc_obj->is_CheckCastPP() && raw_obj->is_Proj() && raw_obj->in(0)->is_Allocate(), "");
4513
4514 AllocateNode* alloc = NULL;
4515 if (ReduceBulkZeroing) {
4516 // We will be completely responsible for initializing this object -
4517 // mark Initialize node as complete.
4518 alloc = AllocateNode::Ideal_allocation(alloc_obj, &_gvn);
4519 // The object was just allocated - there should be no any stores!
4520 guarantee(alloc != NULL && alloc->maybe_set_complete(&_gvn), "");
4521 // Mark as complete_with_arraycopy so that on AllocateNode
4522 // expansion, we know this AllocateNode is initialized by an array
4579 int raw_adr_idx = Compile::AliasIdxRaw;
4580 post_barrier(control(),
4581 memory(raw_adr_type),
4582 alloc_obj,
4583 no_particular_field,
4584 raw_adr_idx,
4585 no_particular_value,
4586 T_OBJECT,
4587 false);
4588 }
4589
4590 // Do not let reads from the cloned object float above the arraycopy.
4591 if (alloc != NULL) {
4592 // Do not let stores that initialize this object be reordered with
4593 // a subsequent store that would make this object accessible by
4594 // other threads.
4595 // Record what AllocateNode this StoreStore protects so that
4596 // escape analysis can go from the MemBarStoreStoreNode to the
4597 // AllocateNode and eliminate the MemBarStoreStoreNode if possible
4598 // based on the escape status of the AllocateNode.
4599 insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out(AllocateNode::RawAddress));
4600 } else {
4601 insert_mem_bar(Op_MemBarCPUOrder);
4602 }
4603 }
4604
4605 //------------------------inline_native_clone----------------------------
4606 // protected native Object java.lang.Object.clone();
4607 //
4608 // Here are the simple edge cases:
4609 // null receiver => normal trap
4610 // virtual and clone was overridden => slow path to out-of-line clone
4611 // not cloneable or finalizer => slow path to out-of-line Object.clone
4612 //
4613 // The general case has two steps, allocation and copying.
4614 // Allocation has two cases, and uses GraphKit::new_instance or new_array.
4615 //
4616 // Copying also has two cases, oop arrays and everything else.
4617 // Oop arrays use arrayof_oop_arraycopy (same as System.arraycopy).
4618 // Everything else uses the tight inline loop supplied by CopyArrayNode.
4619 //
4620 // These steps fold up nicely if and when the cloned object's klass
4621 // can be sharply typed as an object array, a type array, or an instance.
4642 ciInstanceKlass* spec_ik = obj_type->speculative_type()->as_instance_klass();
4643 if (spec_ik->nof_nonstatic_fields() <= ArrayCopyLoadStoreMaxElem &&
4644 !spec_ik->has_injected_fields()) {
4645 ciKlass* k = obj_type->klass();
4646 if (!k->is_instance_klass() ||
4647 k->as_instance_klass()->is_interface() ||
4648 k->as_instance_klass()->has_subklass()) {
4649 obj = maybe_cast_profiled_obj(obj, obj_type->speculative_type(), false);
4650 }
4651 }
4652 }
4653
4654 Node* obj_klass = load_object_klass(obj);
4655 const TypeKlassPtr* tklass = _gvn.type(obj_klass)->isa_klassptr();
4656 const TypeOopPtr* toop = ((tklass != NULL)
4657 ? tklass->as_instance_type()
4658 : TypeInstPtr::NOTNULL);
4659
4660 // Conservatively insert a memory barrier on all memory slices.
4661 // Do not let writes into the original float below the clone.
4662 insert_mem_bar(Op_MemBarCPUOrder);
4663
4664 // paths into result_reg:
4665 enum {
4666 _slow_path = 1, // out-of-line call to clone method (virtual or not)
4667 _objArray_path, // plain array allocation, plus arrayof_oop_arraycopy
4668 _array_path, // plain array allocation, plus arrayof_long_arraycopy
4669 _instance_path, // plain instance allocation, plus arrayof_long_arraycopy
4670 PATH_LIMIT
4671 };
4672 RegionNode* result_reg = new RegionNode(PATH_LIMIT);
4673 result_val = new PhiNode(result_reg, TypeInstPtr::NOTNULL);
4674 PhiNode* result_i_o = new PhiNode(result_reg, Type::ABIO);
4675 PhiNode* result_mem = new PhiNode(result_reg, Type::MEMORY, TypePtr::BOTTOM);
4676 record_for_igvn(result_reg);
4677
4678 const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
4679 int raw_adr_idx = Compile::AliasIdxRaw;
4680
4681 Node* array_ctl = generate_array_guard(obj_klass, (RegionNode*)NULL);
4682 if (array_ctl != NULL) {
5949 Node* reference_obj = null_check_receiver();
5950 if (stopped()) return true;
5951
5952 Node* adr = basic_plus_adr(reference_obj, reference_obj, referent_offset);
5953
5954 ciInstanceKlass* klass = env()->Object_klass();
5955 const TypeOopPtr* object_type = TypeOopPtr::make_from_klass(klass);
5956
5957 Node* no_ctrl = NULL;
5958 Node* result = make_load(no_ctrl, adr, object_type, T_OBJECT, MemNode::unordered);
5959
5960 // Use the pre-barrier to record the value in the referent field
5961 pre_barrier(false /* do_load */,
5962 control(),
5963 NULL /* obj */, NULL /* adr */, max_juint /* alias_idx */, NULL /* val */, NULL /* val_type */,
5964 result /* pre_val */,
5965 T_OBJECT);
5966
5967 // Add memory barrier to prevent commoning reads from this field
5968 // across safepoint since GC can change its value.
5969 insert_mem_bar(Op_MemBarCPUOrder);
5970
5971 set_result(result);
5972 return true;
5973 }
5974
5975
5976 Node * LibraryCallKit::load_field_from_object(Node * fromObj, const char * fieldName, const char * fieldTypeString,
5977 bool is_exact=true, bool is_static=false,
5978 ciInstanceKlass * fromKls=NULL) {
5979 if (fromKls == NULL) {
5980 const TypeInstPtr* tinst = _gvn.type(fromObj)->isa_instptr();
5981 assert(tinst != NULL, "obj is null");
5982 assert(tinst->klass()->is_loaded(), "obj is not loaded");
5983 assert(!is_exact || tinst->klass_is_exact(), "klass not exact");
5984 fromKls = tinst->klass()->as_instance_klass();
5985 } else {
5986 assert(is_static, "only for static field access");
5987 }
5988 ciField* field = fromKls->get_field_by_name(ciSymbol::make(fieldName),
5989 ciSymbol::make(fieldTypeString),
6000 // Next code copied from Parse::do_get_xxx():
6001
6002 // Compute address and memory type.
6003 int offset = field->offset_in_bytes();
6004 bool is_vol = field->is_volatile();
6005 ciType* field_klass = field->type();
6006 assert(field_klass->is_loaded(), "should be loaded");
6007 const TypePtr* adr_type = C->alias_type(field)->adr_type();
6008 Node *adr = basic_plus_adr(fromObj, fromObj, offset);
6009 BasicType bt = field->layout_type();
6010
6011 // Build the resultant type of the load
6012 const Type *type;
6013 if (bt == T_OBJECT) {
6014 type = TypeOopPtr::make_from_klass(field_klass->as_klass());
6015 } else {
6016 type = Type::get_const_basic_type(bt);
6017 }
6018
6019 if (support_IRIW_for_not_multiple_copy_atomic_cpu && is_vol) {
6020 insert_mem_bar(Op_MemBarVolatile); // StoreLoad barrier
6021 }
6022 // Build the load.
6023 MemNode::MemOrd mo = is_vol ? MemNode::acquire : MemNode::unordered;
6024 Node* loadedField = make_load(NULL, adr, type, bt, adr_type, mo, LoadNode::DependsOnlyOnTest, is_vol);
6025 // If reference is volatile, prevent following memory ops from
6026 // floating up past the volatile read. Also prevents commoning
6027 // another volatile read.
6028 if (is_vol) {
6029 // Memory barrier includes bogus read of value to force load BEFORE membar
6030 insert_mem_bar(Op_MemBarAcquire, loadedField);
6031 }
6032 return loadedField;
6033 }
6034
6035 Node * LibraryCallKit::field_address_from_object(Node * fromObj, const char * fieldName, const char * fieldTypeString,
6036 bool is_exact = true, bool is_static = false,
6037 ciInstanceKlass * fromKls = NULL) {
6038 if (fromKls == NULL) {
6039 const TypeInstPtr* tinst = _gvn.type(fromObj)->isa_instptr();
6040 assert(tinst != NULL, "obj is null");
6041 assert(tinst->klass()->is_loaded(), "obj is not loaded");
6042 assert(!is_exact || tinst->klass_is_exact(), "klass not exact");
6043 fromKls = tinst->klass()->as_instance_klass();
6044 }
6045 else {
6046 assert(is_static, "only for static field access");
6047 }
6048 ciField* field = fromKls->get_field_by_name(ciSymbol::make(fieldName),
6049 ciSymbol::make(fieldTypeString),
6050 is_static);
|
190 Node* generate_objArray_guard(Node* kls, RegionNode* region) {
191 return generate_array_guard_common(kls, region, true, false);
192 }
193 Node* generate_non_objArray_guard(Node* kls, RegionNode* region) {
194 return generate_array_guard_common(kls, region, true, true);
195 }
196 Node* generate_array_guard_common(Node* kls, RegionNode* region,
197 bool obj_array, bool not_array);
198 Node* generate_virtual_guard(Node* obj_klass, RegionNode* slow_region);
199 CallJavaNode* generate_method_call(vmIntrinsics::ID method_id,
200 bool is_virtual = false, bool is_static = false);
201 CallJavaNode* generate_method_call_static(vmIntrinsics::ID method_id) {
202 return generate_method_call(method_id, false, true);
203 }
204 CallJavaNode* generate_method_call_virtual(vmIntrinsics::ID method_id) {
205 return generate_method_call(method_id, true, false);
206 }
207 Node * load_field_from_object(Node * fromObj, const char * fieldName, const char * fieldTypeString, bool is_exact, bool is_static, ciInstanceKlass * fromKls);
208 Node * field_address_from_object(Node * fromObj, const char * fieldName, const char * fieldTypeString, bool is_exact, bool is_static, ciInstanceKlass * fromKls);
209
210 Node* make_string_method_node(Opcodes opcode, Node* str1_start, Node* cnt1, Node* str2_start, Node* cnt2, StrIntrinsicNode::ArgEnc ae);
211 bool inline_string_compareTo(StrIntrinsicNode::ArgEnc ae);
212 bool inline_string_indexOf(StrIntrinsicNode::ArgEnc ae);
213 bool inline_string_indexOfI(StrIntrinsicNode::ArgEnc ae);
214 Node* make_indexOf_node(Node* src_start, Node* src_count, Node* tgt_start, Node* tgt_count,
215 RegionNode* region, Node* phi, StrIntrinsicNode::ArgEnc ae);
216 bool inline_string_indexOfChar();
217 bool inline_string_equals(StrIntrinsicNode::ArgEnc ae);
218 bool inline_string_toBytesU();
219 bool inline_string_getCharsU();
220 bool inline_string_copy(bool compress);
221 bool inline_string_char_access(bool is_store);
222 Node* round_double_node(Node* n);
223 bool runtime_math(const TypeFunc* call_type, address funcAddr, const char* funcName);
224 bool inline_math_native(vmIntrinsics::ID id);
225 bool inline_trig(vmIntrinsics::ID id);
226 bool inline_math(vmIntrinsics::ID id);
227 template <typename OverflowOp>
228 bool inline_math_overflow(Node* arg1, Node* arg2);
229 void inline_math_mathExact(Node* math, Node* test);
230 bool inline_math_addExactI(bool is_increment);
463 } else {
464 // Root compile
465 C->print_inlining_stream()->print("Did not generate predicate for intrinsic %s%s at bci:%d in",
466 vmIntrinsics::name_at(intrinsic_id()),
467 (is_virtual() ? " (virtual)" : ""), bci);
468 }
469 }
470 C->gather_intrinsic_statistics(intrinsic_id(), is_virtual(), Compile::_intrinsic_failed);
471 return NULL;
472 }
473
474 bool LibraryCallKit::try_to_inline(int predicate) {
475 // Handle symbolic names for otherwise undistinguished boolean switches:
476 const bool is_store = true;
477 const bool is_compress = true;
478 const bool is_static = true;
479 const bool is_volatile = true;
480
481 if (!jvms()->has_method()) {
482 // Root JVMState has a null method.
483 assert(map()->memory()->Opcode() == Opcodes::Op_Parm, "");
484 // Insert the memory aliasing node
485 set_all_memory(reset_memory());
486 }
487 assert(merged_memory(), "");
488
489
490 switch (intrinsic_id()) {
491 case vmIntrinsics::_hashCode: return inline_native_hashcode(intrinsic()->is_virtual(), !is_static);
492 case vmIntrinsics::_identityHashCode: return inline_native_hashcode(/*!virtual*/ false, is_static);
493 case vmIntrinsics::_getClass: return inline_native_getClass();
494
495 case vmIntrinsics::_dsin:
496 case vmIntrinsics::_dcos:
497 case vmIntrinsics::_dtan:
498 case vmIntrinsics::_dabs:
499 case vmIntrinsics::_datan2:
500 case vmIntrinsics::_dsqrt:
501 case vmIntrinsics::_dexp:
502 case vmIntrinsics::_dlog:
503 case vmIntrinsics::_dlog10:
825
826 case vmIntrinsics::_hasNegatives:
827 return inline_hasNegatives();
828
829 default:
830 // If you get here, it may be that someone has added a new intrinsic
831 // to the list in vmSymbols.hpp without implementing it here.
832 #ifndef PRODUCT
833 if ((PrintMiscellaneous && (Verbose || WizardMode)) || PrintOpto) {
834 tty->print_cr("*** Warning: Unimplemented intrinsic %s(%d)",
835 vmIntrinsics::name_at(intrinsic_id()), intrinsic_id());
836 }
837 #endif
838 return false;
839 }
840 }
841
842 Node* LibraryCallKit::try_to_predicate(int predicate) {
843 if (!jvms()->has_method()) {
844 // Root JVMState has a null method.
845 assert(map()->memory()->Opcode() == Opcodes::Op_Parm, "");
846 // Insert the memory aliasing node
847 set_all_memory(reset_memory());
848 }
849 assert(merged_memory(), "");
850
851 switch (intrinsic_id()) {
852 case vmIntrinsics::_cipherBlockChaining_encryptAESCrypt:
853 return inline_cipherBlockChaining_AESCrypt_predicate(false);
854 case vmIntrinsics::_cipherBlockChaining_decryptAESCrypt:
855 return inline_cipherBlockChaining_AESCrypt_predicate(true);
856 case vmIntrinsics::_counterMode_AESCrypt:
857 return inline_counterMode_AESCrypt_predicate();
858 case vmIntrinsics::_digestBase_implCompressMB:
859 return inline_digestBase_implCompressMB_predicate(predicate);
860
861 default:
862 // If you get here, it may be that someone has added a new intrinsic
863 // to the list in vmSymbols.hpp without implementing it here.
864 #ifndef PRODUCT
865 if ((PrintMiscellaneous && (Verbose || WizardMode)) || PrintOpto) {
1005 }
1006 }
1007
1008 //--------------------------generate_current_thread--------------------
1009 Node* LibraryCallKit::generate_current_thread(Node* &tls_output) {
1010 ciKlass* thread_klass = env()->Thread_klass();
1011 const Type* thread_type = TypeOopPtr::make_from_klass(thread_klass)->cast_to_ptr_type(TypePtr::NotNull);
1012 Node* thread = _gvn.transform(new ThreadLocalNode());
1013 Node* p = basic_plus_adr(top()/*!oop*/, thread, in_bytes(JavaThread::threadObj_offset()));
1014 Node* threadObj = make_load(NULL, p, thread_type, T_OBJECT, MemNode::unordered);
1015 tls_output = thread;
1016 return threadObj;
1017 }
1018
1019
1020 //------------------------------make_string_method_node------------------------
1021 // Helper method for String intrinsic functions. This version is called with
1022 // str1 and str2 pointing to byte[] nodes containing Latin1 or UTF16 encoded
1023 // characters (depending on 'is_byte'). cnt1 and cnt2 are pointing to Int nodes
1024 // containing the lengths of str1 and str2.
1025 Node* LibraryCallKit::make_string_method_node(Opcodes opcode, Node* str1_start, Node* cnt1, Node* str2_start, Node* cnt2, StrIntrinsicNode::ArgEnc ae) {
1026 Node* result = NULL;
1027 switch (opcode) {
1028 case Opcodes::Op_StrIndexOf:
1029 result = new StrIndexOfNode(control(), memory(TypeAryPtr::BYTES),
1030 str1_start, cnt1, str2_start, cnt2, ae);
1031 break;
1032 case Opcodes::Op_StrComp:
1033 result = new StrCompNode(control(), memory(TypeAryPtr::BYTES),
1034 str1_start, cnt1, str2_start, cnt2, ae);
1035 break;
1036 case Opcodes::Op_StrEquals:
1037 // We already know that cnt1 == cnt2 here (checked in 'inline_string_equals').
1038 // Use the constant length if there is one because optimized match rule may exist.
1039 result = new StrEqualsNode(control(), memory(TypeAryPtr::BYTES),
1040 str1_start, str2_start, cnt2->is_Con() ? cnt2 : cnt1, ae);
1041 break;
1042 default:
1043 ShouldNotReachHere();
1044 return NULL;
1045 }
1046
1047 // All these intrinsics have checks.
1048 C->set_has_split_ifs(true); // Has chance for split-if optimization
1049
1050 return _gvn.transform(result);
1051 }
1052
1053 //------------------------------inline_string_compareTo------------------------
1054 bool LibraryCallKit::inline_string_compareTo(StrIntrinsicNode::ArgEnc ae) {
1055 Node* arg1 = argument(0);
1056 Node* arg2 = argument(1);
1057
1058 // Get start addr and length of first argument
1059 Node* arg1_start = array_element_address(arg1, intcon(0), T_BYTE);
1060 Node* arg1_cnt = load_array_length(arg1);
1061
1062 // Get start addr and length of second argument
1063 Node* arg2_start = array_element_address(arg2, intcon(0), T_BYTE);
1064 Node* arg2_cnt = load_array_length(arg2);
1065
1066 Node* result = make_string_method_node(Opcodes::Op_StrComp, arg1_start, arg1_cnt, arg2_start, arg2_cnt, ae);
1067 set_result(result);
1068 return true;
1069 }
1070
1071 //------------------------------inline_string_equals------------------------
1072 bool LibraryCallKit::inline_string_equals(StrIntrinsicNode::ArgEnc ae) {
1073 Node* arg1 = argument(0);
1074 Node* arg2 = argument(1);
1075
1076 // paths (plus control) merge
1077 RegionNode* region = new RegionNode(3);
1078 Node* phi = new PhiNode(region, TypeInt::BOOL);
1079
1080 if (!stopped()) {
1081 // Get start addr and length of first argument
1082 Node* arg1_start = array_element_address(arg1, intcon(0), T_BYTE);
1083 Node* arg1_cnt = load_array_length(arg1);
1084
1085 // Get start addr and length of second argument
1086 Node* arg2_start = array_element_address(arg2, intcon(0), T_BYTE);
1087 Node* arg2_cnt = load_array_length(arg2);
1088
1089 // Check for arg1_cnt != arg2_cnt
1090 Node* cmp = _gvn.transform(new CmpINode(arg1_cnt, arg2_cnt));
1091 Node* bol = _gvn.transform(new BoolNode(cmp, BoolTest::ne));
1092 Node* if_ne = generate_slow_guard(bol, NULL);
1093 if (if_ne != NULL) {
1094 phi->init_req(2, intcon(0));
1095 region->init_req(2, if_ne);
1096 }
1097
1098 // Check for count == 0 is done by assembler code for StrEquals.
1099
1100 if (!stopped()) {
1101 Node* equals = make_string_method_node(Opcodes::Op_StrEquals, arg1_start, arg1_cnt, arg2_start, arg2_cnt, ae);
1102 phi->init_req(1, equals);
1103 region->init_req(1, control());
1104 }
1105 }
1106
1107 // post merge
1108 set_control(_gvn.transform(region));
1109 record_for_igvn(region);
1110
1111 set_result(_gvn.transform(phi));
1112 return true;
1113 }
1114
1115 //------------------------------inline_array_equals----------------------------
1116 bool LibraryCallKit::inline_array_equals(StrIntrinsicNode::ArgEnc ae) {
1117 assert(ae == StrIntrinsicNode::UU || ae == StrIntrinsicNode::LL, "unsupported array types");
1118 Node* arg1 = argument(0);
1119 Node* arg2 = argument(1);
1120
1121 const TypeAryPtr* mtype = (ae == StrIntrinsicNode::UU) ? TypeAryPtr::CHARS : TypeAryPtr::BYTES;
1179 PreserveJVMState pjvms(this);
1180 set_control(_gvn.transform(new IfFalseNode(rc)));
1181 uncommon_trap(Deoptimization::Reason_range_check,
1182 Deoptimization::Action_make_not_entrant);
1183 }
1184
1185 if (stopped()) {
1186 return false;
1187 }
1188
1189 Node* result = new CastIINode(index, TypeInt::make(0, _gvn.type(length)->is_int()->_hi, Type::WidenMax));
1190 result->set_req(0, control());
1191 result = _gvn.transform(result);
1192 set_result(result);
1193 replace_in_map(index, result);
1194 return true;
1195 }
1196
1197 //------------------------------inline_string_indexOf------------------------
1198 bool LibraryCallKit::inline_string_indexOf(StrIntrinsicNode::ArgEnc ae) {
1199 if (!Matcher::match_rule_supported(Opcodes::Op_StrIndexOf)) {
1200 return false;
1201 }
1202 Node* src = argument(0);
1203 Node* tgt = argument(1);
1204
1205 // Make the merge point
1206 RegionNode* result_rgn = new RegionNode(4);
1207 Node* result_phi = new PhiNode(result_rgn, TypeInt::INT);
1208
1209 // Get start addr and length of source string
1210 Node* src_start = array_element_address(src, intcon(0), T_BYTE);
1211 Node* src_count = load_array_length(src);
1212
1213 // Get start addr and length of substring
1214 Node* tgt_start = array_element_address(tgt, intcon(0), T_BYTE);
1215 Node* tgt_count = load_array_length(tgt);
1216
1217 if (ae == StrIntrinsicNode::UU || ae == StrIntrinsicNode::UL) {
1218 // Divide src size by 2 if String is UTF16 encoded
1219 src_count = _gvn.transform(new RShiftINode(src_count, intcon(1)));
1223 tgt_count = _gvn.transform(new RShiftINode(tgt_count, intcon(1)));
1224 }
1225
1226 Node* result = make_indexOf_node(src_start, src_count, tgt_start, tgt_count, result_rgn, result_phi, ae);
1227 if (result != NULL) {
1228 result_phi->init_req(3, result);
1229 result_rgn->init_req(3, control());
1230 }
1231 set_control(_gvn.transform(result_rgn));
1232 record_for_igvn(result_rgn);
1233 set_result(_gvn.transform(result_phi));
1234
1235 return true;
1236 }
1237
1238 //-----------------------------inline_string_indexOf-----------------------
1239 bool LibraryCallKit::inline_string_indexOfI(StrIntrinsicNode::ArgEnc ae) {
1240 if (too_many_traps(Deoptimization::Reason_intrinsic)) {
1241 return false;
1242 }
1243 if (!Matcher::match_rule_supported(Opcodes::Op_StrIndexOf)) {
1244 return false;
1245 }
1246 assert(callee()->signature()->size() == 5, "String.indexOf() has 5 arguments");
1247 Node* src = argument(0); // byte[]
1248 Node* src_count = argument(1); // char count
1249 Node* tgt = argument(2); // byte[]
1250 Node* tgt_count = argument(3); // char count
1251 Node* from_index = argument(4); // char index
1252
1253 // Multiply byte array index by 2 if String is UTF16 encoded
1254 Node* src_offset = (ae == StrIntrinsicNode::LL) ? from_index : _gvn.transform(new LShiftINode(from_index, intcon(1)));
1255 src_count = _gvn.transform(new SubINode(src_count, from_index));
1256 Node* src_start = array_element_address(src, src_offset, T_BYTE);
1257 Node* tgt_start = array_element_address(tgt, intcon(0), T_BYTE);
1258
1259 // Range checks
1260 generate_string_range_check(src, src_offset, src_count, ae != StrIntrinsicNode::LL);
1261 generate_string_range_check(tgt, intcon(0), tgt_count, ae == StrIntrinsicNode::UU);
1262 if (stopped()) {
1263 return true;
1298 RegionNode* region, Node* phi, StrIntrinsicNode::ArgEnc ae) {
1299 // Check for substr count > string count
1300 Node* cmp = _gvn.transform(new CmpINode(tgt_count, src_count));
1301 Node* bol = _gvn.transform(new BoolNode(cmp, BoolTest::gt));
1302 Node* if_gt = generate_slow_guard(bol, NULL);
1303 if (if_gt != NULL) {
1304 phi->init_req(1, intcon(-1));
1305 region->init_req(1, if_gt);
1306 }
1307 if (!stopped()) {
1308 // Check for substr count == 0
1309 cmp = _gvn.transform(new CmpINode(tgt_count, intcon(0)));
1310 bol = _gvn.transform(new BoolNode(cmp, BoolTest::eq));
1311 Node* if_zero = generate_slow_guard(bol, NULL);
1312 if (if_zero != NULL) {
1313 phi->init_req(2, intcon(0));
1314 region->init_req(2, if_zero);
1315 }
1316 }
1317 if (!stopped()) {
1318 return make_string_method_node(Opcodes::Op_StrIndexOf, src_start, src_count, tgt_start, tgt_count, ae);
1319 }
1320 return NULL;
1321 }
1322
1323 //-----------------------------inline_string_indexOfChar-----------------------
1324 bool LibraryCallKit::inline_string_indexOfChar() {
1325 if (too_many_traps(Deoptimization::Reason_intrinsic)) {
1326 return false;
1327 }
1328 if (!Matcher::match_rule_supported(Opcodes::Op_StrIndexOfChar)) {
1329 return false;
1330 }
1331 assert(callee()->signature()->size() == 4, "String.indexOfChar() has 4 arguments");
1332 Node* src = argument(0); // byte[]
1333 Node* tgt = argument(1); // tgt is int ch
1334 Node* from_index = argument(2);
1335 Node* max = argument(3);
1336
1337 Node* src_offset = _gvn.transform(new LShiftINode(from_index, intcon(1)));
1338 Node* src_start = array_element_address(src, src_offset, T_BYTE);
1339 Node* src_count = _gvn.transform(new SubINode(max, from_index));
1340
1341 // Range checks
1342 generate_string_range_check(src, src_offset, src_count, true);
1343 if (stopped()) {
1344 return true;
1345 }
1346
1347 RegionNode* region = new RegionNode(3);
1348 Node* phi = new PhiNode(region, TypeInt::INT);
1428 } else {
1429 inflate_string(src_start, dst_start, TypeAryPtr::get_array_body_type(dst_elem), length);
1430 }
1431
1432 if (alloc != NULL) {
1433 if (alloc->maybe_set_complete(&_gvn)) {
1434 // "You break it, you buy it."
1435 InitializeNode* init = alloc->initialization();
1436 assert(init->is_complete(), "we just did this");
1437 init->set_complete_with_arraycopy();
1438 assert(dst->is_CheckCastPP(), "sanity");
1439 assert(dst->in(0)->in(0) == init, "dest pinned");
1440 }
1441 // Do not let stores that initialize this object be reordered with
1442 // a subsequent store that would make this object accessible by
1443 // other threads.
1444 // Record what AllocateNode this StoreStore protects so that
1445 // escape analysis can go from the MemBarStoreStoreNode to the
1446 // AllocateNode and eliminate the MemBarStoreStoreNode if possible
1447 // based on the escape status of the AllocateNode.
1448 insert_mem_bar(Opcodes::Op_MemBarStoreStore, alloc->proj_out(AllocateNode::RawAddress));
1449 }
1450 if (compress) {
1451 set_result(_gvn.transform(count));
1452 }
1453 return true;
1454 }
1455
1456 #ifdef _LP64
1457 #define XTOP ,top() /*additional argument*/
1458 #else //_LP64
1459 #define XTOP /*no additional argument*/
1460 #endif //_LP64
1461
1462 //------------------------inline_string_toBytesU--------------------------
1463 // public static byte[] StringUTF16.toBytes(char[] value, int off, int len)
1464 bool LibraryCallKit::inline_string_toBytesU() {
1465 if (too_many_traps(Deoptimization::Reason_intrinsic)) {
1466 return false;
1467 }
1468 // Get the arguments.
1520 OptoRuntime::fast_arraycopy_Type(),
1521 copyfunc_addr, copyfunc_name, TypeRawPtr::BOTTOM,
1522 src_start, dst_start, ConvI2X(length) XTOP);
1523 // Do not let reads from the cloned object float above the arraycopy.
1524 if (alloc != NULL) {
1525 if (alloc->maybe_set_complete(&_gvn)) {
1526 // "You break it, you buy it."
1527 InitializeNode* init = alloc->initialization();
1528 assert(init->is_complete(), "we just did this");
1529 init->set_complete_with_arraycopy();
1530 assert(newcopy->is_CheckCastPP(), "sanity");
1531 assert(newcopy->in(0)->in(0) == init, "dest pinned");
1532 }
1533 // Do not let stores that initialize this object be reordered with
1534 // a subsequent store that would make this object accessible by
1535 // other threads.
1536 // Record what AllocateNode this StoreStore protects so that
1537 // escape analysis can go from the MemBarStoreStoreNode to the
1538 // AllocateNode and eliminate the MemBarStoreStoreNode if possible
1539 // based on the escape status of the AllocateNode.
1540 insert_mem_bar(Opcodes::Op_MemBarStoreStore, alloc->proj_out(AllocateNode::RawAddress));
1541 } else {
1542 insert_mem_bar(Opcodes::Op_MemBarCPUOrder);
1543 }
1544 } // original reexecute is set back here
1545
1546 C->set_has_split_ifs(true); // Has chance for split-if optimization
1547 if (!stopped()) {
1548 set_result(newcopy);
1549 }
1550 return true;
1551 }
1552
1553 //------------------------inline_string_getCharsU--------------------------
1554 // public void StringUTF16.getChars(byte[] src, int srcBegin, int srcEnd, char dst[], int dstBegin)
1555 bool LibraryCallKit::inline_string_getCharsU() {
1556 if (too_many_traps(Deoptimization::Reason_intrinsic)) {
1557 return false;
1558 }
1559
1560 // Get the arguments.
1561 Node* src = argument(0);
1562 Node* src_begin = argument(1);
1604 OptoRuntime::fast_arraycopy_Type(),
1605 copyfunc_addr, copyfunc_name, TypeRawPtr::BOTTOM,
1606 src_start, dst_start, ConvI2X(length) XTOP);
1607 // Do not let reads from the cloned object float above the arraycopy.
1608 if (alloc != NULL) {
1609 if (alloc->maybe_set_complete(&_gvn)) {
1610 // "You break it, you buy it."
1611 InitializeNode* init = alloc->initialization();
1612 assert(init->is_complete(), "we just did this");
1613 init->set_complete_with_arraycopy();
1614 assert(dst->is_CheckCastPP(), "sanity");
1615 assert(dst->in(0)->in(0) == init, "dest pinned");
1616 }
1617 // Do not let stores that initialize this object be reordered with
1618 // a subsequent store that would make this object accessible by
1619 // other threads.
1620 // Record what AllocateNode this StoreStore protects so that
1621 // escape analysis can go from the MemBarStoreStoreNode to the
1622 // AllocateNode and eliminate the MemBarStoreStoreNode if possible
1623 // based on the escape status of the AllocateNode.
1624 insert_mem_bar(Opcodes::Op_MemBarStoreStore, alloc->proj_out(AllocateNode::RawAddress));
1625 } else {
1626 insert_mem_bar(Opcodes::Op_MemBarCPUOrder);
1627 }
1628 }
1629
1630 C->set_has_split_ifs(true); // Has chance for split-if optimization
1631 return true;
1632 }
1633
1634 //----------------------inline_string_char_access----------------------------
1635 // Store/Load char to/from byte[] array.
1636 // static void StringUTF16.putChar(byte[] val, int index, int c)
1637 // static char StringUTF16.getChar(byte[] val, int index)
1638 bool LibraryCallKit::inline_string_char_access(bool is_store) {
1639 Node* value = argument(0);
1640 Node* index = argument(1);
1641 Node* ch = is_store ? argument(2) : NULL;
1642
1643 // This intrinsic accesses byte[] array as char[] array. Computing the offsets
1644 // correctly requires matched array shapes.
1645 assert (arrayOopDesc::base_offset_in_bytes(T_CHAR) == arrayOopDesc::base_offset_in_bytes(T_BYTE),
1646 "sanity: byte[] and char[] bases agree");
1812 runtime_math(OptoRuntime::Math_D_D_Type(), StubRoutines::dsin(), "dsin") :
1813 runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dsin), "SIN");
1814 case vmIntrinsics::_dcos:
1815 return StubRoutines::dcos() != NULL ?
1816 runtime_math(OptoRuntime::Math_D_D_Type(), StubRoutines::dcos(), "dcos") :
1817 runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dcos), "COS");
1818 case vmIntrinsics::_dtan:
1819 return StubRoutines::dtan() != NULL ?
1820 runtime_math(OptoRuntime::Math_D_D_Type(), StubRoutines::dtan(), "dtan") :
1821 runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dtan), "TAN");
1822 case vmIntrinsics::_dlog:
1823 return StubRoutines::dlog() != NULL ?
1824 runtime_math(OptoRuntime::Math_D_D_Type(), StubRoutines::dlog(), "dlog") :
1825 runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dlog), "LOG");
1826 case vmIntrinsics::_dlog10:
1827 return StubRoutines::dlog10() != NULL ?
1828 runtime_math(OptoRuntime::Math_D_D_Type(), StubRoutines::dlog10(), "dlog10") :
1829 runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dlog10), "LOG10");
1830
1831 // These intrinsics are supported on all hardware
1832 case vmIntrinsics::_dsqrt: return Matcher::match_rule_supported(Opcodes::Op_SqrtD) ? inline_math(id) : false;
1833 case vmIntrinsics::_dabs: return Matcher::has_match_rule(Opcodes::Op_AbsD) ? inline_math(id) : false;
1834
1835 case vmIntrinsics::_dexp:
1836 return StubRoutines::dexp() != NULL ?
1837 runtime_math(OptoRuntime::Math_D_D_Type(), StubRoutines::dexp(), "dexp") :
1838 runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dexp), "EXP");
1839 case vmIntrinsics::_dpow:
1840 return StubRoutines::dpow() != NULL ?
1841 runtime_math(OptoRuntime::Math_DD_D_Type(), StubRoutines::dpow(), "dpow") :
1842 runtime_math(OptoRuntime::Math_DD_D_Type(), FN_PTR(SharedRuntime::dpow), "POW");
1843 #undef FN_PTR
1844
1845 // These intrinsics are not yet correctly implemented
1846 case vmIntrinsics::_datan2:
1847 return false;
1848
1849 default:
1850 fatal_unexpected_iid(id);
1851 return false;
1852 }
1853 }
1957 }
1958
1959 bool want_max = (id == vmIntrinsics::_max);
1960
1961 const TypeInt* txvalue = _gvn.type(xvalue)->isa_int();
1962 const TypeInt* tyvalue = _gvn.type(yvalue)->isa_int();
1963 if (txvalue == NULL || tyvalue == NULL) return top();
1964 // This is not really necessary, but it is consistent with a
1965 // hypothetical MaxINode::Value method:
1966 int widen = MAX2(txvalue->_widen, tyvalue->_widen);
1967
1968 // %%% This folding logic should (ideally) be in a different place.
1969 // Some should be inside IfNode, and there to be a more reliable
1970 // transformation of ?: style patterns into cmoves. We also want
1971 // more powerful optimizations around cmove and min/max.
1972
1973 // Try to find a dominating comparison of these guys.
1974 // It can simplify the index computation for Arrays.copyOf
1975 // and similar uses of System.arraycopy.
1976 // First, compute the normalized version of CmpI(x, y).
1977 Opcodes cmp_op = Opcodes::Op_CmpI;
1978 Node* xkey = xvalue;
1979 Node* ykey = yvalue;
1980 Node* ideal_cmpxy = _gvn.transform(new CmpINode(xkey, ykey));
1981 if (ideal_cmpxy->is_Cmp()) {
1982 // E.g., if we have CmpI(length - offset, count),
1983 // it might idealize to CmpI(length, count + offset)
1984 cmp_op = ideal_cmpxy->Opcode();
1985 xkey = ideal_cmpxy->in(1);
1986 ykey = ideal_cmpxy->in(2);
1987 }
1988
1989 // Start by locating any relevant comparisons.
1990 Node* start_from = (xkey->outcnt() < ykey->outcnt()) ? xkey : ykey;
1991 Node* cmpxy = NULL;
1992 Node* cmpyx = NULL;
1993 for (DUIterator_Fast kmax, k = start_from->fast_outs(kmax); k < kmax; k++) {
1994 Node* cmp = start_from->fast_out(k);
1995 if (cmp->outcnt() > 0 && // must have prior uses
1996 cmp->in(0) == NULL && // must be context-independent
1997 cmp->Opcode() == cmp_op) { // right kind of compare
2270
2271 // Update IdealKit memory and control from graphKit.
2272 __ sync_kit(this);
2273
2274 Node* one = __ ConI(1);
2275 // is_instof == 0 if base_oop == NULL
2276 __ if_then(is_instof, BoolTest::eq, one, unlikely); {
2277
2278 // Update graphKit from IdeakKit.
2279 sync_kit(ideal);
2280
2281 // Use the pre-barrier to record the value in the referent field
2282 pre_barrier(false /* do_load */,
2283 __ ctrl(),
2284 NULL /* obj */, NULL /* adr */, max_juint /* alias_idx */, NULL /* val */, NULL /* val_type */,
2285 pre_val /* pre_val */,
2286 T_OBJECT);
2287 if (need_mem_bar) {
2288 // Add memory barrier to prevent commoning reads from this field
2289 // across safepoint since GC can change its value.
2290 insert_mem_bar(Opcodes::Op_MemBarCPUOrder);
2291 }
2292 // Update IdealKit from graphKit.
2293 __ sync_kit(this);
2294
2295 } __ end_if(); // _ref_type != ref_none
2296 } __ end_if(); // offset == referent_offset
2297
2298 // Final sync IdealKit and GraphKit.
2299 final_sync(ideal);
2300 #undef __
2301 }
2302
2303
2304 const TypeOopPtr* LibraryCallKit::sharpen_unsafe_type(Compile::AliasType* alias_type, const TypePtr *adr_type) {
2305 // Attempt to infer a sharper value type from the offset and base type.
2306 ciKlass* sharpened_klass = NULL;
2307
2308 // See if it is an instance field, with an object type.
2309 if (alias_type->field() != NULL) {
2310 if (alias_type->field()->type()->is_klass()) {
2495 // Heap pointers get a null-check from the interpreter,
2496 // as a courtesy. However, this is not guaranteed by Unsafe,
2497 // and it is not possible to fully distinguish unintended nulls
2498 // from intended ones in this API.
2499
2500 // We need to emit leading and trailing CPU membars (see below) in
2501 // addition to memory membars for special access modes. This is a little
2502 // too strong, but avoids the need to insert per-alias-type
2503 // volatile membars (for stores; compare Parse::do_put_xxx), which
2504 // we cannot do effectively here because we probably only have a
2505 // rough approximation of type.
2506
2507 switch(kind) {
2508 case Relaxed:
2509 case Opaque:
2510 case Acquire:
2511 break;
2512 case Release:
2513 case Volatile:
2514 if (is_store) {
2515 insert_mem_bar(Opcodes::Op_MemBarRelease);
2516 } else {
2517 if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
2518 insert_mem_bar(Opcodes::Op_MemBarVolatile);
2519 }
2520 }
2521 break;
2522 default:
2523 ShouldNotReachHere();
2524 }
2525
2526 // Memory barrier to prevent normal and 'unsafe' accesses from
2527 // bypassing each other. Happens after null checks, so the
2528 // exception paths do not take memory state from the memory barrier,
2529 // so there's no problems making a strong assert about mixing users
2530 // of safe & unsafe memory.
2531 if (need_mem_bar) insert_mem_bar(Opcodes::Op_MemBarCPUOrder);
2532
2533 if (!is_store) {
2534 Node* p = NULL;
2535 // Try to constant fold a load from a constant field
2536 ciField* field = alias_type->field();
2537 if (heap_base_oop != top() && field != NULL && field->is_constant() && !mismatched) {
2538 // final or stable field
2539 p = make_constant_from_field(field, heap_base_oop);
2540 }
2541 if (p == NULL) {
2542 // To be valid, unsafe loads may depend on other conditions than
2543 // the one that guards them: pin the Load node
2544 p = make_load(control(), adr, value_type, type, adr_type, mo, LoadNode::Pinned, requires_atomic_access, unaligned, mismatched);
2545 // load value
2546 switch (type) {
2547 case T_BOOLEAN:
2548 case T_CHAR:
2549 case T_BYTE:
2550 case T_SHORT:
2551 case T_INT:
2610 // Update IdealKit memory.
2611 __ sync_kit(this);
2612 } __ else_(); {
2613 __ store(__ ctrl(), adr, val, type, alias_type->index(), mo, requires_atomic_access, mismatched);
2614 } __ end_if();
2615 // Final sync IdealKit and GraphKit.
2616 final_sync(ideal);
2617 #undef __
2618 }
2619 }
2620 }
2621
2622 switch(kind) {
2623 case Relaxed:
2624 case Opaque:
2625 case Release:
2626 break;
2627 case Acquire:
2628 case Volatile:
2629 if (!is_store) {
2630 insert_mem_bar(Opcodes::Op_MemBarAcquire);
2631 } else {
2632 if (!support_IRIW_for_not_multiple_copy_atomic_cpu) {
2633 insert_mem_bar(Opcodes::Op_MemBarVolatile);
2634 }
2635 }
2636 break;
2637 default:
2638 ShouldNotReachHere();
2639 }
2640
2641 if (need_mem_bar) insert_mem_bar(Opcodes::Op_MemBarCPUOrder);
2642
2643 return true;
2644 }
2645
2646 //----------------------------inline_unsafe_load_store----------------------------
2647 // This method serves a couple of different customers (depending on LoadStoreKind):
2648 //
2649 // LS_cmp_swap:
2650 //
2651 // boolean compareAndSwapObject(Object o, long offset, Object expected, Object x);
2652 // boolean compareAndSwapInt( Object o, long offset, int expected, int x);
2653 // boolean compareAndSwapLong( Object o, long offset, long expected, long x);
2654 //
2655 // LS_cmp_swap_weak:
2656 //
2657 // boolean weakCompareAndSwapObject( Object o, long offset, Object expected, Object x);
2658 // boolean weakCompareAndSwapObjectAcquire(Object o, long offset, Object expected, Object x);
2659 // boolean weakCompareAndSwapObjectRelease(Object o, long offset, Object expected, Object x);
2660 //
2661 // boolean weakCompareAndSwapInt( Object o, long offset, int expected, int x);
2827 case LS_cmp_swap:
2828 case LS_cmp_swap_weak:
2829 case LS_get_add:
2830 break;
2831 default:
2832 ShouldNotReachHere();
2833 }
2834
2835 int alias_idx = C->get_alias_index(adr_type);
2836
2837 // Memory-model-wise, a LoadStore acts like a little synchronized
2838 // block, so needs barriers on each side. These don't translate
2839 // into actual barriers on most machines, but we still need rest of
2840 // compiler to respect ordering.
2841
2842 switch (access_kind) {
2843 case Relaxed:
2844 case Acquire:
2845 break;
2846 case Release:
2847 insert_mem_bar(Opcodes::Op_MemBarRelease);
2848 break;
2849 case Volatile:
2850 if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
2851 insert_mem_bar(Opcodes::Op_MemBarVolatile);
2852 } else {
2853 insert_mem_bar(Opcodes::Op_MemBarRelease);
2854 }
2855 break;
2856 default:
2857 ShouldNotReachHere();
2858 }
2859 insert_mem_bar(Opcodes::Op_MemBarCPUOrder);
2860
2861 // Figure out the memory ordering.
2862 MemNode::MemOrd mo = access_kind_to_memord(access_kind);
2863
2864 // 4984716: MemBars must be inserted before this
2865 // memory node in order to avoid a false
2866 // dependency which will confuse the scheduler.
2867 Node *mem = memory(alias_idx);
2868
2869 // For now, we handle only those cases that actually exist: ints,
2870 // longs, and Object. Adding others should be straightforward.
2871 Node* load_store = NULL;
2872 switch(type) {
2873 case T_BYTE:
2874 switch(kind) {
2875 case LS_get_add:
2876 load_store = _gvn.transform(new GetAndAddBNode(control(), mem, adr, newval, adr_type));
2877 break;
2878 case LS_get_set:
2879 load_store = _gvn.transform(new GetAndSetBNode(control(), mem, adr, newval, adr_type));
3079 set_memory(proj, alias_idx);
3080
3081 if (type == T_OBJECT && (kind == LS_get_set || kind == LS_cmp_exchange)) {
3082 #ifdef _LP64
3083 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
3084 load_store = _gvn.transform(new DecodeNNode(load_store, load_store->get_ptr_type()));
3085 }
3086 #endif
3087 if (can_move_pre_barrier()) {
3088 // Don't need to load pre_val. The old value is returned by load_store.
3089 // The pre_barrier can execute after the xchg as long as no safepoint
3090 // gets inserted between them.
3091 pre_barrier(false /* do_load */,
3092 control(), NULL, NULL, max_juint, NULL, NULL,
3093 load_store /* pre_val */,
3094 T_OBJECT);
3095 }
3096 }
3097
3098 // Add the trailing membar surrounding the access
3099 insert_mem_bar(Opcodes::Op_MemBarCPUOrder);
3100
3101 switch (access_kind) {
3102 case Relaxed:
3103 case Release:
3104 break; // do nothing
3105 case Acquire:
3106 case Volatile:
3107 insert_mem_bar(Opcodes::Op_MemBarAcquire);
3108 // !support_IRIW_for_not_multiple_copy_atomic_cpu handled in platform code
3109 break;
3110 default:
3111 ShouldNotReachHere();
3112 }
3113
3114 assert(type2size[load_store->bottom_type()->basic_type()] == type2size[rtype], "result type should match");
3115 set_result(load_store);
3116 return true;
3117 }
3118
3119 MemNode::MemOrd LibraryCallKit::access_kind_to_memord_LS(AccessKind kind, bool is_store) {
3120 MemNode::MemOrd mo = MemNode::unset;
3121 switch(kind) {
3122 case Opaque:
3123 case Relaxed: mo = MemNode::unordered; break;
3124 case Acquire: mo = MemNode::acquire; break;
3125 case Release: mo = MemNode::release; break;
3126 case Volatile: mo = is_store ? MemNode::release : MemNode::acquire; break;
3127 default:
3132 }
3133
3134 MemNode::MemOrd LibraryCallKit::access_kind_to_memord(AccessKind kind) {
3135 MemNode::MemOrd mo = MemNode::unset;
3136 switch(kind) {
3137 case Opaque:
3138 case Relaxed: mo = MemNode::unordered; break;
3139 case Acquire: mo = MemNode::acquire; break;
3140 case Release: mo = MemNode::release; break;
3141 case Volatile: mo = MemNode::seqcst; break;
3142 default:
3143 ShouldNotReachHere();
3144 }
3145 guarantee(mo != MemNode::unset, "Should select memory ordering");
3146 return mo;
3147 }
3148
3149 bool LibraryCallKit::inline_unsafe_fence(vmIntrinsics::ID id) {
3150 // Regardless of form, don't allow previous ld/st to move down,
3151 // then issue acquire, release, or volatile mem_bar.
3152 insert_mem_bar(Opcodes::Op_MemBarCPUOrder);
3153 switch(id) {
3154 case vmIntrinsics::_loadFence:
3155 insert_mem_bar(Opcodes::Op_LoadFence);
3156 return true;
3157 case vmIntrinsics::_storeFence:
3158 insert_mem_bar(Opcodes::Op_StoreFence);
3159 return true;
3160 case vmIntrinsics::_fullFence:
3161 insert_mem_bar(Opcodes::Op_MemBarVolatile);
3162 return true;
3163 default:
3164 fatal_unexpected_iid(id);
3165 return false;
3166 }
3167 }
3168
3169 bool LibraryCallKit::inline_onspinwait() {
3170 insert_mem_bar(Opcodes::Op_OnSpinWait);
3171 return true;
3172 }
3173
3174 bool LibraryCallKit::klass_needs_init_guard(Node* kls) {
3175 if (!kls->is_Con()) {
3176 return true;
3177 }
3178 const TypeKlassPtr* klsptr = kls->bottom_type()->isa_klassptr();
3179 if (klsptr == NULL) {
3180 return true;
3181 }
3182 ciInstanceKlass* ik = klsptr->klass()->as_instance_klass();
3183 // don't need a guard for a klass that is already initialized
3184 return !ik->is_initialized();
3185 }
3186
3187 //----------------------------inline_unsafe_allocate---------------------------
3188 // public native Object Unsafe.allocateInstance(Class<?> cls);
3189 bool LibraryCallKit::inline_unsafe_allocate() {
3190 if (callee()->is_static()) return false; // caller must have the capability!
3247 // (!TLS._osthread._interrupted || WINDOWS_ONLY(false) NOT_WINDOWS(!clear_int)))
3248 // ? TLS._osthread._interrupted : /*slow path:*/ t.isInterrupted(clear_int)
3249 // So, in the common case that the interrupt bit is false,
3250 // we avoid making a call into the VM. Even if the interrupt bit
3251 // is true, if the clear_int argument is false, we avoid the VM call.
3252 // However, if the receiver is not currentThread, we must call the VM,
3253 // because there must be some locking done around the operation.
3254
3255 // We only go to the fast case code if we pass two guards.
3256 // Paths which do not pass are accumulated in the slow_region.
3257
3258 enum {
3259 no_int_result_path = 1, // t == Thread.current() && !TLS._osthread._interrupted
3260 no_clear_result_path = 2, // t == Thread.current() && TLS._osthread._interrupted && !clear_int
3261 slow_result_path = 3, // slow path: t.isInterrupted(clear_int)
3262 PATH_LIMIT
3263 };
3264
3265 // Ensure that it's not possible to move the load of TLS._osthread._interrupted flag
3266 // out of the function.
3267 insert_mem_bar(Opcodes::Op_MemBarCPUOrder);
3268
3269 RegionNode* result_rgn = new RegionNode(PATH_LIMIT);
3270 PhiNode* result_val = new PhiNode(result_rgn, TypeInt::BOOL);
3271
3272 RegionNode* slow_region = new RegionNode(1);
3273 record_for_igvn(slow_region);
3274
3275 // (a) Receiving thread must be the current thread.
3276 Node* rec_thr = argument(0);
3277 Node* tls_ptr = NULL;
3278 Node* cur_thr = generate_current_thread(tls_ptr);
3279 Node* cmp_thr = _gvn.transform(new CmpPNode(cur_thr, rec_thr));
3280 Node* bol_thr = _gvn.transform(new BoolNode(cmp_thr, BoolTest::ne));
3281
3282 generate_slow_guard(bol_thr, slow_region);
3283
3284 // (b) Interrupt bit on TLS must be false.
3285 Node* p = basic_plus_adr(top()/*!oop*/, tls_ptr, in_bytes(JavaThread::osthread_offset()));
3286 Node* osthread = make_load(NULL, p, TypeRawPtr::NOTNULL, T_ADDRESS, MemNode::unordered);
3287 p = basic_plus_adr(top()/*!oop*/, osthread, in_bytes(OSThread::interrupted_offset()));
4471 if (callee()->is_static()) return false; // caller must have the capability!
4472 null_check_receiver(); // null-check receiver
4473 if (stopped()) return true;
4474
4475 C->set_has_unsafe_access(true); // Mark eventual nmethod as "unsafe".
4476
4477 Node* src_ptr = argument(1); // type: oop
4478 Node* src_off = ConvL2X(argument(2)); // type: long
4479 Node* dst_ptr = argument(4); // type: oop
4480 Node* dst_off = ConvL2X(argument(5)); // type: long
4481 Node* size = ConvL2X(argument(7)); // type: long
4482
4483 assert(Unsafe_field_offset_to_byte_offset(11) == 11,
4484 "fieldOffset must be byte-scaled");
4485
4486 Node* src = make_unsafe_address(src_ptr, src_off);
4487 Node* dst = make_unsafe_address(dst_ptr, dst_off);
4488
4489 // Conservatively insert a memory barrier on all memory slices.
4490 // Do not let writes of the copy source or destination float below the copy.
4491 insert_mem_bar(Opcodes::Op_MemBarCPUOrder);
4492
4493 // Call it. Note that the length argument is not scaled.
4494 make_runtime_call(RC_LEAF|RC_NO_FP,
4495 OptoRuntime::fast_arraycopy_Type(),
4496 StubRoutines::unsafe_arraycopy(),
4497 "unsafe_arraycopy",
4498 TypeRawPtr::BOTTOM,
4499 src, dst, size XTOP);
4500
4501 // Do not let reads of the copy destination float above the copy.
4502 insert_mem_bar(Opcodes::Op_MemBarCPUOrder);
4503
4504 return true;
4505 }
4506
4507 //------------------------clone_coping-----------------------------------
4508 // Helper function for inline_native_clone.
4509 void LibraryCallKit::copy_to_clone(Node* obj, Node* alloc_obj, Node* obj_size, bool is_array, bool card_mark) {
4510 assert(obj_size != NULL, "");
4511 Node* raw_obj = alloc_obj->in(1);
4512 assert(alloc_obj->is_CheckCastPP() && raw_obj->is_Proj() && raw_obj->in(0)->is_Allocate(), "");
4513
4514 AllocateNode* alloc = NULL;
4515 if (ReduceBulkZeroing) {
4516 // We will be completely responsible for initializing this object -
4517 // mark Initialize node as complete.
4518 alloc = AllocateNode::Ideal_allocation(alloc_obj, &_gvn);
4519 // The object was just allocated - there should be no any stores!
4520 guarantee(alloc != NULL && alloc->maybe_set_complete(&_gvn), "");
4521 // Mark as complete_with_arraycopy so that on AllocateNode
4522 // expansion, we know this AllocateNode is initialized by an array
4579 int raw_adr_idx = Compile::AliasIdxRaw;
4580 post_barrier(control(),
4581 memory(raw_adr_type),
4582 alloc_obj,
4583 no_particular_field,
4584 raw_adr_idx,
4585 no_particular_value,
4586 T_OBJECT,
4587 false);
4588 }
4589
4590 // Do not let reads from the cloned object float above the arraycopy.
4591 if (alloc != NULL) {
4592 // Do not let stores that initialize this object be reordered with
4593 // a subsequent store that would make this object accessible by
4594 // other threads.
4595 // Record what AllocateNode this StoreStore protects so that
4596 // escape analysis can go from the MemBarStoreStoreNode to the
4597 // AllocateNode and eliminate the MemBarStoreStoreNode if possible
4598 // based on the escape status of the AllocateNode.
4599 insert_mem_bar(Opcodes::Op_MemBarStoreStore, alloc->proj_out(AllocateNode::RawAddress));
4600 } else {
4601 insert_mem_bar(Opcodes::Op_MemBarCPUOrder);
4602 }
4603 }
4604
4605 //------------------------inline_native_clone----------------------------
4606 // protected native Object java.lang.Object.clone();
4607 //
4608 // Here are the simple edge cases:
4609 // null receiver => normal trap
4610 // virtual and clone was overridden => slow path to out-of-line clone
4611 // not cloneable or finalizer => slow path to out-of-line Object.clone
4612 //
4613 // The general case has two steps, allocation and copying.
4614 // Allocation has two cases, and uses GraphKit::new_instance or new_array.
4615 //
4616 // Copying also has two cases, oop arrays and everything else.
4617 // Oop arrays use arrayof_oop_arraycopy (same as System.arraycopy).
4618 // Everything else uses the tight inline loop supplied by CopyArrayNode.
4619 //
4620 // These steps fold up nicely if and when the cloned object's klass
4621 // can be sharply typed as an object array, a type array, or an instance.
4642 ciInstanceKlass* spec_ik = obj_type->speculative_type()->as_instance_klass();
4643 if (spec_ik->nof_nonstatic_fields() <= ArrayCopyLoadStoreMaxElem &&
4644 !spec_ik->has_injected_fields()) {
4645 ciKlass* k = obj_type->klass();
4646 if (!k->is_instance_klass() ||
4647 k->as_instance_klass()->is_interface() ||
4648 k->as_instance_klass()->has_subklass()) {
4649 obj = maybe_cast_profiled_obj(obj, obj_type->speculative_type(), false);
4650 }
4651 }
4652 }
4653
4654 Node* obj_klass = load_object_klass(obj);
4655 const TypeKlassPtr* tklass = _gvn.type(obj_klass)->isa_klassptr();
4656 const TypeOopPtr* toop = ((tklass != NULL)
4657 ? tklass->as_instance_type()
4658 : TypeInstPtr::NOTNULL);
4659
4660 // Conservatively insert a memory barrier on all memory slices.
4661 // Do not let writes into the original float below the clone.
4662 insert_mem_bar(Opcodes::Op_MemBarCPUOrder);
4663
4664 // paths into result_reg:
4665 enum {
4666 _slow_path = 1, // out-of-line call to clone method (virtual or not)
4667 _objArray_path, // plain array allocation, plus arrayof_oop_arraycopy
4668 _array_path, // plain array allocation, plus arrayof_long_arraycopy
4669 _instance_path, // plain instance allocation, plus arrayof_long_arraycopy
4670 PATH_LIMIT
4671 };
4672 RegionNode* result_reg = new RegionNode(PATH_LIMIT);
4673 result_val = new PhiNode(result_reg, TypeInstPtr::NOTNULL);
4674 PhiNode* result_i_o = new PhiNode(result_reg, Type::ABIO);
4675 PhiNode* result_mem = new PhiNode(result_reg, Type::MEMORY, TypePtr::BOTTOM);
4676 record_for_igvn(result_reg);
4677
4678 const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
4679 int raw_adr_idx = Compile::AliasIdxRaw;
4680
4681 Node* array_ctl = generate_array_guard(obj_klass, (RegionNode*)NULL);
4682 if (array_ctl != NULL) {
5949 Node* reference_obj = null_check_receiver();
5950 if (stopped()) return true;
5951
5952 Node* adr = basic_plus_adr(reference_obj, reference_obj, referent_offset);
5953
5954 ciInstanceKlass* klass = env()->Object_klass();
5955 const TypeOopPtr* object_type = TypeOopPtr::make_from_klass(klass);
5956
5957 Node* no_ctrl = NULL;
5958 Node* result = make_load(no_ctrl, adr, object_type, T_OBJECT, MemNode::unordered);
5959
5960 // Use the pre-barrier to record the value in the referent field
5961 pre_barrier(false /* do_load */,
5962 control(),
5963 NULL /* obj */, NULL /* adr */, max_juint /* alias_idx */, NULL /* val */, NULL /* val_type */,
5964 result /* pre_val */,
5965 T_OBJECT);
5966
5967 // Add memory barrier to prevent commoning reads from this field
5968 // across safepoint since GC can change its value.
5969 insert_mem_bar(Opcodes::Op_MemBarCPUOrder);
5970
5971 set_result(result);
5972 return true;
5973 }
5974
5975
5976 Node * LibraryCallKit::load_field_from_object(Node * fromObj, const char * fieldName, const char * fieldTypeString,
5977 bool is_exact=true, bool is_static=false,
5978 ciInstanceKlass * fromKls=NULL) {
5979 if (fromKls == NULL) {
5980 const TypeInstPtr* tinst = _gvn.type(fromObj)->isa_instptr();
5981 assert(tinst != NULL, "obj is null");
5982 assert(tinst->klass()->is_loaded(), "obj is not loaded");
5983 assert(!is_exact || tinst->klass_is_exact(), "klass not exact");
5984 fromKls = tinst->klass()->as_instance_klass();
5985 } else {
5986 assert(is_static, "only for static field access");
5987 }
5988 ciField* field = fromKls->get_field_by_name(ciSymbol::make(fieldName),
5989 ciSymbol::make(fieldTypeString),
6000 // Next code copied from Parse::do_get_xxx():
6001
6002 // Compute address and memory type.
6003 int offset = field->offset_in_bytes();
6004 bool is_vol = field->is_volatile();
6005 ciType* field_klass = field->type();
6006 assert(field_klass->is_loaded(), "should be loaded");
6007 const TypePtr* adr_type = C->alias_type(field)->adr_type();
6008 Node *adr = basic_plus_adr(fromObj, fromObj, offset);
6009 BasicType bt = field->layout_type();
6010
6011 // Build the resultant type of the load
6012 const Type *type;
6013 if (bt == T_OBJECT) {
6014 type = TypeOopPtr::make_from_klass(field_klass->as_klass());
6015 } else {
6016 type = Type::get_const_basic_type(bt);
6017 }
6018
6019 if (support_IRIW_for_not_multiple_copy_atomic_cpu && is_vol) {
6020 insert_mem_bar(Opcodes::Op_MemBarVolatile); // StoreLoad barrier
6021 }
6022 // Build the load.
6023 MemNode::MemOrd mo = is_vol ? MemNode::acquire : MemNode::unordered;
6024 Node* loadedField = make_load(NULL, adr, type, bt, adr_type, mo, LoadNode::DependsOnlyOnTest, is_vol);
6025 // If reference is volatile, prevent following memory ops from
6026 // floating up past the volatile read. Also prevents commoning
6027 // another volatile read.
6028 if (is_vol) {
6029 // Memory barrier includes bogus read of value to force load BEFORE membar
6030 insert_mem_bar(Opcodes::Op_MemBarAcquire, loadedField);
6031 }
6032 return loadedField;
6033 }
6034
6035 Node * LibraryCallKit::field_address_from_object(Node * fromObj, const char * fieldName, const char * fieldTypeString,
6036 bool is_exact = true, bool is_static = false,
6037 ciInstanceKlass * fromKls = NULL) {
6038 if (fromKls == NULL) {
6039 const TypeInstPtr* tinst = _gvn.type(fromObj)->isa_instptr();
6040 assert(tinst != NULL, "obj is null");
6041 assert(tinst->klass()->is_loaded(), "obj is not loaded");
6042 assert(!is_exact || tinst->klass_is_exact(), "klass not exact");
6043 fromKls = tinst->klass()->as_instance_klass();
6044 }
6045 else {
6046 assert(is_static, "only for static field access");
6047 }
6048 ciField* field = fromKls->get_field_by_name(ciSymbol::make(fieldName),
6049 ciSymbol::make(fieldTypeString),
6050 is_static);
|