274 Node* dest, Node* dest_offset,
275 Node* copy_length, bool dest_uninitialized);
276 void generate_unchecked_arraycopy(const TypePtr* adr_type,
277 BasicType basic_elem_type,
278 bool disjoint_bases,
279 Node* src, Node* src_offset,
280 Node* dest, Node* dest_offset,
281 Node* copy_length, bool dest_uninitialized);
282 typedef enum { LS_xadd, LS_xchg, LS_cmpxchg } LoadStoreKind;
283 bool inline_unsafe_load_store(BasicType type, LoadStoreKind kind);
284 bool inline_unsafe_ordered_store(BasicType type);
285 bool inline_unsafe_fence(vmIntrinsics::ID id);
286 bool inline_fp_conversions(vmIntrinsics::ID id);
287 bool inline_number_methods(vmIntrinsics::ID id);
288 bool inline_reference_get();
289 bool inline_aescrypt_Block(vmIntrinsics::ID id);
290 bool inline_cipherBlockChaining_AESCrypt(vmIntrinsics::ID id);
291 Node* inline_cipherBlockChaining_AESCrypt_predicate(bool decrypting);
292 Node* get_key_start_from_aescrypt_object(Node* aescrypt_object);
293 bool inline_encodeISOArray();
294 };
295
296
297 //---------------------------make_vm_intrinsic----------------------------
298 CallGenerator* Compile::make_vm_intrinsic(ciMethod* m, bool is_virtual) {
299 vmIntrinsics::ID id = m->intrinsic_id();
300 assert(id != vmIntrinsics::_none, "must be a VM intrinsic");
301
302 if (DisableIntrinsic[0] != '\0'
303 && strstr(DisableIntrinsic, vmIntrinsics::name_at(id)) != NULL) {
304 // disabled by a user request on the command line:
305 // example: -XX:DisableIntrinsic=_hashCode,_getClass
306 return NULL;
307 }
308
309 if (!m->is_loaded()) {
310 // do not attempt to inline unloaded methods
311 return NULL;
312 }
313
471 if (!UseCompressedOops && !Matcher::match_rule_supported(Op_GetAndSetP)) return NULL;
472 if (UseCompressedOops && !Matcher::match_rule_supported(Op_GetAndSetN)) return NULL;
473 break;
474 #else
475 if (!Matcher::match_rule_supported(Op_GetAndSetP)) return NULL;
476 break;
477 #endif
478
479 case vmIntrinsics::_aescrypt_encryptBlock:
480 case vmIntrinsics::_aescrypt_decryptBlock:
481 if (!UseAESIntrinsics) return NULL;
482 break;
483
484 case vmIntrinsics::_cipherBlockChaining_encryptAESCrypt:
485 case vmIntrinsics::_cipherBlockChaining_decryptAESCrypt:
486 if (!UseAESIntrinsics) return NULL;
487 // these two require the predicated logic
488 is_predicted = true;
489 break;
490
491 default:
492 assert(id <= vmIntrinsics::LAST_COMPILER_INLINE, "caller responsibility");
493 assert(id != vmIntrinsics::_Object_init && id != vmIntrinsics::_invoke, "enum out of order?");
494 break;
495 }
496
497 // -XX:-InlineClassNatives disables natives from the Class class.
498 // The flag applies to all reflective calls, notably Array.newArray
499 // (visible to Java programmers as Array.newInstance).
500 if (m->holder()->name() == ciSymbol::java_lang_Class() ||
501 m->holder()->name() == ciSymbol::java_lang_reflect_Array()) {
502 if (!InlineClassNatives) return NULL;
503 }
504
505 // -XX:-InlineThreadNatives disables natives from the Thread class.
506 if (m->holder()->name() == ciSymbol::java_lang_Thread()) {
507 if (!InlineThreadNatives) return NULL;
508 }
509
510 // -XX:-InlineMathNatives disables natives from the Math,Float and Double classes.
790 case vmIntrinsics::_bitCount_l:
791 case vmIntrinsics::_reverseBytes_i:
792 case vmIntrinsics::_reverseBytes_l:
793 case vmIntrinsics::_reverseBytes_s:
794 case vmIntrinsics::_reverseBytes_c: return inline_number_methods(intrinsic_id());
795
796 case vmIntrinsics::_getCallerClass: return inline_native_Reflection_getCallerClass();
797
798 case vmIntrinsics::_Reference_get: return inline_reference_get();
799
800 case vmIntrinsics::_aescrypt_encryptBlock:
801 case vmIntrinsics::_aescrypt_decryptBlock: return inline_aescrypt_Block(intrinsic_id());
802
803 case vmIntrinsics::_cipherBlockChaining_encryptAESCrypt:
804 case vmIntrinsics::_cipherBlockChaining_decryptAESCrypt:
805 return inline_cipherBlockChaining_AESCrypt(intrinsic_id());
806
807 case vmIntrinsics::_encodeISOArray:
808 return inline_encodeISOArray();
809
810 default:
811 // If you get here, it may be that someone has added a new intrinsic
812 // to the list in vmSymbols.hpp without implementing it here.
813 #ifndef PRODUCT
814 if ((PrintMiscellaneous && (Verbose || WizardMode)) || PrintOpto) {
815 tty->print_cr("*** Warning: Unimplemented intrinsic %s(%d)",
816 vmIntrinsics::name_at(intrinsic_id()), intrinsic_id());
817 }
818 #endif
819 return false;
820 }
821 }
822
823 Node* LibraryCallKit::try_to_predicate() {
824 if (!jvms()->has_method()) {
825 // Root JVMState has a null method.
826 assert(map()->memory()->Opcode() == Op_Parm, "");
827 // Insert the memory aliasing node
828 set_all_memory(reset_memory());
829 }
867 // The returned value represents the control for the slow path.
868 // The return value is never 'top'; it is either a valid control
869 // or NULL if it is obvious that the slow path can never be taken.
870 // Also, if region and the slow control are not NULL, the slow edge
871 // is appended to the region.
872 Node* LibraryCallKit::generate_guard(Node* test, RegionNode* region, float true_prob) {
873 if (stopped()) {
874 // Already short circuited.
875 return NULL;
876 }
877
878 // Build an if node and its projections.
879 // If test is true we take the slow path, which we assume is uncommon.
880 if (_gvn.type(test) == TypeInt::ZERO) {
881 // The slow branch is never taken. No need to build this guard.
882 return NULL;
883 }
884
885 IfNode* iff = create_and_map_if(control(), test, true_prob, COUNT_UNKNOWN);
886
887 Node* if_slow = _gvn.transform( new (C) IfTrueNode(iff) );
888 if (if_slow == top()) {
889 // The slow branch is never taken. No need to build this guard.
890 return NULL;
891 }
892
893 if (region != NULL)
894 region->add_req(if_slow);
895
896 Node* if_fast = _gvn.transform( new (C) IfFalseNode(iff) );
897 set_control(if_fast);
898
899 return if_slow;
900 }
901
902 inline Node* LibraryCallKit::generate_slow_guard(Node* test, RegionNode* region) {
903 return generate_guard(test, region, PROB_UNLIKELY_MAG(3));
904 }
905 inline Node* LibraryCallKit::generate_fair_guard(Node* test, RegionNode* region) {
906 return generate_guard(test, region, PROB_FAIR);
907 }
908
909 inline Node* LibraryCallKit::generate_negative_guard(Node* index, RegionNode* region,
910 Node* *pos_index) {
911 if (stopped())
912 return NULL; // already stopped
913 if (_gvn.type(index)->higher_equal(TypeInt::POS)) // [0,maxint]
914 return NULL; // index is already adequately typed
915 Node* cmp_lt = _gvn.transform( new (C) CmpINode(index, intcon(0)) );
916 Node* bol_lt = _gvn.transform( new (C) BoolNode(cmp_lt, BoolTest::lt) );
917 Node* is_neg = generate_guard(bol_lt, region, PROB_MIN);
918 if (is_neg != NULL && pos_index != NULL) {
919 // Emulate effect of Parse::adjust_map_after_if.
920 Node* ccast = new (C) CastIINode(index, TypeInt::POS);
921 ccast->set_req(0, control());
922 (*pos_index) = _gvn.transform(ccast);
923 }
924 return is_neg;
925 }
926
927 inline Node* LibraryCallKit::generate_nonpositive_guard(Node* index, bool never_negative,
928 Node* *pos_index) {
929 if (stopped())
930 return NULL; // already stopped
931 if (_gvn.type(index)->higher_equal(TypeInt::POS1)) // [1,maxint]
932 return NULL; // index is already adequately typed
933 Node* cmp_le = _gvn.transform( new (C) CmpINode(index, intcon(0)) );
934 BoolTest::mask le_or_eq = (never_negative ? BoolTest::eq : BoolTest::le);
935 Node* bol_le = _gvn.transform( new (C) BoolNode(cmp_le, le_or_eq) );
936 Node* is_notp = generate_guard(bol_le, NULL, PROB_MIN);
937 if (is_notp != NULL && pos_index != NULL) {
938 // Emulate effect of Parse::adjust_map_after_if.
939 Node* ccast = new (C) CastIINode(index, TypeInt::POS1);
940 ccast->set_req(0, control());
941 (*pos_index) = _gvn.transform(ccast);
942 }
943 return is_notp;
944 }
945
946 // Make sure that 'position' is a valid limit index, in [0..length].
947 // There are two equivalent plans for checking this:
948 // A. (offset + copyLength) unsigned<= arrayLength
949 // B. offset <= (arrayLength - copyLength)
950 // We require that all of the values above, except for the sum and
951 // difference, are already known to be non-negative.
952 // Plan A is robust in the face of overflow, if offset and copyLength
953 // are both hugely positive.
954 //
955 // Plan B is less direct and intuitive, but it does not overflow at
956 // all, since the difference of two non-negatives is always
957 // representable. Whenever Java methods must perform the equivalent
958 // check they generally use Plan B instead of Plan A.
959 // For the moment we use Plan A.
960 inline Node* LibraryCallKit::generate_limit_guard(Node* offset,
961 Node* subseq_length,
962 Node* array_length,
963 RegionNode* region) {
964 if (stopped())
965 return NULL; // already stopped
966 bool zero_offset = _gvn.type(offset) == TypeInt::ZERO;
967 if (zero_offset && subseq_length->eqv_uncast(array_length))
968 return NULL; // common case of whole-array copy
969 Node* last = subseq_length;
970 if (!zero_offset) // last += offset
971 last = _gvn.transform( new (C) AddINode(last, offset));
972 Node* cmp_lt = _gvn.transform( new (C) CmpUNode(array_length, last) );
973 Node* bol_lt = _gvn.transform( new (C) BoolNode(cmp_lt, BoolTest::lt) );
974 Node* is_over = generate_guard(bol_lt, region, PROB_MIN);
975 return is_over;
976 }
977
978
979 //--------------------------generate_current_thread--------------------
980 Node* LibraryCallKit::generate_current_thread(Node* &tls_output) {
981 ciKlass* thread_klass = env()->Thread_klass();
982 const Type* thread_type = TypeOopPtr::make_from_klass(thread_klass)->cast_to_ptr_type(TypePtr::NotNull);
983 Node* thread = _gvn.transform(new (C) ThreadLocalNode());
984 Node* p = basic_plus_adr(top()/*!oop*/, thread, in_bytes(JavaThread::threadObj_offset()));
985 Node* threadObj = make_load(NULL, p, thread_type, T_OBJECT);
986 tls_output = thread;
987 return threadObj;
988 }
989
990
991 //------------------------------make_string_method_node------------------------
992 // Helper method for String intrinsic functions. This version is called
993 // with str1 and str2 pointing to String object nodes.
1134
1135 Node* no_ctrl = NULL;
1136
1137 // Get start addr of receiver
1138 Node* receiver_val = load_String_value(no_ctrl, receiver);
1139 Node* receiver_offset = load_String_offset(no_ctrl, receiver);
1140 Node* receiver_start = array_element_address(receiver_val, receiver_offset, T_CHAR);
1141
1142 // Get length of receiver
1143 Node* receiver_cnt = load_String_length(no_ctrl, receiver);
1144
1145 // Get start addr of argument
1146 Node* argument_val = load_String_value(no_ctrl, argument);
1147 Node* argument_offset = load_String_offset(no_ctrl, argument);
1148 Node* argument_start = array_element_address(argument_val, argument_offset, T_CHAR);
1149
1150 // Get length of argument
1151 Node* argument_cnt = load_String_length(no_ctrl, argument);
1152
1153 // Check for receiver count != argument count
1154 Node* cmp = _gvn.transform( new(C) CmpINode(receiver_cnt, argument_cnt) );
1155 Node* bol = _gvn.transform( new(C) BoolNode(cmp, BoolTest::ne) );
1156 Node* if_ne = generate_slow_guard(bol, NULL);
1157 if (if_ne != NULL) {
1158 phi->init_req(4, intcon(0));
1159 region->init_req(4, if_ne);
1160 }
1161
1162 // Check for count == 0 is done by assembler code for StrEquals.
1163
1164 if (!stopped()) {
1165 Node* equals = make_string_method_node(Op_StrEquals, receiver_start, receiver_cnt, argument_start, argument_cnt);
1166 phi->init_req(1, equals);
1167 region->init_req(1, control());
1168 }
1169 }
1170
1171 // post merge
1172 set_control(_gvn.transform(region));
1173 record_for_igvn(region);
1174
1175 set_result(_gvn.transform(phi));
1241 // } // using "i += targetCount;" and an "else i++;" causes a jump to jump.
1242 // i++;
1243 // }
1244 // return -1;
1245 // }
1246
1247 //------------------------------string_indexOf------------------------
1248 Node* LibraryCallKit::string_indexOf(Node* string_object, ciTypeArray* target_array, jint targetOffset_i,
1249 jint cache_i, jint md2_i) {
1250
1251 Node* no_ctrl = NULL;
1252 float likely = PROB_LIKELY(0.9);
1253 float unlikely = PROB_UNLIKELY(0.9);
1254
1255 const int nargs = 0; // no arguments to push back for uncommon trap in predicate
1256
1257 Node* source = load_String_value(no_ctrl, string_object);
1258 Node* sourceOffset = load_String_offset(no_ctrl, string_object);
1259 Node* sourceCount = load_String_length(no_ctrl, string_object);
1260
1261 Node* target = _gvn.transform( makecon(TypeOopPtr::make_from_constant(target_array, true)) );
1262 jint target_length = target_array->length();
1263 const TypeAry* target_array_type = TypeAry::make(TypeInt::CHAR, TypeInt::make(0, target_length, Type::WidenMin));
1264 const TypeAryPtr* target_type = TypeAryPtr::make(TypePtr::BotPTR, target_array_type, target_array->klass(), true, Type::OffsetBot);
1265
1266 IdealKit kit(this, false, true);
1267 #define __ kit.
1268 Node* zero = __ ConI(0);
1269 Node* one = __ ConI(1);
1270 Node* cache = __ ConI(cache_i);
1271 Node* md2 = __ ConI(md2_i);
1272 Node* lastChar = __ ConI(target_array->char_at(target_length - 1));
1273 Node* targetCount = __ ConI(target_length);
1274 Node* targetCountLess1 = __ ConI(target_length - 1);
1275 Node* targetOffset = __ ConI(targetOffset_i);
1276 Node* sourceEnd = __ SubI(__ AddI(sourceOffset, sourceCount), targetCountLess1);
1277
1278 IdealVariable rtn(kit), i(kit), j(kit); __ declarations_done();
1279 Node* outer_loop = __ make_label(2 /* goto */);
1280 Node* return_ = __ make_label(1);
1281
1348 Node* result_phi = new (C) PhiNode(result_rgn, TypeInt::INT);
1349 Node* no_ctrl = NULL;
1350
1351 // Get start addr of source string
1352 Node* source = load_String_value(no_ctrl, receiver);
1353 Node* source_offset = load_String_offset(no_ctrl, receiver);
1354 Node* source_start = array_element_address(source, source_offset, T_CHAR);
1355
1356 // Get length of source string
1357 Node* source_cnt = load_String_length(no_ctrl, receiver);
1358
1359 // Get start addr of substring
1360 Node* substr = load_String_value(no_ctrl, arg);
1361 Node* substr_offset = load_String_offset(no_ctrl, arg);
1362 Node* substr_start = array_element_address(substr, substr_offset, T_CHAR);
1363
1364 // Get length of source string
1365 Node* substr_cnt = load_String_length(no_ctrl, arg);
1366
1367 // Check for substr count > string count
1368 Node* cmp = _gvn.transform( new(C) CmpINode(substr_cnt, source_cnt) );
1369 Node* bol = _gvn.transform( new(C) BoolNode(cmp, BoolTest::gt) );
1370 Node* if_gt = generate_slow_guard(bol, NULL);
1371 if (if_gt != NULL) {
1372 result_phi->init_req(2, intcon(-1));
1373 result_rgn->init_req(2, if_gt);
1374 }
1375
1376 if (!stopped()) {
1377 // Check for substr count == 0
1378 cmp = _gvn.transform( new(C) CmpINode(substr_cnt, intcon(0)) );
1379 bol = _gvn.transform( new(C) BoolNode(cmp, BoolTest::eq) );
1380 Node* if_zero = generate_slow_guard(bol, NULL);
1381 if (if_zero != NULL) {
1382 result_phi->init_req(3, intcon(0));
1383 result_rgn->init_req(3, if_zero);
1384 }
1385 }
1386
1387 if (!stopped()) {
1388 result = make_string_method_node(Op_StrIndexOf, source_start, source_cnt, substr_start, substr_cnt);
1389 result_phi->init_req(1, result);
1390 result_rgn->init_req(1, control());
1391 }
1392 set_control(_gvn.transform(result_rgn));
1393 record_for_igvn(result_rgn);
1394 result = _gvn.transform(result_phi);
1395
1396 } else { // Use LibraryCallKit::string_indexOf
1397 // don't intrinsify if argument isn't a constant string.
1398 if (!arg->is_Con()) {
1399 return false;
1535 // if (x <= Math.PI / 2.0) return -fsin(x - Math.PI / 2.0);
1536 // }
1537 // return StrictMath.cos(x);
1538
1539 // Actually, sticking in an 80-bit Intel value into C2 will be tough; it
1540 // requires a special machine instruction to load it. Instead we'll try
1541 // the 'easy' case. If we really need the extra range +/- PI/2 we'll
1542 // probably do the math inside the SIN encoding.
1543
1544 // Make the merge point
1545 RegionNode* r = new (C) RegionNode(3);
1546 Node* phi = new (C) PhiNode(r, Type::DOUBLE);
1547
1548 // Flatten arg so we need only 1 test
1549 Node *abs = _gvn.transform(new (C) AbsDNode(arg));
1550 // Node for PI/4 constant
1551 Node *pi4 = makecon(TypeD::make(pi_4));
1552 // Check PI/4 : abs(arg)
1553 Node *cmp = _gvn.transform(new (C) CmpDNode(pi4,abs));
1554 // Check: If PI/4 < abs(arg) then go slow
1555 Node *bol = _gvn.transform( new (C) BoolNode( cmp, BoolTest::lt ) );
1556 // Branch either way
1557 IfNode *iff = create_and_xform_if(control(),bol, PROB_STATIC_FREQUENT, COUNT_UNKNOWN);
1558 set_control(opt_iff(r,iff));
1559
1560 // Set fast path result
1561 phi->init_req(2, n);
1562
1563 // Slow path - non-blocking leaf call
1564 Node* call = NULL;
1565 switch (id) {
1566 case vmIntrinsics::_dsin:
1567 call = make_runtime_call(RC_LEAF, OptoRuntime::Math_D_D_Type(),
1568 CAST_FROM_FN_PTR(address, SharedRuntime::dsin),
1569 "Sin", NULL, arg, top());
1570 break;
1571 case vmIntrinsics::_dcos:
1572 call = make_runtime_call(RC_LEAF, OptoRuntime::Math_D_D_Type(),
1573 CAST_FROM_FN_PTR(address, SharedRuntime::dcos),
1574 "Cos", NULL, arg, top());
1575 break;
1600 //result=(result.isNaN())? funcAddr():result;
1601 // Check: If isNaN() by checking result!=result? then either trap
1602 // or go to runtime
1603 Node* cmpisnan = _gvn.transform(new (C) CmpDNode(result, result));
1604 // Build the boolean node
1605 Node* bolisnum = _gvn.transform(new (C) BoolNode(cmpisnan, BoolTest::eq));
1606
1607 if (!too_many_traps(Deoptimization::Reason_intrinsic)) {
1608 { BuildCutout unless(this, bolisnum, PROB_STATIC_FREQUENT);
1609 // The pow or exp intrinsic returned a NaN, which requires a call
1610 // to the runtime. Recompile with the runtime call.
1611 uncommon_trap(Deoptimization::Reason_intrinsic,
1612 Deoptimization::Action_make_not_entrant);
1613 }
1614 set_result(result);
1615 } else {
1616 // If this inlining ever returned NaN in the past, we compile a call
1617 // to the runtime to properly handle corner cases
1618
1619 IfNode* iff = create_and_xform_if(control(), bolisnum, PROB_STATIC_FREQUENT, COUNT_UNKNOWN);
1620 Node* if_slow = _gvn.transform( new (C) IfFalseNode(iff) );
1621 Node* if_fast = _gvn.transform( new (C) IfTrueNode(iff) );
1622
1623 if (!if_slow->is_top()) {
1624 RegionNode* result_region = new (C) RegionNode(3);
1625 PhiNode* result_val = new (C) PhiNode(result_region, Type::DOUBLE);
1626
1627 result_region->init_req(1, if_fast);
1628 result_val->init_req(1, result);
1629
1630 set_control(if_slow);
1631
1632 const TypePtr* no_memory_effects = NULL;
1633 Node* rt = make_runtime_call(RC_LEAF, call_type, funcAddr, funcName,
1634 no_memory_effects,
1635 x, top(), y, y ? top() : NULL);
1636 Node* value = _gvn.transform(new (C) ProjNode(rt, TypeFunc::Parms+0));
1637 #ifdef ASSERT
1638 Node* value_top = _gvn.transform(new (C) ProjNode(rt, TypeFunc::Parms+1));
1639 assert(value_top == top(), "second value must be top");
1640 #endif
1641
1687 Node* result = NULL;
1688
1689 if (!too_many_traps(Deoptimization::Reason_intrinsic)) {
1690 // Short form: skip the fancy tests and just check for NaN result.
1691 result = _gvn.transform(new (C) PowDNode(C, control(), x, y));
1692 } else {
1693 // If this inlining ever returned NaN in the past, include all
1694 // checks + call to the runtime.
1695
1696 // Set the merge point for If node with condition of (x <= 0.0)
1697 // There are four possible paths to region node and phi node
1698 RegionNode *r = new (C) RegionNode(4);
1699 Node *phi = new (C) PhiNode(r, Type::DOUBLE);
1700
1701 // Build the first if node: if (x <= 0.0)
1702 // Node for 0 constant
1703 Node *zeronode = makecon(TypeD::ZERO);
1704 // Check x:0
1705 Node *cmp = _gvn.transform(new (C) CmpDNode(x, zeronode));
1706 // Check: If (x<=0) then go complex path
1707 Node *bol1 = _gvn.transform( new (C) BoolNode( cmp, BoolTest::le ) );
1708 // Branch either way
1709 IfNode *if1 = create_and_xform_if(control(),bol1, PROB_STATIC_INFREQUENT, COUNT_UNKNOWN);
1710 // Fast path taken; set region slot 3
1711 Node *fast_taken = _gvn.transform( new (C) IfFalseNode(if1) );
1712 r->init_req(3,fast_taken); // Capture fast-control
1713
1714 // Fast path not-taken, i.e. slow path
1715 Node *complex_path = _gvn.transform( new (C) IfTrueNode(if1) );
1716
1717 // Set fast path result
1718 Node *fast_result = _gvn.transform( new (C) PowDNode(C, control(), x, y) );
1719 phi->init_req(3, fast_result);
1720
1721 // Complex path
1722 // Build the second if node (if y is long)
1723 // Node for (long)y
1724 Node *longy = _gvn.transform( new (C) ConvD2LNode(y));
1725 // Node for (double)((long) y)
1726 Node *doublelongy= _gvn.transform( new (C) ConvL2DNode(longy));
1727 // Check (double)((long) y) : y
1728 Node *cmplongy= _gvn.transform(new (C) CmpDNode(doublelongy, y));
1729 // Check if (y isn't long) then go to slow path
1730
1731 Node *bol2 = _gvn.transform( new (C) BoolNode( cmplongy, BoolTest::ne ) );
1732 // Branch either way
1733 IfNode *if2 = create_and_xform_if(complex_path,bol2, PROB_STATIC_INFREQUENT, COUNT_UNKNOWN);
1734 Node* ylong_path = _gvn.transform( new (C) IfFalseNode(if2));
1735
1736 Node *slow_path = _gvn.transform( new (C) IfTrueNode(if2) );
1737
1738 // Calculate DPow(abs(x), y)*(1 & (long)y)
1739 // Node for constant 1
1740 Node *conone = longcon(1);
1741 // 1& (long)y
1742 Node *signnode= _gvn.transform( new (C) AndLNode(conone, longy) );
1743
1744 // A huge number is always even. Detect a huge number by checking
1745 // if y + 1 == y and set integer to be tested for parity to 0.
1746 // Required for corner case:
1747 // (long)9.223372036854776E18 = max_jlong
1748 // (double)(long)9.223372036854776E18 = 9.223372036854776E18
1749 // max_jlong is odd but 9.223372036854776E18 is even
1750 Node* yplus1 = _gvn.transform( new (C) AddDNode(y, makecon(TypeD::make(1))));
1751 Node *cmpyplus1= _gvn.transform(new (C) CmpDNode(yplus1, y));
1752 Node *bolyplus1 = _gvn.transform( new (C) BoolNode( cmpyplus1, BoolTest::eq ) );
1753 Node* correctedsign = NULL;
1754 if (ConditionalMoveLimit != 0) {
1755 correctedsign = _gvn.transform( CMoveNode::make(C, NULL, bolyplus1, signnode, longcon(0), TypeLong::LONG));
1756 } else {
1757 IfNode *ifyplus1 = create_and_xform_if(ylong_path,bolyplus1, PROB_FAIR, COUNT_UNKNOWN);
1758 RegionNode *r = new (C) RegionNode(3);
1759 Node *phi = new (C) PhiNode(r, TypeLong::LONG);
1760 r->init_req(1, _gvn.transform( new (C) IfFalseNode(ifyplus1)));
1761 r->init_req(2, _gvn.transform( new (C) IfTrueNode(ifyplus1)));
1762 phi->init_req(1, signnode);
1763 phi->init_req(2, longcon(0));
1764 correctedsign = _gvn.transform(phi);
1765 ylong_path = _gvn.transform(r);
1766 record_for_igvn(r);
1767 }
1768
1769 // zero node
1770 Node *conzero = longcon(0);
1771 // Check (1&(long)y)==0?
1772 Node *cmpeq1 = _gvn.transform(new (C) CmpLNode(correctedsign, conzero));
1773 // Check if (1&(long)y)!=0?, if so the result is negative
1774 Node *bol3 = _gvn.transform( new (C) BoolNode( cmpeq1, BoolTest::ne ) );
1775 // abs(x)
1776 Node *absx=_gvn.transform( new (C) AbsDNode(x));
1777 // abs(x)^y
1778 Node *absxpowy = _gvn.transform( new (C) PowDNode(C, control(), absx, y) );
1779 // -abs(x)^y
1780 Node *negabsxpowy = _gvn.transform(new (C) NegDNode (absxpowy));
1781 // (1&(long)y)==1?-DPow(abs(x), y):DPow(abs(x), y)
1782 Node *signresult = NULL;
1783 if (ConditionalMoveLimit != 0) {
1784 signresult = _gvn.transform( CMoveNode::make(C, NULL, bol3, absxpowy, negabsxpowy, Type::DOUBLE));
1785 } else {
1786 IfNode *ifyeven = create_and_xform_if(ylong_path,bol3, PROB_FAIR, COUNT_UNKNOWN);
1787 RegionNode *r = new (C) RegionNode(3);
1788 Node *phi = new (C) PhiNode(r, Type::DOUBLE);
1789 r->init_req(1, _gvn.transform( new (C) IfFalseNode(ifyeven)));
1790 r->init_req(2, _gvn.transform( new (C) IfTrueNode(ifyeven)));
1791 phi->init_req(1, absxpowy);
1792 phi->init_req(2, negabsxpowy);
1793 signresult = _gvn.transform(phi);
1794 ylong_path = _gvn.transform(r);
1795 record_for_igvn(r);
1796 }
1797 // Set complex path fast result
1798 r->init_req(2, ylong_path);
1799 phi->init_req(2, signresult);
1800
1801 static const jlong nan_bits = CONST64(0x7ff8000000000000);
1802 Node *slow_result = makecon(TypeD::make(*(double*)&nan_bits)); // return NaN
1803 r->init_req(1,slow_path);
1804 phi->init_req(1,slow_result);
1805
1806 // Post merge
1807 set_control(_gvn.transform(r));
1808 record_for_igvn(r);
1809 result = _gvn.transform(phi);
1810 }
1903
1904 const TypeInt* txvalue = _gvn.type(xvalue)->isa_int();
1905 const TypeInt* tyvalue = _gvn.type(yvalue)->isa_int();
1906 if (txvalue == NULL || tyvalue == NULL) return top();
1907 // This is not really necessary, but it is consistent with a
1908 // hypothetical MaxINode::Value method:
1909 int widen = MAX2(txvalue->_widen, tyvalue->_widen);
1910
1911 // %%% This folding logic should (ideally) be in a different place.
1912 // Some should be inside IfNode, and there to be a more reliable
1913 // transformation of ?: style patterns into cmoves. We also want
1914 // more powerful optimizations around cmove and min/max.
1915
1916 // Try to find a dominating comparison of these guys.
1917 // It can simplify the index computation for Arrays.copyOf
1918 // and similar uses of System.arraycopy.
1919 // First, compute the normalized version of CmpI(x, y).
1920 int cmp_op = Op_CmpI;
1921 Node* xkey = xvalue;
1922 Node* ykey = yvalue;
1923 Node* ideal_cmpxy = _gvn.transform( new(C) CmpINode(xkey, ykey) );
1924 if (ideal_cmpxy->is_Cmp()) {
1925 // E.g., if we have CmpI(length - offset, count),
1926 // it might idealize to CmpI(length, count + offset)
1927 cmp_op = ideal_cmpxy->Opcode();
1928 xkey = ideal_cmpxy->in(1);
1929 ykey = ideal_cmpxy->in(2);
1930 }
1931
1932 // Start by locating any relevant comparisons.
1933 Node* start_from = (xkey->outcnt() < ykey->outcnt()) ? xkey : ykey;
1934 Node* cmpxy = NULL;
1935 Node* cmpyx = NULL;
1936 for (DUIterator_Fast kmax, k = start_from->fast_outs(kmax); k < kmax; k++) {
1937 Node* cmp = start_from->fast_out(k);
1938 if (cmp->outcnt() > 0 && // must have prior uses
1939 cmp->in(0) == NULL && // must be context-independent
1940 cmp->Opcode() == cmp_op) { // right kind of compare
1941 if (cmp->in(1) == xkey && cmp->in(2) == ykey) cmpxy = cmp;
1942 if (cmp->in(1) == ykey && cmp->in(2) == xkey) cmpyx = cmp;
1943 }
1996 if (cmp == NULL) continue;
1997 for (DUIterator_Fast jmax, j = cmp->fast_outs(jmax); j < jmax; j++) {
1998 Node* bol = cmp->fast_out(j);
1999 if (!bol->is_Bool()) continue;
2000 BoolTest::mask btest = bol->as_Bool()->_test._test;
2001 if (btest == BoolTest::eq || btest == BoolTest::ne) continue;
2002 if (cmp->in(1) == ykey) btest = BoolTest(btest).commute();
2003 if (bol->outcnt() > (best_bol == NULL ? 0 : best_bol->outcnt())) {
2004 best_bol = bol->as_Bool();
2005 best_btest = btest;
2006 }
2007 }
2008 }
2009
2010 Node* answer_if_true = NULL;
2011 Node* answer_if_false = NULL;
2012 switch (best_btest) {
2013 default:
2014 if (cmpxy == NULL)
2015 cmpxy = ideal_cmpxy;
2016 best_bol = _gvn.transform( new(C) BoolNode(cmpxy, BoolTest::lt) );
2017 // and fall through:
2018 case BoolTest::lt: // x < y
2019 case BoolTest::le: // x <= y
2020 answer_if_true = (want_max ? yvalue : xvalue);
2021 answer_if_false = (want_max ? xvalue : yvalue);
2022 break;
2023 case BoolTest::gt: // x > y
2024 case BoolTest::ge: // x >= y
2025 answer_if_true = (want_max ? xvalue : yvalue);
2026 answer_if_false = (want_max ? yvalue : xvalue);
2027 break;
2028 }
2029
2030 jint hi, lo;
2031 if (want_max) {
2032 // We can sharpen the minimum.
2033 hi = MAX2(txvalue->_hi, tyvalue->_hi);
2034 lo = MAX2(txvalue->_lo, tyvalue->_lo);
2035 } else {
2036 // We can sharpen the maximum.
2056 switch (id) {
2057 case vmIntrinsics::_min:
2058 result_val = _gvn.transform(new (C, 3) MinINode(x,y)); break;
2059 case vmIntrinsics::_max:
2060 result_val = _gvn.transform(new (C, 3) MaxINode(x,y)); break;
2061 default:
2062 ShouldNotReachHere();
2063 }
2064 */
2065 }
2066
2067 inline int
2068 LibraryCallKit::classify_unsafe_addr(Node* &base, Node* &offset) {
2069 const TypePtr* base_type = TypePtr::NULL_PTR;
2070 if (base != NULL) base_type = _gvn.type(base)->isa_ptr();
2071 if (base_type == NULL) {
2072 // Unknown type.
2073 return Type::AnyPtr;
2074 } else if (base_type == TypePtr::NULL_PTR) {
2075 // Since this is a NULL+long form, we have to switch to a rawptr.
2076 base = _gvn.transform( new (C) CastX2PNode(offset) );
2077 offset = MakeConX(0);
2078 return Type::RawPtr;
2079 } else if (base_type->base() == Type::RawPtr) {
2080 return Type::RawPtr;
2081 } else if (base_type->isa_oopptr()) {
2082 // Base is never null => always a heap address.
2083 if (base_type->ptr() == TypePtr::NotNull) {
2084 return Type::OopPtr;
2085 }
2086 // Offset is small => always a heap address.
2087 const TypeX* offset_type = _gvn.type(offset)->isa_intptr_t();
2088 if (offset_type != NULL &&
2089 base_type->offset() == 0 && // (should always be?)
2090 offset_type->_lo >= 0 &&
2091 !MacroAssembler::needs_explicit_null_check(offset_type->_hi)) {
2092 return Type::OopPtr;
2093 }
2094 // Otherwise, it might either be oop+off or NULL+addr.
2095 return Type::AnyPtr;
2096 } else {
2450 p = ConvX2L(p);
2451 break;
2452 default:
2453 fatal(err_msg_res("unexpected type %d: %s", type, type2name(type)));
2454 break;
2455 }
2456 // The load node has the control of the preceding MemBarCPUOrder. All
2457 // following nodes will have the control of the MemBarCPUOrder inserted at
2458 // the end of this method. So, pushing the load onto the stack at a later
2459 // point is fine.
2460 set_result(p);
2461 } else {
2462 // place effect of store into memory
2463 switch (type) {
2464 case T_DOUBLE:
2465 val = dstore_rounding(val);
2466 break;
2467 case T_ADDRESS:
2468 // Repackage the long as a pointer.
2469 val = ConvL2X(val);
2470 val = _gvn.transform( new (C) CastX2PNode(val) );
2471 break;
2472 }
2473
2474 if (type != T_OBJECT ) {
2475 (void) store_to_memory(control(), adr, val, type, adr_type, is_volatile);
2476 } else {
2477 // Possibly an oop being stored to Java heap or native memory
2478 if (!TypePtr::NULL_PTR->higher_equal(_gvn.type(heap_base_oop))) {
2479 // oop to Java heap.
2480 (void) store_oop_to_unknown(control(), heap_base_oop, adr, adr_type, val, type);
2481 } else {
2482 // We can't tell at compile time if we are storing in the Java heap or outside
2483 // of it. So we need to emit code to conditionally do the proper type of
2484 // store.
2485
2486 IdealKit ideal(this);
2487 #define __ ideal.
2488 // QQQ who knows what probability is here??
2489 __ if_then(heap_base_oop, BoolTest::ne, null(), PROB_UNLIKELY(0.999)); {
2490 // Sync IdealKit and graphKit.
2758 } else
2759 #endif
2760 {
2761 if (kind == LS_xchg) {
2762 load_store = _gvn.transform(new (C) GetAndSetPNode(control(), mem, adr, newval, adr_type, value_type->is_oopptr()));
2763 } else {
2764 assert(kind == LS_cmpxchg, "wrong LoadStore operation");
2765 load_store = _gvn.transform(new (C) CompareAndSwapPNode(control(), mem, adr, newval, oldval));
2766 }
2767 }
2768 post_barrier(control(), load_store, base, adr, alias_idx, newval, T_OBJECT, true);
2769 break;
2770 default:
2771 fatal(err_msg_res("unexpected type %d: %s", type, type2name(type)));
2772 break;
2773 }
2774
2775 // SCMemProjNodes represent the memory state of a LoadStore. Their
2776 // main role is to prevent LoadStore nodes from being optimized away
2777 // when their results aren't used.
2778 Node* proj = _gvn.transform( new (C) SCMemProjNode(load_store));
2779 set_memory(proj, alias_idx);
2780
2781 // Add the trailing membar surrounding the access
2782 insert_mem_bar(Op_MemBarCPUOrder);
2783 insert_mem_bar(Op_MemBarAcquire);
2784
2785 #ifdef _LP64
2786 if (type == T_OBJECT && adr->bottom_type()->is_ptr_to_narrowoop() && kind == LS_xchg) {
2787 load_store = _gvn.transform(new (C) DecodeNNode(load_store, load_store->get_ptr_type()));
2788 }
2789 #endif
2790
2791 assert(type2size[load_store->bottom_type()->basic_type()] == type2size[rtype], "result type should match");
2792 set_result(load_store);
2793 return true;
2794 }
2795
2796 //----------------------------inline_unsafe_ordered_store----------------------
2797 // public native void sun.misc.Unsafe.putOrderedObject(Object o, long offset, Object x);
2798 // public native void sun.misc.Unsafe.putOrderedInt(Object o, long offset, int x);
2993 no_int_result_path = 1, // t == Thread.current() && !TLS._osthread._interrupted
2994 no_clear_result_path = 2, // t == Thread.current() && TLS._osthread._interrupted && !clear_int
2995 slow_result_path = 3, // slow path: t.isInterrupted(clear_int)
2996 PATH_LIMIT
2997 };
2998
2999 // Ensure that it's not possible to move the load of TLS._osthread._interrupted flag
3000 // out of the function.
3001 insert_mem_bar(Op_MemBarCPUOrder);
3002
3003 RegionNode* result_rgn = new (C) RegionNode(PATH_LIMIT);
3004 PhiNode* result_val = new (C) PhiNode(result_rgn, TypeInt::BOOL);
3005
3006 RegionNode* slow_region = new (C) RegionNode(1);
3007 record_for_igvn(slow_region);
3008
3009 // (a) Receiving thread must be the current thread.
3010 Node* rec_thr = argument(0);
3011 Node* tls_ptr = NULL;
3012 Node* cur_thr = generate_current_thread(tls_ptr);
3013 Node* cmp_thr = _gvn.transform( new (C) CmpPNode(cur_thr, rec_thr) );
3014 Node* bol_thr = _gvn.transform( new (C) BoolNode(cmp_thr, BoolTest::ne) );
3015
3016 generate_slow_guard(bol_thr, slow_region);
3017
3018 // (b) Interrupt bit on TLS must be false.
3019 Node* p = basic_plus_adr(top()/*!oop*/, tls_ptr, in_bytes(JavaThread::osthread_offset()));
3020 Node* osthread = make_load(NULL, p, TypeRawPtr::NOTNULL, T_ADDRESS);
3021 p = basic_plus_adr(top()/*!oop*/, osthread, in_bytes(OSThread::interrupted_offset()));
3022
3023 // Set the control input on the field _interrupted read to prevent it floating up.
3024 Node* int_bit = make_load(control(), p, TypeInt::BOOL, T_INT);
3025 Node* cmp_bit = _gvn.transform( new (C) CmpINode(int_bit, intcon(0)) );
3026 Node* bol_bit = _gvn.transform( new (C) BoolNode(cmp_bit, BoolTest::ne) );
3027
3028 IfNode* iff_bit = create_and_map_if(control(), bol_bit, PROB_UNLIKELY_MAG(3), COUNT_UNKNOWN);
3029
3030 // First fast path: if (!TLS._interrupted) return false;
3031 Node* false_bit = _gvn.transform( new (C) IfFalseNode(iff_bit) );
3032 result_rgn->init_req(no_int_result_path, false_bit);
3033 result_val->init_req(no_int_result_path, intcon(0));
3034
3035 // drop through to next case
3036 set_control( _gvn.transform(new (C) IfTrueNode(iff_bit)) );
3037
3038 // (c) Or, if interrupt bit is set and clear_int is false, use 2nd fast path.
3039 Node* clr_arg = argument(1);
3040 Node* cmp_arg = _gvn.transform( new (C) CmpINode(clr_arg, intcon(0)) );
3041 Node* bol_arg = _gvn.transform( new (C) BoolNode(cmp_arg, BoolTest::ne) );
3042 IfNode* iff_arg = create_and_map_if(control(), bol_arg, PROB_FAIR, COUNT_UNKNOWN);
3043
3044 // Second fast path: ... else if (!clear_int) return true;
3045 Node* false_arg = _gvn.transform( new (C) IfFalseNode(iff_arg) );
3046 result_rgn->init_req(no_clear_result_path, false_arg);
3047 result_val->init_req(no_clear_result_path, intcon(1));
3048
3049 // drop through to next case
3050 set_control( _gvn.transform(new (C) IfTrueNode(iff_arg)) );
3051
3052 // (d) Otherwise, go to the slow path.
3053 slow_region->add_req(control());
3054 set_control( _gvn.transform(slow_region) );
3055
3056 if (stopped()) {
3057 // There is no slow path.
3058 result_rgn->init_req(slow_result_path, top());
3059 result_val->init_req(slow_result_path, top());
3060 } else {
3061 // non-virtual because it is a private non-static
3062 CallJavaNode* slow_call = generate_method_call(vmIntrinsics::_isInterrupted);
3063
3064 Node* slow_val = set_results_for_java_call(slow_call);
3065 // this->control() comes from set_results_for_java_call
3066
3067 Node* fast_io = slow_call->in(TypeFunc::I_O);
3068 Node* fast_mem = slow_call->in(TypeFunc::Memory);
3069
3070 // These two phis are pre-filled with copies of of the fast IO and Memory
3071 PhiNode* result_mem = PhiNode::make(result_rgn, fast_mem, Type::MEMORY, TypePtr::BOTTOM);
3072 PhiNode* result_io = PhiNode::make(result_rgn, fast_io, Type::ABIO);
3073
3074 result_rgn->init_req(slow_result_path, control());
3090 Node* LibraryCallKit::load_mirror_from_klass(Node* klass) {
3091 Node* p = basic_plus_adr(klass, in_bytes(Klass::java_mirror_offset()));
3092 return make_load(NULL, p, TypeInstPtr::MIRROR, T_OBJECT);
3093 }
3094
3095 //-----------------------load_klass_from_mirror_common-------------------------
3096 // Given a java mirror (a java.lang.Class oop), load its corresponding klass oop.
3097 // Test the klass oop for null (signifying a primitive Class like Integer.TYPE),
3098 // and branch to the given path on the region.
3099 // If never_see_null, take an uncommon trap on null, so we can optimistically
3100 // compile for the non-null case.
3101 // If the region is NULL, force never_see_null = true.
3102 Node* LibraryCallKit::load_klass_from_mirror_common(Node* mirror,
3103 bool never_see_null,
3104 RegionNode* region,
3105 int null_path,
3106 int offset) {
3107 if (region == NULL) never_see_null = true;
3108 Node* p = basic_plus_adr(mirror, offset);
3109 const TypeKlassPtr* kls_type = TypeKlassPtr::OBJECT_OR_NULL;
3110 Node* kls = _gvn.transform( LoadKlassNode::make(_gvn, immutable_memory(), p, TypeRawPtr::BOTTOM, kls_type) );
3111 Node* null_ctl = top();
3112 kls = null_check_oop(kls, &null_ctl, never_see_null);
3113 if (region != NULL) {
3114 // Set region->in(null_path) if the mirror is a primitive (e.g, int.class).
3115 region->init_req(null_path, null_ctl);
3116 } else {
3117 assert(null_ctl == top(), "no loose ends");
3118 }
3119 return kls;
3120 }
3121
3122 //--------------------(inline_native_Class_query helpers)---------------------
3123 // Use this for JVM_ACC_INTERFACE, JVM_ACC_IS_CLONEABLE, JVM_ACC_HAS_FINALIZER.
3124 // Fall through if (mods & mask) == bits, take the guard otherwise.
3125 Node* LibraryCallKit::generate_access_flags_guard(Node* kls, int modifier_mask, int modifier_bits, RegionNode* region) {
3126 // Branch around if the given klass has the given modifier bit set.
3127 // Like generate_guard, adds a new path onto the region.
3128 Node* modp = basic_plus_adr(kls, in_bytes(Klass::access_flags_offset()));
3129 Node* mods = make_load(NULL, modp, TypeInt::INT, T_INT);
3130 Node* mask = intcon(modifier_mask);
3131 Node* bits = intcon(modifier_bits);
3132 Node* mbit = _gvn.transform( new (C) AndINode(mods, mask) );
3133 Node* cmp = _gvn.transform( new (C) CmpINode(mbit, bits) );
3134 Node* bol = _gvn.transform( new (C) BoolNode(cmp, BoolTest::ne) );
3135 return generate_fair_guard(bol, region);
3136 }
3137 Node* LibraryCallKit::generate_interface_guard(Node* kls, RegionNode* region) {
3138 return generate_access_flags_guard(kls, JVM_ACC_INTERFACE, 0, region);
3139 }
3140
3141 //-------------------------inline_native_Class_query-------------------
3142 bool LibraryCallKit::inline_native_Class_query(vmIntrinsics::ID id) {
3143 const Type* return_type = TypeInt::BOOL;
3144 Node* prim_return_value = top(); // what happens if it's a primitive class?
3145 bool never_see_null = !too_many_traps(Deoptimization::Reason_null_check);
3146 bool expect_prim = false; // most of these guys expect to work on refs
3147
3148 enum { _normal_path = 1, _prim_path = 2, PATH_LIMIT };
3149
3150 Node* mirror = argument(0);
3151 Node* obj = top();
3152
3153 switch (id) {
3154 case vmIntrinsics::_isInstance:
3265
3266 case vmIntrinsics::_isPrimitive:
3267 query_value = intcon(0); // "normal" path produces false
3268 break;
3269
3270 case vmIntrinsics::_getSuperclass:
3271 // The rules here are somewhat unfortunate, but we can still do better
3272 // with random logic than with a JNI call.
3273 // Interfaces store null or Object as _super, but must report null.
3274 // Arrays store an intermediate super as _super, but must report Object.
3275 // Other types can report the actual _super.
3276 // (To verify this code sequence, check the asserts in JVM_IsInterface.)
3277 if (generate_interface_guard(kls, region) != NULL)
3278 // A guard was added. If the guard is taken, it was an interface.
3279 phi->add_req(null());
3280 if (generate_array_guard(kls, region) != NULL)
3281 // A guard was added. If the guard is taken, it was an array.
3282 phi->add_req(makecon(TypeInstPtr::make(env()->Object_klass()->java_mirror())));
3283 // If we fall through, it's a plain class. Get its _super.
3284 p = basic_plus_adr(kls, in_bytes(Klass::super_offset()));
3285 kls = _gvn.transform( LoadKlassNode::make(_gvn, immutable_memory(), p, TypeRawPtr::BOTTOM, TypeKlassPtr::OBJECT_OR_NULL) );
3286 null_ctl = top();
3287 kls = null_check_oop(kls, &null_ctl);
3288 if (null_ctl != top()) {
3289 // If the guard is taken, Object.superClass is null (both klass and mirror).
3290 region->add_req(null_ctl);
3291 phi ->add_req(null());
3292 }
3293 if (!stopped()) {
3294 query_value = load_mirror_from_klass(kls);
3295 }
3296 break;
3297
3298 case vmIntrinsics::_getComponentType:
3299 if (generate_array_guard(kls, region) != NULL) {
3300 // Be sure to pin the oop load to the guard edge just created:
3301 Node* is_array_ctrl = region->in(region->req()-1);
3302 Node* cma = basic_plus_adr(kls, in_bytes(ArrayKlass::component_mirror_offset()));
3303 Node* cmo = make_load(is_array_ctrl, cma, TypeInstPtr::MIRROR, T_OBJECT);
3304 phi->add_req(cmo);
3305 }
3378 region->init_req(prim_path, null_ctl);
3379 if (stopped()) break;
3380 klasses[which_arg] = kls;
3381 }
3382
3383 if (!stopped()) {
3384 // now we have two reference types, in klasses[0..1]
3385 Node* subk = klasses[1]; // the argument to isAssignableFrom
3386 Node* superk = klasses[0]; // the receiver
3387 region->set_req(_both_ref_path, gen_subtype_check(subk, superk));
3388 // now we have a successful reference subtype check
3389 region->set_req(_ref_subtype_path, control());
3390 }
3391
3392 // If both operands are primitive (both klasses null), then
3393 // we must return true when they are identical primitives.
3394 // It is convenient to test this after the first null klass check.
3395 set_control(region->in(_prim_0_path)); // go back to first null check
3396 if (!stopped()) {
3397 // Since superc is primitive, make a guard for the superc==subc case.
3398 Node* cmp_eq = _gvn.transform( new (C) CmpPNode(args[0], args[1]) );
3399 Node* bol_eq = _gvn.transform( new (C) BoolNode(cmp_eq, BoolTest::eq) );
3400 generate_guard(bol_eq, region, PROB_FAIR);
3401 if (region->req() == PATH_LIMIT+1) {
3402 // A guard was added. If the added guard is taken, superc==subc.
3403 region->swap_edges(PATH_LIMIT, _prim_same_path);
3404 region->del_req(PATH_LIMIT);
3405 }
3406 region->set_req(_prim_0_path, control()); // Not equal after all.
3407 }
3408
3409 // these are the only paths that produce 'true':
3410 phi->set_req(_prim_same_path, intcon(1));
3411 phi->set_req(_ref_subtype_path, intcon(1));
3412
3413 // pull together the cases:
3414 assert(region->req() == PATH_LIMIT, "sane region");
3415 for (uint i = 1; i < region->req(); i++) {
3416 Node* ctl = region->in(i);
3417 if (ctl == NULL || ctl == top()) {
3418 region->set_req(i, top());
3419 phi ->set_req(i, top());
3444 Node* layout_val = get_layout_helper(kls, layout_con);
3445 if (layout_val == NULL) {
3446 bool query = (obj_array
3447 ? Klass::layout_helper_is_objArray(layout_con)
3448 : Klass::layout_helper_is_array(layout_con));
3449 if (query == not_array) {
3450 return NULL; // never a branch
3451 } else { // always a branch
3452 Node* always_branch = control();
3453 if (region != NULL)
3454 region->add_req(always_branch);
3455 set_control(top());
3456 return always_branch;
3457 }
3458 }
3459 // Now test the correct condition.
3460 jint nval = (obj_array
3461 ? ((jint)Klass::_lh_array_tag_type_value
3462 << Klass::_lh_array_tag_shift)
3463 : Klass::_lh_neutral_value);
3464 Node* cmp = _gvn.transform( new(C) CmpINode(layout_val, intcon(nval)) );
3465 BoolTest::mask btest = BoolTest::lt; // correct for testing is_[obj]array
3466 // invert the test if we are looking for a non-array
3467 if (not_array) btest = BoolTest(btest).negate();
3468 Node* bol = _gvn.transform( new(C) BoolNode(cmp, btest) );
3469 return generate_fair_guard(bol, region);
3470 }
3471
3472
3473 //-----------------------inline_native_newArray--------------------------
3474 // private static native Object java.lang.reflect.newArray(Class<?> componentType, int length);
3475 bool LibraryCallKit::inline_native_newArray() {
3476 Node* mirror = argument(0);
3477 Node* count_val = argument(1);
3478
3479 mirror = null_check(mirror);
3480 // If mirror or obj is dead, only null-path is taken.
3481 if (stopped()) return true;
3482
3483 enum { _normal_path = 1, _slow_path = 2, PATH_LIMIT };
3484 RegionNode* result_reg = new(C) RegionNode(PATH_LIMIT);
3485 PhiNode* result_val = new(C) PhiNode(result_reg,
3486 TypeInstPtr::NOTNULL);
3487 PhiNode* result_io = new(C) PhiNode(result_reg, Type::ABIO);
3488 PhiNode* result_mem = new(C) PhiNode(result_reg, Type::MEMORY,
3508 result_reg->set_req(_slow_path, control());
3509 result_val->set_req(_slow_path, slow_result);
3510 result_io ->set_req(_slow_path, i_o());
3511 result_mem->set_req(_slow_path, reset_memory());
3512 }
3513
3514 set_control(normal_ctl);
3515 if (!stopped()) {
3516 // Normal case: The array type has been cached in the java.lang.Class.
3517 // The following call works fine even if the array type is polymorphic.
3518 // It could be a dynamic mix of int[], boolean[], Object[], etc.
3519 Node* obj = new_array(klass_node, count_val, 0); // no arguments to push
3520 result_reg->init_req(_normal_path, control());
3521 result_val->init_req(_normal_path, obj);
3522 result_io ->init_req(_normal_path, i_o());
3523 result_mem->init_req(_normal_path, reset_memory());
3524 }
3525
3526 // Return the combined state.
3527 set_i_o( _gvn.transform(result_io) );
3528 set_all_memory( _gvn.transform(result_mem) );
3529
3530 C->set_has_split_ifs(true); // Has chance for split-if optimization
3531 set_result(result_reg, result_val);
3532 return true;
3533 }
3534
3535 //----------------------inline_native_getLength--------------------------
3536 // public static native int java.lang.reflect.Array.getLength(Object array);
3537 bool LibraryCallKit::inline_native_getLength() {
3538 if (too_many_traps(Deoptimization::Reason_intrinsic)) return false;
3539
3540 Node* array = null_check(argument(0));
3541 // If array is dead, only null-path is taken.
3542 if (stopped()) return true;
3543
3544 // Deoptimize if it is a non-array.
3545 Node* non_array = generate_non_array_guard(load_object_klass(array), NULL);
3546
3547 if (non_array != NULL) {
3548 PreserveJVMState pjvms(this);
3661 }
3662
3663
3664 //----------------------generate_virtual_guard---------------------------
3665 // Helper for hashCode and clone. Peeks inside the vtable to avoid a call.
3666 Node* LibraryCallKit::generate_virtual_guard(Node* obj_klass,
3667 RegionNode* slow_region) {
3668 ciMethod* method = callee();
3669 int vtable_index = method->vtable_index();
3670 // Get the Method* out of the appropriate vtable entry.
3671 int entry_offset = (InstanceKlass::vtable_start_offset() +
3672 vtable_index*vtableEntry::size()) * wordSize +
3673 vtableEntry::method_offset_in_bytes();
3674 Node* entry_addr = basic_plus_adr(obj_klass, entry_offset);
3675 Node* target_call = make_load(NULL, entry_addr, TypePtr::NOTNULL, T_ADDRESS);
3676
3677 // Compare the target method with the expected method (e.g., Object.hashCode).
3678 const TypePtr* native_call_addr = TypeMetadataPtr::make(method);
3679
3680 Node* native_call = makecon(native_call_addr);
3681 Node* chk_native = _gvn.transform( new(C) CmpPNode(target_call, native_call) );
3682 Node* test_native = _gvn.transform( new(C) BoolNode(chk_native, BoolTest::ne) );
3683
3684 return generate_slow_guard(test_native, slow_region);
3685 }
3686
3687 //-----------------------generate_method_call----------------------------
3688 // Use generate_method_call to make a slow-call to the real
3689 // method if the fast path fails. An alternative would be to
3690 // use a stub like OptoRuntime::slow_arraycopy_Java.
3691 // This only works for expanding the current library call,
3692 // not another intrinsic. (E.g., don't use this for making an
3693 // arraycopy call inside of the copyOf intrinsic.)
3694 CallJavaNode*
3695 LibraryCallKit::generate_method_call(vmIntrinsics::ID method_id, bool is_virtual, bool is_static) {
3696 // When compiling the intrinsic method itself, do not use this technique.
3697 guarantee(callee() != C->method(), "cannot make slow-call to self");
3698
3699 ciMethod* method = callee();
3700 // ensure the JVMS we have will be correct for this call
3701 guarantee(method_id == method->intrinsic_id(), "must match");
3702
3783 // paths which do not pass are accumulated in the slow_region.
3784 RegionNode* slow_region = new (C) RegionNode(1);
3785 record_for_igvn(slow_region);
3786
3787 // If this is a virtual call, we generate a funny guard. We pull out
3788 // the vtable entry corresponding to hashCode() from the target object.
3789 // If the target method which we are calling happens to be the native
3790 // Object hashCode() method, we pass the guard. We do not need this
3791 // guard for non-virtual calls -- the caller is known to be the native
3792 // Object hashCode().
3793 if (is_virtual) {
3794 generate_virtual_guard(obj_klass, slow_region);
3795 }
3796
3797 // Get the header out of the object, use LoadMarkNode when available
3798 Node* header_addr = basic_plus_adr(obj, oopDesc::mark_offset_in_bytes());
3799 Node* header = make_load(control(), header_addr, TypeX_X, TypeX_X->basic_type());
3800
3801 // Test the header to see if it is unlocked.
3802 Node *lock_mask = _gvn.MakeConX(markOopDesc::biased_lock_mask_in_place);
3803 Node *lmasked_header = _gvn.transform( new (C) AndXNode(header, lock_mask) );
3804 Node *unlocked_val = _gvn.MakeConX(markOopDesc::unlocked_value);
3805 Node *chk_unlocked = _gvn.transform( new (C) CmpXNode( lmasked_header, unlocked_val));
3806 Node *test_unlocked = _gvn.transform( new (C) BoolNode( chk_unlocked, BoolTest::ne) );
3807
3808 generate_slow_guard(test_unlocked, slow_region);
3809
3810 // Get the hash value and check to see that it has been properly assigned.
3811 // We depend on hash_mask being at most 32 bits and avoid the use of
3812 // hash_mask_in_place because it could be larger than 32 bits in a 64-bit
3813 // vm: see markOop.hpp.
3814 Node *hash_mask = _gvn.intcon(markOopDesc::hash_mask);
3815 Node *hash_shift = _gvn.intcon(markOopDesc::hash_shift);
3816 Node *hshifted_header= _gvn.transform( new (C) URShiftXNode(header, hash_shift) );
3817 // This hack lets the hash bits live anywhere in the mark object now, as long
3818 // as the shift drops the relevant bits into the low 32 bits. Note that
3819 // Java spec says that HashCode is an int so there's no point in capturing
3820 // an 'X'-sized hashcode (32 in 32-bit build or 64 in 64-bit build).
3821 hshifted_header = ConvX2I(hshifted_header);
3822 Node *hash_val = _gvn.transform( new (C) AndINode(hshifted_header, hash_mask) );
3823
3824 Node *no_hash_val = _gvn.intcon(markOopDesc::no_hash);
3825 Node *chk_assigned = _gvn.transform( new (C) CmpINode( hash_val, no_hash_val));
3826 Node *test_assigned = _gvn.transform( new (C) BoolNode( chk_assigned, BoolTest::eq) );
3827
3828 generate_slow_guard(test_assigned, slow_region);
3829
3830 Node* init_mem = reset_memory();
3831 // fill in the rest of the null path:
3832 result_io ->init_req(_null_path, i_o());
3833 result_mem->init_req(_null_path, init_mem);
3834
3835 result_val->init_req(_fast_path, hash_val);
3836 result_reg->init_req(_fast_path, control());
3837 result_io ->init_req(_fast_path, i_o());
3838 result_mem->init_req(_fast_path, init_mem);
3839
3840 // Generate code for the slow case. We make a call to hashCode().
3841 set_control(_gvn.transform(slow_region));
3842 if (!stopped()) {
3843 // No need for PreserveJVMState, because we're using up the present state.
3844 set_all_memory(init_mem);
3845 vmIntrinsics::ID hashCode_id = is_static ? vmIntrinsics::_identityHashCode : vmIntrinsics::_hashCode;
3846 CallJavaNode* slow_call = generate_method_call(hashCode_id, is_virtual, is_static);
3847 Node* slow_result = set_results_for_java_call(slow_call);
3848 // this->control() comes from set_results_for_java_call
3849 result_reg->init_req(_slow_path, control());
3850 result_val->init_req(_slow_path, slow_result);
3851 result_io ->set_req(_slow_path, i_o());
3852 result_mem ->set_req(_slow_path, reset_memory());
3853 }
3854
3855 // Return the combined state.
3856 set_i_o( _gvn.transform(result_io) );
3857 set_all_memory( _gvn.transform(result_mem) );
3858
3859 set_result(result_reg, result_val);
3860 return true;
3861 }
3862
3863 //---------------------------inline_native_getClass----------------------------
3864 // public final native Class<?> java.lang.Object.getClass();
3865 //
3866 // Build special case code for calls to getClass on an object.
3867 bool LibraryCallKit::inline_native_getClass() {
3868 Node* obj = null_check_receiver();
3869 if (stopped()) return true;
3870 set_result(load_mirror_from_klass(load_object_klass(obj)));
3871 return true;
3872 }
3873
3874 //-----------------inline_native_Reflection_getCallerClass---------------------
3875 // public static native Class<?> sun.reflect.Reflection.getCallerClass();
3876 //
3877 // In the presence of deep enough inlining, getCallerClass() becomes a no-op.
3965 case vmIntrinsics::_floatToRawIntBits: result = new (C) MoveF2INode(arg); break;
3966 case vmIntrinsics::_intBitsToFloat: result = new (C) MoveI2FNode(arg); break;
3967 case vmIntrinsics::_doubleToRawLongBits: result = new (C) MoveD2LNode(arg); break;
3968 case vmIntrinsics::_longBitsToDouble: result = new (C) MoveL2DNode(arg); break;
3969
3970 case vmIntrinsics::_doubleToLongBits: {
3971 // two paths (plus control) merge in a wood
3972 RegionNode *r = new (C) RegionNode(3);
3973 Node *phi = new (C) PhiNode(r, TypeLong::LONG);
3974
3975 Node *cmpisnan = _gvn.transform(new (C) CmpDNode(arg, arg));
3976 // Build the boolean node
3977 Node *bolisnan = _gvn.transform(new (C) BoolNode(cmpisnan, BoolTest::ne));
3978
3979 // Branch either way.
3980 // NaN case is less traveled, which makes all the difference.
3981 IfNode *ifisnan = create_and_xform_if(control(), bolisnan, PROB_STATIC_FREQUENT, COUNT_UNKNOWN);
3982 Node *opt_isnan = _gvn.transform(ifisnan);
3983 assert( opt_isnan->is_If(), "Expect an IfNode");
3984 IfNode *opt_ifisnan = (IfNode*)opt_isnan;
3985 Node *iftrue = _gvn.transform( new (C) IfTrueNode(opt_ifisnan) );
3986
3987 set_control(iftrue);
3988
3989 static const jlong nan_bits = CONST64(0x7ff8000000000000);
3990 Node *slow_result = longcon(nan_bits); // return NaN
3991 phi->init_req(1, _gvn.transform( slow_result ));
3992 r->init_req(1, iftrue);
3993
3994 // Else fall through
3995 Node *iffalse = _gvn.transform(new (C) IfFalseNode(opt_ifisnan));
3996 set_control(iffalse);
3997
3998 phi->init_req(2, _gvn.transform(new (C) MoveD2LNode(arg)));
3999 r->init_req(2, iffalse);
4000
4001 // Post merge
4002 set_control(_gvn.transform(r));
4003 record_for_igvn(r);
4004
4005 C->set_has_split_ifs(true); // Has chance for split-if optimization
4006 result = phi;
4007 assert(result->bottom_type()->isa_long(), "must be");
4008 break;
4009 }
4010
4011 case vmIntrinsics::_floatToIntBits: {
4012 // two paths (plus control) merge in a wood
4013 RegionNode *r = new (C) RegionNode(3);
4014 Node *phi = new (C) PhiNode(r, TypeInt::INT);
4015
4016 Node *cmpisnan = _gvn.transform(new (C) CmpFNode(arg, arg));
4017 // Build the boolean node
4018 Node *bolisnan = _gvn.transform(new (C) BoolNode(cmpisnan, BoolTest::ne));
4019
4020 // Branch either way.
4021 // NaN case is less traveled, which makes all the difference.
4022 IfNode *ifisnan = create_and_xform_if(control(), bolisnan, PROB_STATIC_FREQUENT, COUNT_UNKNOWN);
4023 Node *opt_isnan = _gvn.transform(ifisnan);
4024 assert( opt_isnan->is_If(), "Expect an IfNode");
4025 IfNode *opt_ifisnan = (IfNode*)opt_isnan;
4026 Node *iftrue = _gvn.transform( new (C) IfTrueNode(opt_ifisnan) );
4027
4028 set_control(iftrue);
4029
4030 static const jint nan_bits = 0x7fc00000;
4031 Node *slow_result = makecon(TypeInt::make(nan_bits)); // return NaN
4032 phi->init_req(1, _gvn.transform( slow_result ));
4033 r->init_req(1, iftrue);
4034
4035 // Else fall through
4036 Node *iffalse = _gvn.transform(new (C) IfFalseNode(opt_ifisnan));
4037 set_control(iffalse);
4038
4039 phi->init_req(2, _gvn.transform(new (C) MoveF2INode(arg)));
4040 r->init_req(2, iffalse);
4041
4042 // Post merge
4043 set_control(_gvn.transform(r));
4044 record_for_igvn(r);
4045
4046 C->set_has_split_ifs(true); // Has chance for split-if optimization
4135 // base_off:
4136 // 8 - 32-bit VM
4137 // 12 - 64-bit VM, compressed klass
4138 // 16 - 64-bit VM, normal klass
4139 if (base_off % BytesPerLong != 0) {
4140 assert(UseCompressedKlassPointers, "");
4141 if (is_array) {
4142 // Exclude length to copy by 8 bytes words.
4143 base_off += sizeof(int);
4144 } else {
4145 // Include klass to copy by 8 bytes words.
4146 base_off = instanceOopDesc::klass_offset_in_bytes();
4147 }
4148 assert(base_off % BytesPerLong == 0, "expect 8 bytes alignment");
4149 }
4150 src = basic_plus_adr(src, base_off);
4151 dest = basic_plus_adr(dest, base_off);
4152
4153 // Compute the length also, if needed:
4154 Node* countx = size;
4155 countx = _gvn.transform( new (C) SubXNode(countx, MakeConX(base_off)) );
4156 countx = _gvn.transform( new (C) URShiftXNode(countx, intcon(LogBytesPerLong) ));
4157
4158 const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
4159 bool disjoint_bases = true;
4160 generate_unchecked_arraycopy(raw_adr_type, T_LONG, disjoint_bases,
4161 src, NULL, dest, NULL, countx,
4162 /*dest_uninitialized*/true);
4163
4164 // If necessary, emit some card marks afterwards. (Non-arrays only.)
4165 if (card_mark) {
4166 assert(!is_array, "");
4167 // Put in store barrier for any and all oops we are sticking
4168 // into this object. (We could avoid this if we could prove
4169 // that the object type contains no oop fields at all.)
4170 Node* no_particular_value = NULL;
4171 Node* no_particular_field = NULL;
4172 int raw_adr_idx = Compile::AliasIdxRaw;
4173 post_barrier(control(),
4174 memory(raw_adr_type),
4175 alloc_obj,
4176 no_particular_field,
4340 result_reg->init_req(_instance_path, control());
4341 result_val->init_req(_instance_path, alloc_obj);
4342 result_i_o ->set_req(_instance_path, i_o());
4343 result_mem ->set_req(_instance_path, reset_memory());
4344 }
4345
4346 // Generate code for the slow case. We make a call to clone().
4347 set_control(_gvn.transform(slow_region));
4348 if (!stopped()) {
4349 PreserveJVMState pjvms(this);
4350 CallJavaNode* slow_call = generate_method_call(vmIntrinsics::_clone, is_virtual);
4351 Node* slow_result = set_results_for_java_call(slow_call);
4352 // this->control() comes from set_results_for_java_call
4353 result_reg->init_req(_slow_path, control());
4354 result_val->init_req(_slow_path, slow_result);
4355 result_i_o ->set_req(_slow_path, i_o());
4356 result_mem ->set_req(_slow_path, reset_memory());
4357 }
4358
4359 // Return the combined state.
4360 set_control( _gvn.transform(result_reg) );
4361 set_i_o( _gvn.transform(result_i_o) );
4362 set_all_memory( _gvn.transform(result_mem) );
4363 } // original reexecute is set back here
4364
4365 set_result(_gvn.transform(result_val));
4366 return true;
4367 }
4368
4369 //------------------------------basictype2arraycopy----------------------------
4370 address LibraryCallKit::basictype2arraycopy(BasicType t,
4371 Node* src_offset,
4372 Node* dest_offset,
4373 bool disjoint_bases,
4374 const char* &name,
4375 bool dest_uninitialized) {
4376 const TypeInt* src_offset_inttype = gvn().find_int_type(src_offset);;
4377 const TypeInt* dest_offset_inttype = gvn().find_int_type(dest_offset);;
4378
4379 bool aligned = false;
4380 bool disjoint = disjoint_bases;
4381
4382 // if the offsets are the same, we can treat the memory regions as
4667 // paths have stub or runtime calls as raw memory barriers.
4668 InitializeNode* init = insert_mem_bar_volatile(Op_Initialize,
4669 Compile::AliasIdxRaw,
4670 top())->as_Initialize();
4671 init->set_complete(&_gvn); // (there is no corresponding AllocateNode)
4672 }
4673 }
4674
4675 // Present the results of the fast call.
4676 result_region->init_req(zero_path, control());
4677 result_i_o ->init_req(zero_path, i_o());
4678 result_memory->init_req(zero_path, memory(adr_type));
4679 }
4680
4681 if (!stopped() && dest_uninitialized) {
4682 // We have to initialize the *uncopied* part of the array to zero.
4683 // The copy destination is the slice dest[off..off+len]. The other slices
4684 // are dest_head = dest[0..off] and dest_tail = dest[off+len..dest.length].
4685 Node* dest_size = alloc->in(AllocateNode::AllocSize);
4686 Node* dest_length = alloc->in(AllocateNode::ALength);
4687 Node* dest_tail = _gvn.transform( new(C) AddINode(dest_offset,
4688 copy_length) );
4689
4690 // If there is a head section that needs zeroing, do it now.
4691 if (find_int_con(dest_offset, -1) != 0) {
4692 generate_clear_array(adr_type, dest, basic_elem_type,
4693 intcon(0), dest_offset,
4694 NULL);
4695 }
4696
4697 // Next, perform a dynamic check on the tail length.
4698 // It is often zero, and we can win big if we prove this.
4699 // There are two wins: Avoid generating the ClearArray
4700 // with its attendant messy index arithmetic, and upgrade
4701 // the copy to a more hardware-friendly word size of 64 bits.
4702 Node* tail_ctl = NULL;
4703 if (!stopped() && !dest_tail->eqv_uncast(dest_length)) {
4704 Node* cmp_lt = _gvn.transform( new(C) CmpINode(dest_tail, dest_length) );
4705 Node* bol_lt = _gvn.transform( new(C) BoolNode(cmp_lt, BoolTest::lt) );
4706 tail_ctl = generate_slow_guard(bol_lt, NULL);
4707 assert(tail_ctl != NULL || !stopped(), "must be an outcome");
4708 }
4709
4710 // At this point, let's assume there is no tail.
4711 if (!stopped() && alloc != NULL && basic_elem_type != T_OBJECT) {
4712 // There is no tail. Try an upgrade to a 64-bit copy.
4713 bool didit = false;
4714 { PreserveJVMState pjvms(this);
4715 didit = generate_block_arraycopy(adr_type, basic_elem_type, alloc,
4716 src, src_offset, dest, dest_offset,
4717 dest_size, dest_uninitialized);
4718 if (didit) {
4719 // Present the results of the block-copying fast call.
4720 result_region->init_req(bcopy_path, control());
4721 result_i_o ->init_req(bcopy_path, i_o());
4722 result_memory->init_req(bcopy_path, memory(adr_type));
4723 }
4724 }
4725 if (didit)
4728
4729 // Clear the tail, if any.
4730 if (tail_ctl != NULL) {
4731 Node* notail_ctl = stopped() ? NULL : control();
4732 set_control(tail_ctl);
4733 if (notail_ctl == NULL) {
4734 generate_clear_array(adr_type, dest, basic_elem_type,
4735 dest_tail, NULL,
4736 dest_size);
4737 } else {
4738 // Make a local merge.
4739 Node* done_ctl = new(C) RegionNode(3);
4740 Node* done_mem = new(C) PhiNode(done_ctl, Type::MEMORY, adr_type);
4741 done_ctl->init_req(1, notail_ctl);
4742 done_mem->init_req(1, memory(adr_type));
4743 generate_clear_array(adr_type, dest, basic_elem_type,
4744 dest_tail, NULL,
4745 dest_size);
4746 done_ctl->init_req(2, control());
4747 done_mem->init_req(2, memory(adr_type));
4748 set_control( _gvn.transform(done_ctl) );
4749 set_memory( _gvn.transform(done_mem), adr_type );
4750 }
4751 }
4752 }
4753
4754 BasicType copy_type = basic_elem_type;
4755 assert(basic_elem_type != T_ARRAY, "caller must fix this");
4756 if (!stopped() && copy_type == T_OBJECT) {
4757 // If src and dest have compatible element types, we can copy bits.
4758 // Types S[] and D[] are compatible if D is a supertype of S.
4759 //
4760 // If they are not, we will use checked_oop_disjoint_arraycopy,
4761 // which performs a fast optimistic per-oop check, and backs off
4762 // further to JVM_ArrayCopy on the first per-oop check that fails.
4763 // (Actually, we don't move raw bits only; the GC requires card marks.)
4764
4765 // Get the Klass* for both src and dest
4766 Node* src_klass = load_object_klass(src);
4767 Node* dest_klass = load_object_klass(dest);
4768
4815 src, src_offset, dest, dest_offset,
4816 ConvI2X(copy_length), dest_uninitialized);
4817
4818 // Present the results of the fast call.
4819 result_region->init_req(fast_path, control());
4820 result_i_o ->init_req(fast_path, i_o());
4821 result_memory->init_req(fast_path, memory(adr_type));
4822 }
4823
4824 // Here are all the slow paths up to this point, in one bundle:
4825 slow_control = top();
4826 if (slow_region != NULL)
4827 slow_control = _gvn.transform(slow_region);
4828 DEBUG_ONLY(slow_region = (RegionNode*)badAddress);
4829
4830 set_control(checked_control);
4831 if (!stopped()) {
4832 // Clean up after the checked call.
4833 // The returned value is either 0 or -1^K,
4834 // where K = number of partially transferred array elements.
4835 Node* cmp = _gvn.transform( new(C) CmpINode(checked_value, intcon(0)) );
4836 Node* bol = _gvn.transform( new(C) BoolNode(cmp, BoolTest::eq) );
4837 IfNode* iff = create_and_map_if(control(), bol, PROB_MAX, COUNT_UNKNOWN);
4838
4839 // If it is 0, we are done, so transfer to the end.
4840 Node* checks_done = _gvn.transform( new(C) IfTrueNode(iff) );
4841 result_region->init_req(checked_path, checks_done);
4842 result_i_o ->init_req(checked_path, checked_i_o);
4843 result_memory->init_req(checked_path, checked_mem);
4844
4845 // If it is not zero, merge into the slow call.
4846 set_control( _gvn.transform( new(C) IfFalseNode(iff) ));
4847 RegionNode* slow_reg2 = new(C) RegionNode(3);
4848 PhiNode* slow_i_o2 = new(C) PhiNode(slow_reg2, Type::ABIO);
4849 PhiNode* slow_mem2 = new(C) PhiNode(slow_reg2, Type::MEMORY, adr_type);
4850 record_for_igvn(slow_reg2);
4851 slow_reg2 ->init_req(1, slow_control);
4852 slow_i_o2 ->init_req(1, slow_i_o);
4853 slow_mem2 ->init_req(1, slow_mem);
4854 slow_reg2 ->init_req(2, control());
4855 slow_i_o2 ->init_req(2, checked_i_o);
4856 slow_mem2 ->init_req(2, checked_mem);
4857
4858 slow_control = _gvn.transform(slow_reg2);
4859 slow_i_o = _gvn.transform(slow_i_o2);
4860 slow_mem = _gvn.transform(slow_mem2);
4861
4862 if (alloc != NULL) {
4863 // We'll restart from the very beginning, after zeroing the whole thing.
4864 // This can cause double writes, but that's OK since dest is brand new.
4865 // So we ignore the low 31 bits of the value returned from the stub.
4866 } else {
4867 // We must continue the copy exactly where it failed, or else
4868 // another thread might see the wrong number of writes to dest.
4869 Node* checked_offset = _gvn.transform( new(C) XorINode(checked_value, intcon(-1)) );
4870 Node* slow_offset = new(C) PhiNode(slow_reg2, TypeInt::INT);
4871 slow_offset->init_req(1, intcon(0));
4872 slow_offset->init_req(2, checked_offset);
4873 slow_offset = _gvn.transform(slow_offset);
4874
4875 // Adjust the arguments by the conditionally incoming offset.
4876 Node* src_off_plus = _gvn.transform( new(C) AddINode(src_offset, slow_offset) );
4877 Node* dest_off_plus = _gvn.transform( new(C) AddINode(dest_offset, slow_offset) );
4878 Node* length_minus = _gvn.transform( new(C) SubINode(copy_length, slow_offset) );
4879
4880 // Tweak the node variables to adjust the code produced below:
4881 src_offset = src_off_plus;
4882 dest_offset = dest_off_plus;
4883 copy_length = length_minus;
4884 }
4885 }
4886
4887 set_control(slow_control);
4888 if (!stopped()) {
4889 // Generate the slow path, if needed.
4890 PreserveJVMState pjvms(this); // replace_in_map may trash the map
4891
4892 set_memory(slow_mem, adr_type);
4893 set_i_o(slow_i_o);
4894
4895 if (dest_uninitialized) {
4896 generate_clear_array(adr_type, dest, basic_elem_type,
4897 intcon(0), NULL,
4898 alloc->in(AllocateNode::AllocSize));
4899 }
4900
4901 generate_slow_arraycopy(adr_type,
4902 src, src_offset, dest, dest_offset,
4903 copy_length, /*dest_uninitialized*/false);
4904
4905 result_region->init_req(slow_call_path, control());
4906 result_i_o ->init_req(slow_call_path, i_o());
4907 result_memory->init_req(slow_call_path, memory(adr_type));
4908 }
4909
4910 // Remove unused edges.
4911 for (uint i = 1; i < result_region->req(); i++) {
4912 if (result_region->in(i) == NULL)
4913 result_region->init_req(i, top());
4914 }
4915
4916 // Finished; return the combined state.
4917 set_control( _gvn.transform(result_region) );
4918 set_i_o( _gvn.transform(result_i_o) );
4919 set_memory( _gvn.transform(result_memory), adr_type );
4920
4921 // The memory edges above are precise in order to model effects around
4922 // array copies accurately to allow value numbering of field loads around
4923 // arraycopy. Such field loads, both before and after, are common in Java
4924 // collections and similar classes involving header/array data structures.
4925 //
4926 // But with low number of register or when some registers are used or killed
4927 // by arraycopy calls it causes registers spilling on stack. See 6544710.
4928 // The next memory barrier is added to avoid it. If the arraycopy can be
4929 // optimized away (which it can, sometimes) then we can manually remove
4930 // the membar also.
4931 //
4932 // Do not let reads from the cloned object float above the arraycopy.
4933 if (alloc != NULL) {
4934 // Do not let stores that initialize this object be reordered with
4935 // a subsequent store that would make this object accessible by
4936 // other threads.
4937 // Record what AllocateNode this StoreStore protects so that
5079 end_con = round_to(abase + ((slice_idx_con + slice_len_con) << scale),
5080 BytesPerLong);
5081 }
5082
5083 if (start_con >= 0 && end_con >= 0) {
5084 // Constant start and end. Simple.
5085 mem = ClearArrayNode::clear_memory(control(), mem, dest,
5086 start_con, end_con, &_gvn);
5087 } else if (start_con >= 0 && dest_size != top()) {
5088 // Constant start, pre-rounded end after the tail of the array.
5089 Node* end = dest_size;
5090 mem = ClearArrayNode::clear_memory(control(), mem, dest,
5091 start_con, end, &_gvn);
5092 } else if (start_con >= 0 && slice_len != top()) {
5093 // Constant start, non-constant end. End needs rounding up.
5094 // End offset = round_up(abase + ((slice_idx_con + slice_len) << scale), 8)
5095 intptr_t end_base = abase + (slice_idx_con << scale);
5096 int end_round = (-1 << scale) & (BytesPerLong - 1);
5097 Node* end = ConvI2X(slice_len);
5098 if (scale != 0)
5099 end = _gvn.transform( new(C) LShiftXNode(end, intcon(scale) ));
5100 end_base += end_round;
5101 end = _gvn.transform( new(C) AddXNode(end, MakeConX(end_base)) );
5102 end = _gvn.transform( new(C) AndXNode(end, MakeConX(~end_round)) );
5103 mem = ClearArrayNode::clear_memory(control(), mem, dest,
5104 start_con, end, &_gvn);
5105 } else if (start_con < 0 && dest_size != top()) {
5106 // Non-constant start, pre-rounded end after the tail of the array.
5107 // This is almost certainly a "round-to-end" operation.
5108 Node* start = slice_idx;
5109 start = ConvI2X(start);
5110 if (scale != 0)
5111 start = _gvn.transform( new(C) LShiftXNode( start, intcon(scale) ));
5112 start = _gvn.transform( new(C) AddXNode(start, MakeConX(abase)) );
5113 if ((bump_bit | clear_low) != 0) {
5114 int to_clear = (bump_bit | clear_low);
5115 // Align up mod 8, then store a jint zero unconditionally
5116 // just before the mod-8 boundary.
5117 if (((abase + bump_bit) & ~to_clear) - bump_bit
5118 < arrayOopDesc::length_offset_in_bytes() + BytesPerInt) {
5119 bump_bit = 0;
5120 assert((abase & to_clear) == 0, "array base must be long-aligned");
5121 } else {
5122 // Bump 'start' up to (or past) the next jint boundary:
5123 start = _gvn.transform( new(C) AddXNode(start, MakeConX(bump_bit)) );
5124 assert((abase & clear_low) == 0, "array base must be int-aligned");
5125 }
5126 // Round bumped 'start' down to jlong boundary in body of array.
5127 start = _gvn.transform( new(C) AndXNode(start, MakeConX(~to_clear)) );
5128 if (bump_bit != 0) {
5129 // Store a zero to the immediately preceding jint:
5130 Node* x1 = _gvn.transform( new(C) AddXNode(start, MakeConX(-bump_bit)) );
5131 Node* p1 = basic_plus_adr(dest, x1);
5132 mem = StoreNode::make(_gvn, control(), mem, p1, adr_type, intcon(0), T_INT);
5133 mem = _gvn.transform(mem);
5134 }
5135 }
5136 Node* end = dest_size; // pre-rounded
5137 mem = ClearArrayNode::clear_memory(control(), mem, dest,
5138 start, end, &_gvn);
5139 } else {
5140 // Non-constant start, unrounded non-constant end.
5141 // (Nobody zeroes a random midsection of an array using this routine.)
5142 ShouldNotReachHere(); // fix caller
5143 }
5144
5145 // Done.
5146 set_memory(mem, adr_type);
5147 }
5148
5149
5150 bool
5177 // This is a common case, since abase can be odd mod 8.
5178 if (((src_off | dest_off) & (BytesPerLong-1)) == BytesPerInt &&
5179 ((src_off ^ dest_off) & (BytesPerLong-1)) == 0) {
5180 Node* sptr = basic_plus_adr(src, src_off);
5181 Node* dptr = basic_plus_adr(dest, dest_off);
5182 Node* sval = make_load(control(), sptr, TypeInt::INT, T_INT, adr_type);
5183 store_to_memory(control(), dptr, sval, T_INT, adr_type);
5184 src_off += BytesPerInt;
5185 dest_off += BytesPerInt;
5186 } else {
5187 return false;
5188 }
5189 }
5190 assert(src_off % BytesPerLong == 0, "");
5191 assert(dest_off % BytesPerLong == 0, "");
5192
5193 // Do this copy by giant steps.
5194 Node* sptr = basic_plus_adr(src, src_off);
5195 Node* dptr = basic_plus_adr(dest, dest_off);
5196 Node* countx = dest_size;
5197 countx = _gvn.transform( new (C) SubXNode(countx, MakeConX(dest_off)) );
5198 countx = _gvn.transform( new (C) URShiftXNode(countx, intcon(LogBytesPerLong)) );
5199
5200 bool disjoint_bases = true; // since alloc != NULL
5201 generate_unchecked_arraycopy(adr_type, T_LONG, disjoint_bases,
5202 sptr, NULL, dptr, NULL, countx, dest_uninitialized);
5203
5204 return true;
5205 }
5206
5207
5208 // Helper function; generates code for the slow case.
5209 // We make a call to a runtime method which emulates the native method,
5210 // but without the native wrapper overhead.
5211 void
5212 LibraryCallKit::generate_slow_arraycopy(const TypePtr* adr_type,
5213 Node* src, Node* src_offset,
5214 Node* dest, Node* dest_offset,
5215 Node* copy_length, bool dest_uninitialized) {
5216 assert(!dest_uninitialized, "Invariant");
5217 Node* call = make_runtime_call(RC_NO_LEAF | RC_UNCOMMON,
5218 OptoRuntime::slow_arraycopy_Type(),
5343 // Figure out the size and type of the elements we will be copying.
5344 BasicType src_elem = src_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
5345 BasicType dst_elem = dst_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
5346 if (src_elem != T_CHAR || dst_elem != T_BYTE) {
5347 return false;
5348 }
5349 Node* src_start = array_element_address(src, src_offset, src_elem);
5350 Node* dst_start = array_element_address(dst, dst_offset, dst_elem);
5351 // 'src_start' points to src array + scaled offset
5352 // 'dst_start' points to dst array + scaled offset
5353
5354 const TypeAryPtr* mtype = TypeAryPtr::BYTES;
5355 Node* enc = new (C) EncodeISOArrayNode(control(), memory(mtype), src_start, dst_start, length);
5356 enc = _gvn.transform(enc);
5357 Node* res_mem = _gvn.transform(new (C) SCMemProjNode(enc));
5358 set_memory(res_mem, mtype);
5359 set_result(enc);
5360 return true;
5361 }
5362
5363 //----------------------------inline_reference_get----------------------------
5364 // public T java.lang.ref.Reference.get();
5365 bool LibraryCallKit::inline_reference_get() {
5366 const int referent_offset = java_lang_ref_Reference::referent_offset;
5367 guarantee(referent_offset > 0, "should have already been set");
5368
5369 // Get the argument:
5370 Node* reference_obj = null_check_receiver();
5371 if (stopped()) return true;
5372
5373 Node* adr = basic_plus_adr(reference_obj, reference_obj, referent_offset);
5374
5375 ciInstanceKlass* klass = env()->Object_klass();
5376 const TypeOopPtr* object_type = TypeOopPtr::make_from_klass(klass);
5377
5378 Node* no_ctrl = NULL;
5379 Node* result = make_load(no_ctrl, adr, object_type, T_OBJECT);
5380
5381 // Use the pre-barrier to record the value in the referent field
5382 pre_barrier(false /* do_load */,
|
274 Node* dest, Node* dest_offset,
275 Node* copy_length, bool dest_uninitialized);
276 void generate_unchecked_arraycopy(const TypePtr* adr_type,
277 BasicType basic_elem_type,
278 bool disjoint_bases,
279 Node* src, Node* src_offset,
280 Node* dest, Node* dest_offset,
281 Node* copy_length, bool dest_uninitialized);
282 typedef enum { LS_xadd, LS_xchg, LS_cmpxchg } LoadStoreKind;
283 bool inline_unsafe_load_store(BasicType type, LoadStoreKind kind);
284 bool inline_unsafe_ordered_store(BasicType type);
285 bool inline_unsafe_fence(vmIntrinsics::ID id);
286 bool inline_fp_conversions(vmIntrinsics::ID id);
287 bool inline_number_methods(vmIntrinsics::ID id);
288 bool inline_reference_get();
289 bool inline_aescrypt_Block(vmIntrinsics::ID id);
290 bool inline_cipherBlockChaining_AESCrypt(vmIntrinsics::ID id);
291 Node* inline_cipherBlockChaining_AESCrypt_predicate(bool decrypting);
292 Node* get_key_start_from_aescrypt_object(Node* aescrypt_object);
293 bool inline_encodeISOArray();
294 bool inline_updateCRC32();
295 bool inline_updateBytesCRC32();
296 bool inline_updateByteBufferCRC32();
297 };
298
299
300 //---------------------------make_vm_intrinsic----------------------------
301 CallGenerator* Compile::make_vm_intrinsic(ciMethod* m, bool is_virtual) {
302 vmIntrinsics::ID id = m->intrinsic_id();
303 assert(id != vmIntrinsics::_none, "must be a VM intrinsic");
304
305 if (DisableIntrinsic[0] != '\0'
306 && strstr(DisableIntrinsic, vmIntrinsics::name_at(id)) != NULL) {
307 // disabled by a user request on the command line:
308 // example: -XX:DisableIntrinsic=_hashCode,_getClass
309 return NULL;
310 }
311
312 if (!m->is_loaded()) {
313 // do not attempt to inline unloaded methods
314 return NULL;
315 }
316
474 if (!UseCompressedOops && !Matcher::match_rule_supported(Op_GetAndSetP)) return NULL;
475 if (UseCompressedOops && !Matcher::match_rule_supported(Op_GetAndSetN)) return NULL;
476 break;
477 #else
478 if (!Matcher::match_rule_supported(Op_GetAndSetP)) return NULL;
479 break;
480 #endif
481
482 case vmIntrinsics::_aescrypt_encryptBlock:
483 case vmIntrinsics::_aescrypt_decryptBlock:
484 if (!UseAESIntrinsics) return NULL;
485 break;
486
487 case vmIntrinsics::_cipherBlockChaining_encryptAESCrypt:
488 case vmIntrinsics::_cipherBlockChaining_decryptAESCrypt:
489 if (!UseAESIntrinsics) return NULL;
490 // these two require the predicated logic
491 is_predicted = true;
492 break;
493
494 case vmIntrinsics::_updateCRC32:
495 case vmIntrinsics::_updateBytesCRC32:
496 case vmIntrinsics::_updateByteBufferCRC32:
497 if (!UseCRC32Intrinsics) return NULL;
498 break;
499
500 default:
501 assert(id <= vmIntrinsics::LAST_COMPILER_INLINE, "caller responsibility");
502 assert(id != vmIntrinsics::_Object_init && id != vmIntrinsics::_invoke, "enum out of order?");
503 break;
504 }
505
506 // -XX:-InlineClassNatives disables natives from the Class class.
507 // The flag applies to all reflective calls, notably Array.newArray
508 // (visible to Java programmers as Array.newInstance).
509 if (m->holder()->name() == ciSymbol::java_lang_Class() ||
510 m->holder()->name() == ciSymbol::java_lang_reflect_Array()) {
511 if (!InlineClassNatives) return NULL;
512 }
513
514 // -XX:-InlineThreadNatives disables natives from the Thread class.
515 if (m->holder()->name() == ciSymbol::java_lang_Thread()) {
516 if (!InlineThreadNatives) return NULL;
517 }
518
519 // -XX:-InlineMathNatives disables natives from the Math,Float and Double classes.
799 case vmIntrinsics::_bitCount_l:
800 case vmIntrinsics::_reverseBytes_i:
801 case vmIntrinsics::_reverseBytes_l:
802 case vmIntrinsics::_reverseBytes_s:
803 case vmIntrinsics::_reverseBytes_c: return inline_number_methods(intrinsic_id());
804
805 case vmIntrinsics::_getCallerClass: return inline_native_Reflection_getCallerClass();
806
807 case vmIntrinsics::_Reference_get: return inline_reference_get();
808
809 case vmIntrinsics::_aescrypt_encryptBlock:
810 case vmIntrinsics::_aescrypt_decryptBlock: return inline_aescrypt_Block(intrinsic_id());
811
812 case vmIntrinsics::_cipherBlockChaining_encryptAESCrypt:
813 case vmIntrinsics::_cipherBlockChaining_decryptAESCrypt:
814 return inline_cipherBlockChaining_AESCrypt(intrinsic_id());
815
816 case vmIntrinsics::_encodeISOArray:
817 return inline_encodeISOArray();
818
819 case vmIntrinsics::_updateCRC32:
820 return inline_updateCRC32();
821 case vmIntrinsics::_updateBytesCRC32:
822 return inline_updateBytesCRC32();
823 case vmIntrinsics::_updateByteBufferCRC32:
824 return inline_updateByteBufferCRC32();
825
826 default:
827 // If you get here, it may be that someone has added a new intrinsic
828 // to the list in vmSymbols.hpp without implementing it here.
829 #ifndef PRODUCT
830 if ((PrintMiscellaneous && (Verbose || WizardMode)) || PrintOpto) {
831 tty->print_cr("*** Warning: Unimplemented intrinsic %s(%d)",
832 vmIntrinsics::name_at(intrinsic_id()), intrinsic_id());
833 }
834 #endif
835 return false;
836 }
837 }
838
839 Node* LibraryCallKit::try_to_predicate() {
840 if (!jvms()->has_method()) {
841 // Root JVMState has a null method.
842 assert(map()->memory()->Opcode() == Op_Parm, "");
843 // Insert the memory aliasing node
844 set_all_memory(reset_memory());
845 }
883 // The returned value represents the control for the slow path.
884 // The return value is never 'top'; it is either a valid control
885 // or NULL if it is obvious that the slow path can never be taken.
886 // Also, if region and the slow control are not NULL, the slow edge
887 // is appended to the region.
888 Node* LibraryCallKit::generate_guard(Node* test, RegionNode* region, float true_prob) {
889 if (stopped()) {
890 // Already short circuited.
891 return NULL;
892 }
893
894 // Build an if node and its projections.
895 // If test is true we take the slow path, which we assume is uncommon.
896 if (_gvn.type(test) == TypeInt::ZERO) {
897 // The slow branch is never taken. No need to build this guard.
898 return NULL;
899 }
900
901 IfNode* iff = create_and_map_if(control(), test, true_prob, COUNT_UNKNOWN);
902
903 Node* if_slow = _gvn.transform(new (C) IfTrueNode(iff));
904 if (if_slow == top()) {
905 // The slow branch is never taken. No need to build this guard.
906 return NULL;
907 }
908
909 if (region != NULL)
910 region->add_req(if_slow);
911
912 Node* if_fast = _gvn.transform(new (C) IfFalseNode(iff));
913 set_control(if_fast);
914
915 return if_slow;
916 }
917
918 inline Node* LibraryCallKit::generate_slow_guard(Node* test, RegionNode* region) {
919 return generate_guard(test, region, PROB_UNLIKELY_MAG(3));
920 }
921 inline Node* LibraryCallKit::generate_fair_guard(Node* test, RegionNode* region) {
922 return generate_guard(test, region, PROB_FAIR);
923 }
924
925 inline Node* LibraryCallKit::generate_negative_guard(Node* index, RegionNode* region,
926 Node* *pos_index) {
927 if (stopped())
928 return NULL; // already stopped
929 if (_gvn.type(index)->higher_equal(TypeInt::POS)) // [0,maxint]
930 return NULL; // index is already adequately typed
931 Node* cmp_lt = _gvn.transform(new (C) CmpINode(index, intcon(0)));
932 Node* bol_lt = _gvn.transform(new (C) BoolNode(cmp_lt, BoolTest::lt));
933 Node* is_neg = generate_guard(bol_lt, region, PROB_MIN);
934 if (is_neg != NULL && pos_index != NULL) {
935 // Emulate effect of Parse::adjust_map_after_if.
936 Node* ccast = new (C) CastIINode(index, TypeInt::POS);
937 ccast->set_req(0, control());
938 (*pos_index) = _gvn.transform(ccast);
939 }
940 return is_neg;
941 }
942
943 inline Node* LibraryCallKit::generate_nonpositive_guard(Node* index, bool never_negative,
944 Node* *pos_index) {
945 if (stopped())
946 return NULL; // already stopped
947 if (_gvn.type(index)->higher_equal(TypeInt::POS1)) // [1,maxint]
948 return NULL; // index is already adequately typed
949 Node* cmp_le = _gvn.transform(new (C) CmpINode(index, intcon(0)));
950 BoolTest::mask le_or_eq = (never_negative ? BoolTest::eq : BoolTest::le);
951 Node* bol_le = _gvn.transform(new (C) BoolNode(cmp_le, le_or_eq));
952 Node* is_notp = generate_guard(bol_le, NULL, PROB_MIN);
953 if (is_notp != NULL && pos_index != NULL) {
954 // Emulate effect of Parse::adjust_map_after_if.
955 Node* ccast = new (C) CastIINode(index, TypeInt::POS1);
956 ccast->set_req(0, control());
957 (*pos_index) = _gvn.transform(ccast);
958 }
959 return is_notp;
960 }
961
962 // Make sure that 'position' is a valid limit index, in [0..length].
963 // There are two equivalent plans for checking this:
964 // A. (offset + copyLength) unsigned<= arrayLength
965 // B. offset <= (arrayLength - copyLength)
966 // We require that all of the values above, except for the sum and
967 // difference, are already known to be non-negative.
968 // Plan A is robust in the face of overflow, if offset and copyLength
969 // are both hugely positive.
970 //
971 // Plan B is less direct and intuitive, but it does not overflow at
972 // all, since the difference of two non-negatives is always
973 // representable. Whenever Java methods must perform the equivalent
974 // check they generally use Plan B instead of Plan A.
975 // For the moment we use Plan A.
976 inline Node* LibraryCallKit::generate_limit_guard(Node* offset,
977 Node* subseq_length,
978 Node* array_length,
979 RegionNode* region) {
980 if (stopped())
981 return NULL; // already stopped
982 bool zero_offset = _gvn.type(offset) == TypeInt::ZERO;
983 if (zero_offset && subseq_length->eqv_uncast(array_length))
984 return NULL; // common case of whole-array copy
985 Node* last = subseq_length;
986 if (!zero_offset) // last += offset
987 last = _gvn.transform(new (C) AddINode(last, offset));
988 Node* cmp_lt = _gvn.transform(new (C) CmpUNode(array_length, last));
989 Node* bol_lt = _gvn.transform(new (C) BoolNode(cmp_lt, BoolTest::lt));
990 Node* is_over = generate_guard(bol_lt, region, PROB_MIN);
991 return is_over;
992 }
993
994
995 //--------------------------generate_current_thread--------------------
996 Node* LibraryCallKit::generate_current_thread(Node* &tls_output) {
997 ciKlass* thread_klass = env()->Thread_klass();
998 const Type* thread_type = TypeOopPtr::make_from_klass(thread_klass)->cast_to_ptr_type(TypePtr::NotNull);
999 Node* thread = _gvn.transform(new (C) ThreadLocalNode());
1000 Node* p = basic_plus_adr(top()/*!oop*/, thread, in_bytes(JavaThread::threadObj_offset()));
1001 Node* threadObj = make_load(NULL, p, thread_type, T_OBJECT);
1002 tls_output = thread;
1003 return threadObj;
1004 }
1005
1006
1007 //------------------------------make_string_method_node------------------------
1008 // Helper method for String intrinsic functions. This version is called
1009 // with str1 and str2 pointing to String object nodes.
1150
1151 Node* no_ctrl = NULL;
1152
1153 // Get start addr of receiver
1154 Node* receiver_val = load_String_value(no_ctrl, receiver);
1155 Node* receiver_offset = load_String_offset(no_ctrl, receiver);
1156 Node* receiver_start = array_element_address(receiver_val, receiver_offset, T_CHAR);
1157
1158 // Get length of receiver
1159 Node* receiver_cnt = load_String_length(no_ctrl, receiver);
1160
1161 // Get start addr of argument
1162 Node* argument_val = load_String_value(no_ctrl, argument);
1163 Node* argument_offset = load_String_offset(no_ctrl, argument);
1164 Node* argument_start = array_element_address(argument_val, argument_offset, T_CHAR);
1165
1166 // Get length of argument
1167 Node* argument_cnt = load_String_length(no_ctrl, argument);
1168
1169 // Check for receiver count != argument count
1170 Node* cmp = _gvn.transform(new(C) CmpINode(receiver_cnt, argument_cnt));
1171 Node* bol = _gvn.transform(new(C) BoolNode(cmp, BoolTest::ne));
1172 Node* if_ne = generate_slow_guard(bol, NULL);
1173 if (if_ne != NULL) {
1174 phi->init_req(4, intcon(0));
1175 region->init_req(4, if_ne);
1176 }
1177
1178 // Check for count == 0 is done by assembler code for StrEquals.
1179
1180 if (!stopped()) {
1181 Node* equals = make_string_method_node(Op_StrEquals, receiver_start, receiver_cnt, argument_start, argument_cnt);
1182 phi->init_req(1, equals);
1183 region->init_req(1, control());
1184 }
1185 }
1186
1187 // post merge
1188 set_control(_gvn.transform(region));
1189 record_for_igvn(region);
1190
1191 set_result(_gvn.transform(phi));
1257 // } // using "i += targetCount;" and an "else i++;" causes a jump to jump.
1258 // i++;
1259 // }
1260 // return -1;
1261 // }
1262
1263 //------------------------------string_indexOf------------------------
1264 Node* LibraryCallKit::string_indexOf(Node* string_object, ciTypeArray* target_array, jint targetOffset_i,
1265 jint cache_i, jint md2_i) {
1266
1267 Node* no_ctrl = NULL;
1268 float likely = PROB_LIKELY(0.9);
1269 float unlikely = PROB_UNLIKELY(0.9);
1270
1271 const int nargs = 0; // no arguments to push back for uncommon trap in predicate
1272
1273 Node* source = load_String_value(no_ctrl, string_object);
1274 Node* sourceOffset = load_String_offset(no_ctrl, string_object);
1275 Node* sourceCount = load_String_length(no_ctrl, string_object);
1276
1277 Node* target = _gvn.transform( makecon(TypeOopPtr::make_from_constant(target_array, true)));
1278 jint target_length = target_array->length();
1279 const TypeAry* target_array_type = TypeAry::make(TypeInt::CHAR, TypeInt::make(0, target_length, Type::WidenMin));
1280 const TypeAryPtr* target_type = TypeAryPtr::make(TypePtr::BotPTR, target_array_type, target_array->klass(), true, Type::OffsetBot);
1281
1282 IdealKit kit(this, false, true);
1283 #define __ kit.
1284 Node* zero = __ ConI(0);
1285 Node* one = __ ConI(1);
1286 Node* cache = __ ConI(cache_i);
1287 Node* md2 = __ ConI(md2_i);
1288 Node* lastChar = __ ConI(target_array->char_at(target_length - 1));
1289 Node* targetCount = __ ConI(target_length);
1290 Node* targetCountLess1 = __ ConI(target_length - 1);
1291 Node* targetOffset = __ ConI(targetOffset_i);
1292 Node* sourceEnd = __ SubI(__ AddI(sourceOffset, sourceCount), targetCountLess1);
1293
1294 IdealVariable rtn(kit), i(kit), j(kit); __ declarations_done();
1295 Node* outer_loop = __ make_label(2 /* goto */);
1296 Node* return_ = __ make_label(1);
1297
1364 Node* result_phi = new (C) PhiNode(result_rgn, TypeInt::INT);
1365 Node* no_ctrl = NULL;
1366
1367 // Get start addr of source string
1368 Node* source = load_String_value(no_ctrl, receiver);
1369 Node* source_offset = load_String_offset(no_ctrl, receiver);
1370 Node* source_start = array_element_address(source, source_offset, T_CHAR);
1371
1372 // Get length of source string
1373 Node* source_cnt = load_String_length(no_ctrl, receiver);
1374
1375 // Get start addr of substring
1376 Node* substr = load_String_value(no_ctrl, arg);
1377 Node* substr_offset = load_String_offset(no_ctrl, arg);
1378 Node* substr_start = array_element_address(substr, substr_offset, T_CHAR);
1379
1380 // Get length of source string
1381 Node* substr_cnt = load_String_length(no_ctrl, arg);
1382
1383 // Check for substr count > string count
1384 Node* cmp = _gvn.transform(new(C) CmpINode(substr_cnt, source_cnt));
1385 Node* bol = _gvn.transform(new(C) BoolNode(cmp, BoolTest::gt));
1386 Node* if_gt = generate_slow_guard(bol, NULL);
1387 if (if_gt != NULL) {
1388 result_phi->init_req(2, intcon(-1));
1389 result_rgn->init_req(2, if_gt);
1390 }
1391
1392 if (!stopped()) {
1393 // Check for substr count == 0
1394 cmp = _gvn.transform(new(C) CmpINode(substr_cnt, intcon(0)));
1395 bol = _gvn.transform(new(C) BoolNode(cmp, BoolTest::eq));
1396 Node* if_zero = generate_slow_guard(bol, NULL);
1397 if (if_zero != NULL) {
1398 result_phi->init_req(3, intcon(0));
1399 result_rgn->init_req(3, if_zero);
1400 }
1401 }
1402
1403 if (!stopped()) {
1404 result = make_string_method_node(Op_StrIndexOf, source_start, source_cnt, substr_start, substr_cnt);
1405 result_phi->init_req(1, result);
1406 result_rgn->init_req(1, control());
1407 }
1408 set_control(_gvn.transform(result_rgn));
1409 record_for_igvn(result_rgn);
1410 result = _gvn.transform(result_phi);
1411
1412 } else { // Use LibraryCallKit::string_indexOf
1413 // don't intrinsify if argument isn't a constant string.
1414 if (!arg->is_Con()) {
1415 return false;
1551 // if (x <= Math.PI / 2.0) return -fsin(x - Math.PI / 2.0);
1552 // }
1553 // return StrictMath.cos(x);
1554
1555 // Actually, sticking in an 80-bit Intel value into C2 will be tough; it
1556 // requires a special machine instruction to load it. Instead we'll try
1557 // the 'easy' case. If we really need the extra range +/- PI/2 we'll
1558 // probably do the math inside the SIN encoding.
1559
1560 // Make the merge point
1561 RegionNode* r = new (C) RegionNode(3);
1562 Node* phi = new (C) PhiNode(r, Type::DOUBLE);
1563
1564 // Flatten arg so we need only 1 test
1565 Node *abs = _gvn.transform(new (C) AbsDNode(arg));
1566 // Node for PI/4 constant
1567 Node *pi4 = makecon(TypeD::make(pi_4));
1568 // Check PI/4 : abs(arg)
1569 Node *cmp = _gvn.transform(new (C) CmpDNode(pi4,abs));
1570 // Check: If PI/4 < abs(arg) then go slow
1571 Node *bol = _gvn.transform(new (C) BoolNode( cmp, BoolTest::lt ));
1572 // Branch either way
1573 IfNode *iff = create_and_xform_if(control(),bol, PROB_STATIC_FREQUENT, COUNT_UNKNOWN);
1574 set_control(opt_iff(r,iff));
1575
1576 // Set fast path result
1577 phi->init_req(2, n);
1578
1579 // Slow path - non-blocking leaf call
1580 Node* call = NULL;
1581 switch (id) {
1582 case vmIntrinsics::_dsin:
1583 call = make_runtime_call(RC_LEAF, OptoRuntime::Math_D_D_Type(),
1584 CAST_FROM_FN_PTR(address, SharedRuntime::dsin),
1585 "Sin", NULL, arg, top());
1586 break;
1587 case vmIntrinsics::_dcos:
1588 call = make_runtime_call(RC_LEAF, OptoRuntime::Math_D_D_Type(),
1589 CAST_FROM_FN_PTR(address, SharedRuntime::dcos),
1590 "Cos", NULL, arg, top());
1591 break;
1616 //result=(result.isNaN())? funcAddr():result;
1617 // Check: If isNaN() by checking result!=result? then either trap
1618 // or go to runtime
1619 Node* cmpisnan = _gvn.transform(new (C) CmpDNode(result, result));
1620 // Build the boolean node
1621 Node* bolisnum = _gvn.transform(new (C) BoolNode(cmpisnan, BoolTest::eq));
1622
1623 if (!too_many_traps(Deoptimization::Reason_intrinsic)) {
1624 { BuildCutout unless(this, bolisnum, PROB_STATIC_FREQUENT);
1625 // The pow or exp intrinsic returned a NaN, which requires a call
1626 // to the runtime. Recompile with the runtime call.
1627 uncommon_trap(Deoptimization::Reason_intrinsic,
1628 Deoptimization::Action_make_not_entrant);
1629 }
1630 set_result(result);
1631 } else {
1632 // If this inlining ever returned NaN in the past, we compile a call
1633 // to the runtime to properly handle corner cases
1634
1635 IfNode* iff = create_and_xform_if(control(), bolisnum, PROB_STATIC_FREQUENT, COUNT_UNKNOWN);
1636 Node* if_slow = _gvn.transform(new (C) IfFalseNode(iff));
1637 Node* if_fast = _gvn.transform(new (C) IfTrueNode(iff));
1638
1639 if (!if_slow->is_top()) {
1640 RegionNode* result_region = new (C) RegionNode(3);
1641 PhiNode* result_val = new (C) PhiNode(result_region, Type::DOUBLE);
1642
1643 result_region->init_req(1, if_fast);
1644 result_val->init_req(1, result);
1645
1646 set_control(if_slow);
1647
1648 const TypePtr* no_memory_effects = NULL;
1649 Node* rt = make_runtime_call(RC_LEAF, call_type, funcAddr, funcName,
1650 no_memory_effects,
1651 x, top(), y, y ? top() : NULL);
1652 Node* value = _gvn.transform(new (C) ProjNode(rt, TypeFunc::Parms+0));
1653 #ifdef ASSERT
1654 Node* value_top = _gvn.transform(new (C) ProjNode(rt, TypeFunc::Parms+1));
1655 assert(value_top == top(), "second value must be top");
1656 #endif
1657
1703 Node* result = NULL;
1704
1705 if (!too_many_traps(Deoptimization::Reason_intrinsic)) {
1706 // Short form: skip the fancy tests and just check for NaN result.
1707 result = _gvn.transform(new (C) PowDNode(C, control(), x, y));
1708 } else {
1709 // If this inlining ever returned NaN in the past, include all
1710 // checks + call to the runtime.
1711
1712 // Set the merge point for If node with condition of (x <= 0.0)
1713 // There are four possible paths to region node and phi node
1714 RegionNode *r = new (C) RegionNode(4);
1715 Node *phi = new (C) PhiNode(r, Type::DOUBLE);
1716
1717 // Build the first if node: if (x <= 0.0)
1718 // Node for 0 constant
1719 Node *zeronode = makecon(TypeD::ZERO);
1720 // Check x:0
1721 Node *cmp = _gvn.transform(new (C) CmpDNode(x, zeronode));
1722 // Check: If (x<=0) then go complex path
1723 Node *bol1 = _gvn.transform(new (C) BoolNode( cmp, BoolTest::le ));
1724 // Branch either way
1725 IfNode *if1 = create_and_xform_if(control(),bol1, PROB_STATIC_INFREQUENT, COUNT_UNKNOWN);
1726 // Fast path taken; set region slot 3
1727 Node *fast_taken = _gvn.transform(new (C) IfFalseNode(if1));
1728 r->init_req(3,fast_taken); // Capture fast-control
1729
1730 // Fast path not-taken, i.e. slow path
1731 Node *complex_path = _gvn.transform(new (C) IfTrueNode(if1));
1732
1733 // Set fast path result
1734 Node *fast_result = _gvn.transform(new (C) PowDNode(C, control(), x, y));
1735 phi->init_req(3, fast_result);
1736
1737 // Complex path
1738 // Build the second if node (if y is long)
1739 // Node for (long)y
1740 Node *longy = _gvn.transform(new (C) ConvD2LNode(y));
1741 // Node for (double)((long) y)
1742 Node *doublelongy= _gvn.transform(new (C) ConvL2DNode(longy));
1743 // Check (double)((long) y) : y
1744 Node *cmplongy= _gvn.transform(new (C) CmpDNode(doublelongy, y));
1745 // Check if (y isn't long) then go to slow path
1746
1747 Node *bol2 = _gvn.transform(new (C) BoolNode( cmplongy, BoolTest::ne ));
1748 // Branch either way
1749 IfNode *if2 = create_and_xform_if(complex_path,bol2, PROB_STATIC_INFREQUENT, COUNT_UNKNOWN);
1750 Node* ylong_path = _gvn.transform(new (C) IfFalseNode(if2));
1751
1752 Node *slow_path = _gvn.transform(new (C) IfTrueNode(if2));
1753
1754 // Calculate DPow(abs(x), y)*(1 & (long)y)
1755 // Node for constant 1
1756 Node *conone = longcon(1);
1757 // 1& (long)y
1758 Node *signnode= _gvn.transform(new (C) AndLNode(conone, longy));
1759
1760 // A huge number is always even. Detect a huge number by checking
1761 // if y + 1 == y and set integer to be tested for parity to 0.
1762 // Required for corner case:
1763 // (long)9.223372036854776E18 = max_jlong
1764 // (double)(long)9.223372036854776E18 = 9.223372036854776E18
1765 // max_jlong is odd but 9.223372036854776E18 is even
1766 Node* yplus1 = _gvn.transform(new (C) AddDNode(y, makecon(TypeD::make(1))));
1767 Node *cmpyplus1= _gvn.transform(new (C) CmpDNode(yplus1, y));
1768 Node *bolyplus1 = _gvn.transform(new (C) BoolNode( cmpyplus1, BoolTest::eq ));
1769 Node* correctedsign = NULL;
1770 if (ConditionalMoveLimit != 0) {
1771 correctedsign = _gvn.transform( CMoveNode::make(C, NULL, bolyplus1, signnode, longcon(0), TypeLong::LONG));
1772 } else {
1773 IfNode *ifyplus1 = create_and_xform_if(ylong_path,bolyplus1, PROB_FAIR, COUNT_UNKNOWN);
1774 RegionNode *r = new (C) RegionNode(3);
1775 Node *phi = new (C) PhiNode(r, TypeLong::LONG);
1776 r->init_req(1, _gvn.transform(new (C) IfFalseNode(ifyplus1)));
1777 r->init_req(2, _gvn.transform(new (C) IfTrueNode(ifyplus1)));
1778 phi->init_req(1, signnode);
1779 phi->init_req(2, longcon(0));
1780 correctedsign = _gvn.transform(phi);
1781 ylong_path = _gvn.transform(r);
1782 record_for_igvn(r);
1783 }
1784
1785 // zero node
1786 Node *conzero = longcon(0);
1787 // Check (1&(long)y)==0?
1788 Node *cmpeq1 = _gvn.transform(new (C) CmpLNode(correctedsign, conzero));
1789 // Check if (1&(long)y)!=0?, if so the result is negative
1790 Node *bol3 = _gvn.transform(new (C) BoolNode( cmpeq1, BoolTest::ne ));
1791 // abs(x)
1792 Node *absx=_gvn.transform(new (C) AbsDNode(x));
1793 // abs(x)^y
1794 Node *absxpowy = _gvn.transform(new (C) PowDNode(C, control(), absx, y));
1795 // -abs(x)^y
1796 Node *negabsxpowy = _gvn.transform(new (C) NegDNode (absxpowy));
1797 // (1&(long)y)==1?-DPow(abs(x), y):DPow(abs(x), y)
1798 Node *signresult = NULL;
1799 if (ConditionalMoveLimit != 0) {
1800 signresult = _gvn.transform( CMoveNode::make(C, NULL, bol3, absxpowy, negabsxpowy, Type::DOUBLE));
1801 } else {
1802 IfNode *ifyeven = create_and_xform_if(ylong_path,bol3, PROB_FAIR, COUNT_UNKNOWN);
1803 RegionNode *r = new (C) RegionNode(3);
1804 Node *phi = new (C) PhiNode(r, Type::DOUBLE);
1805 r->init_req(1, _gvn.transform(new (C) IfFalseNode(ifyeven)));
1806 r->init_req(2, _gvn.transform(new (C) IfTrueNode(ifyeven)));
1807 phi->init_req(1, absxpowy);
1808 phi->init_req(2, negabsxpowy);
1809 signresult = _gvn.transform(phi);
1810 ylong_path = _gvn.transform(r);
1811 record_for_igvn(r);
1812 }
1813 // Set complex path fast result
1814 r->init_req(2, ylong_path);
1815 phi->init_req(2, signresult);
1816
1817 static const jlong nan_bits = CONST64(0x7ff8000000000000);
1818 Node *slow_result = makecon(TypeD::make(*(double*)&nan_bits)); // return NaN
1819 r->init_req(1,slow_path);
1820 phi->init_req(1,slow_result);
1821
1822 // Post merge
1823 set_control(_gvn.transform(r));
1824 record_for_igvn(r);
1825 result = _gvn.transform(phi);
1826 }
1919
1920 const TypeInt* txvalue = _gvn.type(xvalue)->isa_int();
1921 const TypeInt* tyvalue = _gvn.type(yvalue)->isa_int();
1922 if (txvalue == NULL || tyvalue == NULL) return top();
1923 // This is not really necessary, but it is consistent with a
1924 // hypothetical MaxINode::Value method:
1925 int widen = MAX2(txvalue->_widen, tyvalue->_widen);
1926
1927 // %%% This folding logic should (ideally) be in a different place.
1928 // Some should be inside IfNode, and there to be a more reliable
1929 // transformation of ?: style patterns into cmoves. We also want
1930 // more powerful optimizations around cmove and min/max.
1931
1932 // Try to find a dominating comparison of these guys.
1933 // It can simplify the index computation for Arrays.copyOf
1934 // and similar uses of System.arraycopy.
1935 // First, compute the normalized version of CmpI(x, y).
1936 int cmp_op = Op_CmpI;
1937 Node* xkey = xvalue;
1938 Node* ykey = yvalue;
1939 Node* ideal_cmpxy = _gvn.transform(new(C) CmpINode(xkey, ykey));
1940 if (ideal_cmpxy->is_Cmp()) {
1941 // E.g., if we have CmpI(length - offset, count),
1942 // it might idealize to CmpI(length, count + offset)
1943 cmp_op = ideal_cmpxy->Opcode();
1944 xkey = ideal_cmpxy->in(1);
1945 ykey = ideal_cmpxy->in(2);
1946 }
1947
1948 // Start by locating any relevant comparisons.
1949 Node* start_from = (xkey->outcnt() < ykey->outcnt()) ? xkey : ykey;
1950 Node* cmpxy = NULL;
1951 Node* cmpyx = NULL;
1952 for (DUIterator_Fast kmax, k = start_from->fast_outs(kmax); k < kmax; k++) {
1953 Node* cmp = start_from->fast_out(k);
1954 if (cmp->outcnt() > 0 && // must have prior uses
1955 cmp->in(0) == NULL && // must be context-independent
1956 cmp->Opcode() == cmp_op) { // right kind of compare
1957 if (cmp->in(1) == xkey && cmp->in(2) == ykey) cmpxy = cmp;
1958 if (cmp->in(1) == ykey && cmp->in(2) == xkey) cmpyx = cmp;
1959 }
2012 if (cmp == NULL) continue;
2013 for (DUIterator_Fast jmax, j = cmp->fast_outs(jmax); j < jmax; j++) {
2014 Node* bol = cmp->fast_out(j);
2015 if (!bol->is_Bool()) continue;
2016 BoolTest::mask btest = bol->as_Bool()->_test._test;
2017 if (btest == BoolTest::eq || btest == BoolTest::ne) continue;
2018 if (cmp->in(1) == ykey) btest = BoolTest(btest).commute();
2019 if (bol->outcnt() > (best_bol == NULL ? 0 : best_bol->outcnt())) {
2020 best_bol = bol->as_Bool();
2021 best_btest = btest;
2022 }
2023 }
2024 }
2025
2026 Node* answer_if_true = NULL;
2027 Node* answer_if_false = NULL;
2028 switch (best_btest) {
2029 default:
2030 if (cmpxy == NULL)
2031 cmpxy = ideal_cmpxy;
2032 best_bol = _gvn.transform(new(C) BoolNode(cmpxy, BoolTest::lt));
2033 // and fall through:
2034 case BoolTest::lt: // x < y
2035 case BoolTest::le: // x <= y
2036 answer_if_true = (want_max ? yvalue : xvalue);
2037 answer_if_false = (want_max ? xvalue : yvalue);
2038 break;
2039 case BoolTest::gt: // x > y
2040 case BoolTest::ge: // x >= y
2041 answer_if_true = (want_max ? xvalue : yvalue);
2042 answer_if_false = (want_max ? yvalue : xvalue);
2043 break;
2044 }
2045
2046 jint hi, lo;
2047 if (want_max) {
2048 // We can sharpen the minimum.
2049 hi = MAX2(txvalue->_hi, tyvalue->_hi);
2050 lo = MAX2(txvalue->_lo, tyvalue->_lo);
2051 } else {
2052 // We can sharpen the maximum.
2072 switch (id) {
2073 case vmIntrinsics::_min:
2074 result_val = _gvn.transform(new (C, 3) MinINode(x,y)); break;
2075 case vmIntrinsics::_max:
2076 result_val = _gvn.transform(new (C, 3) MaxINode(x,y)); break;
2077 default:
2078 ShouldNotReachHere();
2079 }
2080 */
2081 }
2082
2083 inline int
2084 LibraryCallKit::classify_unsafe_addr(Node* &base, Node* &offset) {
2085 const TypePtr* base_type = TypePtr::NULL_PTR;
2086 if (base != NULL) base_type = _gvn.type(base)->isa_ptr();
2087 if (base_type == NULL) {
2088 // Unknown type.
2089 return Type::AnyPtr;
2090 } else if (base_type == TypePtr::NULL_PTR) {
2091 // Since this is a NULL+long form, we have to switch to a rawptr.
2092 base = _gvn.transform(new (C) CastX2PNode(offset));
2093 offset = MakeConX(0);
2094 return Type::RawPtr;
2095 } else if (base_type->base() == Type::RawPtr) {
2096 return Type::RawPtr;
2097 } else if (base_type->isa_oopptr()) {
2098 // Base is never null => always a heap address.
2099 if (base_type->ptr() == TypePtr::NotNull) {
2100 return Type::OopPtr;
2101 }
2102 // Offset is small => always a heap address.
2103 const TypeX* offset_type = _gvn.type(offset)->isa_intptr_t();
2104 if (offset_type != NULL &&
2105 base_type->offset() == 0 && // (should always be?)
2106 offset_type->_lo >= 0 &&
2107 !MacroAssembler::needs_explicit_null_check(offset_type->_hi)) {
2108 return Type::OopPtr;
2109 }
2110 // Otherwise, it might either be oop+off or NULL+addr.
2111 return Type::AnyPtr;
2112 } else {
2466 p = ConvX2L(p);
2467 break;
2468 default:
2469 fatal(err_msg_res("unexpected type %d: %s", type, type2name(type)));
2470 break;
2471 }
2472 // The load node has the control of the preceding MemBarCPUOrder. All
2473 // following nodes will have the control of the MemBarCPUOrder inserted at
2474 // the end of this method. So, pushing the load onto the stack at a later
2475 // point is fine.
2476 set_result(p);
2477 } else {
2478 // place effect of store into memory
2479 switch (type) {
2480 case T_DOUBLE:
2481 val = dstore_rounding(val);
2482 break;
2483 case T_ADDRESS:
2484 // Repackage the long as a pointer.
2485 val = ConvL2X(val);
2486 val = _gvn.transform(new (C) CastX2PNode(val));
2487 break;
2488 }
2489
2490 if (type != T_OBJECT ) {
2491 (void) store_to_memory(control(), adr, val, type, adr_type, is_volatile);
2492 } else {
2493 // Possibly an oop being stored to Java heap or native memory
2494 if (!TypePtr::NULL_PTR->higher_equal(_gvn.type(heap_base_oop))) {
2495 // oop to Java heap.
2496 (void) store_oop_to_unknown(control(), heap_base_oop, adr, adr_type, val, type);
2497 } else {
2498 // We can't tell at compile time if we are storing in the Java heap or outside
2499 // of it. So we need to emit code to conditionally do the proper type of
2500 // store.
2501
2502 IdealKit ideal(this);
2503 #define __ ideal.
2504 // QQQ who knows what probability is here??
2505 __ if_then(heap_base_oop, BoolTest::ne, null(), PROB_UNLIKELY(0.999)); {
2506 // Sync IdealKit and graphKit.
2774 } else
2775 #endif
2776 {
2777 if (kind == LS_xchg) {
2778 load_store = _gvn.transform(new (C) GetAndSetPNode(control(), mem, adr, newval, adr_type, value_type->is_oopptr()));
2779 } else {
2780 assert(kind == LS_cmpxchg, "wrong LoadStore operation");
2781 load_store = _gvn.transform(new (C) CompareAndSwapPNode(control(), mem, adr, newval, oldval));
2782 }
2783 }
2784 post_barrier(control(), load_store, base, adr, alias_idx, newval, T_OBJECT, true);
2785 break;
2786 default:
2787 fatal(err_msg_res("unexpected type %d: %s", type, type2name(type)));
2788 break;
2789 }
2790
2791 // SCMemProjNodes represent the memory state of a LoadStore. Their
2792 // main role is to prevent LoadStore nodes from being optimized away
2793 // when their results aren't used.
2794 Node* proj = _gvn.transform(new (C) SCMemProjNode(load_store));
2795 set_memory(proj, alias_idx);
2796
2797 // Add the trailing membar surrounding the access
2798 insert_mem_bar(Op_MemBarCPUOrder);
2799 insert_mem_bar(Op_MemBarAcquire);
2800
2801 #ifdef _LP64
2802 if (type == T_OBJECT && adr->bottom_type()->is_ptr_to_narrowoop() && kind == LS_xchg) {
2803 load_store = _gvn.transform(new (C) DecodeNNode(load_store, load_store->get_ptr_type()));
2804 }
2805 #endif
2806
2807 assert(type2size[load_store->bottom_type()->basic_type()] == type2size[rtype], "result type should match");
2808 set_result(load_store);
2809 return true;
2810 }
2811
2812 //----------------------------inline_unsafe_ordered_store----------------------
2813 // public native void sun.misc.Unsafe.putOrderedObject(Object o, long offset, Object x);
2814 // public native void sun.misc.Unsafe.putOrderedInt(Object o, long offset, int x);
3009 no_int_result_path = 1, // t == Thread.current() && !TLS._osthread._interrupted
3010 no_clear_result_path = 2, // t == Thread.current() && TLS._osthread._interrupted && !clear_int
3011 slow_result_path = 3, // slow path: t.isInterrupted(clear_int)
3012 PATH_LIMIT
3013 };
3014
3015 // Ensure that it's not possible to move the load of TLS._osthread._interrupted flag
3016 // out of the function.
3017 insert_mem_bar(Op_MemBarCPUOrder);
3018
3019 RegionNode* result_rgn = new (C) RegionNode(PATH_LIMIT);
3020 PhiNode* result_val = new (C) PhiNode(result_rgn, TypeInt::BOOL);
3021
3022 RegionNode* slow_region = new (C) RegionNode(1);
3023 record_for_igvn(slow_region);
3024
3025 // (a) Receiving thread must be the current thread.
3026 Node* rec_thr = argument(0);
3027 Node* tls_ptr = NULL;
3028 Node* cur_thr = generate_current_thread(tls_ptr);
3029 Node* cmp_thr = _gvn.transform(new (C) CmpPNode(cur_thr, rec_thr));
3030 Node* bol_thr = _gvn.transform(new (C) BoolNode(cmp_thr, BoolTest::ne));
3031
3032 generate_slow_guard(bol_thr, slow_region);
3033
3034 // (b) Interrupt bit on TLS must be false.
3035 Node* p = basic_plus_adr(top()/*!oop*/, tls_ptr, in_bytes(JavaThread::osthread_offset()));
3036 Node* osthread = make_load(NULL, p, TypeRawPtr::NOTNULL, T_ADDRESS);
3037 p = basic_plus_adr(top()/*!oop*/, osthread, in_bytes(OSThread::interrupted_offset()));
3038
3039 // Set the control input on the field _interrupted read to prevent it floating up.
3040 Node* int_bit = make_load(control(), p, TypeInt::BOOL, T_INT);
3041 Node* cmp_bit = _gvn.transform(new (C) CmpINode(int_bit, intcon(0)));
3042 Node* bol_bit = _gvn.transform(new (C) BoolNode(cmp_bit, BoolTest::ne));
3043
3044 IfNode* iff_bit = create_and_map_if(control(), bol_bit, PROB_UNLIKELY_MAG(3), COUNT_UNKNOWN);
3045
3046 // First fast path: if (!TLS._interrupted) return false;
3047 Node* false_bit = _gvn.transform(new (C) IfFalseNode(iff_bit));
3048 result_rgn->init_req(no_int_result_path, false_bit);
3049 result_val->init_req(no_int_result_path, intcon(0));
3050
3051 // drop through to next case
3052 set_control( _gvn.transform(new (C) IfTrueNode(iff_bit)));
3053
3054 // (c) Or, if interrupt bit is set and clear_int is false, use 2nd fast path.
3055 Node* clr_arg = argument(1);
3056 Node* cmp_arg = _gvn.transform(new (C) CmpINode(clr_arg, intcon(0)));
3057 Node* bol_arg = _gvn.transform(new (C) BoolNode(cmp_arg, BoolTest::ne));
3058 IfNode* iff_arg = create_and_map_if(control(), bol_arg, PROB_FAIR, COUNT_UNKNOWN);
3059
3060 // Second fast path: ... else if (!clear_int) return true;
3061 Node* false_arg = _gvn.transform(new (C) IfFalseNode(iff_arg));
3062 result_rgn->init_req(no_clear_result_path, false_arg);
3063 result_val->init_req(no_clear_result_path, intcon(1));
3064
3065 // drop through to next case
3066 set_control( _gvn.transform(new (C) IfTrueNode(iff_arg)));
3067
3068 // (d) Otherwise, go to the slow path.
3069 slow_region->add_req(control());
3070 set_control( _gvn.transform(slow_region));
3071
3072 if (stopped()) {
3073 // There is no slow path.
3074 result_rgn->init_req(slow_result_path, top());
3075 result_val->init_req(slow_result_path, top());
3076 } else {
3077 // non-virtual because it is a private non-static
3078 CallJavaNode* slow_call = generate_method_call(vmIntrinsics::_isInterrupted);
3079
3080 Node* slow_val = set_results_for_java_call(slow_call);
3081 // this->control() comes from set_results_for_java_call
3082
3083 Node* fast_io = slow_call->in(TypeFunc::I_O);
3084 Node* fast_mem = slow_call->in(TypeFunc::Memory);
3085
3086 // These two phis are pre-filled with copies of of the fast IO and Memory
3087 PhiNode* result_mem = PhiNode::make(result_rgn, fast_mem, Type::MEMORY, TypePtr::BOTTOM);
3088 PhiNode* result_io = PhiNode::make(result_rgn, fast_io, Type::ABIO);
3089
3090 result_rgn->init_req(slow_result_path, control());
3106 Node* LibraryCallKit::load_mirror_from_klass(Node* klass) {
3107 Node* p = basic_plus_adr(klass, in_bytes(Klass::java_mirror_offset()));
3108 return make_load(NULL, p, TypeInstPtr::MIRROR, T_OBJECT);
3109 }
3110
3111 //-----------------------load_klass_from_mirror_common-------------------------
3112 // Given a java mirror (a java.lang.Class oop), load its corresponding klass oop.
3113 // Test the klass oop for null (signifying a primitive Class like Integer.TYPE),
3114 // and branch to the given path on the region.
3115 // If never_see_null, take an uncommon trap on null, so we can optimistically
3116 // compile for the non-null case.
3117 // If the region is NULL, force never_see_null = true.
3118 Node* LibraryCallKit::load_klass_from_mirror_common(Node* mirror,
3119 bool never_see_null,
3120 RegionNode* region,
3121 int null_path,
3122 int offset) {
3123 if (region == NULL) never_see_null = true;
3124 Node* p = basic_plus_adr(mirror, offset);
3125 const TypeKlassPtr* kls_type = TypeKlassPtr::OBJECT_OR_NULL;
3126 Node* kls = _gvn.transform( LoadKlassNode::make(_gvn, immutable_memory(), p, TypeRawPtr::BOTTOM, kls_type));
3127 Node* null_ctl = top();
3128 kls = null_check_oop(kls, &null_ctl, never_see_null);
3129 if (region != NULL) {
3130 // Set region->in(null_path) if the mirror is a primitive (e.g, int.class).
3131 region->init_req(null_path, null_ctl);
3132 } else {
3133 assert(null_ctl == top(), "no loose ends");
3134 }
3135 return kls;
3136 }
3137
3138 //--------------------(inline_native_Class_query helpers)---------------------
3139 // Use this for JVM_ACC_INTERFACE, JVM_ACC_IS_CLONEABLE, JVM_ACC_HAS_FINALIZER.
3140 // Fall through if (mods & mask) == bits, take the guard otherwise.
3141 Node* LibraryCallKit::generate_access_flags_guard(Node* kls, int modifier_mask, int modifier_bits, RegionNode* region) {
3142 // Branch around if the given klass has the given modifier bit set.
3143 // Like generate_guard, adds a new path onto the region.
3144 Node* modp = basic_plus_adr(kls, in_bytes(Klass::access_flags_offset()));
3145 Node* mods = make_load(NULL, modp, TypeInt::INT, T_INT);
3146 Node* mask = intcon(modifier_mask);
3147 Node* bits = intcon(modifier_bits);
3148 Node* mbit = _gvn.transform(new (C) AndINode(mods, mask));
3149 Node* cmp = _gvn.transform(new (C) CmpINode(mbit, bits));
3150 Node* bol = _gvn.transform(new (C) BoolNode(cmp, BoolTest::ne));
3151 return generate_fair_guard(bol, region);
3152 }
3153 Node* LibraryCallKit::generate_interface_guard(Node* kls, RegionNode* region) {
3154 return generate_access_flags_guard(kls, JVM_ACC_INTERFACE, 0, region);
3155 }
3156
3157 //-------------------------inline_native_Class_query-------------------
3158 bool LibraryCallKit::inline_native_Class_query(vmIntrinsics::ID id) {
3159 const Type* return_type = TypeInt::BOOL;
3160 Node* prim_return_value = top(); // what happens if it's a primitive class?
3161 bool never_see_null = !too_many_traps(Deoptimization::Reason_null_check);
3162 bool expect_prim = false; // most of these guys expect to work on refs
3163
3164 enum { _normal_path = 1, _prim_path = 2, PATH_LIMIT };
3165
3166 Node* mirror = argument(0);
3167 Node* obj = top();
3168
3169 switch (id) {
3170 case vmIntrinsics::_isInstance:
3281
3282 case vmIntrinsics::_isPrimitive:
3283 query_value = intcon(0); // "normal" path produces false
3284 break;
3285
3286 case vmIntrinsics::_getSuperclass:
3287 // The rules here are somewhat unfortunate, but we can still do better
3288 // with random logic than with a JNI call.
3289 // Interfaces store null or Object as _super, but must report null.
3290 // Arrays store an intermediate super as _super, but must report Object.
3291 // Other types can report the actual _super.
3292 // (To verify this code sequence, check the asserts in JVM_IsInterface.)
3293 if (generate_interface_guard(kls, region) != NULL)
3294 // A guard was added. If the guard is taken, it was an interface.
3295 phi->add_req(null());
3296 if (generate_array_guard(kls, region) != NULL)
3297 // A guard was added. If the guard is taken, it was an array.
3298 phi->add_req(makecon(TypeInstPtr::make(env()->Object_klass()->java_mirror())));
3299 // If we fall through, it's a plain class. Get its _super.
3300 p = basic_plus_adr(kls, in_bytes(Klass::super_offset()));
3301 kls = _gvn.transform( LoadKlassNode::make(_gvn, immutable_memory(), p, TypeRawPtr::BOTTOM, TypeKlassPtr::OBJECT_OR_NULL));
3302 null_ctl = top();
3303 kls = null_check_oop(kls, &null_ctl);
3304 if (null_ctl != top()) {
3305 // If the guard is taken, Object.superClass is null (both klass and mirror).
3306 region->add_req(null_ctl);
3307 phi ->add_req(null());
3308 }
3309 if (!stopped()) {
3310 query_value = load_mirror_from_klass(kls);
3311 }
3312 break;
3313
3314 case vmIntrinsics::_getComponentType:
3315 if (generate_array_guard(kls, region) != NULL) {
3316 // Be sure to pin the oop load to the guard edge just created:
3317 Node* is_array_ctrl = region->in(region->req()-1);
3318 Node* cma = basic_plus_adr(kls, in_bytes(ArrayKlass::component_mirror_offset()));
3319 Node* cmo = make_load(is_array_ctrl, cma, TypeInstPtr::MIRROR, T_OBJECT);
3320 phi->add_req(cmo);
3321 }
3394 region->init_req(prim_path, null_ctl);
3395 if (stopped()) break;
3396 klasses[which_arg] = kls;
3397 }
3398
3399 if (!stopped()) {
3400 // now we have two reference types, in klasses[0..1]
3401 Node* subk = klasses[1]; // the argument to isAssignableFrom
3402 Node* superk = klasses[0]; // the receiver
3403 region->set_req(_both_ref_path, gen_subtype_check(subk, superk));
3404 // now we have a successful reference subtype check
3405 region->set_req(_ref_subtype_path, control());
3406 }
3407
3408 // If both operands are primitive (both klasses null), then
3409 // we must return true when they are identical primitives.
3410 // It is convenient to test this after the first null klass check.
3411 set_control(region->in(_prim_0_path)); // go back to first null check
3412 if (!stopped()) {
3413 // Since superc is primitive, make a guard for the superc==subc case.
3414 Node* cmp_eq = _gvn.transform(new (C) CmpPNode(args[0], args[1]));
3415 Node* bol_eq = _gvn.transform(new (C) BoolNode(cmp_eq, BoolTest::eq));
3416 generate_guard(bol_eq, region, PROB_FAIR);
3417 if (region->req() == PATH_LIMIT+1) {
3418 // A guard was added. If the added guard is taken, superc==subc.
3419 region->swap_edges(PATH_LIMIT, _prim_same_path);
3420 region->del_req(PATH_LIMIT);
3421 }
3422 region->set_req(_prim_0_path, control()); // Not equal after all.
3423 }
3424
3425 // these are the only paths that produce 'true':
3426 phi->set_req(_prim_same_path, intcon(1));
3427 phi->set_req(_ref_subtype_path, intcon(1));
3428
3429 // pull together the cases:
3430 assert(region->req() == PATH_LIMIT, "sane region");
3431 for (uint i = 1; i < region->req(); i++) {
3432 Node* ctl = region->in(i);
3433 if (ctl == NULL || ctl == top()) {
3434 region->set_req(i, top());
3435 phi ->set_req(i, top());
3460 Node* layout_val = get_layout_helper(kls, layout_con);
3461 if (layout_val == NULL) {
3462 bool query = (obj_array
3463 ? Klass::layout_helper_is_objArray(layout_con)
3464 : Klass::layout_helper_is_array(layout_con));
3465 if (query == not_array) {
3466 return NULL; // never a branch
3467 } else { // always a branch
3468 Node* always_branch = control();
3469 if (region != NULL)
3470 region->add_req(always_branch);
3471 set_control(top());
3472 return always_branch;
3473 }
3474 }
3475 // Now test the correct condition.
3476 jint nval = (obj_array
3477 ? ((jint)Klass::_lh_array_tag_type_value
3478 << Klass::_lh_array_tag_shift)
3479 : Klass::_lh_neutral_value);
3480 Node* cmp = _gvn.transform(new(C) CmpINode(layout_val, intcon(nval)));
3481 BoolTest::mask btest = BoolTest::lt; // correct for testing is_[obj]array
3482 // invert the test if we are looking for a non-array
3483 if (not_array) btest = BoolTest(btest).negate();
3484 Node* bol = _gvn.transform(new(C) BoolNode(cmp, btest));
3485 return generate_fair_guard(bol, region);
3486 }
3487
3488
3489 //-----------------------inline_native_newArray--------------------------
3490 // private static native Object java.lang.reflect.newArray(Class<?> componentType, int length);
3491 bool LibraryCallKit::inline_native_newArray() {
3492 Node* mirror = argument(0);
3493 Node* count_val = argument(1);
3494
3495 mirror = null_check(mirror);
3496 // If mirror or obj is dead, only null-path is taken.
3497 if (stopped()) return true;
3498
3499 enum { _normal_path = 1, _slow_path = 2, PATH_LIMIT };
3500 RegionNode* result_reg = new(C) RegionNode(PATH_LIMIT);
3501 PhiNode* result_val = new(C) PhiNode(result_reg,
3502 TypeInstPtr::NOTNULL);
3503 PhiNode* result_io = new(C) PhiNode(result_reg, Type::ABIO);
3504 PhiNode* result_mem = new(C) PhiNode(result_reg, Type::MEMORY,
3524 result_reg->set_req(_slow_path, control());
3525 result_val->set_req(_slow_path, slow_result);
3526 result_io ->set_req(_slow_path, i_o());
3527 result_mem->set_req(_slow_path, reset_memory());
3528 }
3529
3530 set_control(normal_ctl);
3531 if (!stopped()) {
3532 // Normal case: The array type has been cached in the java.lang.Class.
3533 // The following call works fine even if the array type is polymorphic.
3534 // It could be a dynamic mix of int[], boolean[], Object[], etc.
3535 Node* obj = new_array(klass_node, count_val, 0); // no arguments to push
3536 result_reg->init_req(_normal_path, control());
3537 result_val->init_req(_normal_path, obj);
3538 result_io ->init_req(_normal_path, i_o());
3539 result_mem->init_req(_normal_path, reset_memory());
3540 }
3541
3542 // Return the combined state.
3543 set_i_o( _gvn.transform(result_io) );
3544 set_all_memory( _gvn.transform(result_mem));
3545
3546 C->set_has_split_ifs(true); // Has chance for split-if optimization
3547 set_result(result_reg, result_val);
3548 return true;
3549 }
3550
3551 //----------------------inline_native_getLength--------------------------
3552 // public static native int java.lang.reflect.Array.getLength(Object array);
3553 bool LibraryCallKit::inline_native_getLength() {
3554 if (too_many_traps(Deoptimization::Reason_intrinsic)) return false;
3555
3556 Node* array = null_check(argument(0));
3557 // If array is dead, only null-path is taken.
3558 if (stopped()) return true;
3559
3560 // Deoptimize if it is a non-array.
3561 Node* non_array = generate_non_array_guard(load_object_klass(array), NULL);
3562
3563 if (non_array != NULL) {
3564 PreserveJVMState pjvms(this);
3677 }
3678
3679
3680 //----------------------generate_virtual_guard---------------------------
3681 // Helper for hashCode and clone. Peeks inside the vtable to avoid a call.
3682 Node* LibraryCallKit::generate_virtual_guard(Node* obj_klass,
3683 RegionNode* slow_region) {
3684 ciMethod* method = callee();
3685 int vtable_index = method->vtable_index();
3686 // Get the Method* out of the appropriate vtable entry.
3687 int entry_offset = (InstanceKlass::vtable_start_offset() +
3688 vtable_index*vtableEntry::size()) * wordSize +
3689 vtableEntry::method_offset_in_bytes();
3690 Node* entry_addr = basic_plus_adr(obj_klass, entry_offset);
3691 Node* target_call = make_load(NULL, entry_addr, TypePtr::NOTNULL, T_ADDRESS);
3692
3693 // Compare the target method with the expected method (e.g., Object.hashCode).
3694 const TypePtr* native_call_addr = TypeMetadataPtr::make(method);
3695
3696 Node* native_call = makecon(native_call_addr);
3697 Node* chk_native = _gvn.transform(new(C) CmpPNode(target_call, native_call));
3698 Node* test_native = _gvn.transform(new(C) BoolNode(chk_native, BoolTest::ne));
3699
3700 return generate_slow_guard(test_native, slow_region);
3701 }
3702
3703 //-----------------------generate_method_call----------------------------
3704 // Use generate_method_call to make a slow-call to the real
3705 // method if the fast path fails. An alternative would be to
3706 // use a stub like OptoRuntime::slow_arraycopy_Java.
3707 // This only works for expanding the current library call,
3708 // not another intrinsic. (E.g., don't use this for making an
3709 // arraycopy call inside of the copyOf intrinsic.)
3710 CallJavaNode*
3711 LibraryCallKit::generate_method_call(vmIntrinsics::ID method_id, bool is_virtual, bool is_static) {
3712 // When compiling the intrinsic method itself, do not use this technique.
3713 guarantee(callee() != C->method(), "cannot make slow-call to self");
3714
3715 ciMethod* method = callee();
3716 // ensure the JVMS we have will be correct for this call
3717 guarantee(method_id == method->intrinsic_id(), "must match");
3718
3799 // paths which do not pass are accumulated in the slow_region.
3800 RegionNode* slow_region = new (C) RegionNode(1);
3801 record_for_igvn(slow_region);
3802
3803 // If this is a virtual call, we generate a funny guard. We pull out
3804 // the vtable entry corresponding to hashCode() from the target object.
3805 // If the target method which we are calling happens to be the native
3806 // Object hashCode() method, we pass the guard. We do not need this
3807 // guard for non-virtual calls -- the caller is known to be the native
3808 // Object hashCode().
3809 if (is_virtual) {
3810 generate_virtual_guard(obj_klass, slow_region);
3811 }
3812
3813 // Get the header out of the object, use LoadMarkNode when available
3814 Node* header_addr = basic_plus_adr(obj, oopDesc::mark_offset_in_bytes());
3815 Node* header = make_load(control(), header_addr, TypeX_X, TypeX_X->basic_type());
3816
3817 // Test the header to see if it is unlocked.
3818 Node *lock_mask = _gvn.MakeConX(markOopDesc::biased_lock_mask_in_place);
3819 Node *lmasked_header = _gvn.transform(new (C) AndXNode(header, lock_mask));
3820 Node *unlocked_val = _gvn.MakeConX(markOopDesc::unlocked_value);
3821 Node *chk_unlocked = _gvn.transform(new (C) CmpXNode( lmasked_header, unlocked_val));
3822 Node *test_unlocked = _gvn.transform(new (C) BoolNode( chk_unlocked, BoolTest::ne));
3823
3824 generate_slow_guard(test_unlocked, slow_region);
3825
3826 // Get the hash value and check to see that it has been properly assigned.
3827 // We depend on hash_mask being at most 32 bits and avoid the use of
3828 // hash_mask_in_place because it could be larger than 32 bits in a 64-bit
3829 // vm: see markOop.hpp.
3830 Node *hash_mask = _gvn.intcon(markOopDesc::hash_mask);
3831 Node *hash_shift = _gvn.intcon(markOopDesc::hash_shift);
3832 Node *hshifted_header= _gvn.transform(new (C) URShiftXNode(header, hash_shift));
3833 // This hack lets the hash bits live anywhere in the mark object now, as long
3834 // as the shift drops the relevant bits into the low 32 bits. Note that
3835 // Java spec says that HashCode is an int so there's no point in capturing
3836 // an 'X'-sized hashcode (32 in 32-bit build or 64 in 64-bit build).
3837 hshifted_header = ConvX2I(hshifted_header);
3838 Node *hash_val = _gvn.transform(new (C) AndINode(hshifted_header, hash_mask));
3839
3840 Node *no_hash_val = _gvn.intcon(markOopDesc::no_hash);
3841 Node *chk_assigned = _gvn.transform(new (C) CmpINode( hash_val, no_hash_val));
3842 Node *test_assigned = _gvn.transform(new (C) BoolNode( chk_assigned, BoolTest::eq));
3843
3844 generate_slow_guard(test_assigned, slow_region);
3845
3846 Node* init_mem = reset_memory();
3847 // fill in the rest of the null path:
3848 result_io ->init_req(_null_path, i_o());
3849 result_mem->init_req(_null_path, init_mem);
3850
3851 result_val->init_req(_fast_path, hash_val);
3852 result_reg->init_req(_fast_path, control());
3853 result_io ->init_req(_fast_path, i_o());
3854 result_mem->init_req(_fast_path, init_mem);
3855
3856 // Generate code for the slow case. We make a call to hashCode().
3857 set_control(_gvn.transform(slow_region));
3858 if (!stopped()) {
3859 // No need for PreserveJVMState, because we're using up the present state.
3860 set_all_memory(init_mem);
3861 vmIntrinsics::ID hashCode_id = is_static ? vmIntrinsics::_identityHashCode : vmIntrinsics::_hashCode;
3862 CallJavaNode* slow_call = generate_method_call(hashCode_id, is_virtual, is_static);
3863 Node* slow_result = set_results_for_java_call(slow_call);
3864 // this->control() comes from set_results_for_java_call
3865 result_reg->init_req(_slow_path, control());
3866 result_val->init_req(_slow_path, slow_result);
3867 result_io ->set_req(_slow_path, i_o());
3868 result_mem ->set_req(_slow_path, reset_memory());
3869 }
3870
3871 // Return the combined state.
3872 set_i_o( _gvn.transform(result_io) );
3873 set_all_memory( _gvn.transform(result_mem));
3874
3875 set_result(result_reg, result_val);
3876 return true;
3877 }
3878
3879 //---------------------------inline_native_getClass----------------------------
3880 // public final native Class<?> java.lang.Object.getClass();
3881 //
3882 // Build special case code for calls to getClass on an object.
3883 bool LibraryCallKit::inline_native_getClass() {
3884 Node* obj = null_check_receiver();
3885 if (stopped()) return true;
3886 set_result(load_mirror_from_klass(load_object_klass(obj)));
3887 return true;
3888 }
3889
3890 //-----------------inline_native_Reflection_getCallerClass---------------------
3891 // public static native Class<?> sun.reflect.Reflection.getCallerClass();
3892 //
3893 // In the presence of deep enough inlining, getCallerClass() becomes a no-op.
3981 case vmIntrinsics::_floatToRawIntBits: result = new (C) MoveF2INode(arg); break;
3982 case vmIntrinsics::_intBitsToFloat: result = new (C) MoveI2FNode(arg); break;
3983 case vmIntrinsics::_doubleToRawLongBits: result = new (C) MoveD2LNode(arg); break;
3984 case vmIntrinsics::_longBitsToDouble: result = new (C) MoveL2DNode(arg); break;
3985
3986 case vmIntrinsics::_doubleToLongBits: {
3987 // two paths (plus control) merge in a wood
3988 RegionNode *r = new (C) RegionNode(3);
3989 Node *phi = new (C) PhiNode(r, TypeLong::LONG);
3990
3991 Node *cmpisnan = _gvn.transform(new (C) CmpDNode(arg, arg));
3992 // Build the boolean node
3993 Node *bolisnan = _gvn.transform(new (C) BoolNode(cmpisnan, BoolTest::ne));
3994
3995 // Branch either way.
3996 // NaN case is less traveled, which makes all the difference.
3997 IfNode *ifisnan = create_and_xform_if(control(), bolisnan, PROB_STATIC_FREQUENT, COUNT_UNKNOWN);
3998 Node *opt_isnan = _gvn.transform(ifisnan);
3999 assert( opt_isnan->is_If(), "Expect an IfNode");
4000 IfNode *opt_ifisnan = (IfNode*)opt_isnan;
4001 Node *iftrue = _gvn.transform(new (C) IfTrueNode(opt_ifisnan));
4002
4003 set_control(iftrue);
4004
4005 static const jlong nan_bits = CONST64(0x7ff8000000000000);
4006 Node *slow_result = longcon(nan_bits); // return NaN
4007 phi->init_req(1, _gvn.transform( slow_result ));
4008 r->init_req(1, iftrue);
4009
4010 // Else fall through
4011 Node *iffalse = _gvn.transform(new (C) IfFalseNode(opt_ifisnan));
4012 set_control(iffalse);
4013
4014 phi->init_req(2, _gvn.transform(new (C) MoveD2LNode(arg)));
4015 r->init_req(2, iffalse);
4016
4017 // Post merge
4018 set_control(_gvn.transform(r));
4019 record_for_igvn(r);
4020
4021 C->set_has_split_ifs(true); // Has chance for split-if optimization
4022 result = phi;
4023 assert(result->bottom_type()->isa_long(), "must be");
4024 break;
4025 }
4026
4027 case vmIntrinsics::_floatToIntBits: {
4028 // two paths (plus control) merge in a wood
4029 RegionNode *r = new (C) RegionNode(3);
4030 Node *phi = new (C) PhiNode(r, TypeInt::INT);
4031
4032 Node *cmpisnan = _gvn.transform(new (C) CmpFNode(arg, arg));
4033 // Build the boolean node
4034 Node *bolisnan = _gvn.transform(new (C) BoolNode(cmpisnan, BoolTest::ne));
4035
4036 // Branch either way.
4037 // NaN case is less traveled, which makes all the difference.
4038 IfNode *ifisnan = create_and_xform_if(control(), bolisnan, PROB_STATIC_FREQUENT, COUNT_UNKNOWN);
4039 Node *opt_isnan = _gvn.transform(ifisnan);
4040 assert( opt_isnan->is_If(), "Expect an IfNode");
4041 IfNode *opt_ifisnan = (IfNode*)opt_isnan;
4042 Node *iftrue = _gvn.transform(new (C) IfTrueNode(opt_ifisnan));
4043
4044 set_control(iftrue);
4045
4046 static const jint nan_bits = 0x7fc00000;
4047 Node *slow_result = makecon(TypeInt::make(nan_bits)); // return NaN
4048 phi->init_req(1, _gvn.transform( slow_result ));
4049 r->init_req(1, iftrue);
4050
4051 // Else fall through
4052 Node *iffalse = _gvn.transform(new (C) IfFalseNode(opt_ifisnan));
4053 set_control(iffalse);
4054
4055 phi->init_req(2, _gvn.transform(new (C) MoveF2INode(arg)));
4056 r->init_req(2, iffalse);
4057
4058 // Post merge
4059 set_control(_gvn.transform(r));
4060 record_for_igvn(r);
4061
4062 C->set_has_split_ifs(true); // Has chance for split-if optimization
4151 // base_off:
4152 // 8 - 32-bit VM
4153 // 12 - 64-bit VM, compressed klass
4154 // 16 - 64-bit VM, normal klass
4155 if (base_off % BytesPerLong != 0) {
4156 assert(UseCompressedKlassPointers, "");
4157 if (is_array) {
4158 // Exclude length to copy by 8 bytes words.
4159 base_off += sizeof(int);
4160 } else {
4161 // Include klass to copy by 8 bytes words.
4162 base_off = instanceOopDesc::klass_offset_in_bytes();
4163 }
4164 assert(base_off % BytesPerLong == 0, "expect 8 bytes alignment");
4165 }
4166 src = basic_plus_adr(src, base_off);
4167 dest = basic_plus_adr(dest, base_off);
4168
4169 // Compute the length also, if needed:
4170 Node* countx = size;
4171 countx = _gvn.transform(new (C) SubXNode(countx, MakeConX(base_off)));
4172 countx = _gvn.transform(new (C) URShiftXNode(countx, intcon(LogBytesPerLong) ));
4173
4174 const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
4175 bool disjoint_bases = true;
4176 generate_unchecked_arraycopy(raw_adr_type, T_LONG, disjoint_bases,
4177 src, NULL, dest, NULL, countx,
4178 /*dest_uninitialized*/true);
4179
4180 // If necessary, emit some card marks afterwards. (Non-arrays only.)
4181 if (card_mark) {
4182 assert(!is_array, "");
4183 // Put in store barrier for any and all oops we are sticking
4184 // into this object. (We could avoid this if we could prove
4185 // that the object type contains no oop fields at all.)
4186 Node* no_particular_value = NULL;
4187 Node* no_particular_field = NULL;
4188 int raw_adr_idx = Compile::AliasIdxRaw;
4189 post_barrier(control(),
4190 memory(raw_adr_type),
4191 alloc_obj,
4192 no_particular_field,
4356 result_reg->init_req(_instance_path, control());
4357 result_val->init_req(_instance_path, alloc_obj);
4358 result_i_o ->set_req(_instance_path, i_o());
4359 result_mem ->set_req(_instance_path, reset_memory());
4360 }
4361
4362 // Generate code for the slow case. We make a call to clone().
4363 set_control(_gvn.transform(slow_region));
4364 if (!stopped()) {
4365 PreserveJVMState pjvms(this);
4366 CallJavaNode* slow_call = generate_method_call(vmIntrinsics::_clone, is_virtual);
4367 Node* slow_result = set_results_for_java_call(slow_call);
4368 // this->control() comes from set_results_for_java_call
4369 result_reg->init_req(_slow_path, control());
4370 result_val->init_req(_slow_path, slow_result);
4371 result_i_o ->set_req(_slow_path, i_o());
4372 result_mem ->set_req(_slow_path, reset_memory());
4373 }
4374
4375 // Return the combined state.
4376 set_control( _gvn.transform(result_reg));
4377 set_i_o( _gvn.transform(result_i_o));
4378 set_all_memory( _gvn.transform(result_mem));
4379 } // original reexecute is set back here
4380
4381 set_result(_gvn.transform(result_val));
4382 return true;
4383 }
4384
4385 //------------------------------basictype2arraycopy----------------------------
4386 address LibraryCallKit::basictype2arraycopy(BasicType t,
4387 Node* src_offset,
4388 Node* dest_offset,
4389 bool disjoint_bases,
4390 const char* &name,
4391 bool dest_uninitialized) {
4392 const TypeInt* src_offset_inttype = gvn().find_int_type(src_offset);;
4393 const TypeInt* dest_offset_inttype = gvn().find_int_type(dest_offset);;
4394
4395 bool aligned = false;
4396 bool disjoint = disjoint_bases;
4397
4398 // if the offsets are the same, we can treat the memory regions as
4683 // paths have stub or runtime calls as raw memory barriers.
4684 InitializeNode* init = insert_mem_bar_volatile(Op_Initialize,
4685 Compile::AliasIdxRaw,
4686 top())->as_Initialize();
4687 init->set_complete(&_gvn); // (there is no corresponding AllocateNode)
4688 }
4689 }
4690
4691 // Present the results of the fast call.
4692 result_region->init_req(zero_path, control());
4693 result_i_o ->init_req(zero_path, i_o());
4694 result_memory->init_req(zero_path, memory(adr_type));
4695 }
4696
4697 if (!stopped() && dest_uninitialized) {
4698 // We have to initialize the *uncopied* part of the array to zero.
4699 // The copy destination is the slice dest[off..off+len]. The other slices
4700 // are dest_head = dest[0..off] and dest_tail = dest[off+len..dest.length].
4701 Node* dest_size = alloc->in(AllocateNode::AllocSize);
4702 Node* dest_length = alloc->in(AllocateNode::ALength);
4703 Node* dest_tail = _gvn.transform(new(C) AddINode(dest_offset,
4704 copy_length));
4705
4706 // If there is a head section that needs zeroing, do it now.
4707 if (find_int_con(dest_offset, -1) != 0) {
4708 generate_clear_array(adr_type, dest, basic_elem_type,
4709 intcon(0), dest_offset,
4710 NULL);
4711 }
4712
4713 // Next, perform a dynamic check on the tail length.
4714 // It is often zero, and we can win big if we prove this.
4715 // There are two wins: Avoid generating the ClearArray
4716 // with its attendant messy index arithmetic, and upgrade
4717 // the copy to a more hardware-friendly word size of 64 bits.
4718 Node* tail_ctl = NULL;
4719 if (!stopped() && !dest_tail->eqv_uncast(dest_length)) {
4720 Node* cmp_lt = _gvn.transform(new(C) CmpINode(dest_tail, dest_length));
4721 Node* bol_lt = _gvn.transform(new(C) BoolNode(cmp_lt, BoolTest::lt));
4722 tail_ctl = generate_slow_guard(bol_lt, NULL);
4723 assert(tail_ctl != NULL || !stopped(), "must be an outcome");
4724 }
4725
4726 // At this point, let's assume there is no tail.
4727 if (!stopped() && alloc != NULL && basic_elem_type != T_OBJECT) {
4728 // There is no tail. Try an upgrade to a 64-bit copy.
4729 bool didit = false;
4730 { PreserveJVMState pjvms(this);
4731 didit = generate_block_arraycopy(adr_type, basic_elem_type, alloc,
4732 src, src_offset, dest, dest_offset,
4733 dest_size, dest_uninitialized);
4734 if (didit) {
4735 // Present the results of the block-copying fast call.
4736 result_region->init_req(bcopy_path, control());
4737 result_i_o ->init_req(bcopy_path, i_o());
4738 result_memory->init_req(bcopy_path, memory(adr_type));
4739 }
4740 }
4741 if (didit)
4744
4745 // Clear the tail, if any.
4746 if (tail_ctl != NULL) {
4747 Node* notail_ctl = stopped() ? NULL : control();
4748 set_control(tail_ctl);
4749 if (notail_ctl == NULL) {
4750 generate_clear_array(adr_type, dest, basic_elem_type,
4751 dest_tail, NULL,
4752 dest_size);
4753 } else {
4754 // Make a local merge.
4755 Node* done_ctl = new(C) RegionNode(3);
4756 Node* done_mem = new(C) PhiNode(done_ctl, Type::MEMORY, adr_type);
4757 done_ctl->init_req(1, notail_ctl);
4758 done_mem->init_req(1, memory(adr_type));
4759 generate_clear_array(adr_type, dest, basic_elem_type,
4760 dest_tail, NULL,
4761 dest_size);
4762 done_ctl->init_req(2, control());
4763 done_mem->init_req(2, memory(adr_type));
4764 set_control( _gvn.transform(done_ctl));
4765 set_memory( _gvn.transform(done_mem), adr_type );
4766 }
4767 }
4768 }
4769
4770 BasicType copy_type = basic_elem_type;
4771 assert(basic_elem_type != T_ARRAY, "caller must fix this");
4772 if (!stopped() && copy_type == T_OBJECT) {
4773 // If src and dest have compatible element types, we can copy bits.
4774 // Types S[] and D[] are compatible if D is a supertype of S.
4775 //
4776 // If they are not, we will use checked_oop_disjoint_arraycopy,
4777 // which performs a fast optimistic per-oop check, and backs off
4778 // further to JVM_ArrayCopy on the first per-oop check that fails.
4779 // (Actually, we don't move raw bits only; the GC requires card marks.)
4780
4781 // Get the Klass* for both src and dest
4782 Node* src_klass = load_object_klass(src);
4783 Node* dest_klass = load_object_klass(dest);
4784
4831 src, src_offset, dest, dest_offset,
4832 ConvI2X(copy_length), dest_uninitialized);
4833
4834 // Present the results of the fast call.
4835 result_region->init_req(fast_path, control());
4836 result_i_o ->init_req(fast_path, i_o());
4837 result_memory->init_req(fast_path, memory(adr_type));
4838 }
4839
4840 // Here are all the slow paths up to this point, in one bundle:
4841 slow_control = top();
4842 if (slow_region != NULL)
4843 slow_control = _gvn.transform(slow_region);
4844 DEBUG_ONLY(slow_region = (RegionNode*)badAddress);
4845
4846 set_control(checked_control);
4847 if (!stopped()) {
4848 // Clean up after the checked call.
4849 // The returned value is either 0 or -1^K,
4850 // where K = number of partially transferred array elements.
4851 Node* cmp = _gvn.transform(new(C) CmpINode(checked_value, intcon(0)));
4852 Node* bol = _gvn.transform(new(C) BoolNode(cmp, BoolTest::eq));
4853 IfNode* iff = create_and_map_if(control(), bol, PROB_MAX, COUNT_UNKNOWN);
4854
4855 // If it is 0, we are done, so transfer to the end.
4856 Node* checks_done = _gvn.transform(new(C) IfTrueNode(iff));
4857 result_region->init_req(checked_path, checks_done);
4858 result_i_o ->init_req(checked_path, checked_i_o);
4859 result_memory->init_req(checked_path, checked_mem);
4860
4861 // If it is not zero, merge into the slow call.
4862 set_control( _gvn.transform(new(C) IfFalseNode(iff) ));
4863 RegionNode* slow_reg2 = new(C) RegionNode(3);
4864 PhiNode* slow_i_o2 = new(C) PhiNode(slow_reg2, Type::ABIO);
4865 PhiNode* slow_mem2 = new(C) PhiNode(slow_reg2, Type::MEMORY, adr_type);
4866 record_for_igvn(slow_reg2);
4867 slow_reg2 ->init_req(1, slow_control);
4868 slow_i_o2 ->init_req(1, slow_i_o);
4869 slow_mem2 ->init_req(1, slow_mem);
4870 slow_reg2 ->init_req(2, control());
4871 slow_i_o2 ->init_req(2, checked_i_o);
4872 slow_mem2 ->init_req(2, checked_mem);
4873
4874 slow_control = _gvn.transform(slow_reg2);
4875 slow_i_o = _gvn.transform(slow_i_o2);
4876 slow_mem = _gvn.transform(slow_mem2);
4877
4878 if (alloc != NULL) {
4879 // We'll restart from the very beginning, after zeroing the whole thing.
4880 // This can cause double writes, but that's OK since dest is brand new.
4881 // So we ignore the low 31 bits of the value returned from the stub.
4882 } else {
4883 // We must continue the copy exactly where it failed, or else
4884 // another thread might see the wrong number of writes to dest.
4885 Node* checked_offset = _gvn.transform(new(C) XorINode(checked_value, intcon(-1)));
4886 Node* slow_offset = new(C) PhiNode(slow_reg2, TypeInt::INT);
4887 slow_offset->init_req(1, intcon(0));
4888 slow_offset->init_req(2, checked_offset);
4889 slow_offset = _gvn.transform(slow_offset);
4890
4891 // Adjust the arguments by the conditionally incoming offset.
4892 Node* src_off_plus = _gvn.transform(new(C) AddINode(src_offset, slow_offset));
4893 Node* dest_off_plus = _gvn.transform(new(C) AddINode(dest_offset, slow_offset));
4894 Node* length_minus = _gvn.transform(new(C) SubINode(copy_length, slow_offset));
4895
4896 // Tweak the node variables to adjust the code produced below:
4897 src_offset = src_off_plus;
4898 dest_offset = dest_off_plus;
4899 copy_length = length_minus;
4900 }
4901 }
4902
4903 set_control(slow_control);
4904 if (!stopped()) {
4905 // Generate the slow path, if needed.
4906 PreserveJVMState pjvms(this); // replace_in_map may trash the map
4907
4908 set_memory(slow_mem, adr_type);
4909 set_i_o(slow_i_o);
4910
4911 if (dest_uninitialized) {
4912 generate_clear_array(adr_type, dest, basic_elem_type,
4913 intcon(0), NULL,
4914 alloc->in(AllocateNode::AllocSize));
4915 }
4916
4917 generate_slow_arraycopy(adr_type,
4918 src, src_offset, dest, dest_offset,
4919 copy_length, /*dest_uninitialized*/false);
4920
4921 result_region->init_req(slow_call_path, control());
4922 result_i_o ->init_req(slow_call_path, i_o());
4923 result_memory->init_req(slow_call_path, memory(adr_type));
4924 }
4925
4926 // Remove unused edges.
4927 for (uint i = 1; i < result_region->req(); i++) {
4928 if (result_region->in(i) == NULL)
4929 result_region->init_req(i, top());
4930 }
4931
4932 // Finished; return the combined state.
4933 set_control( _gvn.transform(result_region));
4934 set_i_o( _gvn.transform(result_i_o) );
4935 set_memory( _gvn.transform(result_memory), adr_type );
4936
4937 // The memory edges above are precise in order to model effects around
4938 // array copies accurately to allow value numbering of field loads around
4939 // arraycopy. Such field loads, both before and after, are common in Java
4940 // collections and similar classes involving header/array data structures.
4941 //
4942 // But with low number of register or when some registers are used or killed
4943 // by arraycopy calls it causes registers spilling on stack. See 6544710.
4944 // The next memory barrier is added to avoid it. If the arraycopy can be
4945 // optimized away (which it can, sometimes) then we can manually remove
4946 // the membar also.
4947 //
4948 // Do not let reads from the cloned object float above the arraycopy.
4949 if (alloc != NULL) {
4950 // Do not let stores that initialize this object be reordered with
4951 // a subsequent store that would make this object accessible by
4952 // other threads.
4953 // Record what AllocateNode this StoreStore protects so that
5095 end_con = round_to(abase + ((slice_idx_con + slice_len_con) << scale),
5096 BytesPerLong);
5097 }
5098
5099 if (start_con >= 0 && end_con >= 0) {
5100 // Constant start and end. Simple.
5101 mem = ClearArrayNode::clear_memory(control(), mem, dest,
5102 start_con, end_con, &_gvn);
5103 } else if (start_con >= 0 && dest_size != top()) {
5104 // Constant start, pre-rounded end after the tail of the array.
5105 Node* end = dest_size;
5106 mem = ClearArrayNode::clear_memory(control(), mem, dest,
5107 start_con, end, &_gvn);
5108 } else if (start_con >= 0 && slice_len != top()) {
5109 // Constant start, non-constant end. End needs rounding up.
5110 // End offset = round_up(abase + ((slice_idx_con + slice_len) << scale), 8)
5111 intptr_t end_base = abase + (slice_idx_con << scale);
5112 int end_round = (-1 << scale) & (BytesPerLong - 1);
5113 Node* end = ConvI2X(slice_len);
5114 if (scale != 0)
5115 end = _gvn.transform(new(C) LShiftXNode(end, intcon(scale) ));
5116 end_base += end_round;
5117 end = _gvn.transform(new(C) AddXNode(end, MakeConX(end_base)));
5118 end = _gvn.transform(new(C) AndXNode(end, MakeConX(~end_round)));
5119 mem = ClearArrayNode::clear_memory(control(), mem, dest,
5120 start_con, end, &_gvn);
5121 } else if (start_con < 0 && dest_size != top()) {
5122 // Non-constant start, pre-rounded end after the tail of the array.
5123 // This is almost certainly a "round-to-end" operation.
5124 Node* start = slice_idx;
5125 start = ConvI2X(start);
5126 if (scale != 0)
5127 start = _gvn.transform(new(C) LShiftXNode( start, intcon(scale) ));
5128 start = _gvn.transform(new(C) AddXNode(start, MakeConX(abase)));
5129 if ((bump_bit | clear_low) != 0) {
5130 int to_clear = (bump_bit | clear_low);
5131 // Align up mod 8, then store a jint zero unconditionally
5132 // just before the mod-8 boundary.
5133 if (((abase + bump_bit) & ~to_clear) - bump_bit
5134 < arrayOopDesc::length_offset_in_bytes() + BytesPerInt) {
5135 bump_bit = 0;
5136 assert((abase & to_clear) == 0, "array base must be long-aligned");
5137 } else {
5138 // Bump 'start' up to (or past) the next jint boundary:
5139 start = _gvn.transform(new(C) AddXNode(start, MakeConX(bump_bit)));
5140 assert((abase & clear_low) == 0, "array base must be int-aligned");
5141 }
5142 // Round bumped 'start' down to jlong boundary in body of array.
5143 start = _gvn.transform(new(C) AndXNode(start, MakeConX(~to_clear)));
5144 if (bump_bit != 0) {
5145 // Store a zero to the immediately preceding jint:
5146 Node* x1 = _gvn.transform(new(C) AddXNode(start, MakeConX(-bump_bit)));
5147 Node* p1 = basic_plus_adr(dest, x1);
5148 mem = StoreNode::make(_gvn, control(), mem, p1, adr_type, intcon(0), T_INT);
5149 mem = _gvn.transform(mem);
5150 }
5151 }
5152 Node* end = dest_size; // pre-rounded
5153 mem = ClearArrayNode::clear_memory(control(), mem, dest,
5154 start, end, &_gvn);
5155 } else {
5156 // Non-constant start, unrounded non-constant end.
5157 // (Nobody zeroes a random midsection of an array using this routine.)
5158 ShouldNotReachHere(); // fix caller
5159 }
5160
5161 // Done.
5162 set_memory(mem, adr_type);
5163 }
5164
5165
5166 bool
5193 // This is a common case, since abase can be odd mod 8.
5194 if (((src_off | dest_off) & (BytesPerLong-1)) == BytesPerInt &&
5195 ((src_off ^ dest_off) & (BytesPerLong-1)) == 0) {
5196 Node* sptr = basic_plus_adr(src, src_off);
5197 Node* dptr = basic_plus_adr(dest, dest_off);
5198 Node* sval = make_load(control(), sptr, TypeInt::INT, T_INT, adr_type);
5199 store_to_memory(control(), dptr, sval, T_INT, adr_type);
5200 src_off += BytesPerInt;
5201 dest_off += BytesPerInt;
5202 } else {
5203 return false;
5204 }
5205 }
5206 assert(src_off % BytesPerLong == 0, "");
5207 assert(dest_off % BytesPerLong == 0, "");
5208
5209 // Do this copy by giant steps.
5210 Node* sptr = basic_plus_adr(src, src_off);
5211 Node* dptr = basic_plus_adr(dest, dest_off);
5212 Node* countx = dest_size;
5213 countx = _gvn.transform(new (C) SubXNode(countx, MakeConX(dest_off)));
5214 countx = _gvn.transform(new (C) URShiftXNode(countx, intcon(LogBytesPerLong)));
5215
5216 bool disjoint_bases = true; // since alloc != NULL
5217 generate_unchecked_arraycopy(adr_type, T_LONG, disjoint_bases,
5218 sptr, NULL, dptr, NULL, countx, dest_uninitialized);
5219
5220 return true;
5221 }
5222
5223
5224 // Helper function; generates code for the slow case.
5225 // We make a call to a runtime method which emulates the native method,
5226 // but without the native wrapper overhead.
5227 void
5228 LibraryCallKit::generate_slow_arraycopy(const TypePtr* adr_type,
5229 Node* src, Node* src_offset,
5230 Node* dest, Node* dest_offset,
5231 Node* copy_length, bool dest_uninitialized) {
5232 assert(!dest_uninitialized, "Invariant");
5233 Node* call = make_runtime_call(RC_NO_LEAF | RC_UNCOMMON,
5234 OptoRuntime::slow_arraycopy_Type(),
5359 // Figure out the size and type of the elements we will be copying.
5360 BasicType src_elem = src_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
5361 BasicType dst_elem = dst_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
5362 if (src_elem != T_CHAR || dst_elem != T_BYTE) {
5363 return false;
5364 }
5365 Node* src_start = array_element_address(src, src_offset, src_elem);
5366 Node* dst_start = array_element_address(dst, dst_offset, dst_elem);
5367 // 'src_start' points to src array + scaled offset
5368 // 'dst_start' points to dst array + scaled offset
5369
5370 const TypeAryPtr* mtype = TypeAryPtr::BYTES;
5371 Node* enc = new (C) EncodeISOArrayNode(control(), memory(mtype), src_start, dst_start, length);
5372 enc = _gvn.transform(enc);
5373 Node* res_mem = _gvn.transform(new (C) SCMemProjNode(enc));
5374 set_memory(res_mem, mtype);
5375 set_result(enc);
5376 return true;
5377 }
5378
5379 /**
5380 * Calculate CRC32 for byte.
5381 * int java.util.zip.CRC32.update(int crc, int b)
5382 */
5383 bool LibraryCallKit::inline_updateCRC32() {
5384 assert(UseCRC32Intrinsics, "need AVX and LCMUL instructions support");
5385 assert(callee()->signature()->size() == 2, "update has 2 parameters");
5386 // no receiver since it is static method
5387 Node* crc = argument(0); // type: int
5388 Node* b = argument(1); // type: int
5389
5390 /*
5391 * int c = ~ crc;
5392 * b = timesXtoThe32[(b ^ c) & 0xFF];
5393 * b = b ^ (c >>> 8);
5394 * crc = ~b;
5395 */
5396
5397 Node* M1 = intcon(-1);
5398 crc = _gvn.transform(new (C) XorINode(crc, M1));
5399 Node* result = _gvn.transform(new (C) XorINode(crc, b));
5400 result = _gvn.transform(new (C) AndINode(result, intcon(0xFF)));
5401
5402 Node* base = makecon(TypeRawPtr::make(StubRoutines::crc_table_addr()));
5403 Node* offset = _gvn.transform(new (C) LShiftINode(result, intcon(0x2)));
5404 Node* adr = basic_plus_adr(top(), base, ConvI2X(offset));
5405 result = make_load(control(), adr, TypeInt::INT, T_INT);
5406
5407 crc = _gvn.transform(new (C) URShiftINode(crc, intcon(8)));
5408 result = _gvn.transform(new (C) XorINode(crc, result));
5409 result = _gvn.transform(new (C) XorINode(result, M1));
5410 set_result(result);
5411 return true;
5412 }
5413
5414 /**
5415 * Calculate CRC32 for byte[] array.
5416 * int java.util.zip.CRC32.updateBytes(int crc, byte[] buf, int off, int len)
5417 */
5418 bool LibraryCallKit::inline_updateBytesCRC32() {
5419 assert(UseCRC32Intrinsics, "need AVX and LCMUL instructions support");
5420 assert(callee()->signature()->size() == 4, "updateBytes has 4 parameters");
5421 // no receiver since it is static method
5422 Node* crc = argument(0); // type: int
5423 Node* src = argument(1); // type: oop
5424 Node* offset = argument(2); // type: int
5425 Node* length = argument(3); // type: int
5426
5427 const Type* src_type = src->Value(&_gvn);
5428 const TypeAryPtr* top_src = src_type->isa_aryptr();
5429 if (top_src == NULL || top_src->klass() == NULL) {
5430 // failed array check
5431 return false;
5432 }
5433
5434 // Figure out the size and type of the elements we will be copying.
5435 BasicType src_elem = src_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
5436 if (src_elem != T_BYTE) {
5437 return false;
5438 }
5439
5440 // 'src_start' points to src array + scaled offset
5441 Node* src_start = array_element_address(src, offset, src_elem);
5442
5443 // We assume that range check is done by caller.
5444 // TODO: generate range check (offset+length < src.length) in debug VM.
5445
5446 // Call the stub.
5447 address stubAddr = StubRoutines::updateBytesCRC32();
5448 const char *stubName = "updateBytesCRC32";
5449
5450 Node* call = make_runtime_call(RC_LEAF|RC_NO_FP, OptoRuntime::updateBytesCRC32_Type(),
5451 stubAddr, stubName, TypePtr::BOTTOM,
5452 crc, src_start, length);
5453 Node* result = _gvn.transform(new (C) ProjNode(call, TypeFunc::Parms));
5454 set_result(result);
5455 return true;
5456 }
5457
5458 /**
5459 * Calculate CRC32 for ByteBuffer.
5460 * int java.util.zip.CRC32.updateByteBuffer(int crc, long buf, int off, int len)
5461 */
5462 bool LibraryCallKit::inline_updateByteBufferCRC32() {
5463 assert(UseCRC32Intrinsics, "need AVX and LCMUL instructions support");
5464 assert(callee()->signature()->size() == 5, "updateByteBuffer has 4 parameters and one is long");
5465 // no receiver since it is static method
5466 Node* crc = argument(0); // type: int
5467 Node* src = argument(1); // type: long
5468 Node* offset = argument(3); // type: int
5469 Node* length = argument(4); // type: int
5470
5471 src = ConvL2X(src); // adjust Java long to machine word
5472 Node* base = _gvn.transform(new (C) CastX2PNode(src));
5473 offset = ConvI2X(offset);
5474
5475 // 'src_start' points to src array + scaled offset
5476 Node* src_start = basic_plus_adr(top(), base, offset);
5477
5478 // Call the stub.
5479 address stubAddr = StubRoutines::updateBytesCRC32();
5480 const char *stubName = "updateBytesCRC32";
5481
5482 Node* call = make_runtime_call(RC_LEAF|RC_NO_FP, OptoRuntime::updateBytesCRC32_Type(),
5483 stubAddr, stubName, TypePtr::BOTTOM,
5484 crc, src_start, length);
5485 Node* result = _gvn.transform(new (C) ProjNode(call, TypeFunc::Parms));
5486 set_result(result);
5487 return true;
5488 }
5489
5490 //----------------------------inline_reference_get----------------------------
5491 // public T java.lang.ref.Reference.get();
5492 bool LibraryCallKit::inline_reference_get() {
5493 const int referent_offset = java_lang_ref_Reference::referent_offset;
5494 guarantee(referent_offset > 0, "should have already been set");
5495
5496 // Get the argument:
5497 Node* reference_obj = null_check_receiver();
5498 if (stopped()) return true;
5499
5500 Node* adr = basic_plus_adr(reference_obj, reference_obj, referent_offset);
5501
5502 ciInstanceKlass* klass = env()->Object_klass();
5503 const TypeOopPtr* object_type = TypeOopPtr::make_from_klass(klass);
5504
5505 Node* no_ctrl = NULL;
5506 Node* result = make_load(no_ctrl, adr, object_type, T_OBJECT);
5507
5508 // Use the pre-barrier to record the value in the referent field
5509 pre_barrier(false /* do_load */,
|