src/share/vm/opto/library_call.cpp
Index Unified diffs Context diffs Sdiffs Patch New Old Previous File Next File 7088419 Sdiff src/share/vm/opto

src/share/vm/opto/library_call.cpp

Print this page




 274                                    Node* dest, Node* dest_offset,
 275                                    Node* copy_length, bool dest_uninitialized);
 276   void generate_unchecked_arraycopy(const TypePtr* adr_type,
 277                                     BasicType basic_elem_type,
 278                                     bool disjoint_bases,
 279                                     Node* src,  Node* src_offset,
 280                                     Node* dest, Node* dest_offset,
 281                                     Node* copy_length, bool dest_uninitialized);
 282   typedef enum { LS_xadd, LS_xchg, LS_cmpxchg } LoadStoreKind;
 283   bool inline_unsafe_load_store(BasicType type,  LoadStoreKind kind);
 284   bool inline_unsafe_ordered_store(BasicType type);
 285   bool inline_unsafe_fence(vmIntrinsics::ID id);
 286   bool inline_fp_conversions(vmIntrinsics::ID id);
 287   bool inline_number_methods(vmIntrinsics::ID id);
 288   bool inline_reference_get();
 289   bool inline_aescrypt_Block(vmIntrinsics::ID id);
 290   bool inline_cipherBlockChaining_AESCrypt(vmIntrinsics::ID id);
 291   Node* inline_cipherBlockChaining_AESCrypt_predicate(bool decrypting);
 292   Node* get_key_start_from_aescrypt_object(Node* aescrypt_object);
 293   bool inline_encodeISOArray();



 294 };
 295 
 296 
 297 //---------------------------make_vm_intrinsic----------------------------
 298 CallGenerator* Compile::make_vm_intrinsic(ciMethod* m, bool is_virtual) {
 299   vmIntrinsics::ID id = m->intrinsic_id();
 300   assert(id != vmIntrinsics::_none, "must be a VM intrinsic");
 301 
 302   if (DisableIntrinsic[0] != '\0'
 303       && strstr(DisableIntrinsic, vmIntrinsics::name_at(id)) != NULL) {
 304     // disabled by a user request on the command line:
 305     // example: -XX:DisableIntrinsic=_hashCode,_getClass
 306     return NULL;
 307   }
 308 
 309   if (!m->is_loaded()) {
 310     // do not attempt to inline unloaded methods
 311     return NULL;
 312   }
 313 


 471     if (!UseCompressedOops && !Matcher::match_rule_supported(Op_GetAndSetP)) return NULL;
 472     if (UseCompressedOops && !Matcher::match_rule_supported(Op_GetAndSetN)) return NULL;
 473     break;
 474 #else
 475     if (!Matcher::match_rule_supported(Op_GetAndSetP)) return NULL;
 476     break;
 477 #endif
 478 
 479   case vmIntrinsics::_aescrypt_encryptBlock:
 480   case vmIntrinsics::_aescrypt_decryptBlock:
 481     if (!UseAESIntrinsics) return NULL;
 482     break;
 483 
 484   case vmIntrinsics::_cipherBlockChaining_encryptAESCrypt:
 485   case vmIntrinsics::_cipherBlockChaining_decryptAESCrypt:
 486     if (!UseAESIntrinsics) return NULL;
 487     // these two require the predicated logic
 488     is_predicted = true;
 489     break;
 490 






 491  default:
 492     assert(id <= vmIntrinsics::LAST_COMPILER_INLINE, "caller responsibility");
 493     assert(id != vmIntrinsics::_Object_init && id != vmIntrinsics::_invoke, "enum out of order?");
 494     break;
 495   }
 496 
 497   // -XX:-InlineClassNatives disables natives from the Class class.
 498   // The flag applies to all reflective calls, notably Array.newArray
 499   // (visible to Java programmers as Array.newInstance).
 500   if (m->holder()->name() == ciSymbol::java_lang_Class() ||
 501       m->holder()->name() == ciSymbol::java_lang_reflect_Array()) {
 502     if (!InlineClassNatives)  return NULL;
 503   }
 504 
 505   // -XX:-InlineThreadNatives disables natives from the Thread class.
 506   if (m->holder()->name() == ciSymbol::java_lang_Thread()) {
 507     if (!InlineThreadNatives)  return NULL;
 508   }
 509 
 510   // -XX:-InlineMathNatives disables natives from the Math,Float and Double classes.


 790   case vmIntrinsics::_bitCount_l:
 791   case vmIntrinsics::_reverseBytes_i:
 792   case vmIntrinsics::_reverseBytes_l:
 793   case vmIntrinsics::_reverseBytes_s:
 794   case vmIntrinsics::_reverseBytes_c:           return inline_number_methods(intrinsic_id());
 795 
 796   case vmIntrinsics::_getCallerClass:           return inline_native_Reflection_getCallerClass();
 797 
 798   case vmIntrinsics::_Reference_get:            return inline_reference_get();
 799 
 800   case vmIntrinsics::_aescrypt_encryptBlock:
 801   case vmIntrinsics::_aescrypt_decryptBlock:    return inline_aescrypt_Block(intrinsic_id());
 802 
 803   case vmIntrinsics::_cipherBlockChaining_encryptAESCrypt:
 804   case vmIntrinsics::_cipherBlockChaining_decryptAESCrypt:
 805     return inline_cipherBlockChaining_AESCrypt(intrinsic_id());
 806 
 807   case vmIntrinsics::_encodeISOArray:
 808     return inline_encodeISOArray();
 809 







 810   default:
 811     // If you get here, it may be that someone has added a new intrinsic
 812     // to the list in vmSymbols.hpp without implementing it here.
 813 #ifndef PRODUCT
 814     if ((PrintMiscellaneous && (Verbose || WizardMode)) || PrintOpto) {
 815       tty->print_cr("*** Warning: Unimplemented intrinsic %s(%d)",
 816                     vmIntrinsics::name_at(intrinsic_id()), intrinsic_id());
 817     }
 818 #endif
 819     return false;
 820   }
 821 }
 822 
 823 Node* LibraryCallKit::try_to_predicate() {
 824   if (!jvms()->has_method()) {
 825     // Root JVMState has a null method.
 826     assert(map()->memory()->Opcode() == Op_Parm, "");
 827     // Insert the memory aliasing node
 828     set_all_memory(reset_memory());
 829   }


 867 // The returned value represents the control for the slow path.
 868 // The return value is never 'top'; it is either a valid control
 869 // or NULL if it is obvious that the slow path can never be taken.
 870 // Also, if region and the slow control are not NULL, the slow edge
 871 // is appended to the region.
 872 Node* LibraryCallKit::generate_guard(Node* test, RegionNode* region, float true_prob) {
 873   if (stopped()) {
 874     // Already short circuited.
 875     return NULL;
 876   }
 877 
 878   // Build an if node and its projections.
 879   // If test is true we take the slow path, which we assume is uncommon.
 880   if (_gvn.type(test) == TypeInt::ZERO) {
 881     // The slow branch is never taken.  No need to build this guard.
 882     return NULL;
 883   }
 884 
 885   IfNode* iff = create_and_map_if(control(), test, true_prob, COUNT_UNKNOWN);
 886 
 887   Node* if_slow = _gvn.transform( new (C) IfTrueNode(iff) );
 888   if (if_slow == top()) {
 889     // The slow branch is never taken.  No need to build this guard.
 890     return NULL;
 891   }
 892 
 893   if (region != NULL)
 894     region->add_req(if_slow);
 895 
 896   Node* if_fast = _gvn.transform( new (C) IfFalseNode(iff) );
 897   set_control(if_fast);
 898 
 899   return if_slow;
 900 }
 901 
 902 inline Node* LibraryCallKit::generate_slow_guard(Node* test, RegionNode* region) {
 903   return generate_guard(test, region, PROB_UNLIKELY_MAG(3));
 904 }
 905 inline Node* LibraryCallKit::generate_fair_guard(Node* test, RegionNode* region) {
 906   return generate_guard(test, region, PROB_FAIR);
 907 }
 908 
 909 inline Node* LibraryCallKit::generate_negative_guard(Node* index, RegionNode* region,
 910                                                      Node* *pos_index) {
 911   if (stopped())
 912     return NULL;                // already stopped
 913   if (_gvn.type(index)->higher_equal(TypeInt::POS)) // [0,maxint]
 914     return NULL;                // index is already adequately typed
 915   Node* cmp_lt = _gvn.transform( new (C) CmpINode(index, intcon(0)) );
 916   Node* bol_lt = _gvn.transform( new (C) BoolNode(cmp_lt, BoolTest::lt) );
 917   Node* is_neg = generate_guard(bol_lt, region, PROB_MIN);
 918   if (is_neg != NULL && pos_index != NULL) {
 919     // Emulate effect of Parse::adjust_map_after_if.
 920     Node* ccast = new (C) CastIINode(index, TypeInt::POS);
 921     ccast->set_req(0, control());
 922     (*pos_index) = _gvn.transform(ccast);
 923   }
 924   return is_neg;
 925 }
 926 
 927 inline Node* LibraryCallKit::generate_nonpositive_guard(Node* index, bool never_negative,
 928                                                         Node* *pos_index) {
 929   if (stopped())
 930     return NULL;                // already stopped
 931   if (_gvn.type(index)->higher_equal(TypeInt::POS1)) // [1,maxint]
 932     return NULL;                // index is already adequately typed
 933   Node* cmp_le = _gvn.transform( new (C) CmpINode(index, intcon(0)) );
 934   BoolTest::mask le_or_eq = (never_negative ? BoolTest::eq : BoolTest::le);
 935   Node* bol_le = _gvn.transform( new (C) BoolNode(cmp_le, le_or_eq) );
 936   Node* is_notp = generate_guard(bol_le, NULL, PROB_MIN);
 937   if (is_notp != NULL && pos_index != NULL) {
 938     // Emulate effect of Parse::adjust_map_after_if.
 939     Node* ccast = new (C) CastIINode(index, TypeInt::POS1);
 940     ccast->set_req(0, control());
 941     (*pos_index) = _gvn.transform(ccast);
 942   }
 943   return is_notp;
 944 }
 945 
 946 // Make sure that 'position' is a valid limit index, in [0..length].
 947 // There are two equivalent plans for checking this:
 948 //   A. (offset + copyLength)  unsigned<=  arrayLength
 949 //   B. offset  <=  (arrayLength - copyLength)
 950 // We require that all of the values above, except for the sum and
 951 // difference, are already known to be non-negative.
 952 // Plan A is robust in the face of overflow, if offset and copyLength
 953 // are both hugely positive.
 954 //
 955 // Plan B is less direct and intuitive, but it does not overflow at
 956 // all, since the difference of two non-negatives is always
 957 // representable.  Whenever Java methods must perform the equivalent
 958 // check they generally use Plan B instead of Plan A.
 959 // For the moment we use Plan A.
 960 inline Node* LibraryCallKit::generate_limit_guard(Node* offset,
 961                                                   Node* subseq_length,
 962                                                   Node* array_length,
 963                                                   RegionNode* region) {
 964   if (stopped())
 965     return NULL;                // already stopped
 966   bool zero_offset = _gvn.type(offset) == TypeInt::ZERO;
 967   if (zero_offset && subseq_length->eqv_uncast(array_length))
 968     return NULL;                // common case of whole-array copy
 969   Node* last = subseq_length;
 970   if (!zero_offset)             // last += offset
 971     last = _gvn.transform( new (C) AddINode(last, offset));
 972   Node* cmp_lt = _gvn.transform( new (C) CmpUNode(array_length, last) );
 973   Node* bol_lt = _gvn.transform( new (C) BoolNode(cmp_lt, BoolTest::lt) );
 974   Node* is_over = generate_guard(bol_lt, region, PROB_MIN);
 975   return is_over;
 976 }
 977 
 978 
 979 //--------------------------generate_current_thread--------------------
 980 Node* LibraryCallKit::generate_current_thread(Node* &tls_output) {
 981   ciKlass*    thread_klass = env()->Thread_klass();
 982   const Type* thread_type  = TypeOopPtr::make_from_klass(thread_klass)->cast_to_ptr_type(TypePtr::NotNull);
 983   Node* thread = _gvn.transform(new (C) ThreadLocalNode());
 984   Node* p = basic_plus_adr(top()/*!oop*/, thread, in_bytes(JavaThread::threadObj_offset()));
 985   Node* threadObj = make_load(NULL, p, thread_type, T_OBJECT);
 986   tls_output = thread;
 987   return threadObj;
 988 }
 989 
 990 
 991 //------------------------------make_string_method_node------------------------
 992 // Helper method for String intrinsic functions. This version is called
 993 // with str1 and str2 pointing to String object nodes.


1134 
1135     Node* no_ctrl = NULL;
1136 
1137     // Get start addr of receiver
1138     Node* receiver_val    = load_String_value(no_ctrl, receiver);
1139     Node* receiver_offset = load_String_offset(no_ctrl, receiver);
1140     Node* receiver_start = array_element_address(receiver_val, receiver_offset, T_CHAR);
1141 
1142     // Get length of receiver
1143     Node* receiver_cnt  = load_String_length(no_ctrl, receiver);
1144 
1145     // Get start addr of argument
1146     Node* argument_val    = load_String_value(no_ctrl, argument);
1147     Node* argument_offset = load_String_offset(no_ctrl, argument);
1148     Node* argument_start = array_element_address(argument_val, argument_offset, T_CHAR);
1149 
1150     // Get length of argument
1151     Node* argument_cnt  = load_String_length(no_ctrl, argument);
1152 
1153     // Check for receiver count != argument count
1154     Node* cmp = _gvn.transform( new(C) CmpINode(receiver_cnt, argument_cnt) );
1155     Node* bol = _gvn.transform( new(C) BoolNode(cmp, BoolTest::ne) );
1156     Node* if_ne = generate_slow_guard(bol, NULL);
1157     if (if_ne != NULL) {
1158       phi->init_req(4, intcon(0));
1159       region->init_req(4, if_ne);
1160     }
1161 
1162     // Check for count == 0 is done by assembler code for StrEquals.
1163 
1164     if (!stopped()) {
1165       Node* equals = make_string_method_node(Op_StrEquals, receiver_start, receiver_cnt, argument_start, argument_cnt);
1166       phi->init_req(1, equals);
1167       region->init_req(1, control());
1168     }
1169   }
1170 
1171   // post merge
1172   set_control(_gvn.transform(region));
1173   record_for_igvn(region);
1174 
1175   set_result(_gvn.transform(phi));


1241 //     } // using "i += targetCount;" and an "else i++;" causes a jump to jump.
1242 //     i++;
1243 //   }
1244 //   return -1;
1245 // }
1246 
1247 //------------------------------string_indexOf------------------------
1248 Node* LibraryCallKit::string_indexOf(Node* string_object, ciTypeArray* target_array, jint targetOffset_i,
1249                                      jint cache_i, jint md2_i) {
1250 
1251   Node* no_ctrl  = NULL;
1252   float likely   = PROB_LIKELY(0.9);
1253   float unlikely = PROB_UNLIKELY(0.9);
1254 
1255   const int nargs = 0; // no arguments to push back for uncommon trap in predicate
1256 
1257   Node* source        = load_String_value(no_ctrl, string_object);
1258   Node* sourceOffset  = load_String_offset(no_ctrl, string_object);
1259   Node* sourceCount   = load_String_length(no_ctrl, string_object);
1260 
1261   Node* target = _gvn.transform( makecon(TypeOopPtr::make_from_constant(target_array, true)) );
1262   jint target_length = target_array->length();
1263   const TypeAry* target_array_type = TypeAry::make(TypeInt::CHAR, TypeInt::make(0, target_length, Type::WidenMin));
1264   const TypeAryPtr* target_type = TypeAryPtr::make(TypePtr::BotPTR, target_array_type, target_array->klass(), true, Type::OffsetBot);
1265 
1266   IdealKit kit(this, false, true);
1267 #define __ kit.
1268   Node* zero             = __ ConI(0);
1269   Node* one              = __ ConI(1);
1270   Node* cache            = __ ConI(cache_i);
1271   Node* md2              = __ ConI(md2_i);
1272   Node* lastChar         = __ ConI(target_array->char_at(target_length - 1));
1273   Node* targetCount      = __ ConI(target_length);
1274   Node* targetCountLess1 = __ ConI(target_length - 1);
1275   Node* targetOffset     = __ ConI(targetOffset_i);
1276   Node* sourceEnd        = __ SubI(__ AddI(sourceOffset, sourceCount), targetCountLess1);
1277 
1278   IdealVariable rtn(kit), i(kit), j(kit); __ declarations_done();
1279   Node* outer_loop = __ make_label(2 /* goto */);
1280   Node* return_    = __ make_label(1);
1281 


1348     Node*       result_phi = new (C) PhiNode(result_rgn, TypeInt::INT);
1349     Node* no_ctrl  = NULL;
1350 
1351     // Get start addr of source string
1352     Node* source = load_String_value(no_ctrl, receiver);
1353     Node* source_offset = load_String_offset(no_ctrl, receiver);
1354     Node* source_start = array_element_address(source, source_offset, T_CHAR);
1355 
1356     // Get length of source string
1357     Node* source_cnt  = load_String_length(no_ctrl, receiver);
1358 
1359     // Get start addr of substring
1360     Node* substr = load_String_value(no_ctrl, arg);
1361     Node* substr_offset = load_String_offset(no_ctrl, arg);
1362     Node* substr_start = array_element_address(substr, substr_offset, T_CHAR);
1363 
1364     // Get length of source string
1365     Node* substr_cnt  = load_String_length(no_ctrl, arg);
1366 
1367     // Check for substr count > string count
1368     Node* cmp = _gvn.transform( new(C) CmpINode(substr_cnt, source_cnt) );
1369     Node* bol = _gvn.transform( new(C) BoolNode(cmp, BoolTest::gt) );
1370     Node* if_gt = generate_slow_guard(bol, NULL);
1371     if (if_gt != NULL) {
1372       result_phi->init_req(2, intcon(-1));
1373       result_rgn->init_req(2, if_gt);
1374     }
1375 
1376     if (!stopped()) {
1377       // Check for substr count == 0
1378       cmp = _gvn.transform( new(C) CmpINode(substr_cnt, intcon(0)) );
1379       bol = _gvn.transform( new(C) BoolNode(cmp, BoolTest::eq) );
1380       Node* if_zero = generate_slow_guard(bol, NULL);
1381       if (if_zero != NULL) {
1382         result_phi->init_req(3, intcon(0));
1383         result_rgn->init_req(3, if_zero);
1384       }
1385     }
1386 
1387     if (!stopped()) {
1388       result = make_string_method_node(Op_StrIndexOf, source_start, source_cnt, substr_start, substr_cnt);
1389       result_phi->init_req(1, result);
1390       result_rgn->init_req(1, control());
1391     }
1392     set_control(_gvn.transform(result_rgn));
1393     record_for_igvn(result_rgn);
1394     result = _gvn.transform(result_phi);
1395 
1396   } else { // Use LibraryCallKit::string_indexOf
1397     // don't intrinsify if argument isn't a constant string.
1398     if (!arg->is_Con()) {
1399      return false;


1535     //   if (x <=  Math.PI / 2.0) return -fsin(x - Math.PI / 2.0);
1536     // }
1537     // return StrictMath.cos(x);
1538 
1539     // Actually, sticking in an 80-bit Intel value into C2 will be tough; it
1540     // requires a special machine instruction to load it.  Instead we'll try
1541     // the 'easy' case.  If we really need the extra range +/- PI/2 we'll
1542     // probably do the math inside the SIN encoding.
1543 
1544     // Make the merge point
1545     RegionNode* r = new (C) RegionNode(3);
1546     Node* phi = new (C) PhiNode(r, Type::DOUBLE);
1547 
1548     // Flatten arg so we need only 1 test
1549     Node *abs = _gvn.transform(new (C) AbsDNode(arg));
1550     // Node for PI/4 constant
1551     Node *pi4 = makecon(TypeD::make(pi_4));
1552     // Check PI/4 : abs(arg)
1553     Node *cmp = _gvn.transform(new (C) CmpDNode(pi4,abs));
1554     // Check: If PI/4 < abs(arg) then go slow
1555     Node *bol = _gvn.transform( new (C) BoolNode( cmp, BoolTest::lt ) );
1556     // Branch either way
1557     IfNode *iff = create_and_xform_if(control(),bol, PROB_STATIC_FREQUENT, COUNT_UNKNOWN);
1558     set_control(opt_iff(r,iff));
1559 
1560     // Set fast path result
1561     phi->init_req(2, n);
1562 
1563     // Slow path - non-blocking leaf call
1564     Node* call = NULL;
1565     switch (id) {
1566     case vmIntrinsics::_dsin:
1567       call = make_runtime_call(RC_LEAF, OptoRuntime::Math_D_D_Type(),
1568                                CAST_FROM_FN_PTR(address, SharedRuntime::dsin),
1569                                "Sin", NULL, arg, top());
1570       break;
1571     case vmIntrinsics::_dcos:
1572       call = make_runtime_call(RC_LEAF, OptoRuntime::Math_D_D_Type(),
1573                                CAST_FROM_FN_PTR(address, SharedRuntime::dcos),
1574                                "Cos", NULL, arg, top());
1575       break;


1600   //result=(result.isNaN())? funcAddr():result;
1601   // Check: If isNaN() by checking result!=result? then either trap
1602   // or go to runtime
1603   Node* cmpisnan = _gvn.transform(new (C) CmpDNode(result, result));
1604   // Build the boolean node
1605   Node* bolisnum = _gvn.transform(new (C) BoolNode(cmpisnan, BoolTest::eq));
1606 
1607   if (!too_many_traps(Deoptimization::Reason_intrinsic)) {
1608     { BuildCutout unless(this, bolisnum, PROB_STATIC_FREQUENT);
1609       // The pow or exp intrinsic returned a NaN, which requires a call
1610       // to the runtime.  Recompile with the runtime call.
1611       uncommon_trap(Deoptimization::Reason_intrinsic,
1612                     Deoptimization::Action_make_not_entrant);
1613     }
1614     set_result(result);
1615   } else {
1616     // If this inlining ever returned NaN in the past, we compile a call
1617     // to the runtime to properly handle corner cases
1618 
1619     IfNode* iff = create_and_xform_if(control(), bolisnum, PROB_STATIC_FREQUENT, COUNT_UNKNOWN);
1620     Node* if_slow = _gvn.transform( new (C) IfFalseNode(iff) );
1621     Node* if_fast = _gvn.transform( new (C) IfTrueNode(iff) );
1622 
1623     if (!if_slow->is_top()) {
1624       RegionNode* result_region = new (C) RegionNode(3);
1625       PhiNode*    result_val = new (C) PhiNode(result_region, Type::DOUBLE);
1626 
1627       result_region->init_req(1, if_fast);
1628       result_val->init_req(1, result);
1629 
1630       set_control(if_slow);
1631 
1632       const TypePtr* no_memory_effects = NULL;
1633       Node* rt = make_runtime_call(RC_LEAF, call_type, funcAddr, funcName,
1634                                    no_memory_effects,
1635                                    x, top(), y, y ? top() : NULL);
1636       Node* value = _gvn.transform(new (C) ProjNode(rt, TypeFunc::Parms+0));
1637 #ifdef ASSERT
1638       Node* value_top = _gvn.transform(new (C) ProjNode(rt, TypeFunc::Parms+1));
1639       assert(value_top == top(), "second value must be top");
1640 #endif
1641 


1687   Node* result = NULL;
1688 
1689   if (!too_many_traps(Deoptimization::Reason_intrinsic)) {
1690     // Short form: skip the fancy tests and just check for NaN result.
1691     result = _gvn.transform(new (C) PowDNode(C, control(), x, y));
1692   } else {
1693     // If this inlining ever returned NaN in the past, include all
1694     // checks + call to the runtime.
1695 
1696     // Set the merge point for If node with condition of (x <= 0.0)
1697     // There are four possible paths to region node and phi node
1698     RegionNode *r = new (C) RegionNode(4);
1699     Node *phi = new (C) PhiNode(r, Type::DOUBLE);
1700 
1701     // Build the first if node: if (x <= 0.0)
1702     // Node for 0 constant
1703     Node *zeronode = makecon(TypeD::ZERO);
1704     // Check x:0
1705     Node *cmp = _gvn.transform(new (C) CmpDNode(x, zeronode));
1706     // Check: If (x<=0) then go complex path
1707     Node *bol1 = _gvn.transform( new (C) BoolNode( cmp, BoolTest::le ) );
1708     // Branch either way
1709     IfNode *if1 = create_and_xform_if(control(),bol1, PROB_STATIC_INFREQUENT, COUNT_UNKNOWN);
1710     // Fast path taken; set region slot 3
1711     Node *fast_taken = _gvn.transform( new (C) IfFalseNode(if1) );
1712     r->init_req(3,fast_taken); // Capture fast-control
1713 
1714     // Fast path not-taken, i.e. slow path
1715     Node *complex_path = _gvn.transform( new (C) IfTrueNode(if1) );
1716 
1717     // Set fast path result
1718     Node *fast_result = _gvn.transform( new (C) PowDNode(C, control(), x, y) );
1719     phi->init_req(3, fast_result);
1720 
1721     // Complex path
1722     // Build the second if node (if y is long)
1723     // Node for (long)y
1724     Node *longy = _gvn.transform( new (C) ConvD2LNode(y));
1725     // Node for (double)((long) y)
1726     Node *doublelongy= _gvn.transform( new (C) ConvL2DNode(longy));
1727     // Check (double)((long) y) : y
1728     Node *cmplongy= _gvn.transform(new (C) CmpDNode(doublelongy, y));
1729     // Check if (y isn't long) then go to slow path
1730 
1731     Node *bol2 = _gvn.transform( new (C) BoolNode( cmplongy, BoolTest::ne ) );
1732     // Branch either way
1733     IfNode *if2 = create_and_xform_if(complex_path,bol2, PROB_STATIC_INFREQUENT, COUNT_UNKNOWN);
1734     Node* ylong_path = _gvn.transform( new (C) IfFalseNode(if2));
1735 
1736     Node *slow_path = _gvn.transform( new (C) IfTrueNode(if2) );
1737 
1738     // Calculate DPow(abs(x), y)*(1 & (long)y)
1739     // Node for constant 1
1740     Node *conone = longcon(1);
1741     // 1& (long)y
1742     Node *signnode= _gvn.transform( new (C) AndLNode(conone, longy) );
1743 
1744     // A huge number is always even. Detect a huge number by checking
1745     // if y + 1 == y and set integer to be tested for parity to 0.
1746     // Required for corner case:
1747     // (long)9.223372036854776E18 = max_jlong
1748     // (double)(long)9.223372036854776E18 = 9.223372036854776E18
1749     // max_jlong is odd but 9.223372036854776E18 is even
1750     Node* yplus1 = _gvn.transform( new (C) AddDNode(y, makecon(TypeD::make(1))));
1751     Node *cmpyplus1= _gvn.transform(new (C) CmpDNode(yplus1, y));
1752     Node *bolyplus1 = _gvn.transform( new (C) BoolNode( cmpyplus1, BoolTest::eq ) );
1753     Node* correctedsign = NULL;
1754     if (ConditionalMoveLimit != 0) {
1755       correctedsign = _gvn.transform( CMoveNode::make(C, NULL, bolyplus1, signnode, longcon(0), TypeLong::LONG));
1756     } else {
1757       IfNode *ifyplus1 = create_and_xform_if(ylong_path,bolyplus1, PROB_FAIR, COUNT_UNKNOWN);
1758       RegionNode *r = new (C) RegionNode(3);
1759       Node *phi = new (C) PhiNode(r, TypeLong::LONG);
1760       r->init_req(1, _gvn.transform( new (C) IfFalseNode(ifyplus1)));
1761       r->init_req(2, _gvn.transform( new (C) IfTrueNode(ifyplus1)));
1762       phi->init_req(1, signnode);
1763       phi->init_req(2, longcon(0));
1764       correctedsign = _gvn.transform(phi);
1765       ylong_path = _gvn.transform(r);
1766       record_for_igvn(r);
1767     }
1768 
1769     // zero node
1770     Node *conzero = longcon(0);
1771     // Check (1&(long)y)==0?
1772     Node *cmpeq1 = _gvn.transform(new (C) CmpLNode(correctedsign, conzero));
1773     // Check if (1&(long)y)!=0?, if so the result is negative
1774     Node *bol3 = _gvn.transform( new (C) BoolNode( cmpeq1, BoolTest::ne ) );
1775     // abs(x)
1776     Node *absx=_gvn.transform( new (C) AbsDNode(x));
1777     // abs(x)^y
1778     Node *absxpowy = _gvn.transform( new (C) PowDNode(C, control(), absx, y) );
1779     // -abs(x)^y
1780     Node *negabsxpowy = _gvn.transform(new (C) NegDNode (absxpowy));
1781     // (1&(long)y)==1?-DPow(abs(x), y):DPow(abs(x), y)
1782     Node *signresult = NULL;
1783     if (ConditionalMoveLimit != 0) {
1784       signresult = _gvn.transform( CMoveNode::make(C, NULL, bol3, absxpowy, negabsxpowy, Type::DOUBLE));
1785     } else {
1786       IfNode *ifyeven = create_and_xform_if(ylong_path,bol3, PROB_FAIR, COUNT_UNKNOWN);
1787       RegionNode *r = new (C) RegionNode(3);
1788       Node *phi = new (C) PhiNode(r, Type::DOUBLE);
1789       r->init_req(1, _gvn.transform( new (C) IfFalseNode(ifyeven)));
1790       r->init_req(2, _gvn.transform( new (C) IfTrueNode(ifyeven)));
1791       phi->init_req(1, absxpowy);
1792       phi->init_req(2, negabsxpowy);
1793       signresult = _gvn.transform(phi);
1794       ylong_path = _gvn.transform(r);
1795       record_for_igvn(r);
1796     }
1797     // Set complex path fast result
1798     r->init_req(2, ylong_path);
1799     phi->init_req(2, signresult);
1800 
1801     static const jlong nan_bits = CONST64(0x7ff8000000000000);
1802     Node *slow_result = makecon(TypeD::make(*(double*)&nan_bits)); // return NaN
1803     r->init_req(1,slow_path);
1804     phi->init_req(1,slow_result);
1805 
1806     // Post merge
1807     set_control(_gvn.transform(r));
1808     record_for_igvn(r);
1809     result = _gvn.transform(phi);
1810   }


1903 
1904   const TypeInt* txvalue = _gvn.type(xvalue)->isa_int();
1905   const TypeInt* tyvalue = _gvn.type(yvalue)->isa_int();
1906   if (txvalue == NULL || tyvalue == NULL)  return top();
1907   // This is not really necessary, but it is consistent with a
1908   // hypothetical MaxINode::Value method:
1909   int widen = MAX2(txvalue->_widen, tyvalue->_widen);
1910 
1911   // %%% This folding logic should (ideally) be in a different place.
1912   // Some should be inside IfNode, and there to be a more reliable
1913   // transformation of ?: style patterns into cmoves.  We also want
1914   // more powerful optimizations around cmove and min/max.
1915 
1916   // Try to find a dominating comparison of these guys.
1917   // It can simplify the index computation for Arrays.copyOf
1918   // and similar uses of System.arraycopy.
1919   // First, compute the normalized version of CmpI(x, y).
1920   int   cmp_op = Op_CmpI;
1921   Node* xkey = xvalue;
1922   Node* ykey = yvalue;
1923   Node* ideal_cmpxy = _gvn.transform( new(C) CmpINode(xkey, ykey) );
1924   if (ideal_cmpxy->is_Cmp()) {
1925     // E.g., if we have CmpI(length - offset, count),
1926     // it might idealize to CmpI(length, count + offset)
1927     cmp_op = ideal_cmpxy->Opcode();
1928     xkey = ideal_cmpxy->in(1);
1929     ykey = ideal_cmpxy->in(2);
1930   }
1931 
1932   // Start by locating any relevant comparisons.
1933   Node* start_from = (xkey->outcnt() < ykey->outcnt()) ? xkey : ykey;
1934   Node* cmpxy = NULL;
1935   Node* cmpyx = NULL;
1936   for (DUIterator_Fast kmax, k = start_from->fast_outs(kmax); k < kmax; k++) {
1937     Node* cmp = start_from->fast_out(k);
1938     if (cmp->outcnt() > 0 &&            // must have prior uses
1939         cmp->in(0) == NULL &&           // must be context-independent
1940         cmp->Opcode() == cmp_op) {      // right kind of compare
1941       if (cmp->in(1) == xkey && cmp->in(2) == ykey)  cmpxy = cmp;
1942       if (cmp->in(1) == ykey && cmp->in(2) == xkey)  cmpyx = cmp;
1943     }


1996     if (cmp == NULL)  continue;
1997     for (DUIterator_Fast jmax, j = cmp->fast_outs(jmax); j < jmax; j++) {
1998       Node* bol = cmp->fast_out(j);
1999       if (!bol->is_Bool())  continue;
2000       BoolTest::mask btest = bol->as_Bool()->_test._test;
2001       if (btest == BoolTest::eq || btest == BoolTest::ne)  continue;
2002       if (cmp->in(1) == ykey)   btest = BoolTest(btest).commute();
2003       if (bol->outcnt() > (best_bol == NULL ? 0 : best_bol->outcnt())) {
2004         best_bol   = bol->as_Bool();
2005         best_btest = btest;
2006       }
2007     }
2008   }
2009 
2010   Node* answer_if_true  = NULL;
2011   Node* answer_if_false = NULL;
2012   switch (best_btest) {
2013   default:
2014     if (cmpxy == NULL)
2015       cmpxy = ideal_cmpxy;
2016     best_bol = _gvn.transform( new(C) BoolNode(cmpxy, BoolTest::lt) );
2017     // and fall through:
2018   case BoolTest::lt:          // x < y
2019   case BoolTest::le:          // x <= y
2020     answer_if_true  = (want_max ? yvalue : xvalue);
2021     answer_if_false = (want_max ? xvalue : yvalue);
2022     break;
2023   case BoolTest::gt:          // x > y
2024   case BoolTest::ge:          // x >= y
2025     answer_if_true  = (want_max ? xvalue : yvalue);
2026     answer_if_false = (want_max ? yvalue : xvalue);
2027     break;
2028   }
2029 
2030   jint hi, lo;
2031   if (want_max) {
2032     // We can sharpen the minimum.
2033     hi = MAX2(txvalue->_hi, tyvalue->_hi);
2034     lo = MAX2(txvalue->_lo, tyvalue->_lo);
2035   } else {
2036     // We can sharpen the maximum.


2056   switch (id) {
2057   case vmIntrinsics::_min:
2058     result_val = _gvn.transform(new (C, 3) MinINode(x,y)); break;
2059   case vmIntrinsics::_max:
2060     result_val = _gvn.transform(new (C, 3) MaxINode(x,y)); break;
2061   default:
2062     ShouldNotReachHere();
2063   }
2064   */
2065 }
2066 
2067 inline int
2068 LibraryCallKit::classify_unsafe_addr(Node* &base, Node* &offset) {
2069   const TypePtr* base_type = TypePtr::NULL_PTR;
2070   if (base != NULL)  base_type = _gvn.type(base)->isa_ptr();
2071   if (base_type == NULL) {
2072     // Unknown type.
2073     return Type::AnyPtr;
2074   } else if (base_type == TypePtr::NULL_PTR) {
2075     // Since this is a NULL+long form, we have to switch to a rawptr.
2076     base   = _gvn.transform( new (C) CastX2PNode(offset) );
2077     offset = MakeConX(0);
2078     return Type::RawPtr;
2079   } else if (base_type->base() == Type::RawPtr) {
2080     return Type::RawPtr;
2081   } else if (base_type->isa_oopptr()) {
2082     // Base is never null => always a heap address.
2083     if (base_type->ptr() == TypePtr::NotNull) {
2084       return Type::OopPtr;
2085     }
2086     // Offset is small => always a heap address.
2087     const TypeX* offset_type = _gvn.type(offset)->isa_intptr_t();
2088     if (offset_type != NULL &&
2089         base_type->offset() == 0 &&     // (should always be?)
2090         offset_type->_lo >= 0 &&
2091         !MacroAssembler::needs_explicit_null_check(offset_type->_hi)) {
2092       return Type::OopPtr;
2093     }
2094     // Otherwise, it might either be oop+off or NULL+addr.
2095     return Type::AnyPtr;
2096   } else {


2450       p = ConvX2L(p);
2451       break;
2452     default:
2453       fatal(err_msg_res("unexpected type %d: %s", type, type2name(type)));
2454       break;
2455     }
2456     // The load node has the control of the preceding MemBarCPUOrder.  All
2457     // following nodes will have the control of the MemBarCPUOrder inserted at
2458     // the end of this method.  So, pushing the load onto the stack at a later
2459     // point is fine.
2460     set_result(p);
2461   } else {
2462     // place effect of store into memory
2463     switch (type) {
2464     case T_DOUBLE:
2465       val = dstore_rounding(val);
2466       break;
2467     case T_ADDRESS:
2468       // Repackage the long as a pointer.
2469       val = ConvL2X(val);
2470       val = _gvn.transform( new (C) CastX2PNode(val) );
2471       break;
2472     }
2473 
2474     if (type != T_OBJECT ) {
2475       (void) store_to_memory(control(), adr, val, type, adr_type, is_volatile);
2476     } else {
2477       // Possibly an oop being stored to Java heap or native memory
2478       if (!TypePtr::NULL_PTR->higher_equal(_gvn.type(heap_base_oop))) {
2479         // oop to Java heap.
2480         (void) store_oop_to_unknown(control(), heap_base_oop, adr, adr_type, val, type);
2481       } else {
2482         // We can't tell at compile time if we are storing in the Java heap or outside
2483         // of it. So we need to emit code to conditionally do the proper type of
2484         // store.
2485 
2486         IdealKit ideal(this);
2487 #define __ ideal.
2488         // QQQ who knows what probability is here??
2489         __ if_then(heap_base_oop, BoolTest::ne, null(), PROB_UNLIKELY(0.999)); {
2490           // Sync IdealKit and graphKit.


2758     } else
2759 #endif
2760     {
2761       if (kind == LS_xchg) {
2762         load_store = _gvn.transform(new (C) GetAndSetPNode(control(), mem, adr, newval, adr_type, value_type->is_oopptr()));
2763       } else {
2764         assert(kind == LS_cmpxchg, "wrong LoadStore operation");
2765         load_store = _gvn.transform(new (C) CompareAndSwapPNode(control(), mem, adr, newval, oldval));
2766       }
2767     }
2768     post_barrier(control(), load_store, base, adr, alias_idx, newval, T_OBJECT, true);
2769     break;
2770   default:
2771     fatal(err_msg_res("unexpected type %d: %s", type, type2name(type)));
2772     break;
2773   }
2774 
2775   // SCMemProjNodes represent the memory state of a LoadStore. Their
2776   // main role is to prevent LoadStore nodes from being optimized away
2777   // when their results aren't used.
2778   Node* proj = _gvn.transform( new (C) SCMemProjNode(load_store));
2779   set_memory(proj, alias_idx);
2780 
2781   // Add the trailing membar surrounding the access
2782   insert_mem_bar(Op_MemBarCPUOrder);
2783   insert_mem_bar(Op_MemBarAcquire);
2784 
2785 #ifdef _LP64
2786   if (type == T_OBJECT && adr->bottom_type()->is_ptr_to_narrowoop() && kind == LS_xchg) {
2787     load_store = _gvn.transform(new (C) DecodeNNode(load_store, load_store->get_ptr_type()));
2788   }
2789 #endif
2790 
2791   assert(type2size[load_store->bottom_type()->basic_type()] == type2size[rtype], "result type should match");
2792   set_result(load_store);
2793   return true;
2794 }
2795 
2796 //----------------------------inline_unsafe_ordered_store----------------------
2797 // public native void sun.misc.Unsafe.putOrderedObject(Object o, long offset, Object x);
2798 // public native void sun.misc.Unsafe.putOrderedInt(Object o, long offset, int x);


2993     no_int_result_path   = 1, // t == Thread.current() && !TLS._osthread._interrupted
2994     no_clear_result_path = 2, // t == Thread.current() &&  TLS._osthread._interrupted && !clear_int
2995     slow_result_path     = 3, // slow path: t.isInterrupted(clear_int)
2996     PATH_LIMIT
2997   };
2998 
2999   // Ensure that it's not possible to move the load of TLS._osthread._interrupted flag
3000   // out of the function.
3001   insert_mem_bar(Op_MemBarCPUOrder);
3002 
3003   RegionNode* result_rgn = new (C) RegionNode(PATH_LIMIT);
3004   PhiNode*    result_val = new (C) PhiNode(result_rgn, TypeInt::BOOL);
3005 
3006   RegionNode* slow_region = new (C) RegionNode(1);
3007   record_for_igvn(slow_region);
3008 
3009   // (a) Receiving thread must be the current thread.
3010   Node* rec_thr = argument(0);
3011   Node* tls_ptr = NULL;
3012   Node* cur_thr = generate_current_thread(tls_ptr);
3013   Node* cmp_thr = _gvn.transform( new (C) CmpPNode(cur_thr, rec_thr) );
3014   Node* bol_thr = _gvn.transform( new (C) BoolNode(cmp_thr, BoolTest::ne) );
3015 
3016   generate_slow_guard(bol_thr, slow_region);
3017 
3018   // (b) Interrupt bit on TLS must be false.
3019   Node* p = basic_plus_adr(top()/*!oop*/, tls_ptr, in_bytes(JavaThread::osthread_offset()));
3020   Node* osthread = make_load(NULL, p, TypeRawPtr::NOTNULL, T_ADDRESS);
3021   p = basic_plus_adr(top()/*!oop*/, osthread, in_bytes(OSThread::interrupted_offset()));
3022 
3023   // Set the control input on the field _interrupted read to prevent it floating up.
3024   Node* int_bit = make_load(control(), p, TypeInt::BOOL, T_INT);
3025   Node* cmp_bit = _gvn.transform( new (C) CmpINode(int_bit, intcon(0)) );
3026   Node* bol_bit = _gvn.transform( new (C) BoolNode(cmp_bit, BoolTest::ne) );
3027 
3028   IfNode* iff_bit = create_and_map_if(control(), bol_bit, PROB_UNLIKELY_MAG(3), COUNT_UNKNOWN);
3029 
3030   // First fast path:  if (!TLS._interrupted) return false;
3031   Node* false_bit = _gvn.transform( new (C) IfFalseNode(iff_bit) );
3032   result_rgn->init_req(no_int_result_path, false_bit);
3033   result_val->init_req(no_int_result_path, intcon(0));
3034 
3035   // drop through to next case
3036   set_control( _gvn.transform(new (C) IfTrueNode(iff_bit)) );
3037 
3038   // (c) Or, if interrupt bit is set and clear_int is false, use 2nd fast path.
3039   Node* clr_arg = argument(1);
3040   Node* cmp_arg = _gvn.transform( new (C) CmpINode(clr_arg, intcon(0)) );
3041   Node* bol_arg = _gvn.transform( new (C) BoolNode(cmp_arg, BoolTest::ne) );
3042   IfNode* iff_arg = create_and_map_if(control(), bol_arg, PROB_FAIR, COUNT_UNKNOWN);
3043 
3044   // Second fast path:  ... else if (!clear_int) return true;
3045   Node* false_arg = _gvn.transform( new (C) IfFalseNode(iff_arg) );
3046   result_rgn->init_req(no_clear_result_path, false_arg);
3047   result_val->init_req(no_clear_result_path, intcon(1));
3048 
3049   // drop through to next case
3050   set_control( _gvn.transform(new (C) IfTrueNode(iff_arg)) );
3051 
3052   // (d) Otherwise, go to the slow path.
3053   slow_region->add_req(control());
3054   set_control( _gvn.transform(slow_region) );
3055 
3056   if (stopped()) {
3057     // There is no slow path.
3058     result_rgn->init_req(slow_result_path, top());
3059     result_val->init_req(slow_result_path, top());
3060   } else {
3061     // non-virtual because it is a private non-static
3062     CallJavaNode* slow_call = generate_method_call(vmIntrinsics::_isInterrupted);
3063 
3064     Node* slow_val = set_results_for_java_call(slow_call);
3065     // this->control() comes from set_results_for_java_call
3066 
3067     Node* fast_io  = slow_call->in(TypeFunc::I_O);
3068     Node* fast_mem = slow_call->in(TypeFunc::Memory);
3069 
3070     // These two phis are pre-filled with copies of of the fast IO and Memory
3071     PhiNode* result_mem  = PhiNode::make(result_rgn, fast_mem, Type::MEMORY, TypePtr::BOTTOM);
3072     PhiNode* result_io   = PhiNode::make(result_rgn, fast_io,  Type::ABIO);
3073 
3074     result_rgn->init_req(slow_result_path, control());


3090 Node* LibraryCallKit::load_mirror_from_klass(Node* klass) {
3091   Node* p = basic_plus_adr(klass, in_bytes(Klass::java_mirror_offset()));
3092   return make_load(NULL, p, TypeInstPtr::MIRROR, T_OBJECT);
3093 }
3094 
3095 //-----------------------load_klass_from_mirror_common-------------------------
3096 // Given a java mirror (a java.lang.Class oop), load its corresponding klass oop.
3097 // Test the klass oop for null (signifying a primitive Class like Integer.TYPE),
3098 // and branch to the given path on the region.
3099 // If never_see_null, take an uncommon trap on null, so we can optimistically
3100 // compile for the non-null case.
3101 // If the region is NULL, force never_see_null = true.
3102 Node* LibraryCallKit::load_klass_from_mirror_common(Node* mirror,
3103                                                     bool never_see_null,
3104                                                     RegionNode* region,
3105                                                     int null_path,
3106                                                     int offset) {
3107   if (region == NULL)  never_see_null = true;
3108   Node* p = basic_plus_adr(mirror, offset);
3109   const TypeKlassPtr*  kls_type = TypeKlassPtr::OBJECT_OR_NULL;
3110   Node* kls = _gvn.transform( LoadKlassNode::make(_gvn, immutable_memory(), p, TypeRawPtr::BOTTOM, kls_type) );
3111   Node* null_ctl = top();
3112   kls = null_check_oop(kls, &null_ctl, never_see_null);
3113   if (region != NULL) {
3114     // Set region->in(null_path) if the mirror is a primitive (e.g, int.class).
3115     region->init_req(null_path, null_ctl);
3116   } else {
3117     assert(null_ctl == top(), "no loose ends");
3118   }
3119   return kls;
3120 }
3121 
3122 //--------------------(inline_native_Class_query helpers)---------------------
3123 // Use this for JVM_ACC_INTERFACE, JVM_ACC_IS_CLONEABLE, JVM_ACC_HAS_FINALIZER.
3124 // Fall through if (mods & mask) == bits, take the guard otherwise.
3125 Node* LibraryCallKit::generate_access_flags_guard(Node* kls, int modifier_mask, int modifier_bits, RegionNode* region) {
3126   // Branch around if the given klass has the given modifier bit set.
3127   // Like generate_guard, adds a new path onto the region.
3128   Node* modp = basic_plus_adr(kls, in_bytes(Klass::access_flags_offset()));
3129   Node* mods = make_load(NULL, modp, TypeInt::INT, T_INT);
3130   Node* mask = intcon(modifier_mask);
3131   Node* bits = intcon(modifier_bits);
3132   Node* mbit = _gvn.transform( new (C) AndINode(mods, mask) );
3133   Node* cmp  = _gvn.transform( new (C) CmpINode(mbit, bits) );
3134   Node* bol  = _gvn.transform( new (C) BoolNode(cmp, BoolTest::ne) );
3135   return generate_fair_guard(bol, region);
3136 }
3137 Node* LibraryCallKit::generate_interface_guard(Node* kls, RegionNode* region) {
3138   return generate_access_flags_guard(kls, JVM_ACC_INTERFACE, 0, region);
3139 }
3140 
3141 //-------------------------inline_native_Class_query-------------------
3142 bool LibraryCallKit::inline_native_Class_query(vmIntrinsics::ID id) {
3143   const Type* return_type = TypeInt::BOOL;
3144   Node* prim_return_value = top();  // what happens if it's a primitive class?
3145   bool never_see_null = !too_many_traps(Deoptimization::Reason_null_check);
3146   bool expect_prim = false;     // most of these guys expect to work on refs
3147 
3148   enum { _normal_path = 1, _prim_path = 2, PATH_LIMIT };
3149 
3150   Node* mirror = argument(0);
3151   Node* obj    = top();
3152 
3153   switch (id) {
3154   case vmIntrinsics::_isInstance:


3265 
3266   case vmIntrinsics::_isPrimitive:
3267     query_value = intcon(0); // "normal" path produces false
3268     break;
3269 
3270   case vmIntrinsics::_getSuperclass:
3271     // The rules here are somewhat unfortunate, but we can still do better
3272     // with random logic than with a JNI call.
3273     // Interfaces store null or Object as _super, but must report null.
3274     // Arrays store an intermediate super as _super, but must report Object.
3275     // Other types can report the actual _super.
3276     // (To verify this code sequence, check the asserts in JVM_IsInterface.)
3277     if (generate_interface_guard(kls, region) != NULL)
3278       // A guard was added.  If the guard is taken, it was an interface.
3279       phi->add_req(null());
3280     if (generate_array_guard(kls, region) != NULL)
3281       // A guard was added.  If the guard is taken, it was an array.
3282       phi->add_req(makecon(TypeInstPtr::make(env()->Object_klass()->java_mirror())));
3283     // If we fall through, it's a plain class.  Get its _super.
3284     p = basic_plus_adr(kls, in_bytes(Klass::super_offset()));
3285     kls = _gvn.transform( LoadKlassNode::make(_gvn, immutable_memory(), p, TypeRawPtr::BOTTOM, TypeKlassPtr::OBJECT_OR_NULL) );
3286     null_ctl = top();
3287     kls = null_check_oop(kls, &null_ctl);
3288     if (null_ctl != top()) {
3289       // If the guard is taken, Object.superClass is null (both klass and mirror).
3290       region->add_req(null_ctl);
3291       phi   ->add_req(null());
3292     }
3293     if (!stopped()) {
3294       query_value = load_mirror_from_klass(kls);
3295     }
3296     break;
3297 
3298   case vmIntrinsics::_getComponentType:
3299     if (generate_array_guard(kls, region) != NULL) {
3300       // Be sure to pin the oop load to the guard edge just created:
3301       Node* is_array_ctrl = region->in(region->req()-1);
3302       Node* cma = basic_plus_adr(kls, in_bytes(ArrayKlass::component_mirror_offset()));
3303       Node* cmo = make_load(is_array_ctrl, cma, TypeInstPtr::MIRROR, T_OBJECT);
3304       phi->add_req(cmo);
3305     }


3378     region->init_req(prim_path, null_ctl);
3379     if (stopped())  break;
3380     klasses[which_arg] = kls;
3381   }
3382 
3383   if (!stopped()) {
3384     // now we have two reference types, in klasses[0..1]
3385     Node* subk   = klasses[1];  // the argument to isAssignableFrom
3386     Node* superk = klasses[0];  // the receiver
3387     region->set_req(_both_ref_path, gen_subtype_check(subk, superk));
3388     // now we have a successful reference subtype check
3389     region->set_req(_ref_subtype_path, control());
3390   }
3391 
3392   // If both operands are primitive (both klasses null), then
3393   // we must return true when they are identical primitives.
3394   // It is convenient to test this after the first null klass check.
3395   set_control(region->in(_prim_0_path)); // go back to first null check
3396   if (!stopped()) {
3397     // Since superc is primitive, make a guard for the superc==subc case.
3398     Node* cmp_eq = _gvn.transform( new (C) CmpPNode(args[0], args[1]) );
3399     Node* bol_eq = _gvn.transform( new (C) BoolNode(cmp_eq, BoolTest::eq) );
3400     generate_guard(bol_eq, region, PROB_FAIR);
3401     if (region->req() == PATH_LIMIT+1) {
3402       // A guard was added.  If the added guard is taken, superc==subc.
3403       region->swap_edges(PATH_LIMIT, _prim_same_path);
3404       region->del_req(PATH_LIMIT);
3405     }
3406     region->set_req(_prim_0_path, control()); // Not equal after all.
3407   }
3408 
3409   // these are the only paths that produce 'true':
3410   phi->set_req(_prim_same_path,   intcon(1));
3411   phi->set_req(_ref_subtype_path, intcon(1));
3412 
3413   // pull together the cases:
3414   assert(region->req() == PATH_LIMIT, "sane region");
3415   for (uint i = 1; i < region->req(); i++) {
3416     Node* ctl = region->in(i);
3417     if (ctl == NULL || ctl == top()) {
3418       region->set_req(i, top());
3419       phi   ->set_req(i, top());


3444   Node* layout_val = get_layout_helper(kls, layout_con);
3445   if (layout_val == NULL) {
3446     bool query = (obj_array
3447                   ? Klass::layout_helper_is_objArray(layout_con)
3448                   : Klass::layout_helper_is_array(layout_con));
3449     if (query == not_array) {
3450       return NULL;                       // never a branch
3451     } else {                             // always a branch
3452       Node* always_branch = control();
3453       if (region != NULL)
3454         region->add_req(always_branch);
3455       set_control(top());
3456       return always_branch;
3457     }
3458   }
3459   // Now test the correct condition.
3460   jint  nval = (obj_array
3461                 ? ((jint)Klass::_lh_array_tag_type_value
3462                    <<    Klass::_lh_array_tag_shift)
3463                 : Klass::_lh_neutral_value);
3464   Node* cmp = _gvn.transform( new(C) CmpINode(layout_val, intcon(nval)) );
3465   BoolTest::mask btest = BoolTest::lt;  // correct for testing is_[obj]array
3466   // invert the test if we are looking for a non-array
3467   if (not_array)  btest = BoolTest(btest).negate();
3468   Node* bol = _gvn.transform( new(C) BoolNode(cmp, btest) );
3469   return generate_fair_guard(bol, region);
3470 }
3471 
3472 
3473 //-----------------------inline_native_newArray--------------------------
3474 // private static native Object java.lang.reflect.newArray(Class<?> componentType, int length);
3475 bool LibraryCallKit::inline_native_newArray() {
3476   Node* mirror    = argument(0);
3477   Node* count_val = argument(1);
3478 
3479   mirror = null_check(mirror);
3480   // If mirror or obj is dead, only null-path is taken.
3481   if (stopped())  return true;
3482 
3483   enum { _normal_path = 1, _slow_path = 2, PATH_LIMIT };
3484   RegionNode* result_reg = new(C) RegionNode(PATH_LIMIT);
3485   PhiNode*    result_val = new(C) PhiNode(result_reg,
3486                                           TypeInstPtr::NOTNULL);
3487   PhiNode*    result_io  = new(C) PhiNode(result_reg, Type::ABIO);
3488   PhiNode*    result_mem = new(C) PhiNode(result_reg, Type::MEMORY,


3508     result_reg->set_req(_slow_path, control());
3509     result_val->set_req(_slow_path, slow_result);
3510     result_io ->set_req(_slow_path, i_o());
3511     result_mem->set_req(_slow_path, reset_memory());
3512   }
3513 
3514   set_control(normal_ctl);
3515   if (!stopped()) {
3516     // Normal case:  The array type has been cached in the java.lang.Class.
3517     // The following call works fine even if the array type is polymorphic.
3518     // It could be a dynamic mix of int[], boolean[], Object[], etc.
3519     Node* obj = new_array(klass_node, count_val, 0);  // no arguments to push
3520     result_reg->init_req(_normal_path, control());
3521     result_val->init_req(_normal_path, obj);
3522     result_io ->init_req(_normal_path, i_o());
3523     result_mem->init_req(_normal_path, reset_memory());
3524   }
3525 
3526   // Return the combined state.
3527   set_i_o(        _gvn.transform(result_io)  );
3528   set_all_memory( _gvn.transform(result_mem) );
3529 
3530   C->set_has_split_ifs(true); // Has chance for split-if optimization
3531   set_result(result_reg, result_val);
3532   return true;
3533 }
3534 
3535 //----------------------inline_native_getLength--------------------------
3536 // public static native int java.lang.reflect.Array.getLength(Object array);
3537 bool LibraryCallKit::inline_native_getLength() {
3538   if (too_many_traps(Deoptimization::Reason_intrinsic))  return false;
3539 
3540   Node* array = null_check(argument(0));
3541   // If array is dead, only null-path is taken.
3542   if (stopped())  return true;
3543 
3544   // Deoptimize if it is a non-array.
3545   Node* non_array = generate_non_array_guard(load_object_klass(array), NULL);
3546 
3547   if (non_array != NULL) {
3548     PreserveJVMState pjvms(this);


3661 }
3662 
3663 
3664 //----------------------generate_virtual_guard---------------------------
3665 // Helper for hashCode and clone.  Peeks inside the vtable to avoid a call.
3666 Node* LibraryCallKit::generate_virtual_guard(Node* obj_klass,
3667                                              RegionNode* slow_region) {
3668   ciMethod* method = callee();
3669   int vtable_index = method->vtable_index();
3670   // Get the Method* out of the appropriate vtable entry.
3671   int entry_offset  = (InstanceKlass::vtable_start_offset() +
3672                      vtable_index*vtableEntry::size()) * wordSize +
3673                      vtableEntry::method_offset_in_bytes();
3674   Node* entry_addr  = basic_plus_adr(obj_klass, entry_offset);
3675   Node* target_call = make_load(NULL, entry_addr, TypePtr::NOTNULL, T_ADDRESS);
3676 
3677   // Compare the target method with the expected method (e.g., Object.hashCode).
3678   const TypePtr* native_call_addr = TypeMetadataPtr::make(method);
3679 
3680   Node* native_call = makecon(native_call_addr);
3681   Node* chk_native  = _gvn.transform( new(C) CmpPNode(target_call, native_call) );
3682   Node* test_native = _gvn.transform( new(C) BoolNode(chk_native, BoolTest::ne) );
3683 
3684   return generate_slow_guard(test_native, slow_region);
3685 }
3686 
3687 //-----------------------generate_method_call----------------------------
3688 // Use generate_method_call to make a slow-call to the real
3689 // method if the fast path fails.  An alternative would be to
3690 // use a stub like OptoRuntime::slow_arraycopy_Java.
3691 // This only works for expanding the current library call,
3692 // not another intrinsic.  (E.g., don't use this for making an
3693 // arraycopy call inside of the copyOf intrinsic.)
3694 CallJavaNode*
3695 LibraryCallKit::generate_method_call(vmIntrinsics::ID method_id, bool is_virtual, bool is_static) {
3696   // When compiling the intrinsic method itself, do not use this technique.
3697   guarantee(callee() != C->method(), "cannot make slow-call to self");
3698 
3699   ciMethod* method = callee();
3700   // ensure the JVMS we have will be correct for this call
3701   guarantee(method_id == method->intrinsic_id(), "must match");
3702 


3783   // paths which do not pass are accumulated in the slow_region.
3784   RegionNode* slow_region = new (C) RegionNode(1);
3785   record_for_igvn(slow_region);
3786 
3787   // If this is a virtual call, we generate a funny guard.  We pull out
3788   // the vtable entry corresponding to hashCode() from the target object.
3789   // If the target method which we are calling happens to be the native
3790   // Object hashCode() method, we pass the guard.  We do not need this
3791   // guard for non-virtual calls -- the caller is known to be the native
3792   // Object hashCode().
3793   if (is_virtual) {
3794     generate_virtual_guard(obj_klass, slow_region);
3795   }
3796 
3797   // Get the header out of the object, use LoadMarkNode when available
3798   Node* header_addr = basic_plus_adr(obj, oopDesc::mark_offset_in_bytes());
3799   Node* header = make_load(control(), header_addr, TypeX_X, TypeX_X->basic_type());
3800 
3801   // Test the header to see if it is unlocked.
3802   Node *lock_mask      = _gvn.MakeConX(markOopDesc::biased_lock_mask_in_place);
3803   Node *lmasked_header = _gvn.transform( new (C) AndXNode(header, lock_mask) );
3804   Node *unlocked_val   = _gvn.MakeConX(markOopDesc::unlocked_value);
3805   Node *chk_unlocked   = _gvn.transform( new (C) CmpXNode( lmasked_header, unlocked_val));
3806   Node *test_unlocked  = _gvn.transform( new (C) BoolNode( chk_unlocked, BoolTest::ne) );
3807 
3808   generate_slow_guard(test_unlocked, slow_region);
3809 
3810   // Get the hash value and check to see that it has been properly assigned.
3811   // We depend on hash_mask being at most 32 bits and avoid the use of
3812   // hash_mask_in_place because it could be larger than 32 bits in a 64-bit
3813   // vm: see markOop.hpp.
3814   Node *hash_mask      = _gvn.intcon(markOopDesc::hash_mask);
3815   Node *hash_shift     = _gvn.intcon(markOopDesc::hash_shift);
3816   Node *hshifted_header= _gvn.transform( new (C) URShiftXNode(header, hash_shift) );
3817   // This hack lets the hash bits live anywhere in the mark object now, as long
3818   // as the shift drops the relevant bits into the low 32 bits.  Note that
3819   // Java spec says that HashCode is an int so there's no point in capturing
3820   // an 'X'-sized hashcode (32 in 32-bit build or 64 in 64-bit build).
3821   hshifted_header      = ConvX2I(hshifted_header);
3822   Node *hash_val       = _gvn.transform( new (C) AndINode(hshifted_header, hash_mask) );
3823 
3824   Node *no_hash_val    = _gvn.intcon(markOopDesc::no_hash);
3825   Node *chk_assigned   = _gvn.transform( new (C) CmpINode( hash_val, no_hash_val));
3826   Node *test_assigned  = _gvn.transform( new (C) BoolNode( chk_assigned, BoolTest::eq) );
3827 
3828   generate_slow_guard(test_assigned, slow_region);
3829 
3830   Node* init_mem = reset_memory();
3831   // fill in the rest of the null path:
3832   result_io ->init_req(_null_path, i_o());
3833   result_mem->init_req(_null_path, init_mem);
3834 
3835   result_val->init_req(_fast_path, hash_val);
3836   result_reg->init_req(_fast_path, control());
3837   result_io ->init_req(_fast_path, i_o());
3838   result_mem->init_req(_fast_path, init_mem);
3839 
3840   // Generate code for the slow case.  We make a call to hashCode().
3841   set_control(_gvn.transform(slow_region));
3842   if (!stopped()) {
3843     // No need for PreserveJVMState, because we're using up the present state.
3844     set_all_memory(init_mem);
3845     vmIntrinsics::ID hashCode_id = is_static ? vmIntrinsics::_identityHashCode : vmIntrinsics::_hashCode;
3846     CallJavaNode* slow_call = generate_method_call(hashCode_id, is_virtual, is_static);
3847     Node* slow_result = set_results_for_java_call(slow_call);
3848     // this->control() comes from set_results_for_java_call
3849     result_reg->init_req(_slow_path, control());
3850     result_val->init_req(_slow_path, slow_result);
3851     result_io  ->set_req(_slow_path, i_o());
3852     result_mem ->set_req(_slow_path, reset_memory());
3853   }
3854 
3855   // Return the combined state.
3856   set_i_o(        _gvn.transform(result_io)  );
3857   set_all_memory( _gvn.transform(result_mem) );
3858 
3859   set_result(result_reg, result_val);
3860   return true;
3861 }
3862 
3863 //---------------------------inline_native_getClass----------------------------
3864 // public final native Class<?> java.lang.Object.getClass();
3865 //
3866 // Build special case code for calls to getClass on an object.
3867 bool LibraryCallKit::inline_native_getClass() {
3868   Node* obj = null_check_receiver();
3869   if (stopped())  return true;
3870   set_result(load_mirror_from_klass(load_object_klass(obj)));
3871   return true;
3872 }
3873 
3874 //-----------------inline_native_Reflection_getCallerClass---------------------
3875 // public static native Class<?> sun.reflect.Reflection.getCallerClass();
3876 //
3877 // In the presence of deep enough inlining, getCallerClass() becomes a no-op.


3965   case vmIntrinsics::_floatToRawIntBits:    result = new (C) MoveF2INode(arg);  break;
3966   case vmIntrinsics::_intBitsToFloat:       result = new (C) MoveI2FNode(arg);  break;
3967   case vmIntrinsics::_doubleToRawLongBits:  result = new (C) MoveD2LNode(arg);  break;
3968   case vmIntrinsics::_longBitsToDouble:     result = new (C) MoveL2DNode(arg);  break;
3969 
3970   case vmIntrinsics::_doubleToLongBits: {
3971     // two paths (plus control) merge in a wood
3972     RegionNode *r = new (C) RegionNode(3);
3973     Node *phi = new (C) PhiNode(r, TypeLong::LONG);
3974 
3975     Node *cmpisnan = _gvn.transform(new (C) CmpDNode(arg, arg));
3976     // Build the boolean node
3977     Node *bolisnan = _gvn.transform(new (C) BoolNode(cmpisnan, BoolTest::ne));
3978 
3979     // Branch either way.
3980     // NaN case is less traveled, which makes all the difference.
3981     IfNode *ifisnan = create_and_xform_if(control(), bolisnan, PROB_STATIC_FREQUENT, COUNT_UNKNOWN);
3982     Node *opt_isnan = _gvn.transform(ifisnan);
3983     assert( opt_isnan->is_If(), "Expect an IfNode");
3984     IfNode *opt_ifisnan = (IfNode*)opt_isnan;
3985     Node *iftrue = _gvn.transform( new (C) IfTrueNode(opt_ifisnan) );
3986 
3987     set_control(iftrue);
3988 
3989     static const jlong nan_bits = CONST64(0x7ff8000000000000);
3990     Node *slow_result = longcon(nan_bits); // return NaN
3991     phi->init_req(1, _gvn.transform( slow_result ));
3992     r->init_req(1, iftrue);
3993 
3994     // Else fall through
3995     Node *iffalse = _gvn.transform(new (C) IfFalseNode(opt_ifisnan));
3996     set_control(iffalse);
3997 
3998     phi->init_req(2, _gvn.transform(new (C) MoveD2LNode(arg)));
3999     r->init_req(2, iffalse);
4000 
4001     // Post merge
4002     set_control(_gvn.transform(r));
4003     record_for_igvn(r);
4004 
4005     C->set_has_split_ifs(true); // Has chance for split-if optimization
4006     result = phi;
4007     assert(result->bottom_type()->isa_long(), "must be");
4008     break;
4009   }
4010 
4011   case vmIntrinsics::_floatToIntBits: {
4012     // two paths (plus control) merge in a wood
4013     RegionNode *r = new (C) RegionNode(3);
4014     Node *phi = new (C) PhiNode(r, TypeInt::INT);
4015 
4016     Node *cmpisnan = _gvn.transform(new (C) CmpFNode(arg, arg));
4017     // Build the boolean node
4018     Node *bolisnan = _gvn.transform(new (C) BoolNode(cmpisnan, BoolTest::ne));
4019 
4020     // Branch either way.
4021     // NaN case is less traveled, which makes all the difference.
4022     IfNode *ifisnan = create_and_xform_if(control(), bolisnan, PROB_STATIC_FREQUENT, COUNT_UNKNOWN);
4023     Node *opt_isnan = _gvn.transform(ifisnan);
4024     assert( opt_isnan->is_If(), "Expect an IfNode");
4025     IfNode *opt_ifisnan = (IfNode*)opt_isnan;
4026     Node *iftrue = _gvn.transform( new (C) IfTrueNode(opt_ifisnan) );
4027 
4028     set_control(iftrue);
4029 
4030     static const jint nan_bits = 0x7fc00000;
4031     Node *slow_result = makecon(TypeInt::make(nan_bits)); // return NaN
4032     phi->init_req(1, _gvn.transform( slow_result ));
4033     r->init_req(1, iftrue);
4034 
4035     // Else fall through
4036     Node *iffalse = _gvn.transform(new (C) IfFalseNode(opt_ifisnan));
4037     set_control(iffalse);
4038 
4039     phi->init_req(2, _gvn.transform(new (C) MoveF2INode(arg)));
4040     r->init_req(2, iffalse);
4041 
4042     // Post merge
4043     set_control(_gvn.transform(r));
4044     record_for_igvn(r);
4045 
4046     C->set_has_split_ifs(true); // Has chance for split-if optimization


4135   // base_off:
4136   // 8  - 32-bit VM
4137   // 12 - 64-bit VM, compressed klass
4138   // 16 - 64-bit VM, normal klass
4139   if (base_off % BytesPerLong != 0) {
4140     assert(UseCompressedKlassPointers, "");
4141     if (is_array) {
4142       // Exclude length to copy by 8 bytes words.
4143       base_off += sizeof(int);
4144     } else {
4145       // Include klass to copy by 8 bytes words.
4146       base_off = instanceOopDesc::klass_offset_in_bytes();
4147     }
4148     assert(base_off % BytesPerLong == 0, "expect 8 bytes alignment");
4149   }
4150   src  = basic_plus_adr(src,  base_off);
4151   dest = basic_plus_adr(dest, base_off);
4152 
4153   // Compute the length also, if needed:
4154   Node* countx = size;
4155   countx = _gvn.transform( new (C) SubXNode(countx, MakeConX(base_off)) );
4156   countx = _gvn.transform( new (C) URShiftXNode(countx, intcon(LogBytesPerLong) ));
4157 
4158   const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
4159   bool disjoint_bases = true;
4160   generate_unchecked_arraycopy(raw_adr_type, T_LONG, disjoint_bases,
4161                                src, NULL, dest, NULL, countx,
4162                                /*dest_uninitialized*/true);
4163 
4164   // If necessary, emit some card marks afterwards.  (Non-arrays only.)
4165   if (card_mark) {
4166     assert(!is_array, "");
4167     // Put in store barrier for any and all oops we are sticking
4168     // into this object.  (We could avoid this if we could prove
4169     // that the object type contains no oop fields at all.)
4170     Node* no_particular_value = NULL;
4171     Node* no_particular_field = NULL;
4172     int raw_adr_idx = Compile::AliasIdxRaw;
4173     post_barrier(control(),
4174                  memory(raw_adr_type),
4175                  alloc_obj,
4176                  no_particular_field,


4340       result_reg->init_req(_instance_path, control());
4341       result_val->init_req(_instance_path, alloc_obj);
4342       result_i_o ->set_req(_instance_path, i_o());
4343       result_mem ->set_req(_instance_path, reset_memory());
4344     }
4345 
4346     // Generate code for the slow case.  We make a call to clone().
4347     set_control(_gvn.transform(slow_region));
4348     if (!stopped()) {
4349       PreserveJVMState pjvms(this);
4350       CallJavaNode* slow_call = generate_method_call(vmIntrinsics::_clone, is_virtual);
4351       Node* slow_result = set_results_for_java_call(slow_call);
4352       // this->control() comes from set_results_for_java_call
4353       result_reg->init_req(_slow_path, control());
4354       result_val->init_req(_slow_path, slow_result);
4355       result_i_o ->set_req(_slow_path, i_o());
4356       result_mem ->set_req(_slow_path, reset_memory());
4357     }
4358 
4359     // Return the combined state.
4360     set_control(    _gvn.transform(result_reg) );
4361     set_i_o(        _gvn.transform(result_i_o) );
4362     set_all_memory( _gvn.transform(result_mem) );
4363   } // original reexecute is set back here
4364 
4365   set_result(_gvn.transform(result_val));
4366   return true;
4367 }
4368 
4369 //------------------------------basictype2arraycopy----------------------------
4370 address LibraryCallKit::basictype2arraycopy(BasicType t,
4371                                             Node* src_offset,
4372                                             Node* dest_offset,
4373                                             bool disjoint_bases,
4374                                             const char* &name,
4375                                             bool dest_uninitialized) {
4376   const TypeInt* src_offset_inttype  = gvn().find_int_type(src_offset);;
4377   const TypeInt* dest_offset_inttype = gvn().find_int_type(dest_offset);;
4378 
4379   bool aligned = false;
4380   bool disjoint = disjoint_bases;
4381 
4382   // if the offsets are the same, we can treat the memory regions as


4667         // paths have stub or runtime calls as raw memory barriers.
4668         InitializeNode* init = insert_mem_bar_volatile(Op_Initialize,
4669                                                        Compile::AliasIdxRaw,
4670                                                        top())->as_Initialize();
4671         init->set_complete(&_gvn);  // (there is no corresponding AllocateNode)
4672       }
4673     }
4674 
4675     // Present the results of the fast call.
4676     result_region->init_req(zero_path, control());
4677     result_i_o   ->init_req(zero_path, i_o());
4678     result_memory->init_req(zero_path, memory(adr_type));
4679   }
4680 
4681   if (!stopped() && dest_uninitialized) {
4682     // We have to initialize the *uncopied* part of the array to zero.
4683     // The copy destination is the slice dest[off..off+len].  The other slices
4684     // are dest_head = dest[0..off] and dest_tail = dest[off+len..dest.length].
4685     Node* dest_size   = alloc->in(AllocateNode::AllocSize);
4686     Node* dest_length = alloc->in(AllocateNode::ALength);
4687     Node* dest_tail   = _gvn.transform( new(C) AddINode(dest_offset,
4688                                                           copy_length) );
4689 
4690     // If there is a head section that needs zeroing, do it now.
4691     if (find_int_con(dest_offset, -1) != 0) {
4692       generate_clear_array(adr_type, dest, basic_elem_type,
4693                            intcon(0), dest_offset,
4694                            NULL);
4695     }
4696 
4697     // Next, perform a dynamic check on the tail length.
4698     // It is often zero, and we can win big if we prove this.
4699     // There are two wins:  Avoid generating the ClearArray
4700     // with its attendant messy index arithmetic, and upgrade
4701     // the copy to a more hardware-friendly word size of 64 bits.
4702     Node* tail_ctl = NULL;
4703     if (!stopped() && !dest_tail->eqv_uncast(dest_length)) {
4704       Node* cmp_lt   = _gvn.transform( new(C) CmpINode(dest_tail, dest_length) );
4705       Node* bol_lt   = _gvn.transform( new(C) BoolNode(cmp_lt, BoolTest::lt) );
4706       tail_ctl = generate_slow_guard(bol_lt, NULL);
4707       assert(tail_ctl != NULL || !stopped(), "must be an outcome");
4708     }
4709 
4710     // At this point, let's assume there is no tail.
4711     if (!stopped() && alloc != NULL && basic_elem_type != T_OBJECT) {
4712       // There is no tail.  Try an upgrade to a 64-bit copy.
4713       bool didit = false;
4714       { PreserveJVMState pjvms(this);
4715         didit = generate_block_arraycopy(adr_type, basic_elem_type, alloc,
4716                                          src, src_offset, dest, dest_offset,
4717                                          dest_size, dest_uninitialized);
4718         if (didit) {
4719           // Present the results of the block-copying fast call.
4720           result_region->init_req(bcopy_path, control());
4721           result_i_o   ->init_req(bcopy_path, i_o());
4722           result_memory->init_req(bcopy_path, memory(adr_type));
4723         }
4724       }
4725       if (didit)


4728 
4729     // Clear the tail, if any.
4730     if (tail_ctl != NULL) {
4731       Node* notail_ctl = stopped() ? NULL : control();
4732       set_control(tail_ctl);
4733       if (notail_ctl == NULL) {
4734         generate_clear_array(adr_type, dest, basic_elem_type,
4735                              dest_tail, NULL,
4736                              dest_size);
4737       } else {
4738         // Make a local merge.
4739         Node* done_ctl = new(C) RegionNode(3);
4740         Node* done_mem = new(C) PhiNode(done_ctl, Type::MEMORY, adr_type);
4741         done_ctl->init_req(1, notail_ctl);
4742         done_mem->init_req(1, memory(adr_type));
4743         generate_clear_array(adr_type, dest, basic_elem_type,
4744                              dest_tail, NULL,
4745                              dest_size);
4746         done_ctl->init_req(2, control());
4747         done_mem->init_req(2, memory(adr_type));
4748         set_control( _gvn.transform(done_ctl) );
4749         set_memory(  _gvn.transform(done_mem), adr_type );
4750       }
4751     }
4752   }
4753 
4754   BasicType copy_type = basic_elem_type;
4755   assert(basic_elem_type != T_ARRAY, "caller must fix this");
4756   if (!stopped() && copy_type == T_OBJECT) {
4757     // If src and dest have compatible element types, we can copy bits.
4758     // Types S[] and D[] are compatible if D is a supertype of S.
4759     //
4760     // If they are not, we will use checked_oop_disjoint_arraycopy,
4761     // which performs a fast optimistic per-oop check, and backs off
4762     // further to JVM_ArrayCopy on the first per-oop check that fails.
4763     // (Actually, we don't move raw bits only; the GC requires card marks.)
4764 
4765     // Get the Klass* for both src and dest
4766     Node* src_klass  = load_object_klass(src);
4767     Node* dest_klass = load_object_klass(dest);
4768 


4815                                  src, src_offset, dest, dest_offset,
4816                                  ConvI2X(copy_length), dest_uninitialized);
4817 
4818     // Present the results of the fast call.
4819     result_region->init_req(fast_path, control());
4820     result_i_o   ->init_req(fast_path, i_o());
4821     result_memory->init_req(fast_path, memory(adr_type));
4822   }
4823 
4824   // Here are all the slow paths up to this point, in one bundle:
4825   slow_control = top();
4826   if (slow_region != NULL)
4827     slow_control = _gvn.transform(slow_region);
4828   DEBUG_ONLY(slow_region = (RegionNode*)badAddress);
4829 
4830   set_control(checked_control);
4831   if (!stopped()) {
4832     // Clean up after the checked call.
4833     // The returned value is either 0 or -1^K,
4834     // where K = number of partially transferred array elements.
4835     Node* cmp = _gvn.transform( new(C) CmpINode(checked_value, intcon(0)) );
4836     Node* bol = _gvn.transform( new(C) BoolNode(cmp, BoolTest::eq) );
4837     IfNode* iff = create_and_map_if(control(), bol, PROB_MAX, COUNT_UNKNOWN);
4838 
4839     // If it is 0, we are done, so transfer to the end.
4840     Node* checks_done = _gvn.transform( new(C) IfTrueNode(iff) );
4841     result_region->init_req(checked_path, checks_done);
4842     result_i_o   ->init_req(checked_path, checked_i_o);
4843     result_memory->init_req(checked_path, checked_mem);
4844 
4845     // If it is not zero, merge into the slow call.
4846     set_control( _gvn.transform( new(C) IfFalseNode(iff) ));
4847     RegionNode* slow_reg2 = new(C) RegionNode(3);
4848     PhiNode*    slow_i_o2 = new(C) PhiNode(slow_reg2, Type::ABIO);
4849     PhiNode*    slow_mem2 = new(C) PhiNode(slow_reg2, Type::MEMORY, adr_type);
4850     record_for_igvn(slow_reg2);
4851     slow_reg2  ->init_req(1, slow_control);
4852     slow_i_o2  ->init_req(1, slow_i_o);
4853     slow_mem2  ->init_req(1, slow_mem);
4854     slow_reg2  ->init_req(2, control());
4855     slow_i_o2  ->init_req(2, checked_i_o);
4856     slow_mem2  ->init_req(2, checked_mem);
4857 
4858     slow_control = _gvn.transform(slow_reg2);
4859     slow_i_o     = _gvn.transform(slow_i_o2);
4860     slow_mem     = _gvn.transform(slow_mem2);
4861 
4862     if (alloc != NULL) {
4863       // We'll restart from the very beginning, after zeroing the whole thing.
4864       // This can cause double writes, but that's OK since dest is brand new.
4865       // So we ignore the low 31 bits of the value returned from the stub.
4866     } else {
4867       // We must continue the copy exactly where it failed, or else
4868       // another thread might see the wrong number of writes to dest.
4869       Node* checked_offset = _gvn.transform( new(C) XorINode(checked_value, intcon(-1)) );
4870       Node* slow_offset    = new(C) PhiNode(slow_reg2, TypeInt::INT);
4871       slow_offset->init_req(1, intcon(0));
4872       slow_offset->init_req(2, checked_offset);
4873       slow_offset  = _gvn.transform(slow_offset);
4874 
4875       // Adjust the arguments by the conditionally incoming offset.
4876       Node* src_off_plus  = _gvn.transform( new(C) AddINode(src_offset,  slow_offset) );
4877       Node* dest_off_plus = _gvn.transform( new(C) AddINode(dest_offset, slow_offset) );
4878       Node* length_minus  = _gvn.transform( new(C) SubINode(copy_length, slow_offset) );
4879 
4880       // Tweak the node variables to adjust the code produced below:
4881       src_offset  = src_off_plus;
4882       dest_offset = dest_off_plus;
4883       copy_length = length_minus;
4884     }
4885   }
4886 
4887   set_control(slow_control);
4888   if (!stopped()) {
4889     // Generate the slow path, if needed.
4890     PreserveJVMState pjvms(this);   // replace_in_map may trash the map
4891 
4892     set_memory(slow_mem, adr_type);
4893     set_i_o(slow_i_o);
4894 
4895     if (dest_uninitialized) {
4896       generate_clear_array(adr_type, dest, basic_elem_type,
4897                            intcon(0), NULL,
4898                            alloc->in(AllocateNode::AllocSize));
4899     }
4900 
4901     generate_slow_arraycopy(adr_type,
4902                             src, src_offset, dest, dest_offset,
4903                             copy_length, /*dest_uninitialized*/false);
4904 
4905     result_region->init_req(slow_call_path, control());
4906     result_i_o   ->init_req(slow_call_path, i_o());
4907     result_memory->init_req(slow_call_path, memory(adr_type));
4908   }
4909 
4910   // Remove unused edges.
4911   for (uint i = 1; i < result_region->req(); i++) {
4912     if (result_region->in(i) == NULL)
4913       result_region->init_req(i, top());
4914   }
4915 
4916   // Finished; return the combined state.
4917   set_control( _gvn.transform(result_region) );
4918   set_i_o(     _gvn.transform(result_i_o)    );
4919   set_memory(  _gvn.transform(result_memory), adr_type );
4920 
4921   // The memory edges above are precise in order to model effects around
4922   // array copies accurately to allow value numbering of field loads around
4923   // arraycopy.  Such field loads, both before and after, are common in Java
4924   // collections and similar classes involving header/array data structures.
4925   //
4926   // But with low number of register or when some registers are used or killed
4927   // by arraycopy calls it causes registers spilling on stack. See 6544710.
4928   // The next memory barrier is added to avoid it. If the arraycopy can be
4929   // optimized away (which it can, sometimes) then we can manually remove
4930   // the membar also.
4931   //
4932   // Do not let reads from the cloned object float above the arraycopy.
4933   if (alloc != NULL) {
4934     // Do not let stores that initialize this object be reordered with
4935     // a subsequent store that would make this object accessible by
4936     // other threads.
4937     // Record what AllocateNode this StoreStore protects so that


5079     end_con = round_to(abase + ((slice_idx_con + slice_len_con) << scale),
5080                        BytesPerLong);
5081   }
5082 
5083   if (start_con >= 0 && end_con >= 0) {
5084     // Constant start and end.  Simple.
5085     mem = ClearArrayNode::clear_memory(control(), mem, dest,
5086                                        start_con, end_con, &_gvn);
5087   } else if (start_con >= 0 && dest_size != top()) {
5088     // Constant start, pre-rounded end after the tail of the array.
5089     Node* end = dest_size;
5090     mem = ClearArrayNode::clear_memory(control(), mem, dest,
5091                                        start_con, end, &_gvn);
5092   } else if (start_con >= 0 && slice_len != top()) {
5093     // Constant start, non-constant end.  End needs rounding up.
5094     // End offset = round_up(abase + ((slice_idx_con + slice_len) << scale), 8)
5095     intptr_t end_base  = abase + (slice_idx_con << scale);
5096     int      end_round = (-1 << scale) & (BytesPerLong  - 1);
5097     Node*    end       = ConvI2X(slice_len);
5098     if (scale != 0)
5099       end = _gvn.transform( new(C) LShiftXNode(end, intcon(scale) ));
5100     end_base += end_round;
5101     end = _gvn.transform( new(C) AddXNode(end, MakeConX(end_base)) );
5102     end = _gvn.transform( new(C) AndXNode(end, MakeConX(~end_round)) );
5103     mem = ClearArrayNode::clear_memory(control(), mem, dest,
5104                                        start_con, end, &_gvn);
5105   } else if (start_con < 0 && dest_size != top()) {
5106     // Non-constant start, pre-rounded end after the tail of the array.
5107     // This is almost certainly a "round-to-end" operation.
5108     Node* start = slice_idx;
5109     start = ConvI2X(start);
5110     if (scale != 0)
5111       start = _gvn.transform( new(C) LShiftXNode( start, intcon(scale) ));
5112     start = _gvn.transform( new(C) AddXNode(start, MakeConX(abase)) );
5113     if ((bump_bit | clear_low) != 0) {
5114       int to_clear = (bump_bit | clear_low);
5115       // Align up mod 8, then store a jint zero unconditionally
5116       // just before the mod-8 boundary.
5117       if (((abase + bump_bit) & ~to_clear) - bump_bit
5118           < arrayOopDesc::length_offset_in_bytes() + BytesPerInt) {
5119         bump_bit = 0;
5120         assert((abase & to_clear) == 0, "array base must be long-aligned");
5121       } else {
5122         // Bump 'start' up to (or past) the next jint boundary:
5123         start = _gvn.transform( new(C) AddXNode(start, MakeConX(bump_bit)) );
5124         assert((abase & clear_low) == 0, "array base must be int-aligned");
5125       }
5126       // Round bumped 'start' down to jlong boundary in body of array.
5127       start = _gvn.transform( new(C) AndXNode(start, MakeConX(~to_clear)) );
5128       if (bump_bit != 0) {
5129         // Store a zero to the immediately preceding jint:
5130         Node* x1 = _gvn.transform( new(C) AddXNode(start, MakeConX(-bump_bit)) );
5131         Node* p1 = basic_plus_adr(dest, x1);
5132         mem = StoreNode::make(_gvn, control(), mem, p1, adr_type, intcon(0), T_INT);
5133         mem = _gvn.transform(mem);
5134       }
5135     }
5136     Node* end = dest_size; // pre-rounded
5137     mem = ClearArrayNode::clear_memory(control(), mem, dest,
5138                                        start, end, &_gvn);
5139   } else {
5140     // Non-constant start, unrounded non-constant end.
5141     // (Nobody zeroes a random midsection of an array using this routine.)
5142     ShouldNotReachHere();       // fix caller
5143   }
5144 
5145   // Done.
5146   set_memory(mem, adr_type);
5147 }
5148 
5149 
5150 bool


5177     // This is a common case, since abase can be odd mod 8.
5178     if (((src_off | dest_off) & (BytesPerLong-1)) == BytesPerInt &&
5179         ((src_off ^ dest_off) & (BytesPerLong-1)) == 0) {
5180       Node* sptr = basic_plus_adr(src,  src_off);
5181       Node* dptr = basic_plus_adr(dest, dest_off);
5182       Node* sval = make_load(control(), sptr, TypeInt::INT, T_INT, adr_type);
5183       store_to_memory(control(), dptr, sval, T_INT, adr_type);
5184       src_off += BytesPerInt;
5185       dest_off += BytesPerInt;
5186     } else {
5187       return false;
5188     }
5189   }
5190   assert(src_off % BytesPerLong == 0, "");
5191   assert(dest_off % BytesPerLong == 0, "");
5192 
5193   // Do this copy by giant steps.
5194   Node* sptr  = basic_plus_adr(src,  src_off);
5195   Node* dptr  = basic_plus_adr(dest, dest_off);
5196   Node* countx = dest_size;
5197   countx = _gvn.transform( new (C) SubXNode(countx, MakeConX(dest_off)) );
5198   countx = _gvn.transform( new (C) URShiftXNode(countx, intcon(LogBytesPerLong)) );
5199 
5200   bool disjoint_bases = true;   // since alloc != NULL
5201   generate_unchecked_arraycopy(adr_type, T_LONG, disjoint_bases,
5202                                sptr, NULL, dptr, NULL, countx, dest_uninitialized);
5203 
5204   return true;
5205 }
5206 
5207 
5208 // Helper function; generates code for the slow case.
5209 // We make a call to a runtime method which emulates the native method,
5210 // but without the native wrapper overhead.
5211 void
5212 LibraryCallKit::generate_slow_arraycopy(const TypePtr* adr_type,
5213                                         Node* src,  Node* src_offset,
5214                                         Node* dest, Node* dest_offset,
5215                                         Node* copy_length, bool dest_uninitialized) {
5216   assert(!dest_uninitialized, "Invariant");
5217   Node* call = make_runtime_call(RC_NO_LEAF | RC_UNCOMMON,
5218                                  OptoRuntime::slow_arraycopy_Type(),


5343   // Figure out the size and type of the elements we will be copying.
5344   BasicType src_elem = src_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
5345   BasicType dst_elem = dst_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
5346   if (src_elem != T_CHAR || dst_elem != T_BYTE) {
5347     return false;
5348   }
5349   Node* src_start = array_element_address(src, src_offset, src_elem);
5350   Node* dst_start = array_element_address(dst, dst_offset, dst_elem);
5351   // 'src_start' points to src array + scaled offset
5352   // 'dst_start' points to dst array + scaled offset
5353 
5354   const TypeAryPtr* mtype = TypeAryPtr::BYTES;
5355   Node* enc = new (C) EncodeISOArrayNode(control(), memory(mtype), src_start, dst_start, length);
5356   enc = _gvn.transform(enc);
5357   Node* res_mem = _gvn.transform(new (C) SCMemProjNode(enc));
5358   set_memory(res_mem, mtype);
5359   set_result(enc);
5360   return true;
5361 }
5362 















































































































5363 //----------------------------inline_reference_get----------------------------
5364 // public T java.lang.ref.Reference.get();
5365 bool LibraryCallKit::inline_reference_get() {
5366   const int referent_offset = java_lang_ref_Reference::referent_offset;
5367   guarantee(referent_offset > 0, "should have already been set");
5368 
5369   // Get the argument:
5370   Node* reference_obj = null_check_receiver();
5371   if (stopped()) return true;
5372 
5373   Node* adr = basic_plus_adr(reference_obj, reference_obj, referent_offset);
5374 
5375   ciInstanceKlass* klass = env()->Object_klass();
5376   const TypeOopPtr* object_type = TypeOopPtr::make_from_klass(klass);
5377 
5378   Node* no_ctrl = NULL;
5379   Node* result = make_load(no_ctrl, adr, object_type, T_OBJECT);
5380 
5381   // Use the pre-barrier to record the value in the referent field
5382   pre_barrier(false /* do_load */,




 274                                    Node* dest, Node* dest_offset,
 275                                    Node* copy_length, bool dest_uninitialized);
 276   void generate_unchecked_arraycopy(const TypePtr* adr_type,
 277                                     BasicType basic_elem_type,
 278                                     bool disjoint_bases,
 279                                     Node* src,  Node* src_offset,
 280                                     Node* dest, Node* dest_offset,
 281                                     Node* copy_length, bool dest_uninitialized);
 282   typedef enum { LS_xadd, LS_xchg, LS_cmpxchg } LoadStoreKind;
 283   bool inline_unsafe_load_store(BasicType type,  LoadStoreKind kind);
 284   bool inline_unsafe_ordered_store(BasicType type);
 285   bool inline_unsafe_fence(vmIntrinsics::ID id);
 286   bool inline_fp_conversions(vmIntrinsics::ID id);
 287   bool inline_number_methods(vmIntrinsics::ID id);
 288   bool inline_reference_get();
 289   bool inline_aescrypt_Block(vmIntrinsics::ID id);
 290   bool inline_cipherBlockChaining_AESCrypt(vmIntrinsics::ID id);
 291   Node* inline_cipherBlockChaining_AESCrypt_predicate(bool decrypting);
 292   Node* get_key_start_from_aescrypt_object(Node* aescrypt_object);
 293   bool inline_encodeISOArray();
 294   bool inline_updateCRC32();
 295   bool inline_updateBytesCRC32();
 296   bool inline_updateByteBufferCRC32();
 297 };
 298 
 299 
 300 //---------------------------make_vm_intrinsic----------------------------
 301 CallGenerator* Compile::make_vm_intrinsic(ciMethod* m, bool is_virtual) {
 302   vmIntrinsics::ID id = m->intrinsic_id();
 303   assert(id != vmIntrinsics::_none, "must be a VM intrinsic");
 304 
 305   if (DisableIntrinsic[0] != '\0'
 306       && strstr(DisableIntrinsic, vmIntrinsics::name_at(id)) != NULL) {
 307     // disabled by a user request on the command line:
 308     // example: -XX:DisableIntrinsic=_hashCode,_getClass
 309     return NULL;
 310   }
 311 
 312   if (!m->is_loaded()) {
 313     // do not attempt to inline unloaded methods
 314     return NULL;
 315   }
 316 


 474     if (!UseCompressedOops && !Matcher::match_rule_supported(Op_GetAndSetP)) return NULL;
 475     if (UseCompressedOops && !Matcher::match_rule_supported(Op_GetAndSetN)) return NULL;
 476     break;
 477 #else
 478     if (!Matcher::match_rule_supported(Op_GetAndSetP)) return NULL;
 479     break;
 480 #endif
 481 
 482   case vmIntrinsics::_aescrypt_encryptBlock:
 483   case vmIntrinsics::_aescrypt_decryptBlock:
 484     if (!UseAESIntrinsics) return NULL;
 485     break;
 486 
 487   case vmIntrinsics::_cipherBlockChaining_encryptAESCrypt:
 488   case vmIntrinsics::_cipherBlockChaining_decryptAESCrypt:
 489     if (!UseAESIntrinsics) return NULL;
 490     // these two require the predicated logic
 491     is_predicted = true;
 492     break;
 493 
 494   case vmIntrinsics::_updateCRC32:
 495   case vmIntrinsics::_updateBytesCRC32:
 496   case vmIntrinsics::_updateByteBufferCRC32:
 497     if (!UseCRC32Intrinsics) return NULL;
 498     break;
 499 
 500  default:
 501     assert(id <= vmIntrinsics::LAST_COMPILER_INLINE, "caller responsibility");
 502     assert(id != vmIntrinsics::_Object_init && id != vmIntrinsics::_invoke, "enum out of order?");
 503     break;
 504   }
 505 
 506   // -XX:-InlineClassNatives disables natives from the Class class.
 507   // The flag applies to all reflective calls, notably Array.newArray
 508   // (visible to Java programmers as Array.newInstance).
 509   if (m->holder()->name() == ciSymbol::java_lang_Class() ||
 510       m->holder()->name() == ciSymbol::java_lang_reflect_Array()) {
 511     if (!InlineClassNatives)  return NULL;
 512   }
 513 
 514   // -XX:-InlineThreadNatives disables natives from the Thread class.
 515   if (m->holder()->name() == ciSymbol::java_lang_Thread()) {
 516     if (!InlineThreadNatives)  return NULL;
 517   }
 518 
 519   // -XX:-InlineMathNatives disables natives from the Math,Float and Double classes.


 799   case vmIntrinsics::_bitCount_l:
 800   case vmIntrinsics::_reverseBytes_i:
 801   case vmIntrinsics::_reverseBytes_l:
 802   case vmIntrinsics::_reverseBytes_s:
 803   case vmIntrinsics::_reverseBytes_c:           return inline_number_methods(intrinsic_id());
 804 
 805   case vmIntrinsics::_getCallerClass:           return inline_native_Reflection_getCallerClass();
 806 
 807   case vmIntrinsics::_Reference_get:            return inline_reference_get();
 808 
 809   case vmIntrinsics::_aescrypt_encryptBlock:
 810   case vmIntrinsics::_aescrypt_decryptBlock:    return inline_aescrypt_Block(intrinsic_id());
 811 
 812   case vmIntrinsics::_cipherBlockChaining_encryptAESCrypt:
 813   case vmIntrinsics::_cipherBlockChaining_decryptAESCrypt:
 814     return inline_cipherBlockChaining_AESCrypt(intrinsic_id());
 815 
 816   case vmIntrinsics::_encodeISOArray:
 817     return inline_encodeISOArray();
 818 
 819   case vmIntrinsics::_updateCRC32:
 820     return inline_updateCRC32();
 821   case vmIntrinsics::_updateBytesCRC32:
 822     return inline_updateBytesCRC32();
 823   case vmIntrinsics::_updateByteBufferCRC32:
 824     return inline_updateByteBufferCRC32();
 825 
 826   default:
 827     // If you get here, it may be that someone has added a new intrinsic
 828     // to the list in vmSymbols.hpp without implementing it here.
 829 #ifndef PRODUCT
 830     if ((PrintMiscellaneous && (Verbose || WizardMode)) || PrintOpto) {
 831       tty->print_cr("*** Warning: Unimplemented intrinsic %s(%d)",
 832                     vmIntrinsics::name_at(intrinsic_id()), intrinsic_id());
 833     }
 834 #endif
 835     return false;
 836   }
 837 }
 838 
 839 Node* LibraryCallKit::try_to_predicate() {
 840   if (!jvms()->has_method()) {
 841     // Root JVMState has a null method.
 842     assert(map()->memory()->Opcode() == Op_Parm, "");
 843     // Insert the memory aliasing node
 844     set_all_memory(reset_memory());
 845   }


 883 // The returned value represents the control for the slow path.
 884 // The return value is never 'top'; it is either a valid control
 885 // or NULL if it is obvious that the slow path can never be taken.
 886 // Also, if region and the slow control are not NULL, the slow edge
 887 // is appended to the region.
 888 Node* LibraryCallKit::generate_guard(Node* test, RegionNode* region, float true_prob) {
 889   if (stopped()) {
 890     // Already short circuited.
 891     return NULL;
 892   }
 893 
 894   // Build an if node and its projections.
 895   // If test is true we take the slow path, which we assume is uncommon.
 896   if (_gvn.type(test) == TypeInt::ZERO) {
 897     // The slow branch is never taken.  No need to build this guard.
 898     return NULL;
 899   }
 900 
 901   IfNode* iff = create_and_map_if(control(), test, true_prob, COUNT_UNKNOWN);
 902 
 903   Node* if_slow = _gvn.transform(new (C) IfTrueNode(iff));
 904   if (if_slow == top()) {
 905     // The slow branch is never taken.  No need to build this guard.
 906     return NULL;
 907   }
 908 
 909   if (region != NULL)
 910     region->add_req(if_slow);
 911 
 912   Node* if_fast = _gvn.transform(new (C) IfFalseNode(iff));
 913   set_control(if_fast);
 914 
 915   return if_slow;
 916 }
 917 
 918 inline Node* LibraryCallKit::generate_slow_guard(Node* test, RegionNode* region) {
 919   return generate_guard(test, region, PROB_UNLIKELY_MAG(3));
 920 }
 921 inline Node* LibraryCallKit::generate_fair_guard(Node* test, RegionNode* region) {
 922   return generate_guard(test, region, PROB_FAIR);
 923 }
 924 
 925 inline Node* LibraryCallKit::generate_negative_guard(Node* index, RegionNode* region,
 926                                                      Node* *pos_index) {
 927   if (stopped())
 928     return NULL;                // already stopped
 929   if (_gvn.type(index)->higher_equal(TypeInt::POS)) // [0,maxint]
 930     return NULL;                // index is already adequately typed
 931   Node* cmp_lt = _gvn.transform(new (C) CmpINode(index, intcon(0)));
 932   Node* bol_lt = _gvn.transform(new (C) BoolNode(cmp_lt, BoolTest::lt));
 933   Node* is_neg = generate_guard(bol_lt, region, PROB_MIN);
 934   if (is_neg != NULL && pos_index != NULL) {
 935     // Emulate effect of Parse::adjust_map_after_if.
 936     Node* ccast = new (C) CastIINode(index, TypeInt::POS);
 937     ccast->set_req(0, control());
 938     (*pos_index) = _gvn.transform(ccast);
 939   }
 940   return is_neg;
 941 }
 942 
 943 inline Node* LibraryCallKit::generate_nonpositive_guard(Node* index, bool never_negative,
 944                                                         Node* *pos_index) {
 945   if (stopped())
 946     return NULL;                // already stopped
 947   if (_gvn.type(index)->higher_equal(TypeInt::POS1)) // [1,maxint]
 948     return NULL;                // index is already adequately typed
 949   Node* cmp_le = _gvn.transform(new (C) CmpINode(index, intcon(0)));
 950   BoolTest::mask le_or_eq = (never_negative ? BoolTest::eq : BoolTest::le);
 951   Node* bol_le = _gvn.transform(new (C) BoolNode(cmp_le, le_or_eq));
 952   Node* is_notp = generate_guard(bol_le, NULL, PROB_MIN);
 953   if (is_notp != NULL && pos_index != NULL) {
 954     // Emulate effect of Parse::adjust_map_after_if.
 955     Node* ccast = new (C) CastIINode(index, TypeInt::POS1);
 956     ccast->set_req(0, control());
 957     (*pos_index) = _gvn.transform(ccast);
 958   }
 959   return is_notp;
 960 }
 961 
 962 // Make sure that 'position' is a valid limit index, in [0..length].
 963 // There are two equivalent plans for checking this:
 964 //   A. (offset + copyLength)  unsigned<=  arrayLength
 965 //   B. offset  <=  (arrayLength - copyLength)
 966 // We require that all of the values above, except for the sum and
 967 // difference, are already known to be non-negative.
 968 // Plan A is robust in the face of overflow, if offset and copyLength
 969 // are both hugely positive.
 970 //
 971 // Plan B is less direct and intuitive, but it does not overflow at
 972 // all, since the difference of two non-negatives is always
 973 // representable.  Whenever Java methods must perform the equivalent
 974 // check they generally use Plan B instead of Plan A.
 975 // For the moment we use Plan A.
 976 inline Node* LibraryCallKit::generate_limit_guard(Node* offset,
 977                                                   Node* subseq_length,
 978                                                   Node* array_length,
 979                                                   RegionNode* region) {
 980   if (stopped())
 981     return NULL;                // already stopped
 982   bool zero_offset = _gvn.type(offset) == TypeInt::ZERO;
 983   if (zero_offset && subseq_length->eqv_uncast(array_length))
 984     return NULL;                // common case of whole-array copy
 985   Node* last = subseq_length;
 986   if (!zero_offset)             // last += offset
 987     last = _gvn.transform(new (C) AddINode(last, offset));
 988   Node* cmp_lt = _gvn.transform(new (C) CmpUNode(array_length, last));
 989   Node* bol_lt = _gvn.transform(new (C) BoolNode(cmp_lt, BoolTest::lt));
 990   Node* is_over = generate_guard(bol_lt, region, PROB_MIN);
 991   return is_over;
 992 }
 993 
 994 
 995 //--------------------------generate_current_thread--------------------
 996 Node* LibraryCallKit::generate_current_thread(Node* &tls_output) {
 997   ciKlass*    thread_klass = env()->Thread_klass();
 998   const Type* thread_type  = TypeOopPtr::make_from_klass(thread_klass)->cast_to_ptr_type(TypePtr::NotNull);
 999   Node* thread = _gvn.transform(new (C) ThreadLocalNode());
1000   Node* p = basic_plus_adr(top()/*!oop*/, thread, in_bytes(JavaThread::threadObj_offset()));
1001   Node* threadObj = make_load(NULL, p, thread_type, T_OBJECT);
1002   tls_output = thread;
1003   return threadObj;
1004 }
1005 
1006 
1007 //------------------------------make_string_method_node------------------------
1008 // Helper method for String intrinsic functions. This version is called
1009 // with str1 and str2 pointing to String object nodes.


1150 
1151     Node* no_ctrl = NULL;
1152 
1153     // Get start addr of receiver
1154     Node* receiver_val    = load_String_value(no_ctrl, receiver);
1155     Node* receiver_offset = load_String_offset(no_ctrl, receiver);
1156     Node* receiver_start = array_element_address(receiver_val, receiver_offset, T_CHAR);
1157 
1158     // Get length of receiver
1159     Node* receiver_cnt  = load_String_length(no_ctrl, receiver);
1160 
1161     // Get start addr of argument
1162     Node* argument_val    = load_String_value(no_ctrl, argument);
1163     Node* argument_offset = load_String_offset(no_ctrl, argument);
1164     Node* argument_start = array_element_address(argument_val, argument_offset, T_CHAR);
1165 
1166     // Get length of argument
1167     Node* argument_cnt  = load_String_length(no_ctrl, argument);
1168 
1169     // Check for receiver count != argument count
1170     Node* cmp = _gvn.transform(new(C) CmpINode(receiver_cnt, argument_cnt));
1171     Node* bol = _gvn.transform(new(C) BoolNode(cmp, BoolTest::ne));
1172     Node* if_ne = generate_slow_guard(bol, NULL);
1173     if (if_ne != NULL) {
1174       phi->init_req(4, intcon(0));
1175       region->init_req(4, if_ne);
1176     }
1177 
1178     // Check for count == 0 is done by assembler code for StrEquals.
1179 
1180     if (!stopped()) {
1181       Node* equals = make_string_method_node(Op_StrEquals, receiver_start, receiver_cnt, argument_start, argument_cnt);
1182       phi->init_req(1, equals);
1183       region->init_req(1, control());
1184     }
1185   }
1186 
1187   // post merge
1188   set_control(_gvn.transform(region));
1189   record_for_igvn(region);
1190 
1191   set_result(_gvn.transform(phi));


1257 //     } // using "i += targetCount;" and an "else i++;" causes a jump to jump.
1258 //     i++;
1259 //   }
1260 //   return -1;
1261 // }
1262 
1263 //------------------------------string_indexOf------------------------
1264 Node* LibraryCallKit::string_indexOf(Node* string_object, ciTypeArray* target_array, jint targetOffset_i,
1265                                      jint cache_i, jint md2_i) {
1266 
1267   Node* no_ctrl  = NULL;
1268   float likely   = PROB_LIKELY(0.9);
1269   float unlikely = PROB_UNLIKELY(0.9);
1270 
1271   const int nargs = 0; // no arguments to push back for uncommon trap in predicate
1272 
1273   Node* source        = load_String_value(no_ctrl, string_object);
1274   Node* sourceOffset  = load_String_offset(no_ctrl, string_object);
1275   Node* sourceCount   = load_String_length(no_ctrl, string_object);
1276 
1277   Node* target = _gvn.transform( makecon(TypeOopPtr::make_from_constant(target_array, true)));
1278   jint target_length = target_array->length();
1279   const TypeAry* target_array_type = TypeAry::make(TypeInt::CHAR, TypeInt::make(0, target_length, Type::WidenMin));
1280   const TypeAryPtr* target_type = TypeAryPtr::make(TypePtr::BotPTR, target_array_type, target_array->klass(), true, Type::OffsetBot);
1281 
1282   IdealKit kit(this, false, true);
1283 #define __ kit.
1284   Node* zero             = __ ConI(0);
1285   Node* one              = __ ConI(1);
1286   Node* cache            = __ ConI(cache_i);
1287   Node* md2              = __ ConI(md2_i);
1288   Node* lastChar         = __ ConI(target_array->char_at(target_length - 1));
1289   Node* targetCount      = __ ConI(target_length);
1290   Node* targetCountLess1 = __ ConI(target_length - 1);
1291   Node* targetOffset     = __ ConI(targetOffset_i);
1292   Node* sourceEnd        = __ SubI(__ AddI(sourceOffset, sourceCount), targetCountLess1);
1293 
1294   IdealVariable rtn(kit), i(kit), j(kit); __ declarations_done();
1295   Node* outer_loop = __ make_label(2 /* goto */);
1296   Node* return_    = __ make_label(1);
1297 


1364     Node*       result_phi = new (C) PhiNode(result_rgn, TypeInt::INT);
1365     Node* no_ctrl  = NULL;
1366 
1367     // Get start addr of source string
1368     Node* source = load_String_value(no_ctrl, receiver);
1369     Node* source_offset = load_String_offset(no_ctrl, receiver);
1370     Node* source_start = array_element_address(source, source_offset, T_CHAR);
1371 
1372     // Get length of source string
1373     Node* source_cnt  = load_String_length(no_ctrl, receiver);
1374 
1375     // Get start addr of substring
1376     Node* substr = load_String_value(no_ctrl, arg);
1377     Node* substr_offset = load_String_offset(no_ctrl, arg);
1378     Node* substr_start = array_element_address(substr, substr_offset, T_CHAR);
1379 
1380     // Get length of source string
1381     Node* substr_cnt  = load_String_length(no_ctrl, arg);
1382 
1383     // Check for substr count > string count
1384     Node* cmp = _gvn.transform(new(C) CmpINode(substr_cnt, source_cnt));
1385     Node* bol = _gvn.transform(new(C) BoolNode(cmp, BoolTest::gt));
1386     Node* if_gt = generate_slow_guard(bol, NULL);
1387     if (if_gt != NULL) {
1388       result_phi->init_req(2, intcon(-1));
1389       result_rgn->init_req(2, if_gt);
1390     }
1391 
1392     if (!stopped()) {
1393       // Check for substr count == 0
1394       cmp = _gvn.transform(new(C) CmpINode(substr_cnt, intcon(0)));
1395       bol = _gvn.transform(new(C) BoolNode(cmp, BoolTest::eq));
1396       Node* if_zero = generate_slow_guard(bol, NULL);
1397       if (if_zero != NULL) {
1398         result_phi->init_req(3, intcon(0));
1399         result_rgn->init_req(3, if_zero);
1400       }
1401     }
1402 
1403     if (!stopped()) {
1404       result = make_string_method_node(Op_StrIndexOf, source_start, source_cnt, substr_start, substr_cnt);
1405       result_phi->init_req(1, result);
1406       result_rgn->init_req(1, control());
1407     }
1408     set_control(_gvn.transform(result_rgn));
1409     record_for_igvn(result_rgn);
1410     result = _gvn.transform(result_phi);
1411 
1412   } else { // Use LibraryCallKit::string_indexOf
1413     // don't intrinsify if argument isn't a constant string.
1414     if (!arg->is_Con()) {
1415      return false;


1551     //   if (x <=  Math.PI / 2.0) return -fsin(x - Math.PI / 2.0);
1552     // }
1553     // return StrictMath.cos(x);
1554 
1555     // Actually, sticking in an 80-bit Intel value into C2 will be tough; it
1556     // requires a special machine instruction to load it.  Instead we'll try
1557     // the 'easy' case.  If we really need the extra range +/- PI/2 we'll
1558     // probably do the math inside the SIN encoding.
1559 
1560     // Make the merge point
1561     RegionNode* r = new (C) RegionNode(3);
1562     Node* phi = new (C) PhiNode(r, Type::DOUBLE);
1563 
1564     // Flatten arg so we need only 1 test
1565     Node *abs = _gvn.transform(new (C) AbsDNode(arg));
1566     // Node for PI/4 constant
1567     Node *pi4 = makecon(TypeD::make(pi_4));
1568     // Check PI/4 : abs(arg)
1569     Node *cmp = _gvn.transform(new (C) CmpDNode(pi4,abs));
1570     // Check: If PI/4 < abs(arg) then go slow
1571     Node *bol = _gvn.transform(new (C) BoolNode( cmp, BoolTest::lt ));
1572     // Branch either way
1573     IfNode *iff = create_and_xform_if(control(),bol, PROB_STATIC_FREQUENT, COUNT_UNKNOWN);
1574     set_control(opt_iff(r,iff));
1575 
1576     // Set fast path result
1577     phi->init_req(2, n);
1578 
1579     // Slow path - non-blocking leaf call
1580     Node* call = NULL;
1581     switch (id) {
1582     case vmIntrinsics::_dsin:
1583       call = make_runtime_call(RC_LEAF, OptoRuntime::Math_D_D_Type(),
1584                                CAST_FROM_FN_PTR(address, SharedRuntime::dsin),
1585                                "Sin", NULL, arg, top());
1586       break;
1587     case vmIntrinsics::_dcos:
1588       call = make_runtime_call(RC_LEAF, OptoRuntime::Math_D_D_Type(),
1589                                CAST_FROM_FN_PTR(address, SharedRuntime::dcos),
1590                                "Cos", NULL, arg, top());
1591       break;


1616   //result=(result.isNaN())? funcAddr():result;
1617   // Check: If isNaN() by checking result!=result? then either trap
1618   // or go to runtime
1619   Node* cmpisnan = _gvn.transform(new (C) CmpDNode(result, result));
1620   // Build the boolean node
1621   Node* bolisnum = _gvn.transform(new (C) BoolNode(cmpisnan, BoolTest::eq));
1622 
1623   if (!too_many_traps(Deoptimization::Reason_intrinsic)) {
1624     { BuildCutout unless(this, bolisnum, PROB_STATIC_FREQUENT);
1625       // The pow or exp intrinsic returned a NaN, which requires a call
1626       // to the runtime.  Recompile with the runtime call.
1627       uncommon_trap(Deoptimization::Reason_intrinsic,
1628                     Deoptimization::Action_make_not_entrant);
1629     }
1630     set_result(result);
1631   } else {
1632     // If this inlining ever returned NaN in the past, we compile a call
1633     // to the runtime to properly handle corner cases
1634 
1635     IfNode* iff = create_and_xform_if(control(), bolisnum, PROB_STATIC_FREQUENT, COUNT_UNKNOWN);
1636     Node* if_slow = _gvn.transform(new (C) IfFalseNode(iff));
1637     Node* if_fast = _gvn.transform(new (C) IfTrueNode(iff));
1638 
1639     if (!if_slow->is_top()) {
1640       RegionNode* result_region = new (C) RegionNode(3);
1641       PhiNode*    result_val = new (C) PhiNode(result_region, Type::DOUBLE);
1642 
1643       result_region->init_req(1, if_fast);
1644       result_val->init_req(1, result);
1645 
1646       set_control(if_slow);
1647 
1648       const TypePtr* no_memory_effects = NULL;
1649       Node* rt = make_runtime_call(RC_LEAF, call_type, funcAddr, funcName,
1650                                    no_memory_effects,
1651                                    x, top(), y, y ? top() : NULL);
1652       Node* value = _gvn.transform(new (C) ProjNode(rt, TypeFunc::Parms+0));
1653 #ifdef ASSERT
1654       Node* value_top = _gvn.transform(new (C) ProjNode(rt, TypeFunc::Parms+1));
1655       assert(value_top == top(), "second value must be top");
1656 #endif
1657 


1703   Node* result = NULL;
1704 
1705   if (!too_many_traps(Deoptimization::Reason_intrinsic)) {
1706     // Short form: skip the fancy tests and just check for NaN result.
1707     result = _gvn.transform(new (C) PowDNode(C, control(), x, y));
1708   } else {
1709     // If this inlining ever returned NaN in the past, include all
1710     // checks + call to the runtime.
1711 
1712     // Set the merge point for If node with condition of (x <= 0.0)
1713     // There are four possible paths to region node and phi node
1714     RegionNode *r = new (C) RegionNode(4);
1715     Node *phi = new (C) PhiNode(r, Type::DOUBLE);
1716 
1717     // Build the first if node: if (x <= 0.0)
1718     // Node for 0 constant
1719     Node *zeronode = makecon(TypeD::ZERO);
1720     // Check x:0
1721     Node *cmp = _gvn.transform(new (C) CmpDNode(x, zeronode));
1722     // Check: If (x<=0) then go complex path
1723     Node *bol1 = _gvn.transform(new (C) BoolNode( cmp, BoolTest::le ));
1724     // Branch either way
1725     IfNode *if1 = create_and_xform_if(control(),bol1, PROB_STATIC_INFREQUENT, COUNT_UNKNOWN);
1726     // Fast path taken; set region slot 3
1727     Node *fast_taken = _gvn.transform(new (C) IfFalseNode(if1));
1728     r->init_req(3,fast_taken); // Capture fast-control
1729 
1730     // Fast path not-taken, i.e. slow path
1731     Node *complex_path = _gvn.transform(new (C) IfTrueNode(if1));
1732 
1733     // Set fast path result
1734     Node *fast_result = _gvn.transform(new (C) PowDNode(C, control(), x, y));
1735     phi->init_req(3, fast_result);
1736 
1737     // Complex path
1738     // Build the second if node (if y is long)
1739     // Node for (long)y
1740     Node *longy = _gvn.transform(new (C) ConvD2LNode(y));
1741     // Node for (double)((long) y)
1742     Node *doublelongy= _gvn.transform(new (C) ConvL2DNode(longy));
1743     // Check (double)((long) y) : y
1744     Node *cmplongy= _gvn.transform(new (C) CmpDNode(doublelongy, y));
1745     // Check if (y isn't long) then go to slow path
1746 
1747     Node *bol2 = _gvn.transform(new (C) BoolNode( cmplongy, BoolTest::ne ));
1748     // Branch either way
1749     IfNode *if2 = create_and_xform_if(complex_path,bol2, PROB_STATIC_INFREQUENT, COUNT_UNKNOWN);
1750     Node* ylong_path = _gvn.transform(new (C) IfFalseNode(if2));
1751 
1752     Node *slow_path = _gvn.transform(new (C) IfTrueNode(if2));
1753 
1754     // Calculate DPow(abs(x), y)*(1 & (long)y)
1755     // Node for constant 1
1756     Node *conone = longcon(1);
1757     // 1& (long)y
1758     Node *signnode= _gvn.transform(new (C) AndLNode(conone, longy));
1759 
1760     // A huge number is always even. Detect a huge number by checking
1761     // if y + 1 == y and set integer to be tested for parity to 0.
1762     // Required for corner case:
1763     // (long)9.223372036854776E18 = max_jlong
1764     // (double)(long)9.223372036854776E18 = 9.223372036854776E18
1765     // max_jlong is odd but 9.223372036854776E18 is even
1766     Node* yplus1 = _gvn.transform(new (C) AddDNode(y, makecon(TypeD::make(1))));
1767     Node *cmpyplus1= _gvn.transform(new (C) CmpDNode(yplus1, y));
1768     Node *bolyplus1 = _gvn.transform(new (C) BoolNode( cmpyplus1, BoolTest::eq ));
1769     Node* correctedsign = NULL;
1770     if (ConditionalMoveLimit != 0) {
1771       correctedsign = _gvn.transform( CMoveNode::make(C, NULL, bolyplus1, signnode, longcon(0), TypeLong::LONG));
1772     } else {
1773       IfNode *ifyplus1 = create_and_xform_if(ylong_path,bolyplus1, PROB_FAIR, COUNT_UNKNOWN);
1774       RegionNode *r = new (C) RegionNode(3);
1775       Node *phi = new (C) PhiNode(r, TypeLong::LONG);
1776       r->init_req(1, _gvn.transform(new (C) IfFalseNode(ifyplus1)));
1777       r->init_req(2, _gvn.transform(new (C) IfTrueNode(ifyplus1)));
1778       phi->init_req(1, signnode);
1779       phi->init_req(2, longcon(0));
1780       correctedsign = _gvn.transform(phi);
1781       ylong_path = _gvn.transform(r);
1782       record_for_igvn(r);
1783     }
1784 
1785     // zero node
1786     Node *conzero = longcon(0);
1787     // Check (1&(long)y)==0?
1788     Node *cmpeq1 = _gvn.transform(new (C) CmpLNode(correctedsign, conzero));
1789     // Check if (1&(long)y)!=0?, if so the result is negative
1790     Node *bol3 = _gvn.transform(new (C) BoolNode( cmpeq1, BoolTest::ne ));
1791     // abs(x)
1792     Node *absx=_gvn.transform(new (C) AbsDNode(x));
1793     // abs(x)^y
1794     Node *absxpowy = _gvn.transform(new (C) PowDNode(C, control(), absx, y));
1795     // -abs(x)^y
1796     Node *negabsxpowy = _gvn.transform(new (C) NegDNode (absxpowy));
1797     // (1&(long)y)==1?-DPow(abs(x), y):DPow(abs(x), y)
1798     Node *signresult = NULL;
1799     if (ConditionalMoveLimit != 0) {
1800       signresult = _gvn.transform( CMoveNode::make(C, NULL, bol3, absxpowy, negabsxpowy, Type::DOUBLE));
1801     } else {
1802       IfNode *ifyeven = create_and_xform_if(ylong_path,bol3, PROB_FAIR, COUNT_UNKNOWN);
1803       RegionNode *r = new (C) RegionNode(3);
1804       Node *phi = new (C) PhiNode(r, Type::DOUBLE);
1805       r->init_req(1, _gvn.transform(new (C) IfFalseNode(ifyeven)));
1806       r->init_req(2, _gvn.transform(new (C) IfTrueNode(ifyeven)));
1807       phi->init_req(1, absxpowy);
1808       phi->init_req(2, negabsxpowy);
1809       signresult = _gvn.transform(phi);
1810       ylong_path = _gvn.transform(r);
1811       record_for_igvn(r);
1812     }
1813     // Set complex path fast result
1814     r->init_req(2, ylong_path);
1815     phi->init_req(2, signresult);
1816 
1817     static const jlong nan_bits = CONST64(0x7ff8000000000000);
1818     Node *slow_result = makecon(TypeD::make(*(double*)&nan_bits)); // return NaN
1819     r->init_req(1,slow_path);
1820     phi->init_req(1,slow_result);
1821 
1822     // Post merge
1823     set_control(_gvn.transform(r));
1824     record_for_igvn(r);
1825     result = _gvn.transform(phi);
1826   }


1919 
1920   const TypeInt* txvalue = _gvn.type(xvalue)->isa_int();
1921   const TypeInt* tyvalue = _gvn.type(yvalue)->isa_int();
1922   if (txvalue == NULL || tyvalue == NULL)  return top();
1923   // This is not really necessary, but it is consistent with a
1924   // hypothetical MaxINode::Value method:
1925   int widen = MAX2(txvalue->_widen, tyvalue->_widen);
1926 
1927   // %%% This folding logic should (ideally) be in a different place.
1928   // Some should be inside IfNode, and there to be a more reliable
1929   // transformation of ?: style patterns into cmoves.  We also want
1930   // more powerful optimizations around cmove and min/max.
1931 
1932   // Try to find a dominating comparison of these guys.
1933   // It can simplify the index computation for Arrays.copyOf
1934   // and similar uses of System.arraycopy.
1935   // First, compute the normalized version of CmpI(x, y).
1936   int   cmp_op = Op_CmpI;
1937   Node* xkey = xvalue;
1938   Node* ykey = yvalue;
1939   Node* ideal_cmpxy = _gvn.transform(new(C) CmpINode(xkey, ykey));
1940   if (ideal_cmpxy->is_Cmp()) {
1941     // E.g., if we have CmpI(length - offset, count),
1942     // it might idealize to CmpI(length, count + offset)
1943     cmp_op = ideal_cmpxy->Opcode();
1944     xkey = ideal_cmpxy->in(1);
1945     ykey = ideal_cmpxy->in(2);
1946   }
1947 
1948   // Start by locating any relevant comparisons.
1949   Node* start_from = (xkey->outcnt() < ykey->outcnt()) ? xkey : ykey;
1950   Node* cmpxy = NULL;
1951   Node* cmpyx = NULL;
1952   for (DUIterator_Fast kmax, k = start_from->fast_outs(kmax); k < kmax; k++) {
1953     Node* cmp = start_from->fast_out(k);
1954     if (cmp->outcnt() > 0 &&            // must have prior uses
1955         cmp->in(0) == NULL &&           // must be context-independent
1956         cmp->Opcode() == cmp_op) {      // right kind of compare
1957       if (cmp->in(1) == xkey && cmp->in(2) == ykey)  cmpxy = cmp;
1958       if (cmp->in(1) == ykey && cmp->in(2) == xkey)  cmpyx = cmp;
1959     }


2012     if (cmp == NULL)  continue;
2013     for (DUIterator_Fast jmax, j = cmp->fast_outs(jmax); j < jmax; j++) {
2014       Node* bol = cmp->fast_out(j);
2015       if (!bol->is_Bool())  continue;
2016       BoolTest::mask btest = bol->as_Bool()->_test._test;
2017       if (btest == BoolTest::eq || btest == BoolTest::ne)  continue;
2018       if (cmp->in(1) == ykey)   btest = BoolTest(btest).commute();
2019       if (bol->outcnt() > (best_bol == NULL ? 0 : best_bol->outcnt())) {
2020         best_bol   = bol->as_Bool();
2021         best_btest = btest;
2022       }
2023     }
2024   }
2025 
2026   Node* answer_if_true  = NULL;
2027   Node* answer_if_false = NULL;
2028   switch (best_btest) {
2029   default:
2030     if (cmpxy == NULL)
2031       cmpxy = ideal_cmpxy;
2032     best_bol = _gvn.transform(new(C) BoolNode(cmpxy, BoolTest::lt));
2033     // and fall through:
2034   case BoolTest::lt:          // x < y
2035   case BoolTest::le:          // x <= y
2036     answer_if_true  = (want_max ? yvalue : xvalue);
2037     answer_if_false = (want_max ? xvalue : yvalue);
2038     break;
2039   case BoolTest::gt:          // x > y
2040   case BoolTest::ge:          // x >= y
2041     answer_if_true  = (want_max ? xvalue : yvalue);
2042     answer_if_false = (want_max ? yvalue : xvalue);
2043     break;
2044   }
2045 
2046   jint hi, lo;
2047   if (want_max) {
2048     // We can sharpen the minimum.
2049     hi = MAX2(txvalue->_hi, tyvalue->_hi);
2050     lo = MAX2(txvalue->_lo, tyvalue->_lo);
2051   } else {
2052     // We can sharpen the maximum.


2072   switch (id) {
2073   case vmIntrinsics::_min:
2074     result_val = _gvn.transform(new (C, 3) MinINode(x,y)); break;
2075   case vmIntrinsics::_max:
2076     result_val = _gvn.transform(new (C, 3) MaxINode(x,y)); break;
2077   default:
2078     ShouldNotReachHere();
2079   }
2080   */
2081 }
2082 
2083 inline int
2084 LibraryCallKit::classify_unsafe_addr(Node* &base, Node* &offset) {
2085   const TypePtr* base_type = TypePtr::NULL_PTR;
2086   if (base != NULL)  base_type = _gvn.type(base)->isa_ptr();
2087   if (base_type == NULL) {
2088     // Unknown type.
2089     return Type::AnyPtr;
2090   } else if (base_type == TypePtr::NULL_PTR) {
2091     // Since this is a NULL+long form, we have to switch to a rawptr.
2092     base   = _gvn.transform(new (C) CastX2PNode(offset));
2093     offset = MakeConX(0);
2094     return Type::RawPtr;
2095   } else if (base_type->base() == Type::RawPtr) {
2096     return Type::RawPtr;
2097   } else if (base_type->isa_oopptr()) {
2098     // Base is never null => always a heap address.
2099     if (base_type->ptr() == TypePtr::NotNull) {
2100       return Type::OopPtr;
2101     }
2102     // Offset is small => always a heap address.
2103     const TypeX* offset_type = _gvn.type(offset)->isa_intptr_t();
2104     if (offset_type != NULL &&
2105         base_type->offset() == 0 &&     // (should always be?)
2106         offset_type->_lo >= 0 &&
2107         !MacroAssembler::needs_explicit_null_check(offset_type->_hi)) {
2108       return Type::OopPtr;
2109     }
2110     // Otherwise, it might either be oop+off or NULL+addr.
2111     return Type::AnyPtr;
2112   } else {


2466       p = ConvX2L(p);
2467       break;
2468     default:
2469       fatal(err_msg_res("unexpected type %d: %s", type, type2name(type)));
2470       break;
2471     }
2472     // The load node has the control of the preceding MemBarCPUOrder.  All
2473     // following nodes will have the control of the MemBarCPUOrder inserted at
2474     // the end of this method.  So, pushing the load onto the stack at a later
2475     // point is fine.
2476     set_result(p);
2477   } else {
2478     // place effect of store into memory
2479     switch (type) {
2480     case T_DOUBLE:
2481       val = dstore_rounding(val);
2482       break;
2483     case T_ADDRESS:
2484       // Repackage the long as a pointer.
2485       val = ConvL2X(val);
2486       val = _gvn.transform(new (C) CastX2PNode(val));
2487       break;
2488     }
2489 
2490     if (type != T_OBJECT ) {
2491       (void) store_to_memory(control(), adr, val, type, adr_type, is_volatile);
2492     } else {
2493       // Possibly an oop being stored to Java heap or native memory
2494       if (!TypePtr::NULL_PTR->higher_equal(_gvn.type(heap_base_oop))) {
2495         // oop to Java heap.
2496         (void) store_oop_to_unknown(control(), heap_base_oop, adr, adr_type, val, type);
2497       } else {
2498         // We can't tell at compile time if we are storing in the Java heap or outside
2499         // of it. So we need to emit code to conditionally do the proper type of
2500         // store.
2501 
2502         IdealKit ideal(this);
2503 #define __ ideal.
2504         // QQQ who knows what probability is here??
2505         __ if_then(heap_base_oop, BoolTest::ne, null(), PROB_UNLIKELY(0.999)); {
2506           // Sync IdealKit and graphKit.


2774     } else
2775 #endif
2776     {
2777       if (kind == LS_xchg) {
2778         load_store = _gvn.transform(new (C) GetAndSetPNode(control(), mem, adr, newval, adr_type, value_type->is_oopptr()));
2779       } else {
2780         assert(kind == LS_cmpxchg, "wrong LoadStore operation");
2781         load_store = _gvn.transform(new (C) CompareAndSwapPNode(control(), mem, adr, newval, oldval));
2782       }
2783     }
2784     post_barrier(control(), load_store, base, adr, alias_idx, newval, T_OBJECT, true);
2785     break;
2786   default:
2787     fatal(err_msg_res("unexpected type %d: %s", type, type2name(type)));
2788     break;
2789   }
2790 
2791   // SCMemProjNodes represent the memory state of a LoadStore. Their
2792   // main role is to prevent LoadStore nodes from being optimized away
2793   // when their results aren't used.
2794   Node* proj = _gvn.transform(new (C) SCMemProjNode(load_store));
2795   set_memory(proj, alias_idx);
2796 
2797   // Add the trailing membar surrounding the access
2798   insert_mem_bar(Op_MemBarCPUOrder);
2799   insert_mem_bar(Op_MemBarAcquire);
2800 
2801 #ifdef _LP64
2802   if (type == T_OBJECT && adr->bottom_type()->is_ptr_to_narrowoop() && kind == LS_xchg) {
2803     load_store = _gvn.transform(new (C) DecodeNNode(load_store, load_store->get_ptr_type()));
2804   }
2805 #endif
2806 
2807   assert(type2size[load_store->bottom_type()->basic_type()] == type2size[rtype], "result type should match");
2808   set_result(load_store);
2809   return true;
2810 }
2811 
2812 //----------------------------inline_unsafe_ordered_store----------------------
2813 // public native void sun.misc.Unsafe.putOrderedObject(Object o, long offset, Object x);
2814 // public native void sun.misc.Unsafe.putOrderedInt(Object o, long offset, int x);


3009     no_int_result_path   = 1, // t == Thread.current() && !TLS._osthread._interrupted
3010     no_clear_result_path = 2, // t == Thread.current() &&  TLS._osthread._interrupted && !clear_int
3011     slow_result_path     = 3, // slow path: t.isInterrupted(clear_int)
3012     PATH_LIMIT
3013   };
3014 
3015   // Ensure that it's not possible to move the load of TLS._osthread._interrupted flag
3016   // out of the function.
3017   insert_mem_bar(Op_MemBarCPUOrder);
3018 
3019   RegionNode* result_rgn = new (C) RegionNode(PATH_LIMIT);
3020   PhiNode*    result_val = new (C) PhiNode(result_rgn, TypeInt::BOOL);
3021 
3022   RegionNode* slow_region = new (C) RegionNode(1);
3023   record_for_igvn(slow_region);
3024 
3025   // (a) Receiving thread must be the current thread.
3026   Node* rec_thr = argument(0);
3027   Node* tls_ptr = NULL;
3028   Node* cur_thr = generate_current_thread(tls_ptr);
3029   Node* cmp_thr = _gvn.transform(new (C) CmpPNode(cur_thr, rec_thr));
3030   Node* bol_thr = _gvn.transform(new (C) BoolNode(cmp_thr, BoolTest::ne));
3031 
3032   generate_slow_guard(bol_thr, slow_region);
3033 
3034   // (b) Interrupt bit on TLS must be false.
3035   Node* p = basic_plus_adr(top()/*!oop*/, tls_ptr, in_bytes(JavaThread::osthread_offset()));
3036   Node* osthread = make_load(NULL, p, TypeRawPtr::NOTNULL, T_ADDRESS);
3037   p = basic_plus_adr(top()/*!oop*/, osthread, in_bytes(OSThread::interrupted_offset()));
3038 
3039   // Set the control input on the field _interrupted read to prevent it floating up.
3040   Node* int_bit = make_load(control(), p, TypeInt::BOOL, T_INT);
3041   Node* cmp_bit = _gvn.transform(new (C) CmpINode(int_bit, intcon(0)));
3042   Node* bol_bit = _gvn.transform(new (C) BoolNode(cmp_bit, BoolTest::ne));
3043 
3044   IfNode* iff_bit = create_and_map_if(control(), bol_bit, PROB_UNLIKELY_MAG(3), COUNT_UNKNOWN);
3045 
3046   // First fast path:  if (!TLS._interrupted) return false;
3047   Node* false_bit = _gvn.transform(new (C) IfFalseNode(iff_bit));
3048   result_rgn->init_req(no_int_result_path, false_bit);
3049   result_val->init_req(no_int_result_path, intcon(0));
3050 
3051   // drop through to next case
3052   set_control( _gvn.transform(new (C) IfTrueNode(iff_bit)));
3053 
3054   // (c) Or, if interrupt bit is set and clear_int is false, use 2nd fast path.
3055   Node* clr_arg = argument(1);
3056   Node* cmp_arg = _gvn.transform(new (C) CmpINode(clr_arg, intcon(0)));
3057   Node* bol_arg = _gvn.transform(new (C) BoolNode(cmp_arg, BoolTest::ne));
3058   IfNode* iff_arg = create_and_map_if(control(), bol_arg, PROB_FAIR, COUNT_UNKNOWN);
3059 
3060   // Second fast path:  ... else if (!clear_int) return true;
3061   Node* false_arg = _gvn.transform(new (C) IfFalseNode(iff_arg));
3062   result_rgn->init_req(no_clear_result_path, false_arg);
3063   result_val->init_req(no_clear_result_path, intcon(1));
3064 
3065   // drop through to next case
3066   set_control( _gvn.transform(new (C) IfTrueNode(iff_arg)));
3067 
3068   // (d) Otherwise, go to the slow path.
3069   slow_region->add_req(control());
3070   set_control( _gvn.transform(slow_region));
3071 
3072   if (stopped()) {
3073     // There is no slow path.
3074     result_rgn->init_req(slow_result_path, top());
3075     result_val->init_req(slow_result_path, top());
3076   } else {
3077     // non-virtual because it is a private non-static
3078     CallJavaNode* slow_call = generate_method_call(vmIntrinsics::_isInterrupted);
3079 
3080     Node* slow_val = set_results_for_java_call(slow_call);
3081     // this->control() comes from set_results_for_java_call
3082 
3083     Node* fast_io  = slow_call->in(TypeFunc::I_O);
3084     Node* fast_mem = slow_call->in(TypeFunc::Memory);
3085 
3086     // These two phis are pre-filled with copies of of the fast IO and Memory
3087     PhiNode* result_mem  = PhiNode::make(result_rgn, fast_mem, Type::MEMORY, TypePtr::BOTTOM);
3088     PhiNode* result_io   = PhiNode::make(result_rgn, fast_io,  Type::ABIO);
3089 
3090     result_rgn->init_req(slow_result_path, control());


3106 Node* LibraryCallKit::load_mirror_from_klass(Node* klass) {
3107   Node* p = basic_plus_adr(klass, in_bytes(Klass::java_mirror_offset()));
3108   return make_load(NULL, p, TypeInstPtr::MIRROR, T_OBJECT);
3109 }
3110 
3111 //-----------------------load_klass_from_mirror_common-------------------------
3112 // Given a java mirror (a java.lang.Class oop), load its corresponding klass oop.
3113 // Test the klass oop for null (signifying a primitive Class like Integer.TYPE),
3114 // and branch to the given path on the region.
3115 // If never_see_null, take an uncommon trap on null, so we can optimistically
3116 // compile for the non-null case.
3117 // If the region is NULL, force never_see_null = true.
3118 Node* LibraryCallKit::load_klass_from_mirror_common(Node* mirror,
3119                                                     bool never_see_null,
3120                                                     RegionNode* region,
3121                                                     int null_path,
3122                                                     int offset) {
3123   if (region == NULL)  never_see_null = true;
3124   Node* p = basic_plus_adr(mirror, offset);
3125   const TypeKlassPtr*  kls_type = TypeKlassPtr::OBJECT_OR_NULL;
3126   Node* kls = _gvn.transform( LoadKlassNode::make(_gvn, immutable_memory(), p, TypeRawPtr::BOTTOM, kls_type));
3127   Node* null_ctl = top();
3128   kls = null_check_oop(kls, &null_ctl, never_see_null);
3129   if (region != NULL) {
3130     // Set region->in(null_path) if the mirror is a primitive (e.g, int.class).
3131     region->init_req(null_path, null_ctl);
3132   } else {
3133     assert(null_ctl == top(), "no loose ends");
3134   }
3135   return kls;
3136 }
3137 
3138 //--------------------(inline_native_Class_query helpers)---------------------
3139 // Use this for JVM_ACC_INTERFACE, JVM_ACC_IS_CLONEABLE, JVM_ACC_HAS_FINALIZER.
3140 // Fall through if (mods & mask) == bits, take the guard otherwise.
3141 Node* LibraryCallKit::generate_access_flags_guard(Node* kls, int modifier_mask, int modifier_bits, RegionNode* region) {
3142   // Branch around if the given klass has the given modifier bit set.
3143   // Like generate_guard, adds a new path onto the region.
3144   Node* modp = basic_plus_adr(kls, in_bytes(Klass::access_flags_offset()));
3145   Node* mods = make_load(NULL, modp, TypeInt::INT, T_INT);
3146   Node* mask = intcon(modifier_mask);
3147   Node* bits = intcon(modifier_bits);
3148   Node* mbit = _gvn.transform(new (C) AndINode(mods, mask));
3149   Node* cmp  = _gvn.transform(new (C) CmpINode(mbit, bits));
3150   Node* bol  = _gvn.transform(new (C) BoolNode(cmp, BoolTest::ne));
3151   return generate_fair_guard(bol, region);
3152 }
3153 Node* LibraryCallKit::generate_interface_guard(Node* kls, RegionNode* region) {
3154   return generate_access_flags_guard(kls, JVM_ACC_INTERFACE, 0, region);
3155 }
3156 
3157 //-------------------------inline_native_Class_query-------------------
3158 bool LibraryCallKit::inline_native_Class_query(vmIntrinsics::ID id) {
3159   const Type* return_type = TypeInt::BOOL;
3160   Node* prim_return_value = top();  // what happens if it's a primitive class?
3161   bool never_see_null = !too_many_traps(Deoptimization::Reason_null_check);
3162   bool expect_prim = false;     // most of these guys expect to work on refs
3163 
3164   enum { _normal_path = 1, _prim_path = 2, PATH_LIMIT };
3165 
3166   Node* mirror = argument(0);
3167   Node* obj    = top();
3168 
3169   switch (id) {
3170   case vmIntrinsics::_isInstance:


3281 
3282   case vmIntrinsics::_isPrimitive:
3283     query_value = intcon(0); // "normal" path produces false
3284     break;
3285 
3286   case vmIntrinsics::_getSuperclass:
3287     // The rules here are somewhat unfortunate, but we can still do better
3288     // with random logic than with a JNI call.
3289     // Interfaces store null or Object as _super, but must report null.
3290     // Arrays store an intermediate super as _super, but must report Object.
3291     // Other types can report the actual _super.
3292     // (To verify this code sequence, check the asserts in JVM_IsInterface.)
3293     if (generate_interface_guard(kls, region) != NULL)
3294       // A guard was added.  If the guard is taken, it was an interface.
3295       phi->add_req(null());
3296     if (generate_array_guard(kls, region) != NULL)
3297       // A guard was added.  If the guard is taken, it was an array.
3298       phi->add_req(makecon(TypeInstPtr::make(env()->Object_klass()->java_mirror())));
3299     // If we fall through, it's a plain class.  Get its _super.
3300     p = basic_plus_adr(kls, in_bytes(Klass::super_offset()));
3301     kls = _gvn.transform( LoadKlassNode::make(_gvn, immutable_memory(), p, TypeRawPtr::BOTTOM, TypeKlassPtr::OBJECT_OR_NULL));
3302     null_ctl = top();
3303     kls = null_check_oop(kls, &null_ctl);
3304     if (null_ctl != top()) {
3305       // If the guard is taken, Object.superClass is null (both klass and mirror).
3306       region->add_req(null_ctl);
3307       phi   ->add_req(null());
3308     }
3309     if (!stopped()) {
3310       query_value = load_mirror_from_klass(kls);
3311     }
3312     break;
3313 
3314   case vmIntrinsics::_getComponentType:
3315     if (generate_array_guard(kls, region) != NULL) {
3316       // Be sure to pin the oop load to the guard edge just created:
3317       Node* is_array_ctrl = region->in(region->req()-1);
3318       Node* cma = basic_plus_adr(kls, in_bytes(ArrayKlass::component_mirror_offset()));
3319       Node* cmo = make_load(is_array_ctrl, cma, TypeInstPtr::MIRROR, T_OBJECT);
3320       phi->add_req(cmo);
3321     }


3394     region->init_req(prim_path, null_ctl);
3395     if (stopped())  break;
3396     klasses[which_arg] = kls;
3397   }
3398 
3399   if (!stopped()) {
3400     // now we have two reference types, in klasses[0..1]
3401     Node* subk   = klasses[1];  // the argument to isAssignableFrom
3402     Node* superk = klasses[0];  // the receiver
3403     region->set_req(_both_ref_path, gen_subtype_check(subk, superk));
3404     // now we have a successful reference subtype check
3405     region->set_req(_ref_subtype_path, control());
3406   }
3407 
3408   // If both operands are primitive (both klasses null), then
3409   // we must return true when they are identical primitives.
3410   // It is convenient to test this after the first null klass check.
3411   set_control(region->in(_prim_0_path)); // go back to first null check
3412   if (!stopped()) {
3413     // Since superc is primitive, make a guard for the superc==subc case.
3414     Node* cmp_eq = _gvn.transform(new (C) CmpPNode(args[0], args[1]));
3415     Node* bol_eq = _gvn.transform(new (C) BoolNode(cmp_eq, BoolTest::eq));
3416     generate_guard(bol_eq, region, PROB_FAIR);
3417     if (region->req() == PATH_LIMIT+1) {
3418       // A guard was added.  If the added guard is taken, superc==subc.
3419       region->swap_edges(PATH_LIMIT, _prim_same_path);
3420       region->del_req(PATH_LIMIT);
3421     }
3422     region->set_req(_prim_0_path, control()); // Not equal after all.
3423   }
3424 
3425   // these are the only paths that produce 'true':
3426   phi->set_req(_prim_same_path,   intcon(1));
3427   phi->set_req(_ref_subtype_path, intcon(1));
3428 
3429   // pull together the cases:
3430   assert(region->req() == PATH_LIMIT, "sane region");
3431   for (uint i = 1; i < region->req(); i++) {
3432     Node* ctl = region->in(i);
3433     if (ctl == NULL || ctl == top()) {
3434       region->set_req(i, top());
3435       phi   ->set_req(i, top());


3460   Node* layout_val = get_layout_helper(kls, layout_con);
3461   if (layout_val == NULL) {
3462     bool query = (obj_array
3463                   ? Klass::layout_helper_is_objArray(layout_con)
3464                   : Klass::layout_helper_is_array(layout_con));
3465     if (query == not_array) {
3466       return NULL;                       // never a branch
3467     } else {                             // always a branch
3468       Node* always_branch = control();
3469       if (region != NULL)
3470         region->add_req(always_branch);
3471       set_control(top());
3472       return always_branch;
3473     }
3474   }
3475   // Now test the correct condition.
3476   jint  nval = (obj_array
3477                 ? ((jint)Klass::_lh_array_tag_type_value
3478                    <<    Klass::_lh_array_tag_shift)
3479                 : Klass::_lh_neutral_value);
3480   Node* cmp = _gvn.transform(new(C) CmpINode(layout_val, intcon(nval)));
3481   BoolTest::mask btest = BoolTest::lt;  // correct for testing is_[obj]array
3482   // invert the test if we are looking for a non-array
3483   if (not_array)  btest = BoolTest(btest).negate();
3484   Node* bol = _gvn.transform(new(C) BoolNode(cmp, btest));
3485   return generate_fair_guard(bol, region);
3486 }
3487 
3488 
3489 //-----------------------inline_native_newArray--------------------------
3490 // private static native Object java.lang.reflect.newArray(Class<?> componentType, int length);
3491 bool LibraryCallKit::inline_native_newArray() {
3492   Node* mirror    = argument(0);
3493   Node* count_val = argument(1);
3494 
3495   mirror = null_check(mirror);
3496   // If mirror or obj is dead, only null-path is taken.
3497   if (stopped())  return true;
3498 
3499   enum { _normal_path = 1, _slow_path = 2, PATH_LIMIT };
3500   RegionNode* result_reg = new(C) RegionNode(PATH_LIMIT);
3501   PhiNode*    result_val = new(C) PhiNode(result_reg,
3502                                           TypeInstPtr::NOTNULL);
3503   PhiNode*    result_io  = new(C) PhiNode(result_reg, Type::ABIO);
3504   PhiNode*    result_mem = new(C) PhiNode(result_reg, Type::MEMORY,


3524     result_reg->set_req(_slow_path, control());
3525     result_val->set_req(_slow_path, slow_result);
3526     result_io ->set_req(_slow_path, i_o());
3527     result_mem->set_req(_slow_path, reset_memory());
3528   }
3529 
3530   set_control(normal_ctl);
3531   if (!stopped()) {
3532     // Normal case:  The array type has been cached in the java.lang.Class.
3533     // The following call works fine even if the array type is polymorphic.
3534     // It could be a dynamic mix of int[], boolean[], Object[], etc.
3535     Node* obj = new_array(klass_node, count_val, 0);  // no arguments to push
3536     result_reg->init_req(_normal_path, control());
3537     result_val->init_req(_normal_path, obj);
3538     result_io ->init_req(_normal_path, i_o());
3539     result_mem->init_req(_normal_path, reset_memory());
3540   }
3541 
3542   // Return the combined state.
3543   set_i_o(        _gvn.transform(result_io)  );
3544   set_all_memory( _gvn.transform(result_mem));
3545 
3546   C->set_has_split_ifs(true); // Has chance for split-if optimization
3547   set_result(result_reg, result_val);
3548   return true;
3549 }
3550 
3551 //----------------------inline_native_getLength--------------------------
3552 // public static native int java.lang.reflect.Array.getLength(Object array);
3553 bool LibraryCallKit::inline_native_getLength() {
3554   if (too_many_traps(Deoptimization::Reason_intrinsic))  return false;
3555 
3556   Node* array = null_check(argument(0));
3557   // If array is dead, only null-path is taken.
3558   if (stopped())  return true;
3559 
3560   // Deoptimize if it is a non-array.
3561   Node* non_array = generate_non_array_guard(load_object_klass(array), NULL);
3562 
3563   if (non_array != NULL) {
3564     PreserveJVMState pjvms(this);


3677 }
3678 
3679 
3680 //----------------------generate_virtual_guard---------------------------
3681 // Helper for hashCode and clone.  Peeks inside the vtable to avoid a call.
3682 Node* LibraryCallKit::generate_virtual_guard(Node* obj_klass,
3683                                              RegionNode* slow_region) {
3684   ciMethod* method = callee();
3685   int vtable_index = method->vtable_index();
3686   // Get the Method* out of the appropriate vtable entry.
3687   int entry_offset  = (InstanceKlass::vtable_start_offset() +
3688                      vtable_index*vtableEntry::size()) * wordSize +
3689                      vtableEntry::method_offset_in_bytes();
3690   Node* entry_addr  = basic_plus_adr(obj_klass, entry_offset);
3691   Node* target_call = make_load(NULL, entry_addr, TypePtr::NOTNULL, T_ADDRESS);
3692 
3693   // Compare the target method with the expected method (e.g., Object.hashCode).
3694   const TypePtr* native_call_addr = TypeMetadataPtr::make(method);
3695 
3696   Node* native_call = makecon(native_call_addr);
3697   Node* chk_native  = _gvn.transform(new(C) CmpPNode(target_call, native_call));
3698   Node* test_native = _gvn.transform(new(C) BoolNode(chk_native, BoolTest::ne));
3699 
3700   return generate_slow_guard(test_native, slow_region);
3701 }
3702 
3703 //-----------------------generate_method_call----------------------------
3704 // Use generate_method_call to make a slow-call to the real
3705 // method if the fast path fails.  An alternative would be to
3706 // use a stub like OptoRuntime::slow_arraycopy_Java.
3707 // This only works for expanding the current library call,
3708 // not another intrinsic.  (E.g., don't use this for making an
3709 // arraycopy call inside of the copyOf intrinsic.)
3710 CallJavaNode*
3711 LibraryCallKit::generate_method_call(vmIntrinsics::ID method_id, bool is_virtual, bool is_static) {
3712   // When compiling the intrinsic method itself, do not use this technique.
3713   guarantee(callee() != C->method(), "cannot make slow-call to self");
3714 
3715   ciMethod* method = callee();
3716   // ensure the JVMS we have will be correct for this call
3717   guarantee(method_id == method->intrinsic_id(), "must match");
3718 


3799   // paths which do not pass are accumulated in the slow_region.
3800   RegionNode* slow_region = new (C) RegionNode(1);
3801   record_for_igvn(slow_region);
3802 
3803   // If this is a virtual call, we generate a funny guard.  We pull out
3804   // the vtable entry corresponding to hashCode() from the target object.
3805   // If the target method which we are calling happens to be the native
3806   // Object hashCode() method, we pass the guard.  We do not need this
3807   // guard for non-virtual calls -- the caller is known to be the native
3808   // Object hashCode().
3809   if (is_virtual) {
3810     generate_virtual_guard(obj_klass, slow_region);
3811   }
3812 
3813   // Get the header out of the object, use LoadMarkNode when available
3814   Node* header_addr = basic_plus_adr(obj, oopDesc::mark_offset_in_bytes());
3815   Node* header = make_load(control(), header_addr, TypeX_X, TypeX_X->basic_type());
3816 
3817   // Test the header to see if it is unlocked.
3818   Node *lock_mask      = _gvn.MakeConX(markOopDesc::biased_lock_mask_in_place);
3819   Node *lmasked_header = _gvn.transform(new (C) AndXNode(header, lock_mask));
3820   Node *unlocked_val   = _gvn.MakeConX(markOopDesc::unlocked_value);
3821   Node *chk_unlocked   = _gvn.transform(new (C) CmpXNode( lmasked_header, unlocked_val));
3822   Node *test_unlocked  = _gvn.transform(new (C) BoolNode( chk_unlocked, BoolTest::ne));
3823 
3824   generate_slow_guard(test_unlocked, slow_region);
3825 
3826   // Get the hash value and check to see that it has been properly assigned.
3827   // We depend on hash_mask being at most 32 bits and avoid the use of
3828   // hash_mask_in_place because it could be larger than 32 bits in a 64-bit
3829   // vm: see markOop.hpp.
3830   Node *hash_mask      = _gvn.intcon(markOopDesc::hash_mask);
3831   Node *hash_shift     = _gvn.intcon(markOopDesc::hash_shift);
3832   Node *hshifted_header= _gvn.transform(new (C) URShiftXNode(header, hash_shift));
3833   // This hack lets the hash bits live anywhere in the mark object now, as long
3834   // as the shift drops the relevant bits into the low 32 bits.  Note that
3835   // Java spec says that HashCode is an int so there's no point in capturing
3836   // an 'X'-sized hashcode (32 in 32-bit build or 64 in 64-bit build).
3837   hshifted_header      = ConvX2I(hshifted_header);
3838   Node *hash_val       = _gvn.transform(new (C) AndINode(hshifted_header, hash_mask));
3839 
3840   Node *no_hash_val    = _gvn.intcon(markOopDesc::no_hash);
3841   Node *chk_assigned   = _gvn.transform(new (C) CmpINode( hash_val, no_hash_val));
3842   Node *test_assigned  = _gvn.transform(new (C) BoolNode( chk_assigned, BoolTest::eq));
3843 
3844   generate_slow_guard(test_assigned, slow_region);
3845 
3846   Node* init_mem = reset_memory();
3847   // fill in the rest of the null path:
3848   result_io ->init_req(_null_path, i_o());
3849   result_mem->init_req(_null_path, init_mem);
3850 
3851   result_val->init_req(_fast_path, hash_val);
3852   result_reg->init_req(_fast_path, control());
3853   result_io ->init_req(_fast_path, i_o());
3854   result_mem->init_req(_fast_path, init_mem);
3855 
3856   // Generate code for the slow case.  We make a call to hashCode().
3857   set_control(_gvn.transform(slow_region));
3858   if (!stopped()) {
3859     // No need for PreserveJVMState, because we're using up the present state.
3860     set_all_memory(init_mem);
3861     vmIntrinsics::ID hashCode_id = is_static ? vmIntrinsics::_identityHashCode : vmIntrinsics::_hashCode;
3862     CallJavaNode* slow_call = generate_method_call(hashCode_id, is_virtual, is_static);
3863     Node* slow_result = set_results_for_java_call(slow_call);
3864     // this->control() comes from set_results_for_java_call
3865     result_reg->init_req(_slow_path, control());
3866     result_val->init_req(_slow_path, slow_result);
3867     result_io  ->set_req(_slow_path, i_o());
3868     result_mem ->set_req(_slow_path, reset_memory());
3869   }
3870 
3871   // Return the combined state.
3872   set_i_o(        _gvn.transform(result_io)  );
3873   set_all_memory( _gvn.transform(result_mem));
3874 
3875   set_result(result_reg, result_val);
3876   return true;
3877 }
3878 
3879 //---------------------------inline_native_getClass----------------------------
3880 // public final native Class<?> java.lang.Object.getClass();
3881 //
3882 // Build special case code for calls to getClass on an object.
3883 bool LibraryCallKit::inline_native_getClass() {
3884   Node* obj = null_check_receiver();
3885   if (stopped())  return true;
3886   set_result(load_mirror_from_klass(load_object_klass(obj)));
3887   return true;
3888 }
3889 
3890 //-----------------inline_native_Reflection_getCallerClass---------------------
3891 // public static native Class<?> sun.reflect.Reflection.getCallerClass();
3892 //
3893 // In the presence of deep enough inlining, getCallerClass() becomes a no-op.


3981   case vmIntrinsics::_floatToRawIntBits:    result = new (C) MoveF2INode(arg);  break;
3982   case vmIntrinsics::_intBitsToFloat:       result = new (C) MoveI2FNode(arg);  break;
3983   case vmIntrinsics::_doubleToRawLongBits:  result = new (C) MoveD2LNode(arg);  break;
3984   case vmIntrinsics::_longBitsToDouble:     result = new (C) MoveL2DNode(arg);  break;
3985 
3986   case vmIntrinsics::_doubleToLongBits: {
3987     // two paths (plus control) merge in a wood
3988     RegionNode *r = new (C) RegionNode(3);
3989     Node *phi = new (C) PhiNode(r, TypeLong::LONG);
3990 
3991     Node *cmpisnan = _gvn.transform(new (C) CmpDNode(arg, arg));
3992     // Build the boolean node
3993     Node *bolisnan = _gvn.transform(new (C) BoolNode(cmpisnan, BoolTest::ne));
3994 
3995     // Branch either way.
3996     // NaN case is less traveled, which makes all the difference.
3997     IfNode *ifisnan = create_and_xform_if(control(), bolisnan, PROB_STATIC_FREQUENT, COUNT_UNKNOWN);
3998     Node *opt_isnan = _gvn.transform(ifisnan);
3999     assert( opt_isnan->is_If(), "Expect an IfNode");
4000     IfNode *opt_ifisnan = (IfNode*)opt_isnan;
4001     Node *iftrue = _gvn.transform(new (C) IfTrueNode(opt_ifisnan));
4002 
4003     set_control(iftrue);
4004 
4005     static const jlong nan_bits = CONST64(0x7ff8000000000000);
4006     Node *slow_result = longcon(nan_bits); // return NaN
4007     phi->init_req(1, _gvn.transform( slow_result ));
4008     r->init_req(1, iftrue);
4009 
4010     // Else fall through
4011     Node *iffalse = _gvn.transform(new (C) IfFalseNode(opt_ifisnan));
4012     set_control(iffalse);
4013 
4014     phi->init_req(2, _gvn.transform(new (C) MoveD2LNode(arg)));
4015     r->init_req(2, iffalse);
4016 
4017     // Post merge
4018     set_control(_gvn.transform(r));
4019     record_for_igvn(r);
4020 
4021     C->set_has_split_ifs(true); // Has chance for split-if optimization
4022     result = phi;
4023     assert(result->bottom_type()->isa_long(), "must be");
4024     break;
4025   }
4026 
4027   case vmIntrinsics::_floatToIntBits: {
4028     // two paths (plus control) merge in a wood
4029     RegionNode *r = new (C) RegionNode(3);
4030     Node *phi = new (C) PhiNode(r, TypeInt::INT);
4031 
4032     Node *cmpisnan = _gvn.transform(new (C) CmpFNode(arg, arg));
4033     // Build the boolean node
4034     Node *bolisnan = _gvn.transform(new (C) BoolNode(cmpisnan, BoolTest::ne));
4035 
4036     // Branch either way.
4037     // NaN case is less traveled, which makes all the difference.
4038     IfNode *ifisnan = create_and_xform_if(control(), bolisnan, PROB_STATIC_FREQUENT, COUNT_UNKNOWN);
4039     Node *opt_isnan = _gvn.transform(ifisnan);
4040     assert( opt_isnan->is_If(), "Expect an IfNode");
4041     IfNode *opt_ifisnan = (IfNode*)opt_isnan;
4042     Node *iftrue = _gvn.transform(new (C) IfTrueNode(opt_ifisnan));
4043 
4044     set_control(iftrue);
4045 
4046     static const jint nan_bits = 0x7fc00000;
4047     Node *slow_result = makecon(TypeInt::make(nan_bits)); // return NaN
4048     phi->init_req(1, _gvn.transform( slow_result ));
4049     r->init_req(1, iftrue);
4050 
4051     // Else fall through
4052     Node *iffalse = _gvn.transform(new (C) IfFalseNode(opt_ifisnan));
4053     set_control(iffalse);
4054 
4055     phi->init_req(2, _gvn.transform(new (C) MoveF2INode(arg)));
4056     r->init_req(2, iffalse);
4057 
4058     // Post merge
4059     set_control(_gvn.transform(r));
4060     record_for_igvn(r);
4061 
4062     C->set_has_split_ifs(true); // Has chance for split-if optimization


4151   // base_off:
4152   // 8  - 32-bit VM
4153   // 12 - 64-bit VM, compressed klass
4154   // 16 - 64-bit VM, normal klass
4155   if (base_off % BytesPerLong != 0) {
4156     assert(UseCompressedKlassPointers, "");
4157     if (is_array) {
4158       // Exclude length to copy by 8 bytes words.
4159       base_off += sizeof(int);
4160     } else {
4161       // Include klass to copy by 8 bytes words.
4162       base_off = instanceOopDesc::klass_offset_in_bytes();
4163     }
4164     assert(base_off % BytesPerLong == 0, "expect 8 bytes alignment");
4165   }
4166   src  = basic_plus_adr(src,  base_off);
4167   dest = basic_plus_adr(dest, base_off);
4168 
4169   // Compute the length also, if needed:
4170   Node* countx = size;
4171   countx = _gvn.transform(new (C) SubXNode(countx, MakeConX(base_off)));
4172   countx = _gvn.transform(new (C) URShiftXNode(countx, intcon(LogBytesPerLong) ));
4173 
4174   const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
4175   bool disjoint_bases = true;
4176   generate_unchecked_arraycopy(raw_adr_type, T_LONG, disjoint_bases,
4177                                src, NULL, dest, NULL, countx,
4178                                /*dest_uninitialized*/true);
4179 
4180   // If necessary, emit some card marks afterwards.  (Non-arrays only.)
4181   if (card_mark) {
4182     assert(!is_array, "");
4183     // Put in store barrier for any and all oops we are sticking
4184     // into this object.  (We could avoid this if we could prove
4185     // that the object type contains no oop fields at all.)
4186     Node* no_particular_value = NULL;
4187     Node* no_particular_field = NULL;
4188     int raw_adr_idx = Compile::AliasIdxRaw;
4189     post_barrier(control(),
4190                  memory(raw_adr_type),
4191                  alloc_obj,
4192                  no_particular_field,


4356       result_reg->init_req(_instance_path, control());
4357       result_val->init_req(_instance_path, alloc_obj);
4358       result_i_o ->set_req(_instance_path, i_o());
4359       result_mem ->set_req(_instance_path, reset_memory());
4360     }
4361 
4362     // Generate code for the slow case.  We make a call to clone().
4363     set_control(_gvn.transform(slow_region));
4364     if (!stopped()) {
4365       PreserveJVMState pjvms(this);
4366       CallJavaNode* slow_call = generate_method_call(vmIntrinsics::_clone, is_virtual);
4367       Node* slow_result = set_results_for_java_call(slow_call);
4368       // this->control() comes from set_results_for_java_call
4369       result_reg->init_req(_slow_path, control());
4370       result_val->init_req(_slow_path, slow_result);
4371       result_i_o ->set_req(_slow_path, i_o());
4372       result_mem ->set_req(_slow_path, reset_memory());
4373     }
4374 
4375     // Return the combined state.
4376     set_control(    _gvn.transform(result_reg));
4377     set_i_o(        _gvn.transform(result_i_o));
4378     set_all_memory( _gvn.transform(result_mem));
4379   } // original reexecute is set back here
4380 
4381   set_result(_gvn.transform(result_val));
4382   return true;
4383 }
4384 
4385 //------------------------------basictype2arraycopy----------------------------
4386 address LibraryCallKit::basictype2arraycopy(BasicType t,
4387                                             Node* src_offset,
4388                                             Node* dest_offset,
4389                                             bool disjoint_bases,
4390                                             const char* &name,
4391                                             bool dest_uninitialized) {
4392   const TypeInt* src_offset_inttype  = gvn().find_int_type(src_offset);;
4393   const TypeInt* dest_offset_inttype = gvn().find_int_type(dest_offset);;
4394 
4395   bool aligned = false;
4396   bool disjoint = disjoint_bases;
4397 
4398   // if the offsets are the same, we can treat the memory regions as


4683         // paths have stub or runtime calls as raw memory barriers.
4684         InitializeNode* init = insert_mem_bar_volatile(Op_Initialize,
4685                                                        Compile::AliasIdxRaw,
4686                                                        top())->as_Initialize();
4687         init->set_complete(&_gvn);  // (there is no corresponding AllocateNode)
4688       }
4689     }
4690 
4691     // Present the results of the fast call.
4692     result_region->init_req(zero_path, control());
4693     result_i_o   ->init_req(zero_path, i_o());
4694     result_memory->init_req(zero_path, memory(adr_type));
4695   }
4696 
4697   if (!stopped() && dest_uninitialized) {
4698     // We have to initialize the *uncopied* part of the array to zero.
4699     // The copy destination is the slice dest[off..off+len].  The other slices
4700     // are dest_head = dest[0..off] and dest_tail = dest[off+len..dest.length].
4701     Node* dest_size   = alloc->in(AllocateNode::AllocSize);
4702     Node* dest_length = alloc->in(AllocateNode::ALength);
4703     Node* dest_tail   = _gvn.transform(new(C) AddINode(dest_offset,
4704                                                           copy_length));
4705 
4706     // If there is a head section that needs zeroing, do it now.
4707     if (find_int_con(dest_offset, -1) != 0) {
4708       generate_clear_array(adr_type, dest, basic_elem_type,
4709                            intcon(0), dest_offset,
4710                            NULL);
4711     }
4712 
4713     // Next, perform a dynamic check on the tail length.
4714     // It is often zero, and we can win big if we prove this.
4715     // There are two wins:  Avoid generating the ClearArray
4716     // with its attendant messy index arithmetic, and upgrade
4717     // the copy to a more hardware-friendly word size of 64 bits.
4718     Node* tail_ctl = NULL;
4719     if (!stopped() && !dest_tail->eqv_uncast(dest_length)) {
4720       Node* cmp_lt   = _gvn.transform(new(C) CmpINode(dest_tail, dest_length));
4721       Node* bol_lt   = _gvn.transform(new(C) BoolNode(cmp_lt, BoolTest::lt));
4722       tail_ctl = generate_slow_guard(bol_lt, NULL);
4723       assert(tail_ctl != NULL || !stopped(), "must be an outcome");
4724     }
4725 
4726     // At this point, let's assume there is no tail.
4727     if (!stopped() && alloc != NULL && basic_elem_type != T_OBJECT) {
4728       // There is no tail.  Try an upgrade to a 64-bit copy.
4729       bool didit = false;
4730       { PreserveJVMState pjvms(this);
4731         didit = generate_block_arraycopy(adr_type, basic_elem_type, alloc,
4732                                          src, src_offset, dest, dest_offset,
4733                                          dest_size, dest_uninitialized);
4734         if (didit) {
4735           // Present the results of the block-copying fast call.
4736           result_region->init_req(bcopy_path, control());
4737           result_i_o   ->init_req(bcopy_path, i_o());
4738           result_memory->init_req(bcopy_path, memory(adr_type));
4739         }
4740       }
4741       if (didit)


4744 
4745     // Clear the tail, if any.
4746     if (tail_ctl != NULL) {
4747       Node* notail_ctl = stopped() ? NULL : control();
4748       set_control(tail_ctl);
4749       if (notail_ctl == NULL) {
4750         generate_clear_array(adr_type, dest, basic_elem_type,
4751                              dest_tail, NULL,
4752                              dest_size);
4753       } else {
4754         // Make a local merge.
4755         Node* done_ctl = new(C) RegionNode(3);
4756         Node* done_mem = new(C) PhiNode(done_ctl, Type::MEMORY, adr_type);
4757         done_ctl->init_req(1, notail_ctl);
4758         done_mem->init_req(1, memory(adr_type));
4759         generate_clear_array(adr_type, dest, basic_elem_type,
4760                              dest_tail, NULL,
4761                              dest_size);
4762         done_ctl->init_req(2, control());
4763         done_mem->init_req(2, memory(adr_type));
4764         set_control( _gvn.transform(done_ctl));
4765         set_memory(  _gvn.transform(done_mem), adr_type );
4766       }
4767     }
4768   }
4769 
4770   BasicType copy_type = basic_elem_type;
4771   assert(basic_elem_type != T_ARRAY, "caller must fix this");
4772   if (!stopped() && copy_type == T_OBJECT) {
4773     // If src and dest have compatible element types, we can copy bits.
4774     // Types S[] and D[] are compatible if D is a supertype of S.
4775     //
4776     // If they are not, we will use checked_oop_disjoint_arraycopy,
4777     // which performs a fast optimistic per-oop check, and backs off
4778     // further to JVM_ArrayCopy on the first per-oop check that fails.
4779     // (Actually, we don't move raw bits only; the GC requires card marks.)
4780 
4781     // Get the Klass* for both src and dest
4782     Node* src_klass  = load_object_klass(src);
4783     Node* dest_klass = load_object_klass(dest);
4784 


4831                                  src, src_offset, dest, dest_offset,
4832                                  ConvI2X(copy_length), dest_uninitialized);
4833 
4834     // Present the results of the fast call.
4835     result_region->init_req(fast_path, control());
4836     result_i_o   ->init_req(fast_path, i_o());
4837     result_memory->init_req(fast_path, memory(adr_type));
4838   }
4839 
4840   // Here are all the slow paths up to this point, in one bundle:
4841   slow_control = top();
4842   if (slow_region != NULL)
4843     slow_control = _gvn.transform(slow_region);
4844   DEBUG_ONLY(slow_region = (RegionNode*)badAddress);
4845 
4846   set_control(checked_control);
4847   if (!stopped()) {
4848     // Clean up after the checked call.
4849     // The returned value is either 0 or -1^K,
4850     // where K = number of partially transferred array elements.
4851     Node* cmp = _gvn.transform(new(C) CmpINode(checked_value, intcon(0)));
4852     Node* bol = _gvn.transform(new(C) BoolNode(cmp, BoolTest::eq));
4853     IfNode* iff = create_and_map_if(control(), bol, PROB_MAX, COUNT_UNKNOWN);
4854 
4855     // If it is 0, we are done, so transfer to the end.
4856     Node* checks_done = _gvn.transform(new(C) IfTrueNode(iff));
4857     result_region->init_req(checked_path, checks_done);
4858     result_i_o   ->init_req(checked_path, checked_i_o);
4859     result_memory->init_req(checked_path, checked_mem);
4860 
4861     // If it is not zero, merge into the slow call.
4862     set_control( _gvn.transform(new(C) IfFalseNode(iff) ));
4863     RegionNode* slow_reg2 = new(C) RegionNode(3);
4864     PhiNode*    slow_i_o2 = new(C) PhiNode(slow_reg2, Type::ABIO);
4865     PhiNode*    slow_mem2 = new(C) PhiNode(slow_reg2, Type::MEMORY, adr_type);
4866     record_for_igvn(slow_reg2);
4867     slow_reg2  ->init_req(1, slow_control);
4868     slow_i_o2  ->init_req(1, slow_i_o);
4869     slow_mem2  ->init_req(1, slow_mem);
4870     slow_reg2  ->init_req(2, control());
4871     slow_i_o2  ->init_req(2, checked_i_o);
4872     slow_mem2  ->init_req(2, checked_mem);
4873 
4874     slow_control = _gvn.transform(slow_reg2);
4875     slow_i_o     = _gvn.transform(slow_i_o2);
4876     slow_mem     = _gvn.transform(slow_mem2);
4877 
4878     if (alloc != NULL) {
4879       // We'll restart from the very beginning, after zeroing the whole thing.
4880       // This can cause double writes, but that's OK since dest is brand new.
4881       // So we ignore the low 31 bits of the value returned from the stub.
4882     } else {
4883       // We must continue the copy exactly where it failed, or else
4884       // another thread might see the wrong number of writes to dest.
4885       Node* checked_offset = _gvn.transform(new(C) XorINode(checked_value, intcon(-1)));
4886       Node* slow_offset    = new(C) PhiNode(slow_reg2, TypeInt::INT);
4887       slow_offset->init_req(1, intcon(0));
4888       slow_offset->init_req(2, checked_offset);
4889       slow_offset  = _gvn.transform(slow_offset);
4890 
4891       // Adjust the arguments by the conditionally incoming offset.
4892       Node* src_off_plus  = _gvn.transform(new(C) AddINode(src_offset,  slow_offset));
4893       Node* dest_off_plus = _gvn.transform(new(C) AddINode(dest_offset, slow_offset));
4894       Node* length_minus  = _gvn.transform(new(C) SubINode(copy_length, slow_offset));
4895 
4896       // Tweak the node variables to adjust the code produced below:
4897       src_offset  = src_off_plus;
4898       dest_offset = dest_off_plus;
4899       copy_length = length_minus;
4900     }
4901   }
4902 
4903   set_control(slow_control);
4904   if (!stopped()) {
4905     // Generate the slow path, if needed.
4906     PreserveJVMState pjvms(this);   // replace_in_map may trash the map
4907 
4908     set_memory(slow_mem, adr_type);
4909     set_i_o(slow_i_o);
4910 
4911     if (dest_uninitialized) {
4912       generate_clear_array(adr_type, dest, basic_elem_type,
4913                            intcon(0), NULL,
4914                            alloc->in(AllocateNode::AllocSize));
4915     }
4916 
4917     generate_slow_arraycopy(adr_type,
4918                             src, src_offset, dest, dest_offset,
4919                             copy_length, /*dest_uninitialized*/false);
4920 
4921     result_region->init_req(slow_call_path, control());
4922     result_i_o   ->init_req(slow_call_path, i_o());
4923     result_memory->init_req(slow_call_path, memory(adr_type));
4924   }
4925 
4926   // Remove unused edges.
4927   for (uint i = 1; i < result_region->req(); i++) {
4928     if (result_region->in(i) == NULL)
4929       result_region->init_req(i, top());
4930   }
4931 
4932   // Finished; return the combined state.
4933   set_control( _gvn.transform(result_region));
4934   set_i_o(     _gvn.transform(result_i_o)    );
4935   set_memory(  _gvn.transform(result_memory), adr_type );
4936 
4937   // The memory edges above are precise in order to model effects around
4938   // array copies accurately to allow value numbering of field loads around
4939   // arraycopy.  Such field loads, both before and after, are common in Java
4940   // collections and similar classes involving header/array data structures.
4941   //
4942   // But with low number of register or when some registers are used or killed
4943   // by arraycopy calls it causes registers spilling on stack. See 6544710.
4944   // The next memory barrier is added to avoid it. If the arraycopy can be
4945   // optimized away (which it can, sometimes) then we can manually remove
4946   // the membar also.
4947   //
4948   // Do not let reads from the cloned object float above the arraycopy.
4949   if (alloc != NULL) {
4950     // Do not let stores that initialize this object be reordered with
4951     // a subsequent store that would make this object accessible by
4952     // other threads.
4953     // Record what AllocateNode this StoreStore protects so that


5095     end_con = round_to(abase + ((slice_idx_con + slice_len_con) << scale),
5096                        BytesPerLong);
5097   }
5098 
5099   if (start_con >= 0 && end_con >= 0) {
5100     // Constant start and end.  Simple.
5101     mem = ClearArrayNode::clear_memory(control(), mem, dest,
5102                                        start_con, end_con, &_gvn);
5103   } else if (start_con >= 0 && dest_size != top()) {
5104     // Constant start, pre-rounded end after the tail of the array.
5105     Node* end = dest_size;
5106     mem = ClearArrayNode::clear_memory(control(), mem, dest,
5107                                        start_con, end, &_gvn);
5108   } else if (start_con >= 0 && slice_len != top()) {
5109     // Constant start, non-constant end.  End needs rounding up.
5110     // End offset = round_up(abase + ((slice_idx_con + slice_len) << scale), 8)
5111     intptr_t end_base  = abase + (slice_idx_con << scale);
5112     int      end_round = (-1 << scale) & (BytesPerLong  - 1);
5113     Node*    end       = ConvI2X(slice_len);
5114     if (scale != 0)
5115       end = _gvn.transform(new(C) LShiftXNode(end, intcon(scale) ));
5116     end_base += end_round;
5117     end = _gvn.transform(new(C) AddXNode(end, MakeConX(end_base)));
5118     end = _gvn.transform(new(C) AndXNode(end, MakeConX(~end_round)));
5119     mem = ClearArrayNode::clear_memory(control(), mem, dest,
5120                                        start_con, end, &_gvn);
5121   } else if (start_con < 0 && dest_size != top()) {
5122     // Non-constant start, pre-rounded end after the tail of the array.
5123     // This is almost certainly a "round-to-end" operation.
5124     Node* start = slice_idx;
5125     start = ConvI2X(start);
5126     if (scale != 0)
5127       start = _gvn.transform(new(C) LShiftXNode( start, intcon(scale) ));
5128     start = _gvn.transform(new(C) AddXNode(start, MakeConX(abase)));
5129     if ((bump_bit | clear_low) != 0) {
5130       int to_clear = (bump_bit | clear_low);
5131       // Align up mod 8, then store a jint zero unconditionally
5132       // just before the mod-8 boundary.
5133       if (((abase + bump_bit) & ~to_clear) - bump_bit
5134           < arrayOopDesc::length_offset_in_bytes() + BytesPerInt) {
5135         bump_bit = 0;
5136         assert((abase & to_clear) == 0, "array base must be long-aligned");
5137       } else {
5138         // Bump 'start' up to (or past) the next jint boundary:
5139         start = _gvn.transform(new(C) AddXNode(start, MakeConX(bump_bit)));
5140         assert((abase & clear_low) == 0, "array base must be int-aligned");
5141       }
5142       // Round bumped 'start' down to jlong boundary in body of array.
5143       start = _gvn.transform(new(C) AndXNode(start, MakeConX(~to_clear)));
5144       if (bump_bit != 0) {
5145         // Store a zero to the immediately preceding jint:
5146         Node* x1 = _gvn.transform(new(C) AddXNode(start, MakeConX(-bump_bit)));
5147         Node* p1 = basic_plus_adr(dest, x1);
5148         mem = StoreNode::make(_gvn, control(), mem, p1, adr_type, intcon(0), T_INT);
5149         mem = _gvn.transform(mem);
5150       }
5151     }
5152     Node* end = dest_size; // pre-rounded
5153     mem = ClearArrayNode::clear_memory(control(), mem, dest,
5154                                        start, end, &_gvn);
5155   } else {
5156     // Non-constant start, unrounded non-constant end.
5157     // (Nobody zeroes a random midsection of an array using this routine.)
5158     ShouldNotReachHere();       // fix caller
5159   }
5160 
5161   // Done.
5162   set_memory(mem, adr_type);
5163 }
5164 
5165 
5166 bool


5193     // This is a common case, since abase can be odd mod 8.
5194     if (((src_off | dest_off) & (BytesPerLong-1)) == BytesPerInt &&
5195         ((src_off ^ dest_off) & (BytesPerLong-1)) == 0) {
5196       Node* sptr = basic_plus_adr(src,  src_off);
5197       Node* dptr = basic_plus_adr(dest, dest_off);
5198       Node* sval = make_load(control(), sptr, TypeInt::INT, T_INT, adr_type);
5199       store_to_memory(control(), dptr, sval, T_INT, adr_type);
5200       src_off += BytesPerInt;
5201       dest_off += BytesPerInt;
5202     } else {
5203       return false;
5204     }
5205   }
5206   assert(src_off % BytesPerLong == 0, "");
5207   assert(dest_off % BytesPerLong == 0, "");
5208 
5209   // Do this copy by giant steps.
5210   Node* sptr  = basic_plus_adr(src,  src_off);
5211   Node* dptr  = basic_plus_adr(dest, dest_off);
5212   Node* countx = dest_size;
5213   countx = _gvn.transform(new (C) SubXNode(countx, MakeConX(dest_off)));
5214   countx = _gvn.transform(new (C) URShiftXNode(countx, intcon(LogBytesPerLong)));
5215 
5216   bool disjoint_bases = true;   // since alloc != NULL
5217   generate_unchecked_arraycopy(adr_type, T_LONG, disjoint_bases,
5218                                sptr, NULL, dptr, NULL, countx, dest_uninitialized);
5219 
5220   return true;
5221 }
5222 
5223 
5224 // Helper function; generates code for the slow case.
5225 // We make a call to a runtime method which emulates the native method,
5226 // but without the native wrapper overhead.
5227 void
5228 LibraryCallKit::generate_slow_arraycopy(const TypePtr* adr_type,
5229                                         Node* src,  Node* src_offset,
5230                                         Node* dest, Node* dest_offset,
5231                                         Node* copy_length, bool dest_uninitialized) {
5232   assert(!dest_uninitialized, "Invariant");
5233   Node* call = make_runtime_call(RC_NO_LEAF | RC_UNCOMMON,
5234                                  OptoRuntime::slow_arraycopy_Type(),


5359   // Figure out the size and type of the elements we will be copying.
5360   BasicType src_elem = src_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
5361   BasicType dst_elem = dst_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
5362   if (src_elem != T_CHAR || dst_elem != T_BYTE) {
5363     return false;
5364   }
5365   Node* src_start = array_element_address(src, src_offset, src_elem);
5366   Node* dst_start = array_element_address(dst, dst_offset, dst_elem);
5367   // 'src_start' points to src array + scaled offset
5368   // 'dst_start' points to dst array + scaled offset
5369 
5370   const TypeAryPtr* mtype = TypeAryPtr::BYTES;
5371   Node* enc = new (C) EncodeISOArrayNode(control(), memory(mtype), src_start, dst_start, length);
5372   enc = _gvn.transform(enc);
5373   Node* res_mem = _gvn.transform(new (C) SCMemProjNode(enc));
5374   set_memory(res_mem, mtype);
5375   set_result(enc);
5376   return true;
5377 }
5378 
5379 /**
5380  * Calculate CRC32 for byte.
5381  * int java.util.zip.CRC32.update(int crc, int b)
5382  */
5383 bool LibraryCallKit::inline_updateCRC32() {
5384   assert(UseCRC32Intrinsics, "need AVX and LCMUL instructions support");
5385   assert(callee()->signature()->size() == 2, "update has 2 parameters");
5386   // no receiver since it is static method
5387   Node* crc  = argument(0); // type: int
5388   Node* b    = argument(1); // type: int
5389 
5390   /*
5391    *    int c = ~ crc;
5392    *    b = timesXtoThe32[(b ^ c) & 0xFF];
5393    *    b = b ^ (c >>> 8);
5394    *    crc = ~b;
5395    */
5396 
5397   Node* M1 = intcon(-1);
5398   crc = _gvn.transform(new (C) XorINode(crc, M1));
5399   Node* result = _gvn.transform(new (C) XorINode(crc, b));
5400   result = _gvn.transform(new (C) AndINode(result, intcon(0xFF)));
5401 
5402   Node* base = makecon(TypeRawPtr::make(StubRoutines::crc_table_addr()));
5403   Node* offset = _gvn.transform(new (C) LShiftINode(result, intcon(0x2)));
5404   Node* adr = basic_plus_adr(top(), base, ConvI2X(offset));
5405   result = make_load(control(), adr, TypeInt::INT, T_INT);
5406 
5407   crc = _gvn.transform(new (C) URShiftINode(crc, intcon(8)));
5408   result = _gvn.transform(new (C) XorINode(crc, result));
5409   result = _gvn.transform(new (C) XorINode(result, M1));
5410   set_result(result);
5411   return true;
5412 }
5413 
5414 /**
5415  * Calculate CRC32 for byte[] array.
5416  * int java.util.zip.CRC32.updateBytes(int crc, byte[] buf, int off, int len)
5417  */
5418 bool LibraryCallKit::inline_updateBytesCRC32() {
5419   assert(UseCRC32Intrinsics, "need AVX and LCMUL instructions support");
5420   assert(callee()->signature()->size() == 4, "updateBytes has 4 parameters");
5421   // no receiver since it is static method
5422   Node* crc     = argument(0); // type: int
5423   Node* src     = argument(1); // type: oop
5424   Node* offset  = argument(2); // type: int
5425   Node* length  = argument(3); // type: int
5426 
5427   const Type* src_type = src->Value(&_gvn);
5428   const TypeAryPtr* top_src = src_type->isa_aryptr();
5429   if (top_src  == NULL || top_src->klass()  == NULL) {
5430     // failed array check
5431     return false;
5432   }
5433 
5434   // Figure out the size and type of the elements we will be copying.
5435   BasicType src_elem = src_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
5436   if (src_elem != T_BYTE) {
5437     return false;
5438   }
5439 
5440   // 'src_start' points to src array + scaled offset
5441   Node* src_start = array_element_address(src, offset, src_elem);
5442 
5443   // We assume that range check is done by caller.
5444   // TODO: generate range check (offset+length < src.length) in debug VM.
5445 
5446   // Call the stub.
5447   address stubAddr = StubRoutines::updateBytesCRC32();
5448   const char *stubName = "updateBytesCRC32";
5449 
5450   Node* call = make_runtime_call(RC_LEAF|RC_NO_FP, OptoRuntime::updateBytesCRC32_Type(),
5451                                  stubAddr, stubName, TypePtr::BOTTOM,
5452                                  crc, src_start, length);
5453   Node* result = _gvn.transform(new (C) ProjNode(call, TypeFunc::Parms));
5454   set_result(result);
5455   return true;
5456 }
5457 
5458 /**
5459  * Calculate CRC32 for ByteBuffer.
5460  * int java.util.zip.CRC32.updateByteBuffer(int crc, long buf, int off, int len)
5461  */
5462 bool LibraryCallKit::inline_updateByteBufferCRC32() {
5463   assert(UseCRC32Intrinsics, "need AVX and LCMUL instructions support");
5464   assert(callee()->signature()->size() == 5, "updateByteBuffer has 4 parameters and one is long");
5465   // no receiver since it is static method
5466   Node* crc     = argument(0); // type: int
5467   Node* src     = argument(1); // type: long
5468   Node* offset  = argument(3); // type: int
5469   Node* length  = argument(4); // type: int
5470 
5471   src = ConvL2X(src);  // adjust Java long to machine word
5472   Node* base = _gvn.transform(new (C) CastX2PNode(src));
5473   offset = ConvI2X(offset);
5474 
5475   // 'src_start' points to src array + scaled offset
5476   Node* src_start = basic_plus_adr(top(), base, offset);
5477 
5478   // Call the stub.
5479   address stubAddr = StubRoutines::updateBytesCRC32();
5480   const char *stubName = "updateBytesCRC32";
5481 
5482   Node* call = make_runtime_call(RC_LEAF|RC_NO_FP, OptoRuntime::updateBytesCRC32_Type(),
5483                                  stubAddr, stubName, TypePtr::BOTTOM,
5484                                  crc, src_start, length);
5485   Node* result = _gvn.transform(new (C) ProjNode(call, TypeFunc::Parms));
5486   set_result(result);
5487   return true;
5488 }
5489 
5490 //----------------------------inline_reference_get----------------------------
5491 // public T java.lang.ref.Reference.get();
5492 bool LibraryCallKit::inline_reference_get() {
5493   const int referent_offset = java_lang_ref_Reference::referent_offset;
5494   guarantee(referent_offset > 0, "should have already been set");
5495 
5496   // Get the argument:
5497   Node* reference_obj = null_check_receiver();
5498   if (stopped()) return true;
5499 
5500   Node* adr = basic_plus_adr(reference_obj, reference_obj, referent_offset);
5501 
5502   ciInstanceKlass* klass = env()->Object_klass();
5503   const TypeOopPtr* object_type = TypeOopPtr::make_from_klass(klass);
5504 
5505   Node* no_ctrl = NULL;
5506   Node* result = make_load(no_ctrl, adr, object_type, T_OBJECT);
5507 
5508   // Use the pre-barrier to record the value in the referent field
5509   pre_barrier(false /* do_load */,


src/share/vm/opto/library_call.cpp
Index Unified diffs Context diffs Sdiffs Patch New Old Previous File Next File