1 /*
   2  * Copyright (c) 1999, 2015, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "asm/macroAssembler.hpp"
  27 #include "classfile/systemDictionary.hpp"
  28 #include "classfile/vmSymbols.hpp"
  29 #include "compiler/compileBroker.hpp"
  30 #include "compiler/compileLog.hpp"
  31 #include "oops/objArrayKlass.hpp"
  32 #include "opto/addnode.hpp"
  33 #include "opto/arraycopynode.hpp"
  34 #include "opto/c2compiler.hpp"
  35 #include "opto/callGenerator.hpp"
  36 #include "opto/castnode.hpp"
  37 #include "opto/cfgnode.hpp"
  38 #include "opto/convertnode.hpp"
  39 #include "opto/countbitsnode.hpp"
  40 #include "opto/intrinsicnode.hpp"
  41 #include "opto/idealKit.hpp"
  42 #include "opto/mathexactnode.hpp"
  43 #include "opto/movenode.hpp"
  44 #include "opto/mulnode.hpp"
  45 #include "opto/narrowptrnode.hpp"
  46 #include "opto/opaquenode.hpp"
  47 #include "opto/parse.hpp"
  48 #include "opto/runtime.hpp"
  49 #include "opto/subnode.hpp"
  50 #include "prims/nativeLookup.hpp"
  51 #include "runtime/sharedRuntime.hpp"
  52 #include "trace/traceMacros.hpp"
  53 
  54 class LibraryIntrinsic : public InlineCallGenerator {
  55   // Extend the set of intrinsics known to the runtime:
  56  public:
  57  private:
  58   bool             _is_virtual;
  59   bool             _does_virtual_dispatch;
  60   int8_t           _predicates_count;  // Intrinsic is predicated by several conditions
  61   int8_t           _last_predicate; // Last generated predicate
  62   vmIntrinsics::ID _intrinsic_id;
  63 
  64  public:
  65   LibraryIntrinsic(ciMethod* m, bool is_virtual, int predicates_count, bool does_virtual_dispatch, vmIntrinsics::ID id)
  66     : InlineCallGenerator(m),
  67       _is_virtual(is_virtual),
  68       _does_virtual_dispatch(does_virtual_dispatch),
  69       _predicates_count((int8_t)predicates_count),
  70       _last_predicate((int8_t)-1),
  71       _intrinsic_id(id)
  72   {
  73   }
  74   virtual bool is_intrinsic() const { return true; }
  75   virtual bool is_virtual()   const { return _is_virtual; }
  76   virtual bool is_predicated() const { return _predicates_count > 0; }
  77   virtual int  predicates_count() const { return _predicates_count; }
  78   virtual bool does_virtual_dispatch()   const { return _does_virtual_dispatch; }
  79   virtual JVMState* generate(JVMState* jvms);
  80   virtual Node* generate_predicate(JVMState* jvms, int predicate);
  81   vmIntrinsics::ID intrinsic_id() const { return _intrinsic_id; }
  82 };
  83 
  84 
  85 // Local helper class for LibraryIntrinsic:
  86 class LibraryCallKit : public GraphKit {
  87  private:
  88   LibraryIntrinsic* _intrinsic;     // the library intrinsic being called
  89   Node*             _result;        // the result node, if any
  90   int               _reexecute_sp;  // the stack pointer when bytecode needs to be reexecuted
  91 
  92   const TypeOopPtr* sharpen_unsafe_type(Compile::AliasType* alias_type, const TypePtr *adr_type, bool is_native_ptr = false);
  93 
  94  public:
  95   LibraryCallKit(JVMState* jvms, LibraryIntrinsic* intrinsic)
  96     : GraphKit(jvms),
  97       _intrinsic(intrinsic),
  98       _result(NULL)
  99   {
 100     // Check if this is a root compile.  In that case we don't have a caller.
 101     if (!jvms->has_method()) {
 102       _reexecute_sp = sp();
 103     } else {
 104       // Find out how many arguments the interpreter needs when deoptimizing
 105       // and save the stack pointer value so it can used by uncommon_trap.
 106       // We find the argument count by looking at the declared signature.
 107       bool ignored_will_link;
 108       ciSignature* declared_signature = NULL;
 109       ciMethod* ignored_callee = caller()->get_method_at_bci(bci(), ignored_will_link, &declared_signature);
 110       const int nargs = declared_signature->arg_size_for_bc(caller()->java_code_at_bci(bci()));
 111       _reexecute_sp = sp() + nargs;  // "push" arguments back on stack
 112     }
 113   }
 114 
 115   virtual LibraryCallKit* is_LibraryCallKit() const { return (LibraryCallKit*)this; }
 116 
 117   ciMethod*         caller()    const    { return jvms()->method(); }
 118   int               bci()       const    { return jvms()->bci(); }
 119   LibraryIntrinsic* intrinsic() const    { return _intrinsic; }
 120   vmIntrinsics::ID  intrinsic_id() const { return _intrinsic->intrinsic_id(); }
 121   ciMethod*         callee()    const    { return _intrinsic->method(); }
 122 
 123   bool  try_to_inline(int predicate);
 124   Node* try_to_predicate(int predicate);
 125 
 126   void push_result() {
 127     // Push the result onto the stack.
 128     if (!stopped() && result() != NULL) {
 129       BasicType bt = result()->bottom_type()->basic_type();
 130       push_node(bt, result());
 131     }
 132   }
 133 
 134  private:
 135   void fatal_unexpected_iid(vmIntrinsics::ID iid) {
 136     fatal("unexpected intrinsic %d: %s", iid, vmIntrinsics::name_at(iid));
 137   }
 138 
 139   void  set_result(Node* n) { assert(_result == NULL, "only set once"); _result = n; }
 140   void  set_result(RegionNode* region, PhiNode* value);
 141   Node*     result() { return _result; }
 142 
 143   virtual int reexecute_sp() { return _reexecute_sp; }
 144 
 145   // Helper functions to inline natives
 146   Node* generate_guard(Node* test, RegionNode* region, float true_prob);
 147   Node* generate_slow_guard(Node* test, RegionNode* region);
 148   Node* generate_fair_guard(Node* test, RegionNode* region);
 149   Node* generate_negative_guard(Node* index, RegionNode* region,
 150                                 // resulting CastII of index:
 151                                 Node* *pos_index = NULL);
 152   Node* generate_limit_guard(Node* offset, Node* subseq_length,
 153                              Node* array_length,
 154                              RegionNode* region);
 155   void  generate_string_range_check(Node* array, Node* offset,
 156                                     Node* length, bool char_count);
 157   Node* generate_current_thread(Node* &tls_output);
 158   Node* load_mirror_from_klass(Node* klass);
 159   Node* load_klass_from_mirror_common(Node* mirror, bool never_see_null,
 160                                       RegionNode* region, int null_path,
 161                                       int offset);
 162   Node* load_klass_from_mirror(Node* mirror, bool never_see_null,
 163                                RegionNode* region, int null_path) {
 164     int offset = java_lang_Class::klass_offset_in_bytes();
 165     return load_klass_from_mirror_common(mirror, never_see_null,
 166                                          region, null_path,
 167                                          offset);
 168   }
 169   Node* load_array_klass_from_mirror(Node* mirror, bool never_see_null,
 170                                      RegionNode* region, int null_path) {
 171     int offset = java_lang_Class::array_klass_offset_in_bytes();
 172     return load_klass_from_mirror_common(mirror, never_see_null,
 173                                          region, null_path,
 174                                          offset);
 175   }
 176   Node* generate_access_flags_guard(Node* kls,
 177                                     int modifier_mask, int modifier_bits,
 178                                     RegionNode* region);
 179   Node* generate_interface_guard(Node* kls, RegionNode* region);
 180   Node* generate_array_guard(Node* kls, RegionNode* region) {
 181     return generate_array_guard_common(kls, region, false, false);
 182   }
 183   Node* generate_non_array_guard(Node* kls, RegionNode* region) {
 184     return generate_array_guard_common(kls, region, false, true);
 185   }
 186   Node* generate_objArray_guard(Node* kls, RegionNode* region) {
 187     return generate_array_guard_common(kls, region, true, false);
 188   }
 189   Node* generate_non_objArray_guard(Node* kls, RegionNode* region) {
 190     return generate_array_guard_common(kls, region, true, true);
 191   }
 192   Node* generate_array_guard_common(Node* kls, RegionNode* region,
 193                                     bool obj_array, bool not_array);
 194   Node* generate_virtual_guard(Node* obj_klass, RegionNode* slow_region);
 195   CallJavaNode* generate_method_call(vmIntrinsics::ID method_id,
 196                                      bool is_virtual = false, bool is_static = false);
 197   CallJavaNode* generate_method_call_static(vmIntrinsics::ID method_id) {
 198     return generate_method_call(method_id, false, true);
 199   }
 200   CallJavaNode* generate_method_call_virtual(vmIntrinsics::ID method_id) {
 201     return generate_method_call(method_id, true, false);
 202   }
 203   Node * load_field_from_object(Node * fromObj, const char * fieldName, const char * fieldTypeString, bool is_exact, bool is_static, ciInstanceKlass * fromKls);
 204 
 205   Node* make_string_method_node(int opcode, Node* str1_start, Node* cnt1, Node* str2_start, Node* cnt2, StrIntrinsicNode::ArgEnc ae);
 206   bool inline_string_compareTo(StrIntrinsicNode::ArgEnc ae);
 207   bool inline_string_indexOf(StrIntrinsicNode::ArgEnc ae);
 208   bool inline_string_indexOfI(StrIntrinsicNode::ArgEnc ae);
 209   Node* make_indexOf_node(Node* src_start, Node* src_count, Node* tgt_start, Node* tgt_count,
 210                           RegionNode* region, Node* phi, StrIntrinsicNode::ArgEnc ae);
 211   bool inline_string_indexOfChar();
 212   bool inline_string_equals(StrIntrinsicNode::ArgEnc ae);
 213   bool inline_string_toBytesU();
 214   bool inline_string_getCharsU();
 215   bool inline_string_copy(bool compress);
 216   bool inline_string_char_access(bool is_store);
 217   Node* round_double_node(Node* n);
 218   bool runtime_math(const TypeFunc* call_type, address funcAddr, const char* funcName);
 219   bool inline_math_native(vmIntrinsics::ID id);
 220   bool inline_trig(vmIntrinsics::ID id);
 221   bool inline_math(vmIntrinsics::ID id);
 222   template <typename OverflowOp>
 223   bool inline_math_overflow(Node* arg1, Node* arg2);
 224   void inline_math_mathExact(Node* math, Node* test);
 225   bool inline_math_addExactI(bool is_increment);
 226   bool inline_math_addExactL(bool is_increment);
 227   bool inline_math_multiplyExactI();
 228   bool inline_math_multiplyExactL();
 229   bool inline_math_negateExactI();
 230   bool inline_math_negateExactL();
 231   bool inline_math_subtractExactI(bool is_decrement);
 232   bool inline_math_subtractExactL(bool is_decrement);
 233   bool inline_pow();
 234   Node* finish_pow_exp(Node* result, Node* x, Node* y, const TypeFunc* call_type, address funcAddr, const char* funcName);
 235   bool inline_min_max(vmIntrinsics::ID id);
 236   bool inline_notify(vmIntrinsics::ID id);
 237   Node* generate_min_max(vmIntrinsics::ID id, Node* x, Node* y);
 238   // This returns Type::AnyPtr, RawPtr, or OopPtr.
 239   int classify_unsafe_addr(Node* &base, Node* &offset);
 240   Node* make_unsafe_address(Node* base, Node* offset);
 241   // Helper for inline_unsafe_access.
 242   // Generates the guards that check whether the result of
 243   // Unsafe.getObject should be recorded in an SATB log buffer.
 244   void insert_pre_barrier(Node* base_oop, Node* offset, Node* pre_val, bool need_mem_bar);
 245   bool inline_unsafe_access(bool is_native_ptr, bool is_store, BasicType type, bool is_volatile, bool is_unaligned);
 246   static bool klass_needs_init_guard(Node* kls);
 247   bool inline_unsafe_allocate();
 248   bool inline_unsafe_copyMemory();
 249   bool inline_native_currentThread();
 250 #ifdef TRACE_HAVE_INTRINSICS
 251   bool inline_native_classID();
 252   bool inline_native_threadID();
 253 #endif
 254   bool inline_native_time_funcs(address method, const char* funcName);
 255   bool inline_native_isInterrupted();
 256   bool inline_native_Class_query(vmIntrinsics::ID id);
 257   bool inline_native_subtype_check();
 258 
 259   bool inline_native_newArray();
 260   bool inline_native_getLength();
 261   bool inline_array_copyOf(bool is_copyOfRange);
 262   bool inline_array_equals(StrIntrinsicNode::ArgEnc ae);
 263   bool inline_objects_checkIndex();
 264   void copy_to_clone(Node* obj, Node* alloc_obj, Node* obj_size, bool is_array, bool card_mark);
 265   bool inline_native_clone(bool is_virtual);
 266   bool inline_native_Reflection_getCallerClass();
 267   // Helper function for inlining native object hash method
 268   bool inline_native_hashcode(bool is_virtual, bool is_static);
 269   bool inline_native_getClass();
 270 
 271   // Helper functions for inlining arraycopy
 272   bool inline_arraycopy();
 273   AllocateArrayNode* tightly_coupled_allocation(Node* ptr,
 274                                                 RegionNode* slow_region);
 275   JVMState* arraycopy_restore_alloc_state(AllocateArrayNode* alloc, int& saved_reexecute_sp);
 276   void arraycopy_move_allocation_here(AllocateArrayNode* alloc, Node* dest, JVMState* saved_jvms, int saved_reexecute_sp);
 277 
 278   typedef enum { LS_xadd, LS_xchg, LS_cmpxchg } LoadStoreKind;
 279   bool inline_unsafe_load_store(BasicType type,  LoadStoreKind kind);
 280   bool inline_unsafe_ordered_store(BasicType type);
 281   bool inline_unsafe_fence(vmIntrinsics::ID id);
 282   bool inline_onspinwait();
 283   bool inline_fp_conversions(vmIntrinsics::ID id);
 284   bool inline_number_methods(vmIntrinsics::ID id);
 285   bool inline_reference_get();
 286   bool inline_Class_cast();
 287   bool inline_aescrypt_Block(vmIntrinsics::ID id);
 288   bool inline_cipherBlockChaining_AESCrypt(vmIntrinsics::ID id);
 289   Node* inline_cipherBlockChaining_AESCrypt_predicate(bool decrypting);
 290   Node* get_key_start_from_aescrypt_object(Node* aescrypt_object);
 291   Node* get_original_key_start_from_aescrypt_object(Node* aescrypt_object);
 292   bool inline_ghash_processBlocks();
 293   bool inline_sha_implCompress(vmIntrinsics::ID id);
 294   bool inline_digestBase_implCompressMB(int predicate);
 295   bool inline_sha_implCompressMB(Node* digestBaseObj, ciInstanceKlass* instklass_SHA,
 296                                  bool long_state, address stubAddr, const char *stubName,
 297                                  Node* src_start, Node* ofs, Node* limit);
 298   Node* get_state_from_sha_object(Node *sha_object);
 299   Node* get_state_from_sha5_object(Node *sha_object);
 300   Node* inline_digestBase_implCompressMB_predicate(int predicate);
 301   bool inline_encodeISOArray();
 302   bool inline_updateCRC32();
 303   bool inline_updateBytesCRC32();
 304   bool inline_updateByteBufferCRC32();
 305   Node* get_table_from_crc32c_class(ciInstanceKlass *crc32c_class);
 306   bool inline_updateBytesCRC32C();
 307   bool inline_updateDirectByteBufferCRC32C();
 308   bool inline_updateBytesAdler32();
 309   bool inline_updateByteBufferAdler32();
 310   bool inline_multiplyToLen();
 311   bool inline_hasNegatives();
 312   bool inline_squareToLen();
 313   bool inline_mulAdd();
 314   bool inline_montgomeryMultiply();
 315   bool inline_montgomerySquare();
 316 
 317   bool inline_profileBoolean();
 318   bool inline_isCompileConstant();
 319 };
 320 
 321 //---------------------------make_vm_intrinsic----------------------------
 322 CallGenerator* Compile::make_vm_intrinsic(ciMethod* m, bool is_virtual) {
 323   vmIntrinsics::ID id = m->intrinsic_id();
 324   assert(id != vmIntrinsics::_none, "must be a VM intrinsic");
 325 
 326   if (!m->is_loaded()) {
 327     // Do not attempt to inline unloaded methods.
 328     return NULL;
 329   }
 330 
 331   C2Compiler* compiler = (C2Compiler*)CompileBroker::compiler(CompLevel_full_optimization);
 332   bool is_available = false;
 333 
 334   {
 335     // For calling is_intrinsic_supported and is_intrinsic_disabled_by_flag
 336     // the compiler must transition to '_thread_in_vm' state because both
 337     // methods access VM-internal data.
 338     VM_ENTRY_MARK;
 339     methodHandle mh(THREAD, m->get_Method());
 340     is_available = compiler->is_intrinsic_supported(mh, is_virtual) &&
 341                    !C->directive()->is_intrinsic_disabled(mh) &&
 342                    !vmIntrinsics::is_disabled_by_flags(mh);
 343 
 344   }
 345 
 346   if (is_available) {
 347     assert(id <= vmIntrinsics::LAST_COMPILER_INLINE, "caller responsibility");
 348     assert(id != vmIntrinsics::_Object_init && id != vmIntrinsics::_invoke, "enum out of order?");
 349     return new LibraryIntrinsic(m, is_virtual,
 350                                 vmIntrinsics::predicates_needed(id),
 351                                 vmIntrinsics::does_virtual_dispatch(id),
 352                                 (vmIntrinsics::ID) id);
 353   } else {
 354     return NULL;
 355   }
 356 }
 357 
 358 //----------------------register_library_intrinsics-----------------------
 359 // Initialize this file's data structures, for each Compile instance.
 360 void Compile::register_library_intrinsics() {
 361   // Nothing to do here.
 362 }
 363 
 364 JVMState* LibraryIntrinsic::generate(JVMState* jvms) {
 365   LibraryCallKit kit(jvms, this);
 366   Compile* C = kit.C;
 367   int nodes = C->unique();
 368 #ifndef PRODUCT
 369   if ((C->print_intrinsics() || C->print_inlining()) && Verbose) {
 370     char buf[1000];
 371     const char* str = vmIntrinsics::short_name_as_C_string(intrinsic_id(), buf, sizeof(buf));
 372     tty->print_cr("Intrinsic %s", str);
 373   }
 374 #endif
 375   ciMethod* callee = kit.callee();
 376   const int bci    = kit.bci();
 377 
 378   // Try to inline the intrinsic.
 379   if ((CheckIntrinsics ? callee->intrinsic_candidate() : true) &&
 380       kit.try_to_inline(_last_predicate)) {
 381     if (C->print_intrinsics() || C->print_inlining()) {
 382       C->print_inlining(callee, jvms->depth() - 1, bci, is_virtual() ? "(intrinsic, virtual)" : "(intrinsic)");
 383     }
 384     C->gather_intrinsic_statistics(intrinsic_id(), is_virtual(), Compile::_intrinsic_worked);
 385     if (C->log()) {
 386       C->log()->elem("intrinsic id='%s'%s nodes='%d'",
 387                      vmIntrinsics::name_at(intrinsic_id()),
 388                      (is_virtual() ? " virtual='1'" : ""),
 389                      C->unique() - nodes);
 390     }
 391     // Push the result from the inlined method onto the stack.
 392     kit.push_result();
 393     C->print_inlining_update(this);
 394     return kit.transfer_exceptions_into_jvms();
 395   }
 396 
 397   // The intrinsic bailed out
 398   if (C->print_intrinsics() || C->print_inlining()) {
 399     if (jvms->has_method()) {
 400       // Not a root compile.
 401       const char* msg;
 402       if (callee->intrinsic_candidate()) {
 403         msg = is_virtual() ? "failed to inline (intrinsic, virtual)" : "failed to inline (intrinsic)";
 404       } else {
 405         msg = is_virtual() ? "failed to inline (intrinsic, virtual), method not annotated"
 406                            : "failed to inline (intrinsic), method not annotated";
 407       }
 408       C->print_inlining(callee, jvms->depth() - 1, bci, msg);
 409     } else {
 410       // Root compile
 411       tty->print("Did not generate intrinsic %s%s at bci:%d in",
 412                vmIntrinsics::name_at(intrinsic_id()),
 413                (is_virtual() ? " (virtual)" : ""), bci);
 414     }
 415   }
 416   C->gather_intrinsic_statistics(intrinsic_id(), is_virtual(), Compile::_intrinsic_failed);
 417   C->print_inlining_update(this);
 418   return NULL;
 419 }
 420 
 421 Node* LibraryIntrinsic::generate_predicate(JVMState* jvms, int predicate) {
 422   LibraryCallKit kit(jvms, this);
 423   Compile* C = kit.C;
 424   int nodes = C->unique();
 425   _last_predicate = predicate;
 426 #ifndef PRODUCT
 427   assert(is_predicated() && predicate < predicates_count(), "sanity");
 428   if ((C->print_intrinsics() || C->print_inlining()) && Verbose) {
 429     char buf[1000];
 430     const char* str = vmIntrinsics::short_name_as_C_string(intrinsic_id(), buf, sizeof(buf));
 431     tty->print_cr("Predicate for intrinsic %s", str);
 432   }
 433 #endif
 434   ciMethod* callee = kit.callee();
 435   const int bci    = kit.bci();
 436 
 437   Node* slow_ctl = kit.try_to_predicate(predicate);
 438   if (!kit.failing()) {
 439     if (C->print_intrinsics() || C->print_inlining()) {
 440       C->print_inlining(callee, jvms->depth() - 1, bci, is_virtual() ? "(intrinsic, virtual, predicate)" : "(intrinsic, predicate)");
 441     }
 442     C->gather_intrinsic_statistics(intrinsic_id(), is_virtual(), Compile::_intrinsic_worked);
 443     if (C->log()) {
 444       C->log()->elem("predicate_intrinsic id='%s'%s nodes='%d'",
 445                      vmIntrinsics::name_at(intrinsic_id()),
 446                      (is_virtual() ? " virtual='1'" : ""),
 447                      C->unique() - nodes);
 448     }
 449     return slow_ctl; // Could be NULL if the check folds.
 450   }
 451 
 452   // The intrinsic bailed out
 453   if (C->print_intrinsics() || C->print_inlining()) {
 454     if (jvms->has_method()) {
 455       // Not a root compile.
 456       const char* msg = "failed to generate predicate for intrinsic";
 457       C->print_inlining(kit.callee(), jvms->depth() - 1, bci, msg);
 458     } else {
 459       // Root compile
 460       C->print_inlining_stream()->print("Did not generate predicate for intrinsic %s%s at bci:%d in",
 461                                         vmIntrinsics::name_at(intrinsic_id()),
 462                                         (is_virtual() ? " (virtual)" : ""), bci);
 463     }
 464   }
 465   C->gather_intrinsic_statistics(intrinsic_id(), is_virtual(), Compile::_intrinsic_failed);
 466   return NULL;
 467 }
 468 
 469 bool LibraryCallKit::try_to_inline(int predicate) {
 470   // Handle symbolic names for otherwise undistinguished boolean switches:
 471   const bool is_store       = true;
 472   const bool is_compress    = true;
 473   const bool is_native_ptr  = true;
 474   const bool is_static      = true;
 475   const bool is_volatile    = true;
 476 
 477   if (!jvms()->has_method()) {
 478     // Root JVMState has a null method.
 479     assert(map()->memory()->Opcode() == Op_Parm, "");
 480     // Insert the memory aliasing node
 481     set_all_memory(reset_memory());
 482   }
 483   assert(merged_memory(), "");
 484 
 485 
 486   switch (intrinsic_id()) {
 487   case vmIntrinsics::_hashCode:                 return inline_native_hashcode(intrinsic()->is_virtual(), !is_static);
 488   case vmIntrinsics::_identityHashCode:         return inline_native_hashcode(/*!virtual*/ false,         is_static);
 489   case vmIntrinsics::_getClass:                 return inline_native_getClass();
 490 
 491   case vmIntrinsics::_dsin:
 492   case vmIntrinsics::_dcos:
 493   case vmIntrinsics::_dtan:
 494   case vmIntrinsics::_dabs:
 495   case vmIntrinsics::_datan2:
 496   case vmIntrinsics::_dsqrt:
 497   case vmIntrinsics::_dexp:
 498   case vmIntrinsics::_dlog:
 499   case vmIntrinsics::_dlog10:
 500   case vmIntrinsics::_dpow:                     return inline_math_native(intrinsic_id());
 501 
 502   case vmIntrinsics::_min:
 503   case vmIntrinsics::_max:                      return inline_min_max(intrinsic_id());
 504 
 505   case vmIntrinsics::_notify:
 506   case vmIntrinsics::_notifyAll:
 507     if (InlineNotify) {
 508       return inline_notify(intrinsic_id());
 509     }
 510     return false;
 511 
 512   case vmIntrinsics::_addExactI:                return inline_math_addExactI(false /* add */);
 513   case vmIntrinsics::_addExactL:                return inline_math_addExactL(false /* add */);
 514   case vmIntrinsics::_decrementExactI:          return inline_math_subtractExactI(true /* decrement */);
 515   case vmIntrinsics::_decrementExactL:          return inline_math_subtractExactL(true /* decrement */);
 516   case vmIntrinsics::_incrementExactI:          return inline_math_addExactI(true /* increment */);
 517   case vmIntrinsics::_incrementExactL:          return inline_math_addExactL(true /* increment */);
 518   case vmIntrinsics::_multiplyExactI:           return inline_math_multiplyExactI();
 519   case vmIntrinsics::_multiplyExactL:           return inline_math_multiplyExactL();
 520   case vmIntrinsics::_negateExactI:             return inline_math_negateExactI();
 521   case vmIntrinsics::_negateExactL:             return inline_math_negateExactL();
 522   case vmIntrinsics::_subtractExactI:           return inline_math_subtractExactI(false /* subtract */);
 523   case vmIntrinsics::_subtractExactL:           return inline_math_subtractExactL(false /* subtract */);
 524 
 525   case vmIntrinsics::_arraycopy:                return inline_arraycopy();
 526 
 527   case vmIntrinsics::_compareToL:               return inline_string_compareTo(StrIntrinsicNode::LL);
 528   case vmIntrinsics::_compareToU:               return inline_string_compareTo(StrIntrinsicNode::UU);
 529   case vmIntrinsics::_compareToLU:              return inline_string_compareTo(StrIntrinsicNode::LU);
 530   case vmIntrinsics::_compareToUL:              return inline_string_compareTo(StrIntrinsicNode::UL);
 531 
 532   case vmIntrinsics::_indexOfL:                 return inline_string_indexOf(StrIntrinsicNode::LL);
 533   case vmIntrinsics::_indexOfU:                 return inline_string_indexOf(StrIntrinsicNode::UU);
 534   case vmIntrinsics::_indexOfUL:                return inline_string_indexOf(StrIntrinsicNode::UL);
 535   case vmIntrinsics::_indexOfIL:                return inline_string_indexOfI(StrIntrinsicNode::LL);
 536   case vmIntrinsics::_indexOfIU:                return inline_string_indexOfI(StrIntrinsicNode::UU);
 537   case vmIntrinsics::_indexOfIUL:               return inline_string_indexOfI(StrIntrinsicNode::UL);
 538   case vmIntrinsics::_indexOfU_char:            return inline_string_indexOfChar();
 539 
 540   case vmIntrinsics::_equalsL:                  return inline_string_equals(StrIntrinsicNode::LL);
 541   case vmIntrinsics::_equalsU:                  return inline_string_equals(StrIntrinsicNode::UU);
 542 
 543   case vmIntrinsics::_toBytesStringU:           return inline_string_toBytesU();
 544   case vmIntrinsics::_getCharsStringU:          return inline_string_getCharsU();
 545   case vmIntrinsics::_getCharStringU:           return inline_string_char_access(!is_store);
 546   case vmIntrinsics::_putCharStringU:           return inline_string_char_access( is_store);
 547 
 548   case vmIntrinsics::_compressStringC:
 549   case vmIntrinsics::_compressStringB:          return inline_string_copy( is_compress);
 550   case vmIntrinsics::_inflateStringC:
 551   case vmIntrinsics::_inflateStringB:           return inline_string_copy(!is_compress);
 552 
 553   case vmIntrinsics::_getObject:                return inline_unsafe_access(!is_native_ptr, !is_store, T_OBJECT,  !is_volatile, false);
 554   case vmIntrinsics::_getBoolean:               return inline_unsafe_access(!is_native_ptr, !is_store, T_BOOLEAN, !is_volatile, false);
 555   case vmIntrinsics::_getByte:                  return inline_unsafe_access(!is_native_ptr, !is_store, T_BYTE,    !is_volatile, false);
 556   case vmIntrinsics::_getShort:                 return inline_unsafe_access(!is_native_ptr, !is_store, T_SHORT,   !is_volatile, false);
 557   case vmIntrinsics::_getChar:                  return inline_unsafe_access(!is_native_ptr, !is_store, T_CHAR,    !is_volatile, false);
 558   case vmIntrinsics::_getInt:                   return inline_unsafe_access(!is_native_ptr, !is_store, T_INT,     !is_volatile, false);
 559   case vmIntrinsics::_getLong:                  return inline_unsafe_access(!is_native_ptr, !is_store, T_LONG,    !is_volatile, false);
 560   case vmIntrinsics::_getFloat:                 return inline_unsafe_access(!is_native_ptr, !is_store, T_FLOAT,   !is_volatile, false);
 561   case vmIntrinsics::_getDouble:                return inline_unsafe_access(!is_native_ptr, !is_store, T_DOUBLE,  !is_volatile, false);
 562   case vmIntrinsics::_putObject:                return inline_unsafe_access(!is_native_ptr,  is_store, T_OBJECT,  !is_volatile, false);
 563   case vmIntrinsics::_putBoolean:               return inline_unsafe_access(!is_native_ptr,  is_store, T_BOOLEAN, !is_volatile, false);
 564   case vmIntrinsics::_putByte:                  return inline_unsafe_access(!is_native_ptr,  is_store, T_BYTE,    !is_volatile, false);
 565   case vmIntrinsics::_putShort:                 return inline_unsafe_access(!is_native_ptr,  is_store, T_SHORT,   !is_volatile, false);
 566   case vmIntrinsics::_putChar:                  return inline_unsafe_access(!is_native_ptr,  is_store, T_CHAR,    !is_volatile, false);
 567   case vmIntrinsics::_putInt:                   return inline_unsafe_access(!is_native_ptr,  is_store, T_INT,     !is_volatile, false);
 568   case vmIntrinsics::_putLong:                  return inline_unsafe_access(!is_native_ptr,  is_store, T_LONG,    !is_volatile, false);
 569   case vmIntrinsics::_putFloat:                 return inline_unsafe_access(!is_native_ptr,  is_store, T_FLOAT,   !is_volatile, false);
 570   case vmIntrinsics::_putDouble:                return inline_unsafe_access(!is_native_ptr,  is_store, T_DOUBLE,  !is_volatile, false);
 571 
 572   case vmIntrinsics::_getByte_raw:              return inline_unsafe_access( is_native_ptr, !is_store, T_BYTE,    !is_volatile, false);
 573   case vmIntrinsics::_getShort_raw:             return inline_unsafe_access( is_native_ptr, !is_store, T_SHORT,   !is_volatile, false);
 574   case vmIntrinsics::_getChar_raw:              return inline_unsafe_access( is_native_ptr, !is_store, T_CHAR,    !is_volatile, false);
 575   case vmIntrinsics::_getInt_raw:               return inline_unsafe_access( is_native_ptr, !is_store, T_INT,     !is_volatile, false);
 576   case vmIntrinsics::_getLong_raw:              return inline_unsafe_access( is_native_ptr, !is_store, T_LONG,    !is_volatile, false);
 577   case vmIntrinsics::_getFloat_raw:             return inline_unsafe_access( is_native_ptr, !is_store, T_FLOAT,   !is_volatile, false);
 578   case vmIntrinsics::_getDouble_raw:            return inline_unsafe_access( is_native_ptr, !is_store, T_DOUBLE,  !is_volatile, false);
 579   case vmIntrinsics::_getAddress_raw:           return inline_unsafe_access( is_native_ptr, !is_store, T_ADDRESS, !is_volatile, false);
 580 
 581   case vmIntrinsics::_putByte_raw:              return inline_unsafe_access( is_native_ptr,  is_store, T_BYTE,    !is_volatile, false);
 582   case vmIntrinsics::_putShort_raw:             return inline_unsafe_access( is_native_ptr,  is_store, T_SHORT,   !is_volatile, false);
 583   case vmIntrinsics::_putChar_raw:              return inline_unsafe_access( is_native_ptr,  is_store, T_CHAR,    !is_volatile, false);
 584   case vmIntrinsics::_putInt_raw:               return inline_unsafe_access( is_native_ptr,  is_store, T_INT,     !is_volatile, false);
 585   case vmIntrinsics::_putLong_raw:              return inline_unsafe_access( is_native_ptr,  is_store, T_LONG,    !is_volatile, false);
 586   case vmIntrinsics::_putFloat_raw:             return inline_unsafe_access( is_native_ptr,  is_store, T_FLOAT,   !is_volatile, false);
 587   case vmIntrinsics::_putDouble_raw:            return inline_unsafe_access( is_native_ptr,  is_store, T_DOUBLE,  !is_volatile, false);
 588   case vmIntrinsics::_putAddress_raw:           return inline_unsafe_access( is_native_ptr,  is_store, T_ADDRESS, !is_volatile, false);
 589 
 590   case vmIntrinsics::_getObjectVolatile:        return inline_unsafe_access(!is_native_ptr, !is_store, T_OBJECT,   is_volatile, false);
 591   case vmIntrinsics::_getBooleanVolatile:       return inline_unsafe_access(!is_native_ptr, !is_store, T_BOOLEAN,  is_volatile, false);
 592   case vmIntrinsics::_getByteVolatile:          return inline_unsafe_access(!is_native_ptr, !is_store, T_BYTE,     is_volatile, false);
 593   case vmIntrinsics::_getShortVolatile:         return inline_unsafe_access(!is_native_ptr, !is_store, T_SHORT,    is_volatile, false);
 594   case vmIntrinsics::_getCharVolatile:          return inline_unsafe_access(!is_native_ptr, !is_store, T_CHAR,     is_volatile, false);
 595   case vmIntrinsics::_getIntVolatile:           return inline_unsafe_access(!is_native_ptr, !is_store, T_INT,      is_volatile, false);
 596   case vmIntrinsics::_getLongVolatile:          return inline_unsafe_access(!is_native_ptr, !is_store, T_LONG,     is_volatile, false);
 597   case vmIntrinsics::_getFloatVolatile:         return inline_unsafe_access(!is_native_ptr, !is_store, T_FLOAT,    is_volatile, false);
 598   case vmIntrinsics::_getDoubleVolatile:        return inline_unsafe_access(!is_native_ptr, !is_store, T_DOUBLE,   is_volatile, false);
 599 
 600   case vmIntrinsics::_putObjectVolatile:        return inline_unsafe_access(!is_native_ptr,  is_store, T_OBJECT,   is_volatile, false);
 601   case vmIntrinsics::_putBooleanVolatile:       return inline_unsafe_access(!is_native_ptr,  is_store, T_BOOLEAN,  is_volatile, false);
 602   case vmIntrinsics::_putByteVolatile:          return inline_unsafe_access(!is_native_ptr,  is_store, T_BYTE,     is_volatile, false);
 603   case vmIntrinsics::_putShortVolatile:         return inline_unsafe_access(!is_native_ptr,  is_store, T_SHORT,    is_volatile, false);
 604   case vmIntrinsics::_putCharVolatile:          return inline_unsafe_access(!is_native_ptr,  is_store, T_CHAR,     is_volatile, false);
 605   case vmIntrinsics::_putIntVolatile:           return inline_unsafe_access(!is_native_ptr,  is_store, T_INT,      is_volatile, false);
 606   case vmIntrinsics::_putLongVolatile:          return inline_unsafe_access(!is_native_ptr,  is_store, T_LONG,     is_volatile, false);
 607   case vmIntrinsics::_putFloatVolatile:         return inline_unsafe_access(!is_native_ptr,  is_store, T_FLOAT,    is_volatile, false);
 608   case vmIntrinsics::_putDoubleVolatile:        return inline_unsafe_access(!is_native_ptr,  is_store, T_DOUBLE,   is_volatile, false);
 609 
 610   case vmIntrinsics::_getShortUnaligned:        return inline_unsafe_access(!is_native_ptr, !is_store, T_SHORT,   !is_volatile, true);
 611   case vmIntrinsics::_getCharUnaligned:         return inline_unsafe_access(!is_native_ptr, !is_store, T_CHAR,    !is_volatile, true);
 612   case vmIntrinsics::_getIntUnaligned:          return inline_unsafe_access(!is_native_ptr, !is_store, T_INT,     !is_volatile, true);
 613   case vmIntrinsics::_getLongUnaligned:         return inline_unsafe_access(!is_native_ptr, !is_store, T_LONG,    !is_volatile, true);
 614 
 615   case vmIntrinsics::_putShortUnaligned:        return inline_unsafe_access(!is_native_ptr,  is_store, T_SHORT,   !is_volatile, true);
 616   case vmIntrinsics::_putCharUnaligned:         return inline_unsafe_access(!is_native_ptr,  is_store, T_CHAR,    !is_volatile, true);
 617   case vmIntrinsics::_putIntUnaligned:          return inline_unsafe_access(!is_native_ptr,  is_store, T_INT,     !is_volatile, true);
 618   case vmIntrinsics::_putLongUnaligned:         return inline_unsafe_access(!is_native_ptr,  is_store, T_LONG,    !is_volatile, true);
 619 
 620   case vmIntrinsics::_compareAndSwapObject:     return inline_unsafe_load_store(T_OBJECT, LS_cmpxchg);
 621   case vmIntrinsics::_compareAndSwapInt:        return inline_unsafe_load_store(T_INT,    LS_cmpxchg);
 622   case vmIntrinsics::_compareAndSwapLong:       return inline_unsafe_load_store(T_LONG,   LS_cmpxchg);
 623 
 624   case vmIntrinsics::_putOrderedObject:         return inline_unsafe_ordered_store(T_OBJECT);
 625   case vmIntrinsics::_putOrderedInt:            return inline_unsafe_ordered_store(T_INT);
 626   case vmIntrinsics::_putOrderedLong:           return inline_unsafe_ordered_store(T_LONG);
 627 
 628   case vmIntrinsics::_getAndAddInt:             return inline_unsafe_load_store(T_INT,    LS_xadd);
 629   case vmIntrinsics::_getAndAddLong:            return inline_unsafe_load_store(T_LONG,   LS_xadd);
 630   case vmIntrinsics::_getAndSetInt:             return inline_unsafe_load_store(T_INT,    LS_xchg);
 631   case vmIntrinsics::_getAndSetLong:            return inline_unsafe_load_store(T_LONG,   LS_xchg);
 632   case vmIntrinsics::_getAndSetObject:          return inline_unsafe_load_store(T_OBJECT, LS_xchg);
 633 
 634   case vmIntrinsics::_loadFence:
 635   case vmIntrinsics::_storeFence:
 636   case vmIntrinsics::_fullFence:                return inline_unsafe_fence(intrinsic_id());
 637 
 638   case vmIntrinsics::_onSpinWait:             return inline_onspinwait();
 639 
 640   case vmIntrinsics::_currentThread:            return inline_native_currentThread();
 641   case vmIntrinsics::_isInterrupted:            return inline_native_isInterrupted();
 642 
 643 #ifdef TRACE_HAVE_INTRINSICS
 644   case vmIntrinsics::_classID:                  return inline_native_classID();
 645   case vmIntrinsics::_threadID:                 return inline_native_threadID();
 646   case vmIntrinsics::_counterTime:              return inline_native_time_funcs(CAST_FROM_FN_PTR(address, TRACE_TIME_METHOD), "counterTime");
 647 #endif
 648   case vmIntrinsics::_currentTimeMillis:        return inline_native_time_funcs(CAST_FROM_FN_PTR(address, os::javaTimeMillis), "currentTimeMillis");
 649   case vmIntrinsics::_nanoTime:                 return inline_native_time_funcs(CAST_FROM_FN_PTR(address, os::javaTimeNanos), "nanoTime");
 650   case vmIntrinsics::_allocateInstance:         return inline_unsafe_allocate();
 651   case vmIntrinsics::_copyMemory:               return inline_unsafe_copyMemory();
 652   case vmIntrinsics::_newArray:                 return inline_native_newArray();
 653   case vmIntrinsics::_getLength:                return inline_native_getLength();
 654   case vmIntrinsics::_copyOf:                   return inline_array_copyOf(false);
 655   case vmIntrinsics::_copyOfRange:              return inline_array_copyOf(true);
 656   case vmIntrinsics::_equalsB:                  return inline_array_equals(StrIntrinsicNode::LL);
 657   case vmIntrinsics::_equalsC:                  return inline_array_equals(StrIntrinsicNode::UU);
 658   case vmIntrinsics::_Objects_checkIndex:       return inline_objects_checkIndex();
 659   case vmIntrinsics::_clone:                    return inline_native_clone(intrinsic()->is_virtual());
 660 
 661   case vmIntrinsics::_isAssignableFrom:         return inline_native_subtype_check();
 662 
 663   case vmIntrinsics::_isInstance:
 664   case vmIntrinsics::_getModifiers:
 665   case vmIntrinsics::_isInterface:
 666   case vmIntrinsics::_isArray:
 667   case vmIntrinsics::_isPrimitive:
 668   case vmIntrinsics::_getSuperclass:
 669   case vmIntrinsics::_getClassAccessFlags:      return inline_native_Class_query(intrinsic_id());
 670 
 671   case vmIntrinsics::_floatToRawIntBits:
 672   case vmIntrinsics::_floatToIntBits:
 673   case vmIntrinsics::_intBitsToFloat:
 674   case vmIntrinsics::_doubleToRawLongBits:
 675   case vmIntrinsics::_doubleToLongBits:
 676   case vmIntrinsics::_longBitsToDouble:         return inline_fp_conversions(intrinsic_id());
 677 
 678   case vmIntrinsics::_numberOfLeadingZeros_i:
 679   case vmIntrinsics::_numberOfLeadingZeros_l:
 680   case vmIntrinsics::_numberOfTrailingZeros_i:
 681   case vmIntrinsics::_numberOfTrailingZeros_l:
 682   case vmIntrinsics::_bitCount_i:
 683   case vmIntrinsics::_bitCount_l:
 684   case vmIntrinsics::_reverseBytes_i:
 685   case vmIntrinsics::_reverseBytes_l:
 686   case vmIntrinsics::_reverseBytes_s:
 687   case vmIntrinsics::_reverseBytes_c:           return inline_number_methods(intrinsic_id());
 688 
 689   case vmIntrinsics::_getCallerClass:           return inline_native_Reflection_getCallerClass();
 690 
 691   case vmIntrinsics::_Reference_get:            return inline_reference_get();
 692 
 693   case vmIntrinsics::_Class_cast:               return inline_Class_cast();
 694 
 695   case vmIntrinsics::_aescrypt_encryptBlock:
 696   case vmIntrinsics::_aescrypt_decryptBlock:    return inline_aescrypt_Block(intrinsic_id());
 697 
 698   case vmIntrinsics::_cipherBlockChaining_encryptAESCrypt:
 699   case vmIntrinsics::_cipherBlockChaining_decryptAESCrypt:
 700     return inline_cipherBlockChaining_AESCrypt(intrinsic_id());
 701 
 702   case vmIntrinsics::_sha_implCompress:
 703   case vmIntrinsics::_sha2_implCompress:
 704   case vmIntrinsics::_sha5_implCompress:
 705     return inline_sha_implCompress(intrinsic_id());
 706 
 707   case vmIntrinsics::_digestBase_implCompressMB:
 708     return inline_digestBase_implCompressMB(predicate);
 709 
 710   case vmIntrinsics::_multiplyToLen:
 711     return inline_multiplyToLen();
 712 
 713   case vmIntrinsics::_squareToLen:
 714     return inline_squareToLen();
 715 
 716   case vmIntrinsics::_mulAdd:
 717     return inline_mulAdd();
 718 
 719   case vmIntrinsics::_montgomeryMultiply:
 720     return inline_montgomeryMultiply();
 721   case vmIntrinsics::_montgomerySquare:
 722     return inline_montgomerySquare();
 723 
 724   case vmIntrinsics::_ghash_processBlocks:
 725     return inline_ghash_processBlocks();
 726 
 727   case vmIntrinsics::_encodeISOArray:
 728   case vmIntrinsics::_encodeByteISOArray:
 729     return inline_encodeISOArray();
 730 
 731   case vmIntrinsics::_updateCRC32:
 732     return inline_updateCRC32();
 733   case vmIntrinsics::_updateBytesCRC32:
 734     return inline_updateBytesCRC32();
 735   case vmIntrinsics::_updateByteBufferCRC32:
 736     return inline_updateByteBufferCRC32();
 737 
 738   case vmIntrinsics::_updateBytesCRC32C:
 739     return inline_updateBytesCRC32C();
 740   case vmIntrinsics::_updateDirectByteBufferCRC32C:
 741     return inline_updateDirectByteBufferCRC32C();
 742 
 743   case vmIntrinsics::_updateBytesAdler32:
 744     return inline_updateBytesAdler32();
 745   case vmIntrinsics::_updateByteBufferAdler32:
 746     return inline_updateByteBufferAdler32();
 747 
 748   case vmIntrinsics::_profileBoolean:
 749     return inline_profileBoolean();
 750   case vmIntrinsics::_isCompileConstant:
 751     return inline_isCompileConstant();
 752 
 753   case vmIntrinsics::_hasNegatives:
 754     return inline_hasNegatives();
 755 
 756   default:
 757     // If you get here, it may be that someone has added a new intrinsic
 758     // to the list in vmSymbols.hpp without implementing it here.
 759 #ifndef PRODUCT
 760     if ((PrintMiscellaneous && (Verbose || WizardMode)) || PrintOpto) {
 761       tty->print_cr("*** Warning: Unimplemented intrinsic %s(%d)",
 762                     vmIntrinsics::name_at(intrinsic_id()), intrinsic_id());
 763     }
 764 #endif
 765     return false;
 766   }
 767 }
 768 
 769 Node* LibraryCallKit::try_to_predicate(int predicate) {
 770   if (!jvms()->has_method()) {
 771     // Root JVMState has a null method.
 772     assert(map()->memory()->Opcode() == Op_Parm, "");
 773     // Insert the memory aliasing node
 774     set_all_memory(reset_memory());
 775   }
 776   assert(merged_memory(), "");
 777 
 778   switch (intrinsic_id()) {
 779   case vmIntrinsics::_cipherBlockChaining_encryptAESCrypt:
 780     return inline_cipherBlockChaining_AESCrypt_predicate(false);
 781   case vmIntrinsics::_cipherBlockChaining_decryptAESCrypt:
 782     return inline_cipherBlockChaining_AESCrypt_predicate(true);
 783   case vmIntrinsics::_digestBase_implCompressMB:
 784     return inline_digestBase_implCompressMB_predicate(predicate);
 785 
 786   default:
 787     // If you get here, it may be that someone has added a new intrinsic
 788     // to the list in vmSymbols.hpp without implementing it here.
 789 #ifndef PRODUCT
 790     if ((PrintMiscellaneous && (Verbose || WizardMode)) || PrintOpto) {
 791       tty->print_cr("*** Warning: Unimplemented predicate for intrinsic %s(%d)",
 792                     vmIntrinsics::name_at(intrinsic_id()), intrinsic_id());
 793     }
 794 #endif
 795     Node* slow_ctl = control();
 796     set_control(top()); // No fast path instrinsic
 797     return slow_ctl;
 798   }
 799 }
 800 
 801 //------------------------------set_result-------------------------------
 802 // Helper function for finishing intrinsics.
 803 void LibraryCallKit::set_result(RegionNode* region, PhiNode* value) {
 804   record_for_igvn(region);
 805   set_control(_gvn.transform(region));
 806   set_result( _gvn.transform(value));
 807   assert(value->type()->basic_type() == result()->bottom_type()->basic_type(), "sanity");
 808 }
 809 
 810 //------------------------------generate_guard---------------------------
 811 // Helper function for generating guarded fast-slow graph structures.
 812 // The given 'test', if true, guards a slow path.  If the test fails
 813 // then a fast path can be taken.  (We generally hope it fails.)
 814 // In all cases, GraphKit::control() is updated to the fast path.
 815 // The returned value represents the control for the slow path.
 816 // The return value is never 'top'; it is either a valid control
 817 // or NULL if it is obvious that the slow path can never be taken.
 818 // Also, if region and the slow control are not NULL, the slow edge
 819 // is appended to the region.
 820 Node* LibraryCallKit::generate_guard(Node* test, RegionNode* region, float true_prob) {
 821   if (stopped()) {
 822     // Already short circuited.
 823     return NULL;
 824   }
 825 
 826   // Build an if node and its projections.
 827   // If test is true we take the slow path, which we assume is uncommon.
 828   if (_gvn.type(test) == TypeInt::ZERO) {
 829     // The slow branch is never taken.  No need to build this guard.
 830     return NULL;
 831   }
 832 
 833   IfNode* iff = create_and_map_if(control(), test, true_prob, COUNT_UNKNOWN);
 834 
 835   Node* if_slow = _gvn.transform(new IfTrueNode(iff));
 836   if (if_slow == top()) {
 837     // The slow branch is never taken.  No need to build this guard.
 838     return NULL;
 839   }
 840 
 841   if (region != NULL)
 842     region->add_req(if_slow);
 843 
 844   Node* if_fast = _gvn.transform(new IfFalseNode(iff));
 845   set_control(if_fast);
 846 
 847   return if_slow;
 848 }
 849 
 850 inline Node* LibraryCallKit::generate_slow_guard(Node* test, RegionNode* region) {
 851   return generate_guard(test, region, PROB_UNLIKELY_MAG(3));
 852 }
 853 inline Node* LibraryCallKit::generate_fair_guard(Node* test, RegionNode* region) {
 854   return generate_guard(test, region, PROB_FAIR);
 855 }
 856 
 857 inline Node* LibraryCallKit::generate_negative_guard(Node* index, RegionNode* region,
 858                                                      Node* *pos_index) {
 859   if (stopped())
 860     return NULL;                // already stopped
 861   if (_gvn.type(index)->higher_equal(TypeInt::POS)) // [0,maxint]
 862     return NULL;                // index is already adequately typed
 863   Node* cmp_lt = _gvn.transform(new CmpINode(index, intcon(0)));
 864   Node* bol_lt = _gvn.transform(new BoolNode(cmp_lt, BoolTest::lt));
 865   Node* is_neg = generate_guard(bol_lt, region, PROB_MIN);
 866   if (is_neg != NULL && pos_index != NULL) {
 867     // Emulate effect of Parse::adjust_map_after_if.
 868     Node* ccast = new CastIINode(index, TypeInt::POS);
 869     ccast->set_req(0, control());
 870     (*pos_index) = _gvn.transform(ccast);
 871   }
 872   return is_neg;
 873 }
 874 
 875 // Make sure that 'position' is a valid limit index, in [0..length].
 876 // There are two equivalent plans for checking this:
 877 //   A. (offset + copyLength)  unsigned<=  arrayLength
 878 //   B. offset  <=  (arrayLength - copyLength)
 879 // We require that all of the values above, except for the sum and
 880 // difference, are already known to be non-negative.
 881 // Plan A is robust in the face of overflow, if offset and copyLength
 882 // are both hugely positive.
 883 //
 884 // Plan B is less direct and intuitive, but it does not overflow at
 885 // all, since the difference of two non-negatives is always
 886 // representable.  Whenever Java methods must perform the equivalent
 887 // check they generally use Plan B instead of Plan A.
 888 // For the moment we use Plan A.
 889 inline Node* LibraryCallKit::generate_limit_guard(Node* offset,
 890                                                   Node* subseq_length,
 891                                                   Node* array_length,
 892                                                   RegionNode* region) {
 893   if (stopped())
 894     return NULL;                // already stopped
 895   bool zero_offset = _gvn.type(offset) == TypeInt::ZERO;
 896   if (zero_offset && subseq_length->eqv_uncast(array_length))
 897     return NULL;                // common case of whole-array copy
 898   Node* last = subseq_length;
 899   if (!zero_offset)             // last += offset
 900     last = _gvn.transform(new AddINode(last, offset));
 901   Node* cmp_lt = _gvn.transform(new CmpUNode(array_length, last));
 902   Node* bol_lt = _gvn.transform(new BoolNode(cmp_lt, BoolTest::lt));
 903   Node* is_over = generate_guard(bol_lt, region, PROB_MIN);
 904   return is_over;
 905 }
 906 
 907 // Emit range checks for the given String.value byte array
 908 void LibraryCallKit::generate_string_range_check(Node* array, Node* offset, Node* count, bool char_count) {
 909   if (stopped()) {
 910     return; // already stopped
 911   }
 912   RegionNode* bailout = new RegionNode(1);
 913   record_for_igvn(bailout);
 914   if (char_count) {
 915     // Convert char count to byte count
 916     count = _gvn.transform(new LShiftINode(count, intcon(1)));
 917   }
 918 
 919   // Offset and count must not be negative
 920   generate_negative_guard(offset, bailout);
 921   generate_negative_guard(count, bailout);
 922   // Offset + count must not exceed length of array
 923   generate_limit_guard(offset, count, load_array_length(array), bailout);
 924 
 925   if (bailout->req() > 1) {
 926     PreserveJVMState pjvms(this);
 927     set_control(_gvn.transform(bailout));
 928     uncommon_trap(Deoptimization::Reason_intrinsic,
 929                   Deoptimization::Action_maybe_recompile);
 930   }
 931 }
 932 
 933 //--------------------------generate_current_thread--------------------
 934 Node* LibraryCallKit::generate_current_thread(Node* &tls_output) {
 935   ciKlass*    thread_klass = env()->Thread_klass();
 936   const Type* thread_type  = TypeOopPtr::make_from_klass(thread_klass)->cast_to_ptr_type(TypePtr::NotNull);
 937   Node* thread = _gvn.transform(new ThreadLocalNode());
 938   Node* p = basic_plus_adr(top()/*!oop*/, thread, in_bytes(JavaThread::threadObj_offset()));
 939   Node* threadObj = make_load(NULL, p, thread_type, T_OBJECT, MemNode::unordered);
 940   tls_output = thread;
 941   return threadObj;
 942 }
 943 
 944 
 945 //------------------------------make_string_method_node------------------------
 946 // Helper method for String intrinsic functions. This version is called with
 947 // str1 and str2 pointing to byte[] nodes containing Latin1 or UTF16 encoded
 948 // characters (depending on 'is_byte'). cnt1 and cnt2 are pointing to Int nodes
 949 // containing the lengths of str1 and str2.
 950 Node* LibraryCallKit::make_string_method_node(int opcode, Node* str1_start, Node* cnt1, Node* str2_start, Node* cnt2, StrIntrinsicNode::ArgEnc ae) {
 951   Node* result = NULL;
 952   switch (opcode) {
 953   case Op_StrIndexOf:
 954     result = new StrIndexOfNode(control(), memory(TypeAryPtr::BYTES),
 955                                 str1_start, cnt1, str2_start, cnt2, ae);
 956     break;
 957   case Op_StrComp:
 958     result = new StrCompNode(control(), memory(TypeAryPtr::BYTES),
 959                              str1_start, cnt1, str2_start, cnt2, ae);
 960     break;
 961   case Op_StrEquals:
 962     result = new StrEqualsNode(control(), memory(TypeAryPtr::BYTES),
 963                                str1_start, str2_start, cnt1, ae);
 964     break;
 965   default:
 966     ShouldNotReachHere();
 967     return NULL;
 968   }
 969 
 970   // All these intrinsics have checks.
 971   C->set_has_split_ifs(true); // Has chance for split-if optimization
 972 
 973   return _gvn.transform(result);
 974 }
 975 
 976 //------------------------------inline_string_compareTo------------------------
 977 bool LibraryCallKit::inline_string_compareTo(StrIntrinsicNode::ArgEnc ae) {
 978   Node* arg1 = argument(0);
 979   Node* arg2 = argument(1);
 980 
 981   // Get start addr and length of first argument
 982   Node* arg1_start  = array_element_address(arg1, intcon(0), T_BYTE);
 983   Node* arg1_cnt    = load_array_length(arg1);
 984 
 985   // Get start addr and length of second argument
 986   Node* arg2_start  = array_element_address(arg2, intcon(0), T_BYTE);
 987   Node* arg2_cnt    = load_array_length(arg2);
 988 
 989   Node* result = make_string_method_node(Op_StrComp, arg1_start, arg1_cnt, arg2_start, arg2_cnt, ae);
 990   set_result(result);
 991   return true;
 992 }
 993 
 994 //------------------------------inline_string_equals------------------------
 995 bool LibraryCallKit::inline_string_equals(StrIntrinsicNode::ArgEnc ae) {
 996   Node* arg1 = argument(0);
 997   Node* arg2 = argument(1);
 998 
 999   // paths (plus control) merge
1000   RegionNode* region = new RegionNode(3);
1001   Node* phi = new PhiNode(region, TypeInt::BOOL);
1002 
1003   if (!stopped()) {
1004     // Get start addr and length of first argument
1005     Node* arg1_start  = array_element_address(arg1, intcon(0), T_BYTE);
1006     Node* arg1_cnt    = load_array_length(arg1);
1007 
1008     // Get start addr and length of second argument
1009     Node* arg2_start  = array_element_address(arg2, intcon(0), T_BYTE);
1010     Node* arg2_cnt    = load_array_length(arg2);
1011 
1012     // Check for arg1_cnt != arg2_cnt
1013     Node* cmp = _gvn.transform(new CmpINode(arg1_cnt, arg2_cnt));
1014     Node* bol = _gvn.transform(new BoolNode(cmp, BoolTest::ne));
1015     Node* if_ne = generate_slow_guard(bol, NULL);
1016     if (if_ne != NULL) {
1017       phi->init_req(2, intcon(0));
1018       region->init_req(2, if_ne);
1019     }
1020 
1021     // Check for count == 0 is done by assembler code for StrEquals.
1022 
1023     if (!stopped()) {
1024       Node* equals = make_string_method_node(Op_StrEquals, arg1_start, arg1_cnt, arg2_start, arg2_cnt, ae);
1025       phi->init_req(1, equals);
1026       region->init_req(1, control());
1027     }
1028   }
1029 
1030   // post merge
1031   set_control(_gvn.transform(region));
1032   record_for_igvn(region);
1033 
1034   set_result(_gvn.transform(phi));
1035   return true;
1036 }
1037 
1038 //------------------------------inline_array_equals----------------------------
1039 bool LibraryCallKit::inline_array_equals(StrIntrinsicNode::ArgEnc ae) {
1040   assert(ae == StrIntrinsicNode::UU || ae == StrIntrinsicNode::LL, "unsupported array types");
1041   Node* arg1 = argument(0);
1042   Node* arg2 = argument(1);
1043 
1044   const TypeAryPtr* mtype = (ae == StrIntrinsicNode::UU) ? TypeAryPtr::CHARS : TypeAryPtr::BYTES;
1045   set_result(_gvn.transform(new AryEqNode(control(), memory(mtype), arg1, arg2, ae)));
1046   return true;
1047 }
1048 
1049 //------------------------------inline_hasNegatives------------------------------
1050 bool LibraryCallKit::inline_hasNegatives() {
1051   if (too_many_traps(Deoptimization::Reason_intrinsic)) {
1052     return false;
1053   }
1054 
1055   assert(callee()->signature()->size() == 3, "hasNegatives has 3 parameters");
1056   // no receiver since it is static method
1057   Node* ba         = argument(0);
1058   Node* offset     = argument(1);
1059   Node* len        = argument(2);
1060 
1061   // Range checks
1062   generate_string_range_check(ba, offset, len, false);
1063   if (stopped()) {
1064     return true;
1065   }
1066   Node* ba_start = array_element_address(ba, offset, T_BYTE);
1067   Node* result = new HasNegativesNode(control(), memory(TypeAryPtr::BYTES), ba_start, len);
1068   set_result(_gvn.transform(result));
1069   return true;
1070 }
1071 
1072 bool LibraryCallKit::inline_objects_checkIndex() {
1073   Node* index = argument(0);
1074   Node* length = argument(1);
1075   if (too_many_traps(Deoptimization::Reason_intrinsic) || too_many_traps(Deoptimization::Reason_range_check)) {
1076     return false;
1077   }
1078 
1079   Node* len_pos_cmp = _gvn.transform(new CmpINode(length, intcon(0)));
1080   Node* len_pos_bol = _gvn.transform(new BoolNode(len_pos_cmp, BoolTest::ge));
1081 
1082   {
1083     BuildCutout unless(this, len_pos_bol, PROB_MAX);
1084     uncommon_trap(Deoptimization::Reason_intrinsic,
1085                   Deoptimization::Action_make_not_entrant);
1086   }
1087 
1088   if (stopped()) {
1089     return false;
1090   }
1091 
1092   Node* rc_cmp = _gvn.transform(new CmpUNode(index, length));
1093   BoolTest::mask btest = BoolTest::lt;
1094   Node* rc_bool = _gvn.transform(new BoolNode(rc_cmp, btest));
1095   RangeCheckNode* rc = new RangeCheckNode(control(), rc_bool, PROB_MAX, COUNT_UNKNOWN);
1096   _gvn.set_type(rc, rc->Value(&_gvn));
1097   if (!rc_bool->is_Con()) {
1098     record_for_igvn(rc);
1099   }
1100   set_control(_gvn.transform(new IfTrueNode(rc)));
1101   {
1102     PreserveJVMState pjvms(this);
1103     set_control(_gvn.transform(new IfFalseNode(rc)));
1104     uncommon_trap(Deoptimization::Reason_range_check,
1105                   Deoptimization::Action_make_not_entrant);
1106   }
1107 
1108   if (stopped()) {
1109     return false;
1110   }
1111 
1112   Node* result = new CastIINode(index, TypeInt::make(0, _gvn.type(length)->is_int()->_hi, Type::WidenMax));
1113   result->set_req(0, control());
1114   result = _gvn.transform(result);
1115   set_result(result);
1116   replace_in_map(index, result);
1117   return true;
1118 }
1119 
1120 //------------------------------inline_string_indexOf------------------------
1121 bool LibraryCallKit::inline_string_indexOf(StrIntrinsicNode::ArgEnc ae) {
1122   if (!Matcher::has_match_rule(Op_StrIndexOf) || !UseSSE42Intrinsics) {
1123     return false;
1124   }
1125   Node* src = argument(0);
1126   Node* tgt = argument(1);
1127 
1128   // Make the merge point
1129   RegionNode* result_rgn = new RegionNode(4);
1130   Node*       result_phi = new PhiNode(result_rgn, TypeInt::INT);
1131 
1132   // Get start addr and length of source string
1133   Node* src_start = array_element_address(src, intcon(0), T_BYTE);
1134   Node* src_count = load_array_length(src);
1135 
1136   // Get start addr and length of substring
1137   Node* tgt_start = array_element_address(tgt, intcon(0), T_BYTE);
1138   Node* tgt_count = load_array_length(tgt);
1139 
1140   if (ae == StrIntrinsicNode::UU || ae == StrIntrinsicNode::UL) {
1141     // Divide src size by 2 if String is UTF16 encoded
1142     src_count = _gvn.transform(new RShiftINode(src_count, intcon(1)));
1143   }
1144   if (ae == StrIntrinsicNode::UU) {
1145     // Divide substring size by 2 if String is UTF16 encoded
1146     tgt_count = _gvn.transform(new RShiftINode(tgt_count, intcon(1)));
1147   }
1148 
1149   Node* result = make_indexOf_node(src_start, src_count, tgt_start, tgt_count, result_rgn, result_phi, ae);
1150   if (result != NULL) {
1151     result_phi->init_req(3, result);
1152     result_rgn->init_req(3, control());
1153   }
1154   set_control(_gvn.transform(result_rgn));
1155   record_for_igvn(result_rgn);
1156   set_result(_gvn.transform(result_phi));
1157 
1158   return true;
1159 }
1160 
1161 //-----------------------------inline_string_indexOf-----------------------
1162 bool LibraryCallKit::inline_string_indexOfI(StrIntrinsicNode::ArgEnc ae) {
1163   if (too_many_traps(Deoptimization::Reason_intrinsic)) {
1164     return false;
1165   }
1166   if (!Matcher::has_match_rule(Op_StrIndexOf) || !UseSSE42Intrinsics) {
1167     return false;
1168   }
1169   assert(callee()->signature()->size() == 5, "String.indexOf() has 5 arguments");
1170   Node* src         = argument(0); // byte[]
1171   Node* src_count   = argument(1); // char count
1172   Node* tgt         = argument(2); // byte[]
1173   Node* tgt_count   = argument(3); // char count
1174   Node* from_index  = argument(4); // char index
1175 
1176   // Multiply byte array index by 2 if String is UTF16 encoded
1177   Node* src_offset = (ae == StrIntrinsicNode::LL) ? from_index : _gvn.transform(new LShiftINode(from_index, intcon(1)));
1178   src_count = _gvn.transform(new SubINode(src_count, from_index));
1179   Node* src_start = array_element_address(src, src_offset, T_BYTE);
1180   Node* tgt_start = array_element_address(tgt, intcon(0), T_BYTE);
1181 
1182   // Range checks
1183   generate_string_range_check(src, src_offset, src_count, ae != StrIntrinsicNode::LL);
1184   generate_string_range_check(tgt, intcon(0), tgt_count, ae == StrIntrinsicNode::UU);
1185   if (stopped()) {
1186     return true;
1187   }
1188 
1189   RegionNode* region = new RegionNode(5);
1190   Node* phi = new PhiNode(region, TypeInt::INT);
1191 
1192   Node* result = make_indexOf_node(src_start, src_count, tgt_start, tgt_count, region, phi, ae);
1193   if (result != NULL) {
1194     // The result is index relative to from_index if substring was found, -1 otherwise.
1195     // Generate code which will fold into cmove.
1196     Node* cmp = _gvn.transform(new CmpINode(result, intcon(0)));
1197     Node* bol = _gvn.transform(new BoolNode(cmp, BoolTest::lt));
1198 
1199     Node* if_lt = generate_slow_guard(bol, NULL);
1200     if (if_lt != NULL) {
1201       // result == -1
1202       phi->init_req(3, result);
1203       region->init_req(3, if_lt);
1204     }
1205     if (!stopped()) {
1206       result = _gvn.transform(new AddINode(result, from_index));
1207       phi->init_req(4, result);
1208       region->init_req(4, control());
1209     }
1210   }
1211 
1212   set_control(_gvn.transform(region));
1213   record_for_igvn(region);
1214   set_result(_gvn.transform(phi));
1215 
1216   return true;
1217 }
1218 
1219 // Create StrIndexOfNode with fast path checks
1220 Node* LibraryCallKit::make_indexOf_node(Node* src_start, Node* src_count, Node* tgt_start, Node* tgt_count,
1221                                         RegionNode* region, Node* phi, StrIntrinsicNode::ArgEnc ae) {
1222   // Check for substr count > string count
1223   Node* cmp = _gvn.transform(new CmpINode(tgt_count, src_count));
1224   Node* bol = _gvn.transform(new BoolNode(cmp, BoolTest::gt));
1225   Node* if_gt = generate_slow_guard(bol, NULL);
1226   if (if_gt != NULL) {
1227     phi->init_req(1, intcon(-1));
1228     region->init_req(1, if_gt);
1229   }
1230   if (!stopped()) {
1231     // Check for substr count == 0
1232     cmp = _gvn.transform(new CmpINode(tgt_count, intcon(0)));
1233     bol = _gvn.transform(new BoolNode(cmp, BoolTest::eq));
1234     Node* if_zero = generate_slow_guard(bol, NULL);
1235     if (if_zero != NULL) {
1236       phi->init_req(2, intcon(0));
1237       region->init_req(2, if_zero);
1238     }
1239   }
1240   if (!stopped()) {
1241     return make_string_method_node(Op_StrIndexOf, src_start, src_count, tgt_start, tgt_count, ae);
1242   }
1243   return NULL;
1244 }
1245 
1246 //-----------------------------inline_string_indexOfChar-----------------------
1247 bool LibraryCallKit::inline_string_indexOfChar() {
1248   if (too_many_traps(Deoptimization::Reason_intrinsic)) {
1249     return false;
1250   }
1251   if (!Matcher::has_match_rule(Op_StrIndexOfChar) || !(UseSSE > 4)) {
1252     return false;
1253   }
1254   assert(callee()->signature()->size() == 4, "String.indexOfChar() has 4 arguments");
1255   Node* src         = argument(0); // byte[]
1256   Node* tgt         = argument(1); // tgt is int ch
1257   Node* from_index  = argument(2);
1258   Node* max         = argument(3);
1259 
1260   Node* src_offset = _gvn.transform(new LShiftINode(from_index, intcon(1)));
1261   Node* src_start = array_element_address(src, src_offset, T_BYTE);
1262   Node* src_count = _gvn.transform(new SubINode(max, from_index));
1263 
1264   // Range checks
1265   generate_string_range_check(src, src_offset, src_count, true);
1266   if (stopped()) {
1267     return true;
1268   }
1269 
1270   RegionNode* region = new RegionNode(3);
1271   Node* phi = new PhiNode(region, TypeInt::INT);
1272 
1273   Node* result = new StrIndexOfCharNode(control(), memory(TypeAryPtr::BYTES), src_start, src_count, tgt, StrIntrinsicNode::none);
1274   C->set_has_split_ifs(true); // Has chance for split-if optimization
1275   _gvn.transform(result);
1276 
1277   Node* cmp = _gvn.transform(new CmpINode(result, intcon(0)));
1278   Node* bol = _gvn.transform(new BoolNode(cmp, BoolTest::lt));
1279 
1280   Node* if_lt = generate_slow_guard(bol, NULL);
1281   if (if_lt != NULL) {
1282     // result == -1
1283     phi->init_req(2, result);
1284     region->init_req(2, if_lt);
1285   }
1286   if (!stopped()) {
1287     result = _gvn.transform(new AddINode(result, from_index));
1288     phi->init_req(1, result);
1289     region->init_req(1, control());
1290   }
1291   set_control(_gvn.transform(region));
1292   record_for_igvn(region);
1293   set_result(_gvn.transform(phi));
1294 
1295   return true;
1296 }
1297 //---------------------------inline_string_copy---------------------
1298 // compressIt == true --> generate a compressed copy operation (compress char[]/byte[] to byte[])
1299 //   int StringUTF16.compress(char[] src, int srcOff, byte[] dst, int dstOff, int len)
1300 //   int StringUTF16.compress(byte[] src, int srcOff, byte[] dst, int dstOff, int len)
1301 // compressIt == false --> generate an inflated copy operation (inflate byte[] to char[]/byte[])
1302 //   void StringLatin1.inflate(byte[] src, int srcOff, char[] dst, int dstOff, int len)
1303 //   void StringLatin1.inflate(byte[] src, int srcOff, byte[] dst, int dstOff, int len)
1304 bool LibraryCallKit::inline_string_copy(bool compress) {
1305   if (too_many_traps(Deoptimization::Reason_intrinsic)) {
1306     return false;
1307   }
1308   int nargs = 5;  // 2 oops, 3 ints
1309   assert(callee()->signature()->size() == nargs, "string copy has 5 arguments");
1310 
1311   Node* src         = argument(0);
1312   Node* src_offset  = argument(1);
1313   Node* dst         = argument(2);
1314   Node* dst_offset  = argument(3);
1315   Node* length      = argument(4);
1316 
1317   // Check for allocation before we add nodes that would confuse
1318   // tightly_coupled_allocation()
1319   AllocateArrayNode* alloc = tightly_coupled_allocation(dst, NULL);
1320 
1321   // Figure out the size and type of the elements we will be copying.
1322   const Type* src_type = src->Value(&_gvn);
1323   const Type* dst_type = dst->Value(&_gvn);
1324   BasicType src_elem = src_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
1325   BasicType dst_elem = dst_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
1326   assert((compress && dst_elem == T_BYTE && (src_elem == T_BYTE || src_elem == T_CHAR)) ||
1327          (!compress && src_elem == T_BYTE && (dst_elem == T_BYTE || dst_elem == T_CHAR)),
1328          "Unsupported array types for inline_string_copy");
1329 
1330   // Range checks
1331   generate_string_range_check(src, src_offset, length, compress && src_elem == T_BYTE);
1332   generate_string_range_check(dst, dst_offset, length, !compress && dst_elem == T_BYTE);
1333   if (stopped()) {
1334     return true;
1335   }
1336 
1337   // Convert char[] offsets to byte[] offsets
1338   if (compress && src_elem == T_BYTE) {
1339     src_offset = _gvn.transform(new LShiftINode(src_offset, intcon(1)));
1340   } else if (!compress && dst_elem == T_BYTE) {
1341     dst_offset = _gvn.transform(new LShiftINode(dst_offset, intcon(1)));
1342   }
1343 
1344   Node* src_start = array_element_address(src, src_offset, src_elem);
1345   Node* dst_start = array_element_address(dst, dst_offset, dst_elem);
1346   // 'src_start' points to src array + scaled offset
1347   // 'dst_start' points to dst array + scaled offset
1348   Node* count = NULL;
1349   if (compress) {
1350     count = compress_string(src_start, dst_start, length);
1351   } else {
1352     inflate_string(src_start, dst_start, length);
1353   }
1354 
1355   if (alloc != NULL) {
1356     if (alloc->maybe_set_complete(&_gvn)) {
1357       // "You break it, you buy it."
1358       InitializeNode* init = alloc->initialization();
1359       assert(init->is_complete(), "we just did this");
1360       init->set_complete_with_arraycopy();
1361       assert(dst->is_CheckCastPP(), "sanity");
1362       assert(dst->in(0)->in(0) == init, "dest pinned");
1363     }
1364     // Do not let stores that initialize this object be reordered with
1365     // a subsequent store that would make this object accessible by
1366     // other threads.
1367     // Record what AllocateNode this StoreStore protects so that
1368     // escape analysis can go from the MemBarStoreStoreNode to the
1369     // AllocateNode and eliminate the MemBarStoreStoreNode if possible
1370     // based on the escape status of the AllocateNode.
1371     insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out(AllocateNode::RawAddress));
1372   }
1373   if (compress) {
1374     set_result(_gvn.transform(count));
1375   }
1376   return true;
1377 }
1378 
1379 #ifdef _LP64
1380 #define XTOP ,top() /*additional argument*/
1381 #else  //_LP64
1382 #define XTOP        /*no additional argument*/
1383 #endif //_LP64
1384 
1385 //------------------------inline_string_toBytesU--------------------------
1386 // public static byte[] StringUTF16.toBytes(char[] value, int off, int len)
1387 bool LibraryCallKit::inline_string_toBytesU() {
1388   if (too_many_traps(Deoptimization::Reason_intrinsic)) {
1389     return false;
1390   }
1391   // Get the arguments.
1392   Node* value     = argument(0);
1393   Node* offset    = argument(1);
1394   Node* length    = argument(2);
1395 
1396   Node* newcopy = NULL;
1397 
1398   // Set the original stack and the reexecute bit for the interpreter to reexecute
1399   // the bytecode that invokes StringUTF16.toBytes() if deoptimization happens.
1400   { PreserveReexecuteState preexecs(this);
1401     jvms()->set_should_reexecute(true);
1402 
1403     // Check if a null path was taken unconditionally.
1404     value = null_check(value);
1405 
1406     RegionNode* bailout = new RegionNode(1);
1407     record_for_igvn(bailout);
1408 
1409     // Range checks
1410     generate_negative_guard(offset, bailout);
1411     generate_negative_guard(length, bailout);
1412     generate_limit_guard(offset, length, load_array_length(value), bailout);
1413     // Make sure that resulting byte[] length does not overflow Integer.MAX_VALUE
1414     generate_limit_guard(length, intcon(0), intcon(max_jint/2), bailout);
1415 
1416     if (bailout->req() > 1) {
1417       PreserveJVMState pjvms(this);
1418       set_control(_gvn.transform(bailout));
1419       uncommon_trap(Deoptimization::Reason_intrinsic,
1420                     Deoptimization::Action_maybe_recompile);
1421     }
1422     if (stopped()) {
1423       return true;
1424     }
1425 
1426     Node* size = _gvn.transform(new LShiftINode(length, intcon(1)));
1427     Node* klass_node = makecon(TypeKlassPtr::make(ciTypeArrayKlass::make(T_BYTE)));
1428     newcopy = new_array(klass_node, size, 0);  // no arguments to push
1429     AllocateArrayNode* alloc = tightly_coupled_allocation(newcopy, NULL);
1430 
1431     // Calculate starting addresses.
1432     Node* src_start = array_element_address(value, offset, T_CHAR);
1433     Node* dst_start = basic_plus_adr(newcopy, arrayOopDesc::base_offset_in_bytes(T_BYTE));
1434 
1435     // Check if src array address is aligned to HeapWordSize (dst is always aligned)
1436     const TypeInt* toffset = gvn().type(offset)->is_int();
1437     bool aligned = toffset->is_con() && ((toffset->get_con() * type2aelembytes(T_CHAR)) % HeapWordSize == 0);
1438 
1439     // Figure out which arraycopy runtime method to call (disjoint, uninitialized).
1440     const char* copyfunc_name = "arraycopy";
1441     address     copyfunc_addr = StubRoutines::select_arraycopy_function(T_CHAR, aligned, true, copyfunc_name, true);
1442     Node* call = make_runtime_call(RC_LEAF|RC_NO_FP,
1443                       OptoRuntime::fast_arraycopy_Type(),
1444                       copyfunc_addr, copyfunc_name, TypeRawPtr::BOTTOM,
1445                       src_start, dst_start, ConvI2X(length) XTOP);
1446     // Do not let reads from the cloned object float above the arraycopy.
1447     if (alloc != NULL) {
1448       if (alloc->maybe_set_complete(&_gvn)) {
1449         // "You break it, you buy it."
1450         InitializeNode* init = alloc->initialization();
1451         assert(init->is_complete(), "we just did this");
1452         init->set_complete_with_arraycopy();
1453         assert(newcopy->is_CheckCastPP(), "sanity");
1454         assert(newcopy->in(0)->in(0) == init, "dest pinned");
1455       }
1456       // Do not let stores that initialize this object be reordered with
1457       // a subsequent store that would make this object accessible by
1458       // other threads.
1459       // Record what AllocateNode this StoreStore protects so that
1460       // escape analysis can go from the MemBarStoreStoreNode to the
1461       // AllocateNode and eliminate the MemBarStoreStoreNode if possible
1462       // based on the escape status of the AllocateNode.
1463       insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out(AllocateNode::RawAddress));
1464     } else {
1465       insert_mem_bar(Op_MemBarCPUOrder);
1466     }
1467   } // original reexecute is set back here
1468 
1469   C->set_has_split_ifs(true); // Has chance for split-if optimization
1470   if (!stopped()) {
1471     set_result(newcopy);
1472   }
1473   return true;
1474 }
1475 
1476 //------------------------inline_string_getCharsU--------------------------
1477 // public void StringUTF16.getChars(byte[] src, int srcBegin, int srcEnd, char dst[], int dstBegin)
1478 bool LibraryCallKit::inline_string_getCharsU() {
1479   if (too_many_traps(Deoptimization::Reason_intrinsic)) {
1480     return false;
1481   }
1482 
1483   // Get the arguments.
1484   Node* src       = argument(0);
1485   Node* src_begin = argument(1);
1486   Node* src_end   = argument(2); // exclusive offset (i < src_end)
1487   Node* dst       = argument(3);
1488   Node* dst_begin = argument(4);
1489 
1490   // Check for allocation before we add nodes that would confuse
1491   // tightly_coupled_allocation()
1492   AllocateArrayNode* alloc = tightly_coupled_allocation(dst, NULL);
1493 
1494   // Check if a null path was taken unconditionally.
1495   src = null_check(src);
1496   dst = null_check(dst);
1497   if (stopped()) {
1498     return true;
1499   }
1500 
1501   // Get length and convert char[] offset to byte[] offset
1502   Node* length = _gvn.transform(new SubINode(src_end, src_begin));
1503   src_begin = _gvn.transform(new LShiftINode(src_begin, intcon(1)));
1504 
1505   // Range checks
1506   generate_string_range_check(src, src_begin, length, true);
1507   generate_string_range_check(dst, dst_begin, length, false);
1508   if (stopped()) {
1509     return true;
1510   }
1511 
1512   if (!stopped()) {
1513     // Calculate starting addresses.
1514     Node* src_start = array_element_address(src, src_begin, T_BYTE);
1515     Node* dst_start = array_element_address(dst, dst_begin, T_CHAR);
1516 
1517     // Check if array addresses are aligned to HeapWordSize
1518     const TypeInt* tsrc = gvn().type(src_begin)->is_int();
1519     const TypeInt* tdst = gvn().type(dst_begin)->is_int();
1520     bool aligned = tsrc->is_con() && ((tsrc->get_con() * type2aelembytes(T_BYTE)) % HeapWordSize == 0) &&
1521                    tdst->is_con() && ((tdst->get_con() * type2aelembytes(T_CHAR)) % HeapWordSize == 0);
1522 
1523     // Figure out which arraycopy runtime method to call (disjoint, uninitialized).
1524     const char* copyfunc_name = "arraycopy";
1525     address     copyfunc_addr = StubRoutines::select_arraycopy_function(T_CHAR, aligned, true, copyfunc_name, true);
1526     Node* call = make_runtime_call(RC_LEAF|RC_NO_FP,
1527                       OptoRuntime::fast_arraycopy_Type(),
1528                       copyfunc_addr, copyfunc_name, TypeRawPtr::BOTTOM,
1529                       src_start, dst_start, ConvI2X(length) XTOP);
1530     // Do not let reads from the cloned object float above the arraycopy.
1531     if (alloc != NULL) {
1532       if (alloc->maybe_set_complete(&_gvn)) {
1533         // "You break it, you buy it."
1534         InitializeNode* init = alloc->initialization();
1535         assert(init->is_complete(), "we just did this");
1536         init->set_complete_with_arraycopy();
1537         assert(dst->is_CheckCastPP(), "sanity");
1538         assert(dst->in(0)->in(0) == init, "dest pinned");
1539       }
1540       // Do not let stores that initialize this object be reordered with
1541       // a subsequent store that would make this object accessible by
1542       // other threads.
1543       // Record what AllocateNode this StoreStore protects so that
1544       // escape analysis can go from the MemBarStoreStoreNode to the
1545       // AllocateNode and eliminate the MemBarStoreStoreNode if possible
1546       // based on the escape status of the AllocateNode.
1547       insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out(AllocateNode::RawAddress));
1548     } else {
1549       insert_mem_bar(Op_MemBarCPUOrder);
1550     }
1551   }
1552 
1553   C->set_has_split_ifs(true); // Has chance for split-if optimization
1554   return true;
1555 }
1556 
1557 //----------------------inline_string_char_access----------------------------
1558 // Store/Load char to/from byte[] array.
1559 // static void StringUTF16.putChar(byte[] val, int index, int c)
1560 // static char StringUTF16.getChar(byte[] val, int index)
1561 bool LibraryCallKit::inline_string_char_access(bool is_store) {
1562   Node* value  = argument(0);
1563   Node* index  = argument(1);
1564   Node* ch = is_store ? argument(2) : NULL;
1565 
1566   // This intrinsic accesses byte[] array as char[] array. Computing the offsets
1567   // correctly requires matched array shapes.
1568   assert (arrayOopDesc::base_offset_in_bytes(T_CHAR) == arrayOopDesc::base_offset_in_bytes(T_BYTE),
1569           "sanity: byte[] and char[] bases agree");
1570   assert (type2aelembytes(T_CHAR) == type2aelembytes(T_BYTE)*2,
1571           "sanity: byte[] and char[] scales agree");
1572 
1573   Node* adr = array_element_address(value, index, T_CHAR);
1574   if (is_store) {
1575     (void) store_to_memory(control(), adr, ch, T_CHAR, TypeAryPtr::BYTES, MemNode::unordered,
1576                            false, false, true /* mismatched */);
1577   } else {
1578     ch = make_load(control(), adr, TypeInt::CHAR, T_CHAR, MemNode::unordered,
1579                    LoadNode::DependsOnlyOnTest, false, false, true /* mismatched */);
1580     set_result(ch);
1581   }
1582   return true;
1583 }
1584 
1585 //--------------------------round_double_node--------------------------------
1586 // Round a double node if necessary.
1587 Node* LibraryCallKit::round_double_node(Node* n) {
1588   if (Matcher::strict_fp_requires_explicit_rounding && UseSSE <= 1)
1589     n = _gvn.transform(new RoundDoubleNode(0, n));
1590   return n;
1591 }
1592 
1593 //------------------------------inline_math-----------------------------------
1594 // public static double Math.abs(double)
1595 // public static double Math.sqrt(double)
1596 // public static double Math.log(double)
1597 // public static double Math.log10(double)
1598 bool LibraryCallKit::inline_math(vmIntrinsics::ID id) {
1599   Node* arg = round_double_node(argument(0));
1600   Node* n = NULL;
1601   switch (id) {
1602   case vmIntrinsics::_dabs:   n = new AbsDNode(                arg);  break;
1603   case vmIntrinsics::_dsqrt:  n = new SqrtDNode(C, control(),  arg);  break;
1604   case vmIntrinsics::_dlog10: n = new Log10DNode(C, control(), arg);  break;
1605   default:  fatal_unexpected_iid(id);  break;
1606   }
1607   set_result(_gvn.transform(n));
1608   return true;
1609 }
1610 
1611 //------------------------------inline_trig----------------------------------
1612 // Inline sin/cos/tan instructions, if possible.  If rounding is required, do
1613 // argument reduction which will turn into a fast/slow diamond.
1614 bool LibraryCallKit::inline_trig(vmIntrinsics::ID id) {
1615   Node* arg = round_double_node(argument(0));
1616   Node* n = NULL;
1617 
1618   switch (id) {
1619   case vmIntrinsics::_dsin:  n = new SinDNode(C, control(), arg);  break;
1620   case vmIntrinsics::_dcos:  n = new CosDNode(C, control(), arg);  break;
1621   case vmIntrinsics::_dtan:  n = new TanDNode(C, control(), arg);  break;
1622   default:  fatal_unexpected_iid(id);  break;
1623   }
1624   n = _gvn.transform(n);
1625 
1626   // Rounding required?  Check for argument reduction!
1627   if (Matcher::strict_fp_requires_explicit_rounding) {
1628     static const double     pi_4 =  0.7853981633974483;
1629     static const double neg_pi_4 = -0.7853981633974483;
1630     // pi/2 in 80-bit extended precision
1631     // static const unsigned char pi_2_bits_x[] = {0x35,0xc2,0x68,0x21,0xa2,0xda,0x0f,0xc9,0xff,0x3f,0x00,0x00,0x00,0x00,0x00,0x00};
1632     // -pi/2 in 80-bit extended precision
1633     // static const unsigned char neg_pi_2_bits_x[] = {0x35,0xc2,0x68,0x21,0xa2,0xda,0x0f,0xc9,0xff,0xbf,0x00,0x00,0x00,0x00,0x00,0x00};
1634     // Cutoff value for using this argument reduction technique
1635     //static const double    pi_2_minus_epsilon =  1.564660403643354;
1636     //static const double neg_pi_2_plus_epsilon = -1.564660403643354;
1637 
1638     // Pseudocode for sin:
1639     // if (x <= Math.PI / 4.0) {
1640     //   if (x >= -Math.PI / 4.0) return  fsin(x);
1641     //   if (x >= -Math.PI / 2.0) return -fcos(x + Math.PI / 2.0);
1642     // } else {
1643     //   if (x <=  Math.PI / 2.0) return  fcos(x - Math.PI / 2.0);
1644     // }
1645     // return StrictMath.sin(x);
1646 
1647     // Pseudocode for cos:
1648     // if (x <= Math.PI / 4.0) {
1649     //   if (x >= -Math.PI / 4.0) return  fcos(x);
1650     //   if (x >= -Math.PI / 2.0) return  fsin(x + Math.PI / 2.0);
1651     // } else {
1652     //   if (x <=  Math.PI / 2.0) return -fsin(x - Math.PI / 2.0);
1653     // }
1654     // return StrictMath.cos(x);
1655 
1656     // Actually, sticking in an 80-bit Intel value into C2 will be tough; it
1657     // requires a special machine instruction to load it.  Instead we'll try
1658     // the 'easy' case.  If we really need the extra range +/- PI/2 we'll
1659     // probably do the math inside the SIN encoding.
1660 
1661     // Make the merge point
1662     RegionNode* r = new RegionNode(3);
1663     Node* phi = new PhiNode(r, Type::DOUBLE);
1664 
1665     // Flatten arg so we need only 1 test
1666     Node *abs = _gvn.transform(new AbsDNode(arg));
1667     // Node for PI/4 constant
1668     Node *pi4 = makecon(TypeD::make(pi_4));
1669     // Check PI/4 : abs(arg)
1670     Node *cmp = _gvn.transform(new CmpDNode(pi4,abs));
1671     // Check: If PI/4 < abs(arg) then go slow
1672     Node *bol = _gvn.transform(new BoolNode( cmp, BoolTest::lt ));
1673     // Branch either way
1674     IfNode *iff = create_and_xform_if(control(),bol, PROB_STATIC_FREQUENT, COUNT_UNKNOWN);
1675     set_control(opt_iff(r,iff));
1676 
1677     // Set fast path result
1678     phi->init_req(2, n);
1679 
1680     // Slow path - non-blocking leaf call
1681     Node* call = NULL;
1682     switch (id) {
1683     case vmIntrinsics::_dsin:
1684       call = make_runtime_call(RC_LEAF, OptoRuntime::Math_D_D_Type(),
1685                                CAST_FROM_FN_PTR(address, SharedRuntime::dsin),
1686                                "Sin", NULL, arg, top());
1687       break;
1688     case vmIntrinsics::_dcos:
1689       call = make_runtime_call(RC_LEAF, OptoRuntime::Math_D_D_Type(),
1690                                CAST_FROM_FN_PTR(address, SharedRuntime::dcos),
1691                                "Cos", NULL, arg, top());
1692       break;
1693     case vmIntrinsics::_dtan:
1694       call = make_runtime_call(RC_LEAF, OptoRuntime::Math_D_D_Type(),
1695                                CAST_FROM_FN_PTR(address, SharedRuntime::dtan),
1696                                "Tan", NULL, arg, top());
1697       break;
1698     }
1699     assert(control()->in(0) == call, "");
1700     Node* slow_result = _gvn.transform(new ProjNode(call, TypeFunc::Parms));
1701     r->init_req(1, control());
1702     phi->init_req(1, slow_result);
1703 
1704     // Post-merge
1705     set_control(_gvn.transform(r));
1706     record_for_igvn(r);
1707     n = _gvn.transform(phi);
1708 
1709     C->set_has_split_ifs(true); // Has chance for split-if optimization
1710   }
1711   set_result(n);
1712   return true;
1713 }
1714 
1715 Node* LibraryCallKit::finish_pow_exp(Node* result, Node* x, Node* y, const TypeFunc* call_type, address funcAddr, const char* funcName) {
1716   //-------------------
1717   //result=(result.isNaN())? funcAddr():result;
1718   // Check: If isNaN() by checking result!=result? then either trap
1719   // or go to runtime
1720   Node* cmpisnan = _gvn.transform(new CmpDNode(result, result));
1721   // Build the boolean node
1722   Node* bolisnum = _gvn.transform(new BoolNode(cmpisnan, BoolTest::eq));
1723 
1724   if (!too_many_traps(Deoptimization::Reason_intrinsic)) {
1725     { BuildCutout unless(this, bolisnum, PROB_STATIC_FREQUENT);
1726       // The pow or exp intrinsic returned a NaN, which requires a call
1727       // to the runtime.  Recompile with the runtime call.
1728       uncommon_trap(Deoptimization::Reason_intrinsic,
1729                     Deoptimization::Action_make_not_entrant);
1730     }
1731     return result;
1732   } else {
1733     // If this inlining ever returned NaN in the past, we compile a call
1734     // to the runtime to properly handle corner cases
1735 
1736     IfNode* iff = create_and_xform_if(control(), bolisnum, PROB_STATIC_FREQUENT, COUNT_UNKNOWN);
1737     Node* if_slow = _gvn.transform(new IfFalseNode(iff));
1738     Node* if_fast = _gvn.transform(new IfTrueNode(iff));
1739 
1740     if (!if_slow->is_top()) {
1741       RegionNode* result_region = new RegionNode(3);
1742       PhiNode*    result_val = new PhiNode(result_region, Type::DOUBLE);
1743 
1744       result_region->init_req(1, if_fast);
1745       result_val->init_req(1, result);
1746 
1747       set_control(if_slow);
1748 
1749       const TypePtr* no_memory_effects = NULL;
1750       Node* rt = make_runtime_call(RC_LEAF, call_type, funcAddr, funcName,
1751                                    no_memory_effects,
1752                                    x, top(), y, y ? top() : NULL);
1753       Node* value = _gvn.transform(new ProjNode(rt, TypeFunc::Parms+0));
1754 #ifdef ASSERT
1755       Node* value_top = _gvn.transform(new ProjNode(rt, TypeFunc::Parms+1));
1756       assert(value_top == top(), "second value must be top");
1757 #endif
1758 
1759       result_region->init_req(2, control());
1760       result_val->init_req(2, value);
1761       set_control(_gvn.transform(result_region));
1762       return _gvn.transform(result_val);
1763     } else {
1764       return result;
1765     }
1766   }
1767 }
1768 
1769 //------------------------------inline_pow-------------------------------------
1770 // Inline power instructions, if possible.
1771 bool LibraryCallKit::inline_pow() {
1772   // Pseudocode for pow
1773   // if (y == 2) {
1774   //   return x * x;
1775   // } else {
1776   //   if (x <= 0.0) {
1777   //     long longy = (long)y;
1778   //     if ((double)longy == y) { // if y is long
1779   //       if (y + 1 == y) longy = 0; // huge number: even
1780   //       result = ((1&longy) == 0)?-DPow(abs(x), y):DPow(abs(x), y);
1781   //     } else {
1782   //       result = NaN;
1783   //     }
1784   //   } else {
1785   //     result = DPow(x,y);
1786   //   }
1787   //   if (result != result)?  {
1788   //     result = uncommon_trap() or runtime_call();
1789   //   }
1790   //   return result;
1791   // }
1792 
1793   Node* x = round_double_node(argument(0));
1794   Node* y = round_double_node(argument(2));
1795 
1796   Node* result = NULL;
1797 
1798   Node*   const_two_node = makecon(TypeD::make(2.0));
1799   Node*   cmp_node       = _gvn.transform(new CmpDNode(y, const_two_node));
1800   Node*   bool_node      = _gvn.transform(new BoolNode(cmp_node, BoolTest::eq));
1801   IfNode* if_node        = create_and_xform_if(control(), bool_node, PROB_STATIC_INFREQUENT, COUNT_UNKNOWN);
1802   Node*   if_true        = _gvn.transform(new IfTrueNode(if_node));
1803   Node*   if_false       = _gvn.transform(new IfFalseNode(if_node));
1804 
1805   RegionNode* region_node = new RegionNode(3);
1806   region_node->init_req(1, if_true);
1807 
1808   Node* phi_node = new PhiNode(region_node, Type::DOUBLE);
1809   // special case for x^y where y == 2, we can convert it to x * x
1810   phi_node->init_req(1, _gvn.transform(new MulDNode(x, x)));
1811 
1812   // set control to if_false since we will now process the false branch
1813   set_control(if_false);
1814 
1815   if (!too_many_traps(Deoptimization::Reason_intrinsic)) {
1816     // Short form: skip the fancy tests and just check for NaN result.
1817     result = _gvn.transform(new PowDNode(C, control(), x, y));
1818   } else {
1819     // If this inlining ever returned NaN in the past, include all
1820     // checks + call to the runtime.
1821 
1822     // Set the merge point for If node with condition of (x <= 0.0)
1823     // There are four possible paths to region node and phi node
1824     RegionNode *r = new RegionNode(4);
1825     Node *phi = new PhiNode(r, Type::DOUBLE);
1826 
1827     // Build the first if node: if (x <= 0.0)
1828     // Node for 0 constant
1829     Node *zeronode = makecon(TypeD::ZERO);
1830     // Check x:0
1831     Node *cmp = _gvn.transform(new CmpDNode(x, zeronode));
1832     // Check: If (x<=0) then go complex path
1833     Node *bol1 = _gvn.transform(new BoolNode( cmp, BoolTest::le ));
1834     // Branch either way
1835     IfNode *if1 = create_and_xform_if(control(),bol1, PROB_STATIC_INFREQUENT, COUNT_UNKNOWN);
1836     // Fast path taken; set region slot 3
1837     Node *fast_taken = _gvn.transform(new IfFalseNode(if1));
1838     r->init_req(3,fast_taken); // Capture fast-control
1839 
1840     // Fast path not-taken, i.e. slow path
1841     Node *complex_path = _gvn.transform(new IfTrueNode(if1));
1842 
1843     // Set fast path result
1844     Node *fast_result = _gvn.transform(new PowDNode(C, control(), x, y));
1845     phi->init_req(3, fast_result);
1846 
1847     // Complex path
1848     // Build the second if node (if y is long)
1849     // Node for (long)y
1850     Node *longy = _gvn.transform(new ConvD2LNode(y));
1851     // Node for (double)((long) y)
1852     Node *doublelongy= _gvn.transform(new ConvL2DNode(longy));
1853     // Check (double)((long) y) : y
1854     Node *cmplongy= _gvn.transform(new CmpDNode(doublelongy, y));
1855     // Check if (y isn't long) then go to slow path
1856 
1857     Node *bol2 = _gvn.transform(new BoolNode( cmplongy, BoolTest::ne ));
1858     // Branch either way
1859     IfNode *if2 = create_and_xform_if(complex_path,bol2, PROB_STATIC_INFREQUENT, COUNT_UNKNOWN);
1860     Node* ylong_path = _gvn.transform(new IfFalseNode(if2));
1861 
1862     Node *slow_path = _gvn.transform(new IfTrueNode(if2));
1863 
1864     // Calculate DPow(abs(x), y)*(1 & (long)y)
1865     // Node for constant 1
1866     Node *conone = longcon(1);
1867     // 1& (long)y
1868     Node *signnode= _gvn.transform(new AndLNode(conone, longy));
1869 
1870     // A huge number is always even. Detect a huge number by checking
1871     // if y + 1 == y and set integer to be tested for parity to 0.
1872     // Required for corner case:
1873     // (long)9.223372036854776E18 = max_jlong
1874     // (double)(long)9.223372036854776E18 = 9.223372036854776E18
1875     // max_jlong is odd but 9.223372036854776E18 is even
1876     Node* yplus1 = _gvn.transform(new AddDNode(y, makecon(TypeD::make(1))));
1877     Node *cmpyplus1= _gvn.transform(new CmpDNode(yplus1, y));
1878     Node *bolyplus1 = _gvn.transform(new BoolNode( cmpyplus1, BoolTest::eq ));
1879     Node* correctedsign = NULL;
1880     if (ConditionalMoveLimit != 0) {
1881       correctedsign = _gvn.transform(CMoveNode::make(NULL, bolyplus1, signnode, longcon(0), TypeLong::LONG));
1882     } else {
1883       IfNode *ifyplus1 = create_and_xform_if(ylong_path,bolyplus1, PROB_FAIR, COUNT_UNKNOWN);
1884       RegionNode *r = new RegionNode(3);
1885       Node *phi = new PhiNode(r, TypeLong::LONG);
1886       r->init_req(1, _gvn.transform(new IfFalseNode(ifyplus1)));
1887       r->init_req(2, _gvn.transform(new IfTrueNode(ifyplus1)));
1888       phi->init_req(1, signnode);
1889       phi->init_req(2, longcon(0));
1890       correctedsign = _gvn.transform(phi);
1891       ylong_path = _gvn.transform(r);
1892       record_for_igvn(r);
1893     }
1894 
1895     // zero node
1896     Node *conzero = longcon(0);
1897     // Check (1&(long)y)==0?
1898     Node *cmpeq1 = _gvn.transform(new CmpLNode(correctedsign, conzero));
1899     // Check if (1&(long)y)!=0?, if so the result is negative
1900     Node *bol3 = _gvn.transform(new BoolNode( cmpeq1, BoolTest::ne ));
1901     // abs(x)
1902     Node *absx=_gvn.transform(new AbsDNode(x));
1903     // abs(x)^y
1904     Node *absxpowy = _gvn.transform(new PowDNode(C, control(), absx, y));
1905     // -abs(x)^y
1906     Node *negabsxpowy = _gvn.transform(new NegDNode (absxpowy));
1907     // (1&(long)y)==1?-DPow(abs(x), y):DPow(abs(x), y)
1908     Node *signresult = NULL;
1909     if (ConditionalMoveLimit != 0) {
1910       signresult = _gvn.transform(CMoveNode::make(NULL, bol3, absxpowy, negabsxpowy, Type::DOUBLE));
1911     } else {
1912       IfNode *ifyeven = create_and_xform_if(ylong_path,bol3, PROB_FAIR, COUNT_UNKNOWN);
1913       RegionNode *r = new RegionNode(3);
1914       Node *phi = new PhiNode(r, Type::DOUBLE);
1915       r->init_req(1, _gvn.transform(new IfFalseNode(ifyeven)));
1916       r->init_req(2, _gvn.transform(new IfTrueNode(ifyeven)));
1917       phi->init_req(1, absxpowy);
1918       phi->init_req(2, negabsxpowy);
1919       signresult = _gvn.transform(phi);
1920       ylong_path = _gvn.transform(r);
1921       record_for_igvn(r);
1922     }
1923     // Set complex path fast result
1924     r->init_req(2, ylong_path);
1925     phi->init_req(2, signresult);
1926 
1927     static const jlong nan_bits = CONST64(0x7ff8000000000000);
1928     Node *slow_result = makecon(TypeD::make(*(double*)&nan_bits)); // return NaN
1929     r->init_req(1,slow_path);
1930     phi->init_req(1,slow_result);
1931 
1932     // Post merge
1933     set_control(_gvn.transform(r));
1934     record_for_igvn(r);
1935     result = _gvn.transform(phi);
1936   }
1937 
1938   result = finish_pow_exp(result, x, y, OptoRuntime::Math_DD_D_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::dpow), "POW");
1939 
1940   // control from finish_pow_exp is now input to the region node
1941   region_node->set_req(2, control());
1942   // the result from finish_pow_exp is now input to the phi node
1943   phi_node->init_req(2, result);
1944   set_control(_gvn.transform(region_node));
1945   record_for_igvn(region_node);
1946   set_result(_gvn.transform(phi_node));
1947 
1948   C->set_has_split_ifs(true); // Has chance for split-if optimization
1949   return true;
1950 }
1951 
1952 //------------------------------runtime_math-----------------------------
1953 bool LibraryCallKit::runtime_math(const TypeFunc* call_type, address funcAddr, const char* funcName) {
1954   assert(call_type == OptoRuntime::Math_DD_D_Type() || call_type == OptoRuntime::Math_D_D_Type(),
1955          "must be (DD)D or (D)D type");
1956 
1957   // Inputs
1958   Node* a = round_double_node(argument(0));
1959   Node* b = (call_type == OptoRuntime::Math_DD_D_Type()) ? round_double_node(argument(2)) : NULL;
1960 
1961   const TypePtr* no_memory_effects = NULL;
1962   Node* trig = make_runtime_call(RC_LEAF, call_type, funcAddr, funcName,
1963                                  no_memory_effects,
1964                                  a, top(), b, b ? top() : NULL);
1965   Node* value = _gvn.transform(new ProjNode(trig, TypeFunc::Parms+0));
1966 #ifdef ASSERT
1967   Node* value_top = _gvn.transform(new ProjNode(trig, TypeFunc::Parms+1));
1968   assert(value_top == top(), "second value must be top");
1969 #endif
1970 
1971   set_result(value);
1972   return true;
1973 }
1974 
1975 //------------------------------inline_math_native-----------------------------
1976 bool LibraryCallKit::inline_math_native(vmIntrinsics::ID id) {
1977 #define FN_PTR(f) CAST_FROM_FN_PTR(address, f)
1978   switch (id) {
1979     // These intrinsics are not properly supported on all hardware
1980   case vmIntrinsics::_dcos:   return Matcher::has_match_rule(Op_CosD)   ? inline_trig(id) :
1981     runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dcos),   "COS");
1982   case vmIntrinsics::_dsin:   return Matcher::has_match_rule(Op_SinD)   ? inline_trig(id) :
1983     runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dsin),   "SIN");
1984   case vmIntrinsics::_dtan:   return Matcher::has_match_rule(Op_TanD)   ? inline_trig(id) :
1985     runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dtan),   "TAN");
1986 
1987   case vmIntrinsics::_dlog:
1988     return StubRoutines::dlog() != NULL ?
1989     runtime_math(OptoRuntime::Math_D_D_Type(), StubRoutines::dlog(), "dlog") :
1990     runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dlog),   "LOG");
1991   case vmIntrinsics::_dlog10: return Matcher::has_match_rule(Op_Log10D) ? inline_math(id) :
1992     runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dlog10), "LOG10");
1993 
1994     // These intrinsics are supported on all hardware
1995   case vmIntrinsics::_dsqrt:  return Matcher::match_rule_supported(Op_SqrtD) ? inline_math(id) : false;
1996   case vmIntrinsics::_dabs:   return Matcher::has_match_rule(Op_AbsD)   ? inline_math(id) : false;
1997 
1998   case vmIntrinsics::_dexp:
1999     return StubRoutines::dexp() != NULL ?
2000       runtime_math(OptoRuntime::Math_D_D_Type(), StubRoutines::dexp(),  "dexp") :
2001       runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dexp),  "EXP");
2002   case vmIntrinsics::_dpow:   return Matcher::has_match_rule(Op_PowD)   ? inline_pow()    :
2003     runtime_math(OptoRuntime::Math_DD_D_Type(), FN_PTR(SharedRuntime::dpow),  "POW");
2004 #undef FN_PTR
2005 
2006    // These intrinsics are not yet correctly implemented
2007   case vmIntrinsics::_datan2:
2008     return false;
2009 
2010   default:
2011     fatal_unexpected_iid(id);
2012     return false;
2013   }
2014 }
2015 
2016 static bool is_simple_name(Node* n) {
2017   return (n->req() == 1         // constant
2018           || (n->is_Type() && n->as_Type()->type()->singleton())
2019           || n->is_Proj()       // parameter or return value
2020           || n->is_Phi()        // local of some sort
2021           );
2022 }
2023 
2024 //----------------------------inline_notify-----------------------------------*
2025 bool LibraryCallKit::inline_notify(vmIntrinsics::ID id) {
2026   const TypeFunc* ftype = OptoRuntime::monitor_notify_Type();
2027   address func;
2028   if (id == vmIntrinsics::_notify) {
2029     func = OptoRuntime::monitor_notify_Java();
2030   } else {
2031     func = OptoRuntime::monitor_notifyAll_Java();
2032   }
2033   Node* call = make_runtime_call(RC_NO_LEAF, ftype, func, NULL, TypeRawPtr::BOTTOM, argument(0));
2034   make_slow_call_ex(call, env()->Throwable_klass(), false);
2035   return true;
2036 }
2037 
2038 
2039 //----------------------------inline_min_max-----------------------------------
2040 bool LibraryCallKit::inline_min_max(vmIntrinsics::ID id) {
2041   set_result(generate_min_max(id, argument(0), argument(1)));
2042   return true;
2043 }
2044 
2045 void LibraryCallKit::inline_math_mathExact(Node* math, Node *test) {
2046   Node* bol = _gvn.transform( new BoolNode(test, BoolTest::overflow) );
2047   IfNode* check = create_and_map_if(control(), bol, PROB_UNLIKELY_MAG(3), COUNT_UNKNOWN);
2048   Node* fast_path = _gvn.transform( new IfFalseNode(check));
2049   Node* slow_path = _gvn.transform( new IfTrueNode(check) );
2050 
2051   {
2052     PreserveJVMState pjvms(this);
2053     PreserveReexecuteState preexecs(this);
2054     jvms()->set_should_reexecute(true);
2055 
2056     set_control(slow_path);
2057     set_i_o(i_o());
2058 
2059     uncommon_trap(Deoptimization::Reason_intrinsic,
2060                   Deoptimization::Action_none);
2061   }
2062 
2063   set_control(fast_path);
2064   set_result(math);
2065 }
2066 
2067 template <typename OverflowOp>
2068 bool LibraryCallKit::inline_math_overflow(Node* arg1, Node* arg2) {
2069   typedef typename OverflowOp::MathOp MathOp;
2070 
2071   MathOp* mathOp = new MathOp(arg1, arg2);
2072   Node* operation = _gvn.transform( mathOp );
2073   Node* ofcheck = _gvn.transform( new OverflowOp(arg1, arg2) );
2074   inline_math_mathExact(operation, ofcheck);
2075   return true;
2076 }
2077 
2078 bool LibraryCallKit::inline_math_addExactI(bool is_increment) {
2079   return inline_math_overflow<OverflowAddINode>(argument(0), is_increment ? intcon(1) : argument(1));
2080 }
2081 
2082 bool LibraryCallKit::inline_math_addExactL(bool is_increment) {
2083   return inline_math_overflow<OverflowAddLNode>(argument(0), is_increment ? longcon(1) : argument(2));
2084 }
2085 
2086 bool LibraryCallKit::inline_math_subtractExactI(bool is_decrement) {
2087   return inline_math_overflow<OverflowSubINode>(argument(0), is_decrement ? intcon(1) : argument(1));
2088 }
2089 
2090 bool LibraryCallKit::inline_math_subtractExactL(bool is_decrement) {
2091   return inline_math_overflow<OverflowSubLNode>(argument(0), is_decrement ? longcon(1) : argument(2));
2092 }
2093 
2094 bool LibraryCallKit::inline_math_negateExactI() {
2095   return inline_math_overflow<OverflowSubINode>(intcon(0), argument(0));
2096 }
2097 
2098 bool LibraryCallKit::inline_math_negateExactL() {
2099   return inline_math_overflow<OverflowSubLNode>(longcon(0), argument(0));
2100 }
2101 
2102 bool LibraryCallKit::inline_math_multiplyExactI() {
2103   return inline_math_overflow<OverflowMulINode>(argument(0), argument(1));
2104 }
2105 
2106 bool LibraryCallKit::inline_math_multiplyExactL() {
2107   return inline_math_overflow<OverflowMulLNode>(argument(0), argument(2));
2108 }
2109 
2110 Node*
2111 LibraryCallKit::generate_min_max(vmIntrinsics::ID id, Node* x0, Node* y0) {
2112   // These are the candidate return value:
2113   Node* xvalue = x0;
2114   Node* yvalue = y0;
2115 
2116   if (xvalue == yvalue) {
2117     return xvalue;
2118   }
2119 
2120   bool want_max = (id == vmIntrinsics::_max);
2121 
2122   const TypeInt* txvalue = _gvn.type(xvalue)->isa_int();
2123   const TypeInt* tyvalue = _gvn.type(yvalue)->isa_int();
2124   if (txvalue == NULL || tyvalue == NULL)  return top();
2125   // This is not really necessary, but it is consistent with a
2126   // hypothetical MaxINode::Value method:
2127   int widen = MAX2(txvalue->_widen, tyvalue->_widen);
2128 
2129   // %%% This folding logic should (ideally) be in a different place.
2130   // Some should be inside IfNode, and there to be a more reliable
2131   // transformation of ?: style patterns into cmoves.  We also want
2132   // more powerful optimizations around cmove and min/max.
2133 
2134   // Try to find a dominating comparison of these guys.
2135   // It can simplify the index computation for Arrays.copyOf
2136   // and similar uses of System.arraycopy.
2137   // First, compute the normalized version of CmpI(x, y).
2138   int   cmp_op = Op_CmpI;
2139   Node* xkey = xvalue;
2140   Node* ykey = yvalue;
2141   Node* ideal_cmpxy = _gvn.transform(new CmpINode(xkey, ykey));
2142   if (ideal_cmpxy->is_Cmp()) {
2143     // E.g., if we have CmpI(length - offset, count),
2144     // it might idealize to CmpI(length, count + offset)
2145     cmp_op = ideal_cmpxy->Opcode();
2146     xkey = ideal_cmpxy->in(1);
2147     ykey = ideal_cmpxy->in(2);
2148   }
2149 
2150   // Start by locating any relevant comparisons.
2151   Node* start_from = (xkey->outcnt() < ykey->outcnt()) ? xkey : ykey;
2152   Node* cmpxy = NULL;
2153   Node* cmpyx = NULL;
2154   for (DUIterator_Fast kmax, k = start_from->fast_outs(kmax); k < kmax; k++) {
2155     Node* cmp = start_from->fast_out(k);
2156     if (cmp->outcnt() > 0 &&            // must have prior uses
2157         cmp->in(0) == NULL &&           // must be context-independent
2158         cmp->Opcode() == cmp_op) {      // right kind of compare
2159       if (cmp->in(1) == xkey && cmp->in(2) == ykey)  cmpxy = cmp;
2160       if (cmp->in(1) == ykey && cmp->in(2) == xkey)  cmpyx = cmp;
2161     }
2162   }
2163 
2164   const int NCMPS = 2;
2165   Node* cmps[NCMPS] = { cmpxy, cmpyx };
2166   int cmpn;
2167   for (cmpn = 0; cmpn < NCMPS; cmpn++) {
2168     if (cmps[cmpn] != NULL)  break;     // find a result
2169   }
2170   if (cmpn < NCMPS) {
2171     // Look for a dominating test that tells us the min and max.
2172     int depth = 0;                // Limit search depth for speed
2173     Node* dom = control();
2174     for (; dom != NULL; dom = IfNode::up_one_dom(dom, true)) {
2175       if (++depth >= 100)  break;
2176       Node* ifproj = dom;
2177       if (!ifproj->is_Proj())  continue;
2178       Node* iff = ifproj->in(0);
2179       if (!iff->is_If())  continue;
2180       Node* bol = iff->in(1);
2181       if (!bol->is_Bool())  continue;
2182       Node* cmp = bol->in(1);
2183       if (cmp == NULL)  continue;
2184       for (cmpn = 0; cmpn < NCMPS; cmpn++)
2185         if (cmps[cmpn] == cmp)  break;
2186       if (cmpn == NCMPS)  continue;
2187       BoolTest::mask btest = bol->as_Bool()->_test._test;
2188       if (ifproj->is_IfFalse())  btest = BoolTest(btest).negate();
2189       if (cmp->in(1) == ykey)    btest = BoolTest(btest).commute();
2190       // At this point, we know that 'x btest y' is true.
2191       switch (btest) {
2192       case BoolTest::eq:
2193         // They are proven equal, so we can collapse the min/max.
2194         // Either value is the answer.  Choose the simpler.
2195         if (is_simple_name(yvalue) && !is_simple_name(xvalue))
2196           return yvalue;
2197         return xvalue;
2198       case BoolTest::lt:          // x < y
2199       case BoolTest::le:          // x <= y
2200         return (want_max ? yvalue : xvalue);
2201       case BoolTest::gt:          // x > y
2202       case BoolTest::ge:          // x >= y
2203         return (want_max ? xvalue : yvalue);
2204       }
2205     }
2206   }
2207 
2208   // We failed to find a dominating test.
2209   // Let's pick a test that might GVN with prior tests.
2210   Node*          best_bol   = NULL;
2211   BoolTest::mask best_btest = BoolTest::illegal;
2212   for (cmpn = 0; cmpn < NCMPS; cmpn++) {
2213     Node* cmp = cmps[cmpn];
2214     if (cmp == NULL)  continue;
2215     for (DUIterator_Fast jmax, j = cmp->fast_outs(jmax); j < jmax; j++) {
2216       Node* bol = cmp->fast_out(j);
2217       if (!bol->is_Bool())  continue;
2218       BoolTest::mask btest = bol->as_Bool()->_test._test;
2219       if (btest == BoolTest::eq || btest == BoolTest::ne)  continue;
2220       if (cmp->in(1) == ykey)   btest = BoolTest(btest).commute();
2221       if (bol->outcnt() > (best_bol == NULL ? 0 : best_bol->outcnt())) {
2222         best_bol   = bol->as_Bool();
2223         best_btest = btest;
2224       }
2225     }
2226   }
2227 
2228   Node* answer_if_true  = NULL;
2229   Node* answer_if_false = NULL;
2230   switch (best_btest) {
2231   default:
2232     if (cmpxy == NULL)
2233       cmpxy = ideal_cmpxy;
2234     best_bol = _gvn.transform(new BoolNode(cmpxy, BoolTest::lt));
2235     // and fall through:
2236   case BoolTest::lt:          // x < y
2237   case BoolTest::le:          // x <= y
2238     answer_if_true  = (want_max ? yvalue : xvalue);
2239     answer_if_false = (want_max ? xvalue : yvalue);
2240     break;
2241   case BoolTest::gt:          // x > y
2242   case BoolTest::ge:          // x >= y
2243     answer_if_true  = (want_max ? xvalue : yvalue);
2244     answer_if_false = (want_max ? yvalue : xvalue);
2245     break;
2246   }
2247 
2248   jint hi, lo;
2249   if (want_max) {
2250     // We can sharpen the minimum.
2251     hi = MAX2(txvalue->_hi, tyvalue->_hi);
2252     lo = MAX2(txvalue->_lo, tyvalue->_lo);
2253   } else {
2254     // We can sharpen the maximum.
2255     hi = MIN2(txvalue->_hi, tyvalue->_hi);
2256     lo = MIN2(txvalue->_lo, tyvalue->_lo);
2257   }
2258 
2259   // Use a flow-free graph structure, to avoid creating excess control edges
2260   // which could hinder other optimizations.
2261   // Since Math.min/max is often used with arraycopy, we want
2262   // tightly_coupled_allocation to be able to see beyond min/max expressions.
2263   Node* cmov = CMoveNode::make(NULL, best_bol,
2264                                answer_if_false, answer_if_true,
2265                                TypeInt::make(lo, hi, widen));
2266 
2267   return _gvn.transform(cmov);
2268 
2269   /*
2270   // This is not as desirable as it may seem, since Min and Max
2271   // nodes do not have a full set of optimizations.
2272   // And they would interfere, anyway, with 'if' optimizations
2273   // and with CMoveI canonical forms.
2274   switch (id) {
2275   case vmIntrinsics::_min:
2276     result_val = _gvn.transform(new (C, 3) MinINode(x,y)); break;
2277   case vmIntrinsics::_max:
2278     result_val = _gvn.transform(new (C, 3) MaxINode(x,y)); break;
2279   default:
2280     ShouldNotReachHere();
2281   }
2282   */
2283 }
2284 
2285 inline int
2286 LibraryCallKit::classify_unsafe_addr(Node* &base, Node* &offset) {
2287   const TypePtr* base_type = TypePtr::NULL_PTR;
2288   if (base != NULL)  base_type = _gvn.type(base)->isa_ptr();
2289   if (base_type == NULL) {
2290     // Unknown type.
2291     return Type::AnyPtr;
2292   } else if (base_type == TypePtr::NULL_PTR) {
2293     // Since this is a NULL+long form, we have to switch to a rawptr.
2294     base   = _gvn.transform(new CastX2PNode(offset));
2295     offset = MakeConX(0);
2296     return Type::RawPtr;
2297   } else if (base_type->base() == Type::RawPtr) {
2298     return Type::RawPtr;
2299   } else if (base_type->isa_oopptr()) {
2300     // Base is never null => always a heap address.
2301     if (base_type->ptr() == TypePtr::NotNull) {
2302       return Type::OopPtr;
2303     }
2304     // Offset is small => always a heap address.
2305     const TypeX* offset_type = _gvn.type(offset)->isa_intptr_t();
2306     if (offset_type != NULL &&
2307         base_type->offset() == 0 &&     // (should always be?)
2308         offset_type->_lo >= 0 &&
2309         !MacroAssembler::needs_explicit_null_check(offset_type->_hi)) {
2310       return Type::OopPtr;
2311     }
2312     // Otherwise, it might either be oop+off or NULL+addr.
2313     return Type::AnyPtr;
2314   } else {
2315     // No information:
2316     return Type::AnyPtr;
2317   }
2318 }
2319 
2320 inline Node* LibraryCallKit::make_unsafe_address(Node* base, Node* offset) {
2321   int kind = classify_unsafe_addr(base, offset);
2322   if (kind == Type::RawPtr) {
2323     return basic_plus_adr(top(), base, offset);
2324   } else {
2325     return basic_plus_adr(base, offset);
2326   }
2327 }
2328 
2329 //--------------------------inline_number_methods-----------------------------
2330 // inline int     Integer.numberOfLeadingZeros(int)
2331 // inline int        Long.numberOfLeadingZeros(long)
2332 //
2333 // inline int     Integer.numberOfTrailingZeros(int)
2334 // inline int        Long.numberOfTrailingZeros(long)
2335 //
2336 // inline int     Integer.bitCount(int)
2337 // inline int        Long.bitCount(long)
2338 //
2339 // inline char  Character.reverseBytes(char)
2340 // inline short     Short.reverseBytes(short)
2341 // inline int     Integer.reverseBytes(int)
2342 // inline long       Long.reverseBytes(long)
2343 bool LibraryCallKit::inline_number_methods(vmIntrinsics::ID id) {
2344   Node* arg = argument(0);
2345   Node* n = NULL;
2346   switch (id) {
2347   case vmIntrinsics::_numberOfLeadingZeros_i:   n = new CountLeadingZerosINode( arg);  break;
2348   case vmIntrinsics::_numberOfLeadingZeros_l:   n = new CountLeadingZerosLNode( arg);  break;
2349   case vmIntrinsics::_numberOfTrailingZeros_i:  n = new CountTrailingZerosINode(arg);  break;
2350   case vmIntrinsics::_numberOfTrailingZeros_l:  n = new CountTrailingZerosLNode(arg);  break;
2351   case vmIntrinsics::_bitCount_i:               n = new PopCountINode(          arg);  break;
2352   case vmIntrinsics::_bitCount_l:               n = new PopCountLNode(          arg);  break;
2353   case vmIntrinsics::_reverseBytes_c:           n = new ReverseBytesUSNode(0,   arg);  break;
2354   case vmIntrinsics::_reverseBytes_s:           n = new ReverseBytesSNode( 0,   arg);  break;
2355   case vmIntrinsics::_reverseBytes_i:           n = new ReverseBytesINode( 0,   arg);  break;
2356   case vmIntrinsics::_reverseBytes_l:           n = new ReverseBytesLNode( 0,   arg);  break;
2357   default:  fatal_unexpected_iid(id);  break;
2358   }
2359   set_result(_gvn.transform(n));
2360   return true;
2361 }
2362 
2363 //----------------------------inline_unsafe_access----------------------------
2364 
2365 const static BasicType T_ADDRESS_HOLDER = T_LONG;
2366 
2367 // Helper that guards and inserts a pre-barrier.
2368 void LibraryCallKit::insert_pre_barrier(Node* base_oop, Node* offset,
2369                                         Node* pre_val, bool need_mem_bar) {
2370   // We could be accessing the referent field of a reference object. If so, when G1
2371   // is enabled, we need to log the value in the referent field in an SATB buffer.
2372   // This routine performs some compile time filters and generates suitable
2373   // runtime filters that guard the pre-barrier code.
2374   // Also add memory barrier for non volatile load from the referent field
2375   // to prevent commoning of loads across safepoint.
2376   if (!UseG1GC && !need_mem_bar)
2377     return;
2378 
2379   // Some compile time checks.
2380 
2381   // If offset is a constant, is it java_lang_ref_Reference::_reference_offset?
2382   const TypeX* otype = offset->find_intptr_t_type();
2383   if (otype != NULL && otype->is_con() &&
2384       otype->get_con() != java_lang_ref_Reference::referent_offset) {
2385     // Constant offset but not the reference_offset so just return
2386     return;
2387   }
2388 
2389   // We only need to generate the runtime guards for instances.
2390   const TypeOopPtr* btype = base_oop->bottom_type()->isa_oopptr();
2391   if (btype != NULL) {
2392     if (btype->isa_aryptr()) {
2393       // Array type so nothing to do
2394       return;
2395     }
2396 
2397     const TypeInstPtr* itype = btype->isa_instptr();
2398     if (itype != NULL) {
2399       // Can the klass of base_oop be statically determined to be
2400       // _not_ a sub-class of Reference and _not_ Object?
2401       ciKlass* klass = itype->klass();
2402       if ( klass->is_loaded() &&
2403           !klass->is_subtype_of(env()->Reference_klass()) &&
2404           !env()->Object_klass()->is_subtype_of(klass)) {
2405         return;
2406       }
2407     }
2408   }
2409 
2410   // The compile time filters did not reject base_oop/offset so
2411   // we need to generate the following runtime filters
2412   //
2413   // if (offset == java_lang_ref_Reference::_reference_offset) {
2414   //   if (instance_of(base, java.lang.ref.Reference)) {
2415   //     pre_barrier(_, pre_val, ...);
2416   //   }
2417   // }
2418 
2419   float likely   = PROB_LIKELY(  0.999);
2420   float unlikely = PROB_UNLIKELY(0.999);
2421 
2422   IdealKit ideal(this);
2423 #define __ ideal.
2424 
2425   Node* referent_off = __ ConX(java_lang_ref_Reference::referent_offset);
2426 
2427   __ if_then(offset, BoolTest::eq, referent_off, unlikely); {
2428       // Update graphKit memory and control from IdealKit.
2429       sync_kit(ideal);
2430 
2431       Node* ref_klass_con = makecon(TypeKlassPtr::make(env()->Reference_klass()));
2432       Node* is_instof = gen_instanceof(base_oop, ref_klass_con);
2433 
2434       // Update IdealKit memory and control from graphKit.
2435       __ sync_kit(this);
2436 
2437       Node* one = __ ConI(1);
2438       // is_instof == 0 if base_oop == NULL
2439       __ if_then(is_instof, BoolTest::eq, one, unlikely); {
2440 
2441         // Update graphKit from IdeakKit.
2442         sync_kit(ideal);
2443 
2444         // Use the pre-barrier to record the value in the referent field
2445         pre_barrier(false /* do_load */,
2446                     __ ctrl(),
2447                     NULL /* obj */, NULL /* adr */, max_juint /* alias_idx */, NULL /* val */, NULL /* val_type */,
2448                     pre_val /* pre_val */,
2449                     T_OBJECT);
2450         if (need_mem_bar) {
2451           // Add memory barrier to prevent commoning reads from this field
2452           // across safepoint since GC can change its value.
2453           insert_mem_bar(Op_MemBarCPUOrder);
2454         }
2455         // Update IdealKit from graphKit.
2456         __ sync_kit(this);
2457 
2458       } __ end_if(); // _ref_type != ref_none
2459   } __ end_if(); // offset == referent_offset
2460 
2461   // Final sync IdealKit and GraphKit.
2462   final_sync(ideal);
2463 #undef __
2464 }
2465 
2466 
2467 // Interpret Unsafe.fieldOffset cookies correctly:
2468 extern jlong Unsafe_field_offset_to_byte_offset(jlong field_offset);
2469 
2470 const TypeOopPtr* LibraryCallKit::sharpen_unsafe_type(Compile::AliasType* alias_type, const TypePtr *adr_type, bool is_native_ptr) {
2471   // Attempt to infer a sharper value type from the offset and base type.
2472   ciKlass* sharpened_klass = NULL;
2473 
2474   // See if it is an instance field, with an object type.
2475   if (alias_type->field() != NULL) {
2476     assert(!is_native_ptr, "native pointer op cannot use a java address");
2477     if (alias_type->field()->type()->is_klass()) {
2478       sharpened_klass = alias_type->field()->type()->as_klass();
2479     }
2480   }
2481 
2482   // See if it is a narrow oop array.
2483   if (adr_type->isa_aryptr()) {
2484     if (adr_type->offset() >= objArrayOopDesc::base_offset_in_bytes()) {
2485       const TypeOopPtr *elem_type = adr_type->is_aryptr()->elem()->isa_oopptr();
2486       if (elem_type != NULL) {
2487         sharpened_klass = elem_type->klass();
2488       }
2489     }
2490   }
2491 
2492   // The sharpened class might be unloaded if there is no class loader
2493   // contraint in place.
2494   if (sharpened_klass != NULL && sharpened_klass->is_loaded()) {
2495     const TypeOopPtr* tjp = TypeOopPtr::make_from_klass(sharpened_klass);
2496 
2497 #ifndef PRODUCT
2498     if (C->print_intrinsics() || C->print_inlining()) {
2499       tty->print("  from base type: ");  adr_type->dump();
2500       tty->print("  sharpened value: ");  tjp->dump();
2501     }
2502 #endif
2503     // Sharpen the value type.
2504     return tjp;
2505   }
2506   return NULL;
2507 }
2508 
2509 bool LibraryCallKit::inline_unsafe_access(bool is_native_ptr, bool is_store, BasicType type, bool is_volatile, bool unaligned) {
2510   if (callee()->is_static())  return false;  // caller must have the capability!
2511 
2512 #ifndef PRODUCT
2513   {
2514     ResourceMark rm;
2515     // Check the signatures.
2516     ciSignature* sig = callee()->signature();
2517 #ifdef ASSERT
2518     if (!is_store) {
2519       // Object getObject(Object base, int/long offset), etc.
2520       BasicType rtype = sig->return_type()->basic_type();
2521       if (rtype == T_ADDRESS_HOLDER && callee()->name() == ciSymbol::getAddress_name())
2522           rtype = T_ADDRESS;  // it is really a C void*
2523       assert(rtype == type, "getter must return the expected value");
2524       if (!is_native_ptr) {
2525         assert(sig->count() == 2, "oop getter has 2 arguments");
2526         assert(sig->type_at(0)->basic_type() == T_OBJECT, "getter base is object");
2527         assert(sig->type_at(1)->basic_type() == T_LONG, "getter offset is correct");
2528       } else {
2529         assert(sig->count() == 1, "native getter has 1 argument");
2530         assert(sig->type_at(0)->basic_type() == T_LONG, "getter base is long");
2531       }
2532     } else {
2533       // void putObject(Object base, int/long offset, Object x), etc.
2534       assert(sig->return_type()->basic_type() == T_VOID, "putter must not return a value");
2535       if (!is_native_ptr) {
2536         assert(sig->count() == 3, "oop putter has 3 arguments");
2537         assert(sig->type_at(0)->basic_type() == T_OBJECT, "putter base is object");
2538         assert(sig->type_at(1)->basic_type() == T_LONG, "putter offset is correct");
2539       } else {
2540         assert(sig->count() == 2, "native putter has 2 arguments");
2541         assert(sig->type_at(0)->basic_type() == T_LONG, "putter base is long");
2542       }
2543       BasicType vtype = sig->type_at(sig->count()-1)->basic_type();
2544       if (vtype == T_ADDRESS_HOLDER && callee()->name() == ciSymbol::putAddress_name())
2545         vtype = T_ADDRESS;  // it is really a C void*
2546       assert(vtype == type, "putter must accept the expected value");
2547     }
2548 #endif // ASSERT
2549  }
2550 #endif //PRODUCT
2551 
2552   C->set_has_unsafe_access(true);  // Mark eventual nmethod as "unsafe".
2553 
2554   Node* receiver = argument(0);  // type: oop
2555 
2556   // Build address expression.
2557   Node* adr;
2558   Node* heap_base_oop = top();
2559   Node* offset = top();
2560   Node* val;
2561 
2562   if (!is_native_ptr) {
2563     // The base is either a Java object or a value produced by Unsafe.staticFieldBase
2564     Node* base = argument(1);  // type: oop
2565     // The offset is a value produced by Unsafe.staticFieldOffset or Unsafe.objectFieldOffset
2566     offset = argument(2);  // type: long
2567     // We currently rely on the cookies produced by Unsafe.xxxFieldOffset
2568     // to be plain byte offsets, which are also the same as those accepted
2569     // by oopDesc::field_base.
2570     assert(Unsafe_field_offset_to_byte_offset(11) == 11,
2571            "fieldOffset must be byte-scaled");
2572     // 32-bit machines ignore the high half!
2573     offset = ConvL2X(offset);
2574     adr = make_unsafe_address(base, offset);
2575     heap_base_oop = base;
2576     val = is_store ? argument(4) : NULL;
2577   } else {
2578     Node* ptr = argument(1);  // type: long
2579     ptr = ConvL2X(ptr);  // adjust Java long to machine word
2580     adr = make_unsafe_address(NULL, ptr);
2581     val = is_store ? argument(3) : NULL;
2582   }
2583 
2584   const TypePtr *adr_type = _gvn.type(adr)->isa_ptr();
2585 
2586   // First guess at the value type.
2587   const Type *value_type = Type::get_const_basic_type(type);
2588 
2589   // Try to categorize the address.  If it comes up as TypeJavaPtr::BOTTOM,
2590   // there was not enough information to nail it down.
2591   Compile::AliasType* alias_type = C->alias_type(adr_type);
2592   assert(alias_type->index() != Compile::AliasIdxBot, "no bare pointers here");
2593 
2594   // We will need memory barriers unless we can determine a unique
2595   // alias category for this reference.  (Note:  If for some reason
2596   // the barriers get omitted and the unsafe reference begins to "pollute"
2597   // the alias analysis of the rest of the graph, either Compile::can_alias
2598   // or Compile::must_alias will throw a diagnostic assert.)
2599   bool need_mem_bar = (alias_type->adr_type() == TypeOopPtr::BOTTOM);
2600 
2601   // If we are reading the value of the referent field of a Reference
2602   // object (either by using Unsafe directly or through reflection)
2603   // then, if G1 is enabled, we need to record the referent in an
2604   // SATB log buffer using the pre-barrier mechanism.
2605   // Also we need to add memory barrier to prevent commoning reads
2606   // from this field across safepoint since GC can change its value.
2607   bool need_read_barrier = !is_native_ptr && !is_store &&
2608                            offset != top() && heap_base_oop != top();
2609 
2610   if (!is_store && type == T_OBJECT) {
2611     const TypeOopPtr* tjp = sharpen_unsafe_type(alias_type, adr_type, is_native_ptr);
2612     if (tjp != NULL) {
2613       value_type = tjp;
2614     }
2615   }
2616 
2617   receiver = null_check(receiver);
2618   if (stopped()) {
2619     return true;
2620   }
2621   // Heap pointers get a null-check from the interpreter,
2622   // as a courtesy.  However, this is not guaranteed by Unsafe,
2623   // and it is not possible to fully distinguish unintended nulls
2624   // from intended ones in this API.
2625 
2626   if (is_volatile) {
2627     // We need to emit leading and trailing CPU membars (see below) in
2628     // addition to memory membars when is_volatile. This is a little
2629     // too strong, but avoids the need to insert per-alias-type
2630     // volatile membars (for stores; compare Parse::do_put_xxx), which
2631     // we cannot do effectively here because we probably only have a
2632     // rough approximation of type.
2633     need_mem_bar = true;
2634     // For Stores, place a memory ordering barrier now.
2635     if (is_store) {
2636       insert_mem_bar(Op_MemBarRelease);
2637     } else {
2638       if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
2639         insert_mem_bar(Op_MemBarVolatile);
2640       }
2641     }
2642   }
2643 
2644   // Memory barrier to prevent normal and 'unsafe' accesses from
2645   // bypassing each other.  Happens after null checks, so the
2646   // exception paths do not take memory state from the memory barrier,
2647   // so there's no problems making a strong assert about mixing users
2648   // of safe & unsafe memory.
2649   if (need_mem_bar) insert_mem_bar(Op_MemBarCPUOrder);
2650 
2651   assert(alias_type->adr_type() == TypeRawPtr::BOTTOM || alias_type->adr_type() == TypeOopPtr::BOTTOM ||
2652          alias_type->field() != NULL || alias_type->element() != NULL, "field, array element or unknown");
2653   bool mismatched = false;
2654   if (alias_type->element() != NULL || alias_type->field() != NULL) {
2655     BasicType bt;
2656     if (alias_type->element() != NULL) {
2657       const Type* element = alias_type->element();
2658       bt = element->isa_narrowoop() ? T_OBJECT : element->array_element_basic_type();
2659     } else {
2660       bt = alias_type->field()->type()->basic_type();
2661     }
2662     if (bt == T_ARRAY) {
2663       // accessing an array field with getObject is not a mismatch
2664       bt = T_OBJECT;
2665     }
2666     if (bt != type) {
2667       mismatched = true;
2668     }
2669   }
2670   assert(type != T_OBJECT || !unaligned, "unaligned access not supported with object type");
2671 
2672   if (!is_store) {
2673     Node* p = NULL;
2674     // Try to constant fold a load from a constant field
2675     ciField* field = alias_type->field();
2676     if (heap_base_oop != top() &&
2677         field != NULL && field->is_constant() && field->layout_type() == type) {
2678       // final or stable field
2679       const Type* con_type = Type::make_constant(alias_type->field(), heap_base_oop);
2680       if (con_type != NULL) {
2681         p = makecon(con_type);
2682       }
2683     }
2684     if (p == NULL) {
2685       MemNode::MemOrd mo = is_volatile ? MemNode::acquire : MemNode::unordered;
2686       // To be valid, unsafe loads may depend on other conditions than
2687       // the one that guards them: pin the Load node
2688       p = make_load(control(), adr, value_type, type, adr_type, mo, LoadNode::Pinned, is_volatile, unaligned, mismatched);
2689       // load value
2690       switch (type) {
2691       case T_BOOLEAN:
2692       case T_CHAR:
2693       case T_BYTE:
2694       case T_SHORT:
2695       case T_INT:
2696       case T_LONG:
2697       case T_FLOAT:
2698       case T_DOUBLE:
2699         break;
2700       case T_OBJECT:
2701         if (need_read_barrier) {
2702           insert_pre_barrier(heap_base_oop, offset, p, !(is_volatile || need_mem_bar));
2703         }
2704         break;
2705       case T_ADDRESS:
2706         // Cast to an int type.
2707         p = _gvn.transform(new CastP2XNode(NULL, p));
2708         p = ConvX2UL(p);
2709         break;
2710       default:
2711         fatal("unexpected type %d: %s", type, type2name(type));
2712         break;
2713       }
2714     }
2715     // The load node has the control of the preceding MemBarCPUOrder.  All
2716     // following nodes will have the control of the MemBarCPUOrder inserted at
2717     // the end of this method.  So, pushing the load onto the stack at a later
2718     // point is fine.
2719     set_result(p);
2720   } else {
2721     // place effect of store into memory
2722     switch (type) {
2723     case T_DOUBLE:
2724       val = dstore_rounding(val);
2725       break;
2726     case T_ADDRESS:
2727       // Repackage the long as a pointer.
2728       val = ConvL2X(val);
2729       val = _gvn.transform(new CastX2PNode(val));
2730       break;
2731     }
2732 
2733     MemNode::MemOrd mo = is_volatile ? MemNode::release : MemNode::unordered;
2734     if (type != T_OBJECT ) {
2735       (void) store_to_memory(control(), adr, val, type, adr_type, mo, is_volatile, unaligned, mismatched);
2736     } else {
2737       // Possibly an oop being stored to Java heap or native memory
2738       if (!TypePtr::NULL_PTR->higher_equal(_gvn.type(heap_base_oop))) {
2739         // oop to Java heap.
2740         (void) store_oop_to_unknown(control(), heap_base_oop, adr, adr_type, val, type, mo, mismatched);
2741       } else {
2742         // We can't tell at compile time if we are storing in the Java heap or outside
2743         // of it. So we need to emit code to conditionally do the proper type of
2744         // store.
2745 
2746         IdealKit ideal(this);
2747 #define __ ideal.
2748         // QQQ who knows what probability is here??
2749         __ if_then(heap_base_oop, BoolTest::ne, null(), PROB_UNLIKELY(0.999)); {
2750           // Sync IdealKit and graphKit.
2751           sync_kit(ideal);
2752           Node* st = store_oop_to_unknown(control(), heap_base_oop, adr, adr_type, val, type, mo, mismatched);
2753           // Update IdealKit memory.
2754           __ sync_kit(this);
2755         } __ else_(); {
2756           __ store(__ ctrl(), adr, val, type, alias_type->index(), mo, is_volatile, mismatched);
2757         } __ end_if();
2758         // Final sync IdealKit and GraphKit.
2759         final_sync(ideal);
2760 #undef __
2761       }
2762     }
2763   }
2764 
2765   if (is_volatile) {
2766     if (!is_store) {
2767       insert_mem_bar(Op_MemBarAcquire);
2768     } else {
2769       if (!support_IRIW_for_not_multiple_copy_atomic_cpu) {
2770         insert_mem_bar(Op_MemBarVolatile);
2771       }
2772     }
2773   }
2774 
2775   if (need_mem_bar) insert_mem_bar(Op_MemBarCPUOrder);
2776 
2777   return true;
2778 }
2779 
2780 //----------------------------inline_unsafe_load_store----------------------------
2781 // This method serves a couple of different customers (depending on LoadStoreKind):
2782 //
2783 // LS_cmpxchg:
2784 //   public final native boolean compareAndSwapObject(Object o, long offset, Object expected, Object x);
2785 //   public final native boolean compareAndSwapInt(   Object o, long offset, int    expected, int    x);
2786 //   public final native boolean compareAndSwapLong(  Object o, long offset, long   expected, long   x);
2787 //
2788 // LS_xadd:
2789 //   public int  getAndAddInt( Object o, long offset, int  delta)
2790 //   public long getAndAddLong(Object o, long offset, long delta)
2791 //
2792 // LS_xchg:
2793 //   int    getAndSet(Object o, long offset, int    newValue)
2794 //   long   getAndSet(Object o, long offset, long   newValue)
2795 //   Object getAndSet(Object o, long offset, Object newValue)
2796 //
2797 bool LibraryCallKit::inline_unsafe_load_store(BasicType type, LoadStoreKind kind) {
2798   // This basic scheme here is the same as inline_unsafe_access, but
2799   // differs in enough details that combining them would make the code
2800   // overly confusing.  (This is a true fact! I originally combined
2801   // them, but even I was confused by it!) As much code/comments as
2802   // possible are retained from inline_unsafe_access though to make
2803   // the correspondences clearer. - dl
2804 
2805   if (callee()->is_static())  return false;  // caller must have the capability!
2806 
2807 #ifndef PRODUCT
2808   BasicType rtype;
2809   {
2810     ResourceMark rm;
2811     // Check the signatures.
2812     ciSignature* sig = callee()->signature();
2813     rtype = sig->return_type()->basic_type();
2814     if (kind == LS_xadd || kind == LS_xchg) {
2815       // Check the signatures.
2816 #ifdef ASSERT
2817       assert(rtype == type, "get and set must return the expected type");
2818       assert(sig->count() == 3, "get and set has 3 arguments");
2819       assert(sig->type_at(0)->basic_type() == T_OBJECT, "get and set base is object");
2820       assert(sig->type_at(1)->basic_type() == T_LONG, "get and set offset is long");
2821       assert(sig->type_at(2)->basic_type() == type, "get and set must take expected type as new value/delta");
2822 #endif // ASSERT
2823     } else if (kind == LS_cmpxchg) {
2824       // Check the signatures.
2825 #ifdef ASSERT
2826       assert(rtype == T_BOOLEAN, "CAS must return boolean");
2827       assert(sig->count() == 4, "CAS has 4 arguments");
2828       assert(sig->type_at(0)->basic_type() == T_OBJECT, "CAS base is object");
2829       assert(sig->type_at(1)->basic_type() == T_LONG, "CAS offset is long");
2830 #endif // ASSERT
2831     } else {
2832       ShouldNotReachHere();
2833     }
2834   }
2835 #endif //PRODUCT
2836 
2837   C->set_has_unsafe_access(true);  // Mark eventual nmethod as "unsafe".
2838 
2839   // Get arguments:
2840   Node* receiver = NULL;
2841   Node* base     = NULL;
2842   Node* offset   = NULL;
2843   Node* oldval   = NULL;
2844   Node* newval   = NULL;
2845   if (kind == LS_cmpxchg) {
2846     const bool two_slot_type = type2size[type] == 2;
2847     receiver = argument(0);  // type: oop
2848     base     = argument(1);  // type: oop
2849     offset   = argument(2);  // type: long
2850     oldval   = argument(4);  // type: oop, int, or long
2851     newval   = argument(two_slot_type ? 6 : 5);  // type: oop, int, or long
2852   } else if (kind == LS_xadd || kind == LS_xchg){
2853     receiver = argument(0);  // type: oop
2854     base     = argument(1);  // type: oop
2855     offset   = argument(2);  // type: long
2856     oldval   = NULL;
2857     newval   = argument(4);  // type: oop, int, or long
2858   }
2859 
2860   // Null check receiver.
2861   receiver = null_check(receiver);
2862   if (stopped()) {
2863     return true;
2864   }
2865 
2866   // Build field offset expression.
2867   // We currently rely on the cookies produced by Unsafe.xxxFieldOffset
2868   // to be plain byte offsets, which are also the same as those accepted
2869   // by oopDesc::field_base.
2870   assert(Unsafe_field_offset_to_byte_offset(11) == 11, "fieldOffset must be byte-scaled");
2871   // 32-bit machines ignore the high half of long offsets
2872   offset = ConvL2X(offset);
2873   Node* adr = make_unsafe_address(base, offset);
2874   const TypePtr *adr_type = _gvn.type(adr)->isa_ptr();
2875 
2876   // For CAS, unlike inline_unsafe_access, there seems no point in
2877   // trying to refine types. Just use the coarse types here.
2878   const Type *value_type = Type::get_const_basic_type(type);
2879   Compile::AliasType* alias_type = C->alias_type(adr_type);
2880   assert(alias_type->index() != Compile::AliasIdxBot, "no bare pointers here");
2881 
2882   if (kind == LS_xchg && type == T_OBJECT) {
2883     const TypeOopPtr* tjp = sharpen_unsafe_type(alias_type, adr_type);
2884     if (tjp != NULL) {
2885       value_type = tjp;
2886     }
2887   }
2888 
2889   int alias_idx = C->get_alias_index(adr_type);
2890 
2891   // Memory-model-wise, a LoadStore acts like a little synchronized
2892   // block, so needs barriers on each side.  These don't translate
2893   // into actual barriers on most machines, but we still need rest of
2894   // compiler to respect ordering.
2895 
2896   insert_mem_bar(Op_MemBarRelease);
2897   insert_mem_bar(Op_MemBarCPUOrder);
2898 
2899   // 4984716: MemBars must be inserted before this
2900   //          memory node in order to avoid a false
2901   //          dependency which will confuse the scheduler.
2902   Node *mem = memory(alias_idx);
2903 
2904   // For now, we handle only those cases that actually exist: ints,
2905   // longs, and Object. Adding others should be straightforward.
2906   Node* load_store = NULL;
2907   switch(type) {
2908   case T_INT:
2909     if (kind == LS_xadd) {
2910       load_store = _gvn.transform(new GetAndAddINode(control(), mem, adr, newval, adr_type));
2911     } else if (kind == LS_xchg) {
2912       load_store = _gvn.transform(new GetAndSetINode(control(), mem, adr, newval, adr_type));
2913     } else if (kind == LS_cmpxchg) {
2914       load_store = _gvn.transform(new CompareAndSwapINode(control(), mem, adr, newval, oldval));
2915     } else {
2916       ShouldNotReachHere();
2917     }
2918     break;
2919   case T_LONG:
2920     if (kind == LS_xadd) {
2921       load_store = _gvn.transform(new GetAndAddLNode(control(), mem, adr, newval, adr_type));
2922     } else if (kind == LS_xchg) {
2923       load_store = _gvn.transform(new GetAndSetLNode(control(), mem, adr, newval, adr_type));
2924     } else if (kind == LS_cmpxchg) {
2925       load_store = _gvn.transform(new CompareAndSwapLNode(control(), mem, adr, newval, oldval));
2926     } else {
2927       ShouldNotReachHere();
2928     }
2929     break;
2930   case T_OBJECT:
2931     // Transformation of a value which could be NULL pointer (CastPP #NULL)
2932     // could be delayed during Parse (for example, in adjust_map_after_if()).
2933     // Execute transformation here to avoid barrier generation in such case.
2934     if (_gvn.type(newval) == TypePtr::NULL_PTR)
2935       newval = _gvn.makecon(TypePtr::NULL_PTR);
2936 
2937     // Reference stores need a store barrier.
2938     if (kind == LS_xchg) {
2939       // If pre-barrier must execute before the oop store, old value will require do_load here.
2940       if (!can_move_pre_barrier()) {
2941         pre_barrier(true /* do_load*/,
2942                     control(), base, adr, alias_idx, newval, value_type->make_oopptr(),
2943                     NULL /* pre_val*/,
2944                     T_OBJECT);
2945       } // Else move pre_barrier to use load_store value, see below.
2946     } else if (kind == LS_cmpxchg) {
2947       // Same as for newval above:
2948       if (_gvn.type(oldval) == TypePtr::NULL_PTR) {
2949         oldval = _gvn.makecon(TypePtr::NULL_PTR);
2950       }
2951       // The only known value which might get overwritten is oldval.
2952       pre_barrier(false /* do_load */,
2953                   control(), NULL, NULL, max_juint, NULL, NULL,
2954                   oldval /* pre_val */,
2955                   T_OBJECT);
2956     } else {
2957       ShouldNotReachHere();
2958     }
2959 
2960 #ifdef _LP64
2961     if (adr->bottom_type()->is_ptr_to_narrowoop()) {
2962       Node *newval_enc = _gvn.transform(new EncodePNode(newval, newval->bottom_type()->make_narrowoop()));
2963       if (kind == LS_xchg) {
2964         load_store = _gvn.transform(new GetAndSetNNode(control(), mem, adr,
2965                                                        newval_enc, adr_type, value_type->make_narrowoop()));
2966       } else {
2967         assert(kind == LS_cmpxchg, "wrong LoadStore operation");
2968         Node *oldval_enc = _gvn.transform(new EncodePNode(oldval, oldval->bottom_type()->make_narrowoop()));
2969         load_store = _gvn.transform(new CompareAndSwapNNode(control(), mem, adr,
2970                                                                 newval_enc, oldval_enc));
2971       }
2972     } else
2973 #endif
2974     {
2975       if (kind == LS_xchg) {
2976         load_store = _gvn.transform(new GetAndSetPNode(control(), mem, adr, newval, adr_type, value_type->is_oopptr()));
2977       } else {
2978         assert(kind == LS_cmpxchg, "wrong LoadStore operation");
2979         load_store = _gvn.transform(new CompareAndSwapPNode(control(), mem, adr, newval, oldval));
2980       }
2981     }
2982     if (kind == LS_cmpxchg) {
2983       // Emit the post barrier only when the actual store happened.
2984       // This makes sense to check only for compareAndSet that can fail to set the value.
2985       // CAS success path is marked more likely since we anticipate this is a performance
2986       // critical path, while CAS failure path can use the penalty for going through unlikely
2987       // path as backoff. Which is still better than doing a store barrier there.
2988       IdealKit ideal(this);
2989       ideal.if_then(load_store, BoolTest::ne, ideal.ConI(0), PROB_STATIC_FREQUENT); {
2990         sync_kit(ideal);
2991         post_barrier(ideal.ctrl(), load_store, base, adr, alias_idx, newval, T_OBJECT, true);
2992         ideal.sync_kit(this);
2993       } ideal.end_if();
2994       final_sync(ideal);
2995     } else {
2996       post_barrier(control(), load_store, base, adr, alias_idx, newval, T_OBJECT, true);
2997     }
2998     break;
2999   default:
3000     fatal("unexpected type %d: %s", type, type2name(type));
3001     break;
3002   }
3003 
3004   // SCMemProjNodes represent the memory state of a LoadStore. Their
3005   // main role is to prevent LoadStore nodes from being optimized away
3006   // when their results aren't used.
3007   Node* proj = _gvn.transform(new SCMemProjNode(load_store));
3008   set_memory(proj, alias_idx);
3009 
3010   if (type == T_OBJECT && kind == LS_xchg) {
3011 #ifdef _LP64
3012     if (adr->bottom_type()->is_ptr_to_narrowoop()) {
3013       load_store = _gvn.transform(new DecodeNNode(load_store, load_store->get_ptr_type()));
3014     }
3015 #endif
3016     if (can_move_pre_barrier()) {
3017       // Don't need to load pre_val. The old value is returned by load_store.
3018       // The pre_barrier can execute after the xchg as long as no safepoint
3019       // gets inserted between them.
3020       pre_barrier(false /* do_load */,
3021                   control(), NULL, NULL, max_juint, NULL, NULL,
3022                   load_store /* pre_val */,
3023                   T_OBJECT);
3024     }
3025   }
3026 
3027   // Add the trailing membar surrounding the access
3028   insert_mem_bar(Op_MemBarCPUOrder);
3029   insert_mem_bar(Op_MemBarAcquire);
3030 
3031   assert(type2size[load_store->bottom_type()->basic_type()] == type2size[rtype], "result type should match");
3032   set_result(load_store);
3033   return true;
3034 }
3035 
3036 //----------------------------inline_unsafe_ordered_store----------------------
3037 // public native void Unsafe.putOrderedObject(Object o, long offset, Object x);
3038 // public native void Unsafe.putOrderedInt(Object o, long offset, int x);
3039 // public native void Unsafe.putOrderedLong(Object o, long offset, long x);
3040 bool LibraryCallKit::inline_unsafe_ordered_store(BasicType type) {
3041   // This is another variant of inline_unsafe_access, differing in
3042   // that it always issues store-store ("release") barrier and ensures
3043   // store-atomicity (which only matters for "long").
3044 
3045   if (callee()->is_static())  return false;  // caller must have the capability!
3046 
3047 #ifndef PRODUCT
3048   {
3049     ResourceMark rm;
3050     // Check the signatures.
3051     ciSignature* sig = callee()->signature();
3052 #ifdef ASSERT
3053     BasicType rtype = sig->return_type()->basic_type();
3054     assert(rtype == T_VOID, "must return void");
3055     assert(sig->count() == 3, "has 3 arguments");
3056     assert(sig->type_at(0)->basic_type() == T_OBJECT, "base is object");
3057     assert(sig->type_at(1)->basic_type() == T_LONG, "offset is long");
3058 #endif // ASSERT
3059   }
3060 #endif //PRODUCT
3061 
3062   C->set_has_unsafe_access(true);  // Mark eventual nmethod as "unsafe".
3063 
3064   // Get arguments:
3065   Node* receiver = argument(0);  // type: oop
3066   Node* base     = argument(1);  // type: oop
3067   Node* offset   = argument(2);  // type: long
3068   Node* val      = argument(4);  // type: oop, int, or long
3069 
3070   // Null check receiver.
3071   receiver = null_check(receiver);
3072   if (stopped()) {
3073     return true;
3074   }
3075 
3076   // Build field offset expression.
3077   assert(Unsafe_field_offset_to_byte_offset(11) == 11, "fieldOffset must be byte-scaled");
3078   // 32-bit machines ignore the high half of long offsets
3079   offset = ConvL2X(offset);
3080   Node* adr = make_unsafe_address(base, offset);
3081   const TypePtr *adr_type = _gvn.type(adr)->isa_ptr();
3082   const Type *value_type = Type::get_const_basic_type(type);
3083   Compile::AliasType* alias_type = C->alias_type(adr_type);
3084 
3085   insert_mem_bar(Op_MemBarRelease);
3086   insert_mem_bar(Op_MemBarCPUOrder);
3087   // Ensure that the store is atomic for longs:
3088   const bool require_atomic_access = true;
3089   Node* store;
3090   if (type == T_OBJECT) // reference stores need a store barrier.
3091     store = store_oop_to_unknown(control(), base, adr, adr_type, val, type, MemNode::release);
3092   else {
3093     store = store_to_memory(control(), adr, val, type, adr_type, MemNode::release, require_atomic_access);
3094   }
3095   insert_mem_bar(Op_MemBarCPUOrder);
3096   return true;
3097 }
3098 
3099 bool LibraryCallKit::inline_unsafe_fence(vmIntrinsics::ID id) {
3100   // Regardless of form, don't allow previous ld/st to move down,
3101   // then issue acquire, release, or volatile mem_bar.
3102   insert_mem_bar(Op_MemBarCPUOrder);
3103   switch(id) {
3104     case vmIntrinsics::_loadFence:
3105       insert_mem_bar(Op_LoadFence);
3106       return true;
3107     case vmIntrinsics::_storeFence:
3108       insert_mem_bar(Op_StoreFence);
3109       return true;
3110     case vmIntrinsics::_fullFence:
3111       insert_mem_bar(Op_MemBarVolatile);
3112       return true;
3113     default:
3114       fatal_unexpected_iid(id);
3115       return false;
3116   }
3117 }
3118 
3119 bool LibraryCallKit::inline_onspinwait() {
3120   insert_mem_bar(Op_OnSpinWait);
3121   return true;
3122 }
3123 
3124 bool LibraryCallKit::klass_needs_init_guard(Node* kls) {
3125   if (!kls->is_Con()) {
3126     return true;
3127   }
3128   const TypeKlassPtr* klsptr = kls->bottom_type()->isa_klassptr();
3129   if (klsptr == NULL) {
3130     return true;
3131   }
3132   ciInstanceKlass* ik = klsptr->klass()->as_instance_klass();
3133   // don't need a guard for a klass that is already initialized
3134   return !ik->is_initialized();
3135 }
3136 
3137 //----------------------------inline_unsafe_allocate---------------------------
3138 // public native Object Unsafe.allocateInstance(Class<?> cls);
3139 bool LibraryCallKit::inline_unsafe_allocate() {
3140   if (callee()->is_static())  return false;  // caller must have the capability!
3141 
3142   null_check_receiver();  // null-check, then ignore
3143   Node* cls = null_check(argument(1));
3144   if (stopped())  return true;
3145 
3146   Node* kls = load_klass_from_mirror(cls, false, NULL, 0);
3147   kls = null_check(kls);
3148   if (stopped())  return true;  // argument was like int.class
3149 
3150   Node* test = NULL;
3151   if (LibraryCallKit::klass_needs_init_guard(kls)) {
3152     // Note:  The argument might still be an illegal value like
3153     // Serializable.class or Object[].class.   The runtime will handle it.
3154     // But we must make an explicit check for initialization.
3155     Node* insp = basic_plus_adr(kls, in_bytes(InstanceKlass::init_state_offset()));
3156     // Use T_BOOLEAN for InstanceKlass::_init_state so the compiler
3157     // can generate code to load it as unsigned byte.
3158     Node* inst = make_load(NULL, insp, TypeInt::UBYTE, T_BOOLEAN, MemNode::unordered);
3159     Node* bits = intcon(InstanceKlass::fully_initialized);
3160     test = _gvn.transform(new SubINode(inst, bits));
3161     // The 'test' is non-zero if we need to take a slow path.
3162   }
3163 
3164   Node* obj = new_instance(kls, test);
3165   set_result(obj);
3166   return true;
3167 }
3168 
3169 #ifdef TRACE_HAVE_INTRINSICS
3170 /*
3171  * oop -> myklass
3172  * myklass->trace_id |= USED
3173  * return myklass->trace_id & ~0x3
3174  */
3175 bool LibraryCallKit::inline_native_classID() {
3176   null_check_receiver();  // null-check, then ignore
3177   Node* cls = null_check(argument(1), T_OBJECT);
3178   Node* kls = load_klass_from_mirror(cls, false, NULL, 0);
3179   kls = null_check(kls, T_OBJECT);
3180   ByteSize offset = TRACE_ID_OFFSET;
3181   Node* insp = basic_plus_adr(kls, in_bytes(offset));
3182   Node* tvalue = make_load(NULL, insp, TypeLong::LONG, T_LONG, MemNode::unordered);
3183   Node* bits = longcon(~0x03l); // ignore bit 0 & 1
3184   Node* andl = _gvn.transform(new AndLNode(tvalue, bits));
3185   Node* clsused = longcon(0x01l); // set the class bit
3186   Node* orl = _gvn.transform(new OrLNode(tvalue, clsused));
3187 
3188   const TypePtr *adr_type = _gvn.type(insp)->isa_ptr();
3189   store_to_memory(control(), insp, orl, T_LONG, adr_type, MemNode::unordered);
3190   set_result(andl);
3191   return true;
3192 }
3193 
3194 bool LibraryCallKit::inline_native_threadID() {
3195   Node* tls_ptr = NULL;
3196   Node* cur_thr = generate_current_thread(tls_ptr);
3197   Node* p = basic_plus_adr(top()/*!oop*/, tls_ptr, in_bytes(JavaThread::osthread_offset()));
3198   Node* osthread = make_load(NULL, p, TypeRawPtr::NOTNULL, T_ADDRESS, MemNode::unordered);
3199   p = basic_plus_adr(top()/*!oop*/, osthread, in_bytes(OSThread::thread_id_offset()));
3200 
3201   Node* threadid = NULL;
3202   size_t thread_id_size = OSThread::thread_id_size();
3203   if (thread_id_size == (size_t) BytesPerLong) {
3204     threadid = ConvL2I(make_load(control(), p, TypeLong::LONG, T_LONG, MemNode::unordered));
3205   } else if (thread_id_size == (size_t) BytesPerInt) {
3206     threadid = make_load(control(), p, TypeInt::INT, T_INT, MemNode::unordered);
3207   } else {
3208     ShouldNotReachHere();
3209   }
3210   set_result(threadid);
3211   return true;
3212 }
3213 #endif
3214 
3215 //------------------------inline_native_time_funcs--------------
3216 // inline code for System.currentTimeMillis() and System.nanoTime()
3217 // these have the same type and signature
3218 bool LibraryCallKit::inline_native_time_funcs(address funcAddr, const char* funcName) {
3219   const TypeFunc* tf = OptoRuntime::void_long_Type();
3220   const TypePtr* no_memory_effects = NULL;
3221   Node* time = make_runtime_call(RC_LEAF, tf, funcAddr, funcName, no_memory_effects);
3222   Node* value = _gvn.transform(new ProjNode(time, TypeFunc::Parms+0));
3223 #ifdef ASSERT
3224   Node* value_top = _gvn.transform(new ProjNode(time, TypeFunc::Parms+1));
3225   assert(value_top == top(), "second value must be top");
3226 #endif
3227   set_result(value);
3228   return true;
3229 }
3230 
3231 //------------------------inline_native_currentThread------------------
3232 bool LibraryCallKit::inline_native_currentThread() {
3233   Node* junk = NULL;
3234   set_result(generate_current_thread(junk));
3235   return true;
3236 }
3237 
3238 //------------------------inline_native_isInterrupted------------------
3239 // private native boolean java.lang.Thread.isInterrupted(boolean ClearInterrupted);
3240 bool LibraryCallKit::inline_native_isInterrupted() {
3241   // Add a fast path to t.isInterrupted(clear_int):
3242   //   (t == Thread.current() &&
3243   //    (!TLS._osthread._interrupted || WINDOWS_ONLY(false) NOT_WINDOWS(!clear_int)))
3244   //   ? TLS._osthread._interrupted : /*slow path:*/ t.isInterrupted(clear_int)
3245   // So, in the common case that the interrupt bit is false,
3246   // we avoid making a call into the VM.  Even if the interrupt bit
3247   // is true, if the clear_int argument is false, we avoid the VM call.
3248   // However, if the receiver is not currentThread, we must call the VM,
3249   // because there must be some locking done around the operation.
3250 
3251   // We only go to the fast case code if we pass two guards.
3252   // Paths which do not pass are accumulated in the slow_region.
3253 
3254   enum {
3255     no_int_result_path   = 1, // t == Thread.current() && !TLS._osthread._interrupted
3256     no_clear_result_path = 2, // t == Thread.current() &&  TLS._osthread._interrupted && !clear_int
3257     slow_result_path     = 3, // slow path: t.isInterrupted(clear_int)
3258     PATH_LIMIT
3259   };
3260 
3261   // Ensure that it's not possible to move the load of TLS._osthread._interrupted flag
3262   // out of the function.
3263   insert_mem_bar(Op_MemBarCPUOrder);
3264 
3265   RegionNode* result_rgn = new RegionNode(PATH_LIMIT);
3266   PhiNode*    result_val = new PhiNode(result_rgn, TypeInt::BOOL);
3267 
3268   RegionNode* slow_region = new RegionNode(1);
3269   record_for_igvn(slow_region);
3270 
3271   // (a) Receiving thread must be the current thread.
3272   Node* rec_thr = argument(0);
3273   Node* tls_ptr = NULL;
3274   Node* cur_thr = generate_current_thread(tls_ptr);
3275   Node* cmp_thr = _gvn.transform(new CmpPNode(cur_thr, rec_thr));
3276   Node* bol_thr = _gvn.transform(new BoolNode(cmp_thr, BoolTest::ne));
3277 
3278   generate_slow_guard(bol_thr, slow_region);
3279 
3280   // (b) Interrupt bit on TLS must be false.
3281   Node* p = basic_plus_adr(top()/*!oop*/, tls_ptr, in_bytes(JavaThread::osthread_offset()));
3282   Node* osthread = make_load(NULL, p, TypeRawPtr::NOTNULL, T_ADDRESS, MemNode::unordered);
3283   p = basic_plus_adr(top()/*!oop*/, osthread, in_bytes(OSThread::interrupted_offset()));
3284 
3285   // Set the control input on the field _interrupted read to prevent it floating up.
3286   Node* int_bit = make_load(control(), p, TypeInt::BOOL, T_INT, MemNode::unordered);
3287   Node* cmp_bit = _gvn.transform(new CmpINode(int_bit, intcon(0)));
3288   Node* bol_bit = _gvn.transform(new BoolNode(cmp_bit, BoolTest::ne));
3289 
3290   IfNode* iff_bit = create_and_map_if(control(), bol_bit, PROB_UNLIKELY_MAG(3), COUNT_UNKNOWN);
3291 
3292   // First fast path:  if (!TLS._interrupted) return false;
3293   Node* false_bit = _gvn.transform(new IfFalseNode(iff_bit));
3294   result_rgn->init_req(no_int_result_path, false_bit);
3295   result_val->init_req(no_int_result_path, intcon(0));
3296 
3297   // drop through to next case
3298   set_control( _gvn.transform(new IfTrueNode(iff_bit)));
3299 
3300 #ifndef TARGET_OS_FAMILY_windows
3301   // (c) Or, if interrupt bit is set and clear_int is false, use 2nd fast path.
3302   Node* clr_arg = argument(1);
3303   Node* cmp_arg = _gvn.transform(new CmpINode(clr_arg, intcon(0)));
3304   Node* bol_arg = _gvn.transform(new BoolNode(cmp_arg, BoolTest::ne));
3305   IfNode* iff_arg = create_and_map_if(control(), bol_arg, PROB_FAIR, COUNT_UNKNOWN);
3306 
3307   // Second fast path:  ... else if (!clear_int) return true;
3308   Node* false_arg = _gvn.transform(new IfFalseNode(iff_arg));
3309   result_rgn->init_req(no_clear_result_path, false_arg);
3310   result_val->init_req(no_clear_result_path, intcon(1));
3311 
3312   // drop through to next case
3313   set_control( _gvn.transform(new IfTrueNode(iff_arg)));
3314 #else
3315   // To return true on Windows you must read the _interrupted field
3316   // and check the the event state i.e. take the slow path.
3317 #endif // TARGET_OS_FAMILY_windows
3318 
3319   // (d) Otherwise, go to the slow path.
3320   slow_region->add_req(control());
3321   set_control( _gvn.transform(slow_region));
3322 
3323   if (stopped()) {
3324     // There is no slow path.
3325     result_rgn->init_req(slow_result_path, top());
3326     result_val->init_req(slow_result_path, top());
3327   } else {
3328     // non-virtual because it is a private non-static
3329     CallJavaNode* slow_call = generate_method_call(vmIntrinsics::_isInterrupted);
3330 
3331     Node* slow_val = set_results_for_java_call(slow_call);
3332     // this->control() comes from set_results_for_java_call
3333 
3334     Node* fast_io  = slow_call->in(TypeFunc::I_O);
3335     Node* fast_mem = slow_call->in(TypeFunc::Memory);
3336 
3337     // These two phis are pre-filled with copies of of the fast IO and Memory
3338     PhiNode* result_mem  = PhiNode::make(result_rgn, fast_mem, Type::MEMORY, TypePtr::BOTTOM);
3339     PhiNode* result_io   = PhiNode::make(result_rgn, fast_io,  Type::ABIO);
3340 
3341     result_rgn->init_req(slow_result_path, control());
3342     result_io ->init_req(slow_result_path, i_o());
3343     result_mem->init_req(slow_result_path, reset_memory());
3344     result_val->init_req(slow_result_path, slow_val);
3345 
3346     set_all_memory(_gvn.transform(result_mem));
3347     set_i_o(       _gvn.transform(result_io));
3348   }
3349 
3350   C->set_has_split_ifs(true); // Has chance for split-if optimization
3351   set_result(result_rgn, result_val);
3352   return true;
3353 }
3354 
3355 //---------------------------load_mirror_from_klass----------------------------
3356 // Given a klass oop, load its java mirror (a java.lang.Class oop).
3357 Node* LibraryCallKit::load_mirror_from_klass(Node* klass) {
3358   Node* p = basic_plus_adr(klass, in_bytes(Klass::java_mirror_offset()));
3359   return make_load(NULL, p, TypeInstPtr::MIRROR, T_OBJECT, MemNode::unordered);
3360 }
3361 
3362 //-----------------------load_klass_from_mirror_common-------------------------
3363 // Given a java mirror (a java.lang.Class oop), load its corresponding klass oop.
3364 // Test the klass oop for null (signifying a primitive Class like Integer.TYPE),
3365 // and branch to the given path on the region.
3366 // If never_see_null, take an uncommon trap on null, so we can optimistically
3367 // compile for the non-null case.
3368 // If the region is NULL, force never_see_null = true.
3369 Node* LibraryCallKit::load_klass_from_mirror_common(Node* mirror,
3370                                                     bool never_see_null,
3371                                                     RegionNode* region,
3372                                                     int null_path,
3373                                                     int offset) {
3374   if (region == NULL)  never_see_null = true;
3375   Node* p = basic_plus_adr(mirror, offset);
3376   const TypeKlassPtr*  kls_type = TypeKlassPtr::OBJECT_OR_NULL;
3377   Node* kls = _gvn.transform(LoadKlassNode::make(_gvn, NULL, immutable_memory(), p, TypeRawPtr::BOTTOM, kls_type));
3378   Node* null_ctl = top();
3379   kls = null_check_oop(kls, &null_ctl, never_see_null);
3380   if (region != NULL) {
3381     // Set region->in(null_path) if the mirror is a primitive (e.g, int.class).
3382     region->init_req(null_path, null_ctl);
3383   } else {
3384     assert(null_ctl == top(), "no loose ends");
3385   }
3386   return kls;
3387 }
3388 
3389 //--------------------(inline_native_Class_query helpers)---------------------
3390 // Use this for JVM_ACC_INTERFACE, JVM_ACC_IS_CLONEABLE, JVM_ACC_HAS_FINALIZER.
3391 // Fall through if (mods & mask) == bits, take the guard otherwise.
3392 Node* LibraryCallKit::generate_access_flags_guard(Node* kls, int modifier_mask, int modifier_bits, RegionNode* region) {
3393   // Branch around if the given klass has the given modifier bit set.
3394   // Like generate_guard, adds a new path onto the region.
3395   Node* modp = basic_plus_adr(kls, in_bytes(Klass::access_flags_offset()));
3396   Node* mods = make_load(NULL, modp, TypeInt::INT, T_INT, MemNode::unordered);
3397   Node* mask = intcon(modifier_mask);
3398   Node* bits = intcon(modifier_bits);
3399   Node* mbit = _gvn.transform(new AndINode(mods, mask));
3400   Node* cmp  = _gvn.transform(new CmpINode(mbit, bits));
3401   Node* bol  = _gvn.transform(new BoolNode(cmp, BoolTest::ne));
3402   return generate_fair_guard(bol, region);
3403 }
3404 Node* LibraryCallKit::generate_interface_guard(Node* kls, RegionNode* region) {
3405   return generate_access_flags_guard(kls, JVM_ACC_INTERFACE, 0, region);
3406 }
3407 
3408 //-------------------------inline_native_Class_query-------------------
3409 bool LibraryCallKit::inline_native_Class_query(vmIntrinsics::ID id) {
3410   const Type* return_type = TypeInt::BOOL;
3411   Node* prim_return_value = top();  // what happens if it's a primitive class?
3412   bool never_see_null = !too_many_traps(Deoptimization::Reason_null_check);
3413   bool expect_prim = false;     // most of these guys expect to work on refs
3414 
3415   enum { _normal_path = 1, _prim_path = 2, PATH_LIMIT };
3416 
3417   Node* mirror = argument(0);
3418   Node* obj    = top();
3419 
3420   switch (id) {
3421   case vmIntrinsics::_isInstance:
3422     // nothing is an instance of a primitive type
3423     prim_return_value = intcon(0);
3424     obj = argument(1);
3425     break;
3426   case vmIntrinsics::_getModifiers:
3427     prim_return_value = intcon(JVM_ACC_ABSTRACT | JVM_ACC_FINAL | JVM_ACC_PUBLIC);
3428     assert(is_power_of_2((int)JVM_ACC_WRITTEN_FLAGS+1), "change next line");
3429     return_type = TypeInt::make(0, JVM_ACC_WRITTEN_FLAGS, Type::WidenMin);
3430     break;
3431   case vmIntrinsics::_isInterface:
3432     prim_return_value = intcon(0);
3433     break;
3434   case vmIntrinsics::_isArray:
3435     prim_return_value = intcon(0);
3436     expect_prim = true;  // cf. ObjectStreamClass.getClassSignature
3437     break;
3438   case vmIntrinsics::_isPrimitive:
3439     prim_return_value = intcon(1);
3440     expect_prim = true;  // obviously
3441     break;
3442   case vmIntrinsics::_getSuperclass:
3443     prim_return_value = null();
3444     return_type = TypeInstPtr::MIRROR->cast_to_ptr_type(TypePtr::BotPTR);
3445     break;
3446   case vmIntrinsics::_getClassAccessFlags:
3447     prim_return_value = intcon(JVM_ACC_ABSTRACT | JVM_ACC_FINAL | JVM_ACC_PUBLIC);
3448     return_type = TypeInt::INT;  // not bool!  6297094
3449     break;
3450   default:
3451     fatal_unexpected_iid(id);
3452     break;
3453   }
3454 
3455   const TypeInstPtr* mirror_con = _gvn.type(mirror)->isa_instptr();
3456   if (mirror_con == NULL)  return false;  // cannot happen?
3457 
3458 #ifndef PRODUCT
3459   if (C->print_intrinsics() || C->print_inlining()) {
3460     ciType* k = mirror_con->java_mirror_type();
3461     if (k) {
3462       tty->print("Inlining %s on constant Class ", vmIntrinsics::name_at(intrinsic_id()));
3463       k->print_name();
3464       tty->cr();
3465     }
3466   }
3467 #endif
3468 
3469   // Null-check the mirror, and the mirror's klass ptr (in case it is a primitive).
3470   RegionNode* region = new RegionNode(PATH_LIMIT);
3471   record_for_igvn(region);
3472   PhiNode* phi = new PhiNode(region, return_type);
3473 
3474   // The mirror will never be null of Reflection.getClassAccessFlags, however
3475   // it may be null for Class.isInstance or Class.getModifiers. Throw a NPE
3476   // if it is. See bug 4774291.
3477 
3478   // For Reflection.getClassAccessFlags(), the null check occurs in
3479   // the wrong place; see inline_unsafe_access(), above, for a similar
3480   // situation.
3481   mirror = null_check(mirror);
3482   // If mirror or obj is dead, only null-path is taken.
3483   if (stopped())  return true;
3484 
3485   if (expect_prim)  never_see_null = false;  // expect nulls (meaning prims)
3486 
3487   // Now load the mirror's klass metaobject, and null-check it.
3488   // Side-effects region with the control path if the klass is null.
3489   Node* kls = load_klass_from_mirror(mirror, never_see_null, region, _prim_path);
3490   // If kls is null, we have a primitive mirror.
3491   phi->init_req(_prim_path, prim_return_value);
3492   if (stopped()) { set_result(region, phi); return true; }
3493   bool safe_for_replace = (region->in(_prim_path) == top());
3494 
3495   Node* p;  // handy temp
3496   Node* null_ctl;
3497 
3498   // Now that we have the non-null klass, we can perform the real query.
3499   // For constant classes, the query will constant-fold in LoadNode::Value.
3500   Node* query_value = top();
3501   switch (id) {
3502   case vmIntrinsics::_isInstance:
3503     // nothing is an instance of a primitive type
3504     query_value = gen_instanceof(obj, kls, safe_for_replace);
3505     break;
3506   
3507   case vmIntrinsics::_onSpinWait:
3508     break;
3509 
3510   case vmIntrinsics::_getModifiers:
3511     p = basic_plus_adr(kls, in_bytes(Klass::modifier_flags_offset()));
3512     query_value = make_load(NULL, p, TypeInt::INT, T_INT, MemNode::unordered);
3513     break;
3514 
3515   case vmIntrinsics::_isInterface:
3516     // (To verify this code sequence, check the asserts in JVM_IsInterface.)
3517     if (generate_interface_guard(kls, region) != NULL)
3518       // A guard was added.  If the guard is taken, it was an interface.
3519       phi->add_req(intcon(1));
3520     // If we fall through, it's a plain class.
3521     query_value = intcon(0);
3522     break;
3523 
3524   case vmIntrinsics::_isArray:
3525     // (To verify this code sequence, check the asserts in JVM_IsArrayClass.)
3526     if (generate_array_guard(kls, region) != NULL)
3527       // A guard was added.  If the guard is taken, it was an array.
3528       phi->add_req(intcon(1));
3529     // If we fall through, it's a plain class.
3530     query_value = intcon(0);
3531     break;
3532 
3533   case vmIntrinsics::_isPrimitive:
3534     query_value = intcon(0); // "normal" path produces false
3535     break;
3536 
3537   case vmIntrinsics::_getSuperclass:
3538     // The rules here are somewhat unfortunate, but we can still do better
3539     // with random logic than with a JNI call.
3540     // Interfaces store null or Object as _super, but must report null.
3541     // Arrays store an intermediate super as _super, but must report Object.
3542     // Other types can report the actual _super.
3543     // (To verify this code sequence, check the asserts in JVM_IsInterface.)
3544     if (generate_interface_guard(kls, region) != NULL)
3545       // A guard was added.  If the guard is taken, it was an interface.
3546       phi->add_req(null());
3547     if (generate_array_guard(kls, region) != NULL)
3548       // A guard was added.  If the guard is taken, it was an array.
3549       phi->add_req(makecon(TypeInstPtr::make(env()->Object_klass()->java_mirror())));
3550     // If we fall through, it's a plain class.  Get its _super.
3551     p = basic_plus_adr(kls, in_bytes(Klass::super_offset()));
3552     kls = _gvn.transform(LoadKlassNode::make(_gvn, NULL, immutable_memory(), p, TypeRawPtr::BOTTOM, TypeKlassPtr::OBJECT_OR_NULL));
3553     null_ctl = top();
3554     kls = null_check_oop(kls, &null_ctl);
3555     if (null_ctl != top()) {
3556       // If the guard is taken, Object.superClass is null (both klass and mirror).
3557       region->add_req(null_ctl);
3558       phi   ->add_req(null());
3559     }
3560     if (!stopped()) {
3561       query_value = load_mirror_from_klass(kls);
3562     }
3563     break;
3564 
3565   case vmIntrinsics::_getClassAccessFlags:
3566     p = basic_plus_adr(kls, in_bytes(Klass::access_flags_offset()));
3567     query_value = make_load(NULL, p, TypeInt::INT, T_INT, MemNode::unordered);
3568     break;
3569 
3570   default:
3571     fatal_unexpected_iid(id);
3572     break;
3573   }
3574 
3575   // Fall-through is the normal case of a query to a real class.
3576   phi->init_req(1, query_value);
3577   region->init_req(1, control());
3578 
3579   C->set_has_split_ifs(true); // Has chance for split-if optimization
3580   set_result(region, phi);
3581   return true;
3582 }
3583 
3584 //-------------------------inline_Class_cast-------------------
3585 bool LibraryCallKit::inline_Class_cast() {
3586   Node* mirror = argument(0); // Class
3587   Node* obj    = argument(1);
3588   const TypeInstPtr* mirror_con = _gvn.type(mirror)->isa_instptr();
3589   if (mirror_con == NULL) {
3590     return false;  // dead path (mirror->is_top()).
3591   }
3592   if (obj == NULL || obj->is_top()) {
3593     return false;  // dead path
3594   }
3595   const TypeOopPtr* tp = _gvn.type(obj)->isa_oopptr();
3596 
3597   // First, see if Class.cast() can be folded statically.
3598   // java_mirror_type() returns non-null for compile-time Class constants.
3599   ciType* tm = mirror_con->java_mirror_type();
3600   if (tm != NULL && tm->is_klass() &&
3601       tp != NULL && tp->klass() != NULL) {
3602     if (!tp->klass()->is_loaded()) {
3603       // Don't use intrinsic when class is not loaded.
3604       return false;
3605     } else {
3606       int static_res = C->static_subtype_check(tm->as_klass(), tp->klass());
3607       if (static_res == Compile::SSC_always_true) {
3608         // isInstance() is true - fold the code.
3609         set_result(obj);
3610         return true;
3611       } else if (static_res == Compile::SSC_always_false) {
3612         // Don't use intrinsic, have to throw ClassCastException.
3613         // If the reference is null, the non-intrinsic bytecode will
3614         // be optimized appropriately.
3615         return false;
3616       }
3617     }
3618   }
3619 
3620   // Bailout intrinsic and do normal inlining if exception path is frequent.
3621   if (too_many_traps(Deoptimization::Reason_intrinsic)) {
3622     return false;
3623   }
3624 
3625   // Generate dynamic checks.
3626   // Class.cast() is java implementation of _checkcast bytecode.
3627   // Do checkcast (Parse::do_checkcast()) optimizations here.
3628 
3629   mirror = null_check(mirror);
3630   // If mirror is dead, only null-path is taken.
3631   if (stopped()) {
3632     return true;
3633   }
3634 
3635   // Not-subtype or the mirror's klass ptr is NULL (in case it is a primitive).
3636   enum { _bad_type_path = 1, _prim_path = 2, PATH_LIMIT };
3637   RegionNode* region = new RegionNode(PATH_LIMIT);
3638   record_for_igvn(region);
3639 
3640   // Now load the mirror's klass metaobject, and null-check it.
3641   // If kls is null, we have a primitive mirror and
3642   // nothing is an instance of a primitive type.
3643   Node* kls = load_klass_from_mirror(mirror, false, region, _prim_path);
3644 
3645   Node* res = top();
3646   if (!stopped()) {
3647     Node* bad_type_ctrl = top();
3648     // Do checkcast optimizations.
3649     res = gen_checkcast(obj, kls, &bad_type_ctrl);
3650     region->init_req(_bad_type_path, bad_type_ctrl);
3651   }
3652   if (region->in(_prim_path) != top() ||
3653       region->in(_bad_type_path) != top()) {
3654     // Let Interpreter throw ClassCastException.
3655     PreserveJVMState pjvms(this);
3656     set_control(_gvn.transform(region));
3657     uncommon_trap(Deoptimization::Reason_intrinsic,
3658                   Deoptimization::Action_maybe_recompile);
3659   }
3660   if (!stopped()) {
3661     set_result(res);
3662   }
3663   return true;
3664 }
3665 
3666 
3667 //--------------------------inline_native_subtype_check------------------------
3668 // This intrinsic takes the JNI calls out of the heart of
3669 // UnsafeFieldAccessorImpl.set, which improves Field.set, readObject, etc.
3670 bool LibraryCallKit::inline_native_subtype_check() {
3671   // Pull both arguments off the stack.
3672   Node* args[2];                // two java.lang.Class mirrors: superc, subc
3673   args[0] = argument(0);
3674   args[1] = argument(1);
3675   Node* klasses[2];             // corresponding Klasses: superk, subk
3676   klasses[0] = klasses[1] = top();
3677 
3678   enum {
3679     // A full decision tree on {superc is prim, subc is prim}:
3680     _prim_0_path = 1,           // {P,N} => false
3681                                 // {P,P} & superc!=subc => false
3682     _prim_same_path,            // {P,P} & superc==subc => true
3683     _prim_1_path,               // {N,P} => false
3684     _ref_subtype_path,          // {N,N} & subtype check wins => true
3685     _both_ref_path,             // {N,N} & subtype check loses => false
3686     PATH_LIMIT
3687   };
3688 
3689   RegionNode* region = new RegionNode(PATH_LIMIT);
3690   Node*       phi    = new PhiNode(region, TypeInt::BOOL);
3691   record_for_igvn(region);
3692 
3693   const TypePtr* adr_type = TypeRawPtr::BOTTOM;   // memory type of loads
3694   const TypeKlassPtr* kls_type = TypeKlassPtr::OBJECT_OR_NULL;
3695   int class_klass_offset = java_lang_Class::klass_offset_in_bytes();
3696 
3697   // First null-check both mirrors and load each mirror's klass metaobject.
3698   int which_arg;
3699   for (which_arg = 0; which_arg <= 1; which_arg++) {
3700     Node* arg = args[which_arg];
3701     arg = null_check(arg);
3702     if (stopped())  break;
3703     args[which_arg] = arg;
3704 
3705     Node* p = basic_plus_adr(arg, class_klass_offset);
3706     Node* kls = LoadKlassNode::make(_gvn, NULL, immutable_memory(), p, adr_type, kls_type);
3707     klasses[which_arg] = _gvn.transform(kls);
3708   }
3709 
3710   // Having loaded both klasses, test each for null.
3711   bool never_see_null = !too_many_traps(Deoptimization::Reason_null_check);
3712   for (which_arg = 0; which_arg <= 1; which_arg++) {
3713     Node* kls = klasses[which_arg];
3714     Node* null_ctl = top();
3715     kls = null_check_oop(kls, &null_ctl, never_see_null);
3716     int prim_path = (which_arg == 0 ? _prim_0_path : _prim_1_path);
3717     region->init_req(prim_path, null_ctl);
3718     if (stopped())  break;
3719     klasses[which_arg] = kls;
3720   }
3721 
3722   if (!stopped()) {
3723     // now we have two reference types, in klasses[0..1]
3724     Node* subk   = klasses[1];  // the argument to isAssignableFrom
3725     Node* superk = klasses[0];  // the receiver
3726     region->set_req(_both_ref_path, gen_subtype_check(subk, superk));
3727     // now we have a successful reference subtype check
3728     region->set_req(_ref_subtype_path, control());
3729   }
3730 
3731   // If both operands are primitive (both klasses null), then
3732   // we must return true when they are identical primitives.
3733   // It is convenient to test this after the first null klass check.
3734   set_control(region->in(_prim_0_path)); // go back to first null check
3735   if (!stopped()) {
3736     // Since superc is primitive, make a guard for the superc==subc case.
3737     Node* cmp_eq = _gvn.transform(new CmpPNode(args[0], args[1]));
3738     Node* bol_eq = _gvn.transform(new BoolNode(cmp_eq, BoolTest::eq));
3739     generate_guard(bol_eq, region, PROB_FAIR);
3740     if (region->req() == PATH_LIMIT+1) {
3741       // A guard was added.  If the added guard is taken, superc==subc.
3742       region->swap_edges(PATH_LIMIT, _prim_same_path);
3743       region->del_req(PATH_LIMIT);
3744     }
3745     region->set_req(_prim_0_path, control()); // Not equal after all.
3746   }
3747 
3748   // these are the only paths that produce 'true':
3749   phi->set_req(_prim_same_path,   intcon(1));
3750   phi->set_req(_ref_subtype_path, intcon(1));
3751 
3752   // pull together the cases:
3753   assert(region->req() == PATH_LIMIT, "sane region");
3754   for (uint i = 1; i < region->req(); i++) {
3755     Node* ctl = region->in(i);
3756     if (ctl == NULL || ctl == top()) {
3757       region->set_req(i, top());
3758       phi   ->set_req(i, top());
3759     } else if (phi->in(i) == NULL) {
3760       phi->set_req(i, intcon(0)); // all other paths produce 'false'
3761     }
3762   }
3763 
3764   set_control(_gvn.transform(region));
3765   set_result(_gvn.transform(phi));
3766   return true;
3767 }
3768 
3769 //---------------------generate_array_guard_common------------------------
3770 Node* LibraryCallKit::generate_array_guard_common(Node* kls, RegionNode* region,
3771                                                   bool obj_array, bool not_array) {
3772 
3773   if (stopped()) {
3774     return NULL;
3775   }
3776 
3777   // If obj_array/non_array==false/false:
3778   // Branch around if the given klass is in fact an array (either obj or prim).
3779   // If obj_array/non_array==false/true:
3780   // Branch around if the given klass is not an array klass of any kind.
3781   // If obj_array/non_array==true/true:
3782   // Branch around if the kls is not an oop array (kls is int[], String, etc.)
3783   // If obj_array/non_array==true/false:
3784   // Branch around if the kls is an oop array (Object[] or subtype)
3785   //
3786   // Like generate_guard, adds a new path onto the region.
3787   jint  layout_con = 0;
3788   Node* layout_val = get_layout_helper(kls, layout_con);
3789   if (layout_val == NULL) {
3790     bool query = (obj_array
3791                   ? Klass::layout_helper_is_objArray(layout_con)
3792                   : Klass::layout_helper_is_array(layout_con));
3793     if (query == not_array) {
3794       return NULL;                       // never a branch
3795     } else {                             // always a branch
3796       Node* always_branch = control();
3797       if (region != NULL)
3798         region->add_req(always_branch);
3799       set_control(top());
3800       return always_branch;
3801     }
3802   }
3803   // Now test the correct condition.
3804   jint  nval = (obj_array
3805                 ? ((jint)Klass::_lh_array_tag_type_value
3806                    <<    Klass::_lh_array_tag_shift)
3807                 : Klass::_lh_neutral_value);
3808   Node* cmp = _gvn.transform(new CmpINode(layout_val, intcon(nval)));
3809   BoolTest::mask btest = BoolTest::lt;  // correct for testing is_[obj]array
3810   // invert the test if we are looking for a non-array
3811   if (not_array)  btest = BoolTest(btest).negate();
3812   Node* bol = _gvn.transform(new BoolNode(cmp, btest));
3813   return generate_fair_guard(bol, region);
3814 }
3815 
3816 
3817 //-----------------------inline_native_newArray--------------------------
3818 // private static native Object java.lang.reflect.newArray(Class<?> componentType, int length);
3819 bool LibraryCallKit::inline_native_newArray() {
3820   Node* mirror    = argument(0);
3821   Node* count_val = argument(1);
3822 
3823   mirror = null_check(mirror);
3824   // If mirror or obj is dead, only null-path is taken.
3825   if (stopped())  return true;
3826 
3827   enum { _normal_path = 1, _slow_path = 2, PATH_LIMIT };
3828   RegionNode* result_reg = new RegionNode(PATH_LIMIT);
3829   PhiNode*    result_val = new PhiNode(result_reg, TypeInstPtr::NOTNULL);
3830   PhiNode*    result_io  = new PhiNode(result_reg, Type::ABIO);
3831   PhiNode*    result_mem = new PhiNode(result_reg, Type::MEMORY, TypePtr::BOTTOM);
3832 
3833   bool never_see_null = !too_many_traps(Deoptimization::Reason_null_check);
3834   Node* klass_node = load_array_klass_from_mirror(mirror, never_see_null,
3835                                                   result_reg, _slow_path);
3836   Node* normal_ctl   = control();
3837   Node* no_array_ctl = result_reg->in(_slow_path);
3838 
3839   // Generate code for the slow case.  We make a call to newArray().
3840   set_control(no_array_ctl);
3841   if (!stopped()) {
3842     // Either the input type is void.class, or else the
3843     // array klass has not yet been cached.  Either the
3844     // ensuing call will throw an exception, or else it
3845     // will cache the array klass for next time.
3846     PreserveJVMState pjvms(this);
3847     CallJavaNode* slow_call = generate_method_call_static(vmIntrinsics::_newArray);
3848     Node* slow_result = set_results_for_java_call(slow_call);
3849     // this->control() comes from set_results_for_java_call
3850     result_reg->set_req(_slow_path, control());
3851     result_val->set_req(_slow_path, slow_result);
3852     result_io ->set_req(_slow_path, i_o());
3853     result_mem->set_req(_slow_path, reset_memory());
3854   }
3855 
3856   set_control(normal_ctl);
3857   if (!stopped()) {
3858     // Normal case:  The array type has been cached in the java.lang.Class.
3859     // The following call works fine even if the array type is polymorphic.
3860     // It could be a dynamic mix of int[], boolean[], Object[], etc.
3861     Node* obj = new_array(klass_node, count_val, 0);  // no arguments to push
3862     result_reg->init_req(_normal_path, control());
3863     result_val->init_req(_normal_path, obj);
3864     result_io ->init_req(_normal_path, i_o());
3865     result_mem->init_req(_normal_path, reset_memory());
3866   }
3867 
3868   // Return the combined state.
3869   set_i_o(        _gvn.transform(result_io)  );
3870   set_all_memory( _gvn.transform(result_mem));
3871 
3872   C->set_has_split_ifs(true); // Has chance for split-if optimization
3873   set_result(result_reg, result_val);
3874   return true;
3875 }
3876 
3877 //----------------------inline_native_getLength--------------------------
3878 // public static native int java.lang.reflect.Array.getLength(Object array);
3879 bool LibraryCallKit::inline_native_getLength() {
3880   if (too_many_traps(Deoptimization::Reason_intrinsic))  return false;
3881 
3882   Node* array = null_check(argument(0));
3883   // If array is dead, only null-path is taken.
3884   if (stopped())  return true;
3885 
3886   // Deoptimize if it is a non-array.
3887   Node* non_array = generate_non_array_guard(load_object_klass(array), NULL);
3888 
3889   if (non_array != NULL) {
3890     PreserveJVMState pjvms(this);
3891     set_control(non_array);
3892     uncommon_trap(Deoptimization::Reason_intrinsic,
3893                   Deoptimization::Action_maybe_recompile);
3894   }
3895 
3896   // If control is dead, only non-array-path is taken.
3897   if (stopped())  return true;
3898 
3899   // The works fine even if the array type is polymorphic.
3900   // It could be a dynamic mix of int[], boolean[], Object[], etc.
3901   Node* result = load_array_length(array);
3902 
3903   C->set_has_split_ifs(true);  // Has chance for split-if optimization
3904   set_result(result);
3905   return true;
3906 }
3907 
3908 //------------------------inline_array_copyOf----------------------------
3909 // public static <T,U> T[] java.util.Arrays.copyOf(     U[] original, int newLength,         Class<? extends T[]> newType);
3910 // public static <T,U> T[] java.util.Arrays.copyOfRange(U[] original, int from,      int to, Class<? extends T[]> newType);
3911 bool LibraryCallKit::inline_array_copyOf(bool is_copyOfRange) {
3912   if (too_many_traps(Deoptimization::Reason_intrinsic))  return false;
3913 
3914   // Get the arguments.
3915   Node* original          = argument(0);
3916   Node* start             = is_copyOfRange? argument(1): intcon(0);
3917   Node* end               = is_copyOfRange? argument(2): argument(1);
3918   Node* array_type_mirror = is_copyOfRange? argument(3): argument(2);
3919 
3920   Node* newcopy = NULL;
3921 
3922   // Set the original stack and the reexecute bit for the interpreter to reexecute
3923   // the bytecode that invokes Arrays.copyOf if deoptimization happens.
3924   { PreserveReexecuteState preexecs(this);
3925     jvms()->set_should_reexecute(true);
3926 
3927     array_type_mirror = null_check(array_type_mirror);
3928     original          = null_check(original);
3929 
3930     // Check if a null path was taken unconditionally.
3931     if (stopped())  return true;
3932 
3933     Node* orig_length = load_array_length(original);
3934 
3935     Node* klass_node = load_klass_from_mirror(array_type_mirror, false, NULL, 0);
3936     klass_node = null_check(klass_node);
3937 
3938     RegionNode* bailout = new RegionNode(1);
3939     record_for_igvn(bailout);
3940 
3941     // Despite the generic type of Arrays.copyOf, the mirror might be int, int[], etc.
3942     // Bail out if that is so.
3943     Node* not_objArray = generate_non_objArray_guard(klass_node, bailout);
3944     if (not_objArray != NULL) {
3945       // Improve the klass node's type from the new optimistic assumption:
3946       ciKlass* ak = ciArrayKlass::make(env()->Object_klass());
3947       const Type* akls = TypeKlassPtr::make(TypePtr::NotNull, ak, 0/*offset*/);
3948       Node* cast = new CastPPNode(klass_node, akls);
3949       cast->init_req(0, control());
3950       klass_node = _gvn.transform(cast);
3951     }
3952 
3953     // Bail out if either start or end is negative.
3954     generate_negative_guard(start, bailout, &start);
3955     generate_negative_guard(end,   bailout, &end);
3956 
3957     Node* length = end;
3958     if (_gvn.type(start) != TypeInt::ZERO) {
3959       length = _gvn.transform(new SubINode(end, start));
3960     }
3961 
3962     // Bail out if length is negative.
3963     // Without this the new_array would throw
3964     // NegativeArraySizeException but IllegalArgumentException is what
3965     // should be thrown
3966     generate_negative_guard(length, bailout, &length);
3967 
3968     if (bailout->req() > 1) {
3969       PreserveJVMState pjvms(this);
3970       set_control(_gvn.transform(bailout));
3971       uncommon_trap(Deoptimization::Reason_intrinsic,
3972                     Deoptimization::Action_maybe_recompile);
3973     }
3974 
3975     if (!stopped()) {
3976       // How many elements will we copy from the original?
3977       // The answer is MinI(orig_length - start, length).
3978       Node* orig_tail = _gvn.transform(new SubINode(orig_length, start));
3979       Node* moved = generate_min_max(vmIntrinsics::_min, orig_tail, length);
3980 
3981       // Generate a direct call to the right arraycopy function(s).
3982       // We know the copy is disjoint but we might not know if the
3983       // oop stores need checking.
3984       // Extreme case:  Arrays.copyOf((Integer[])x, 10, String[].class).
3985       // This will fail a store-check if x contains any non-nulls.
3986 
3987       // ArrayCopyNode:Ideal may transform the ArrayCopyNode to
3988       // loads/stores but it is legal only if we're sure the
3989       // Arrays.copyOf would succeed. So we need all input arguments
3990       // to the copyOf to be validated, including that the copy to the
3991       // new array won't trigger an ArrayStoreException. That subtype
3992       // check can be optimized if we know something on the type of
3993       // the input array from type speculation.
3994       if (_gvn.type(klass_node)->singleton()) {
3995         ciKlass* subk   = _gvn.type(load_object_klass(original))->is_klassptr()->klass();
3996         ciKlass* superk = _gvn.type(klass_node)->is_klassptr()->klass();
3997 
3998         int test = C->static_subtype_check(superk, subk);
3999         if (test != Compile::SSC_always_true && test != Compile::SSC_always_false) {
4000           const TypeOopPtr* t_original = _gvn.type(original)->is_oopptr();
4001           if (t_original->speculative_type() != NULL) {
4002             original = maybe_cast_profiled_obj(original, t_original->speculative_type(), true);
4003           }
4004         }
4005       }
4006 
4007       bool validated = false;
4008       // Reason_class_check rather than Reason_intrinsic because we
4009       // want to intrinsify even if this traps.
4010       if (!too_many_traps(Deoptimization::Reason_class_check)) {
4011         Node* not_subtype_ctrl = gen_subtype_check(load_object_klass(original),
4012                                                    klass_node);
4013 
4014         if (not_subtype_ctrl != top()) {
4015           PreserveJVMState pjvms(this);
4016           set_control(not_subtype_ctrl);
4017           uncommon_trap(Deoptimization::Reason_class_check,
4018                         Deoptimization::Action_make_not_entrant);
4019           assert(stopped(), "Should be stopped");
4020         }
4021         validated = true;
4022       }
4023 
4024       if (!stopped()) {
4025         newcopy = new_array(klass_node, length, 0);  // no arguments to push
4026 
4027         ArrayCopyNode* ac = ArrayCopyNode::make(this, true, original, start, newcopy, intcon(0), moved, true,
4028                                                 load_object_klass(original), klass_node);
4029         if (!is_copyOfRange) {
4030           ac->set_copyof(validated);
4031         } else {
4032           ac->set_copyofrange(validated);
4033         }
4034         Node* n = _gvn.transform(ac);
4035         if (n == ac) {
4036           ac->connect_outputs(this);
4037         } else {
4038           assert(validated, "shouldn't transform if all arguments not validated");
4039           set_all_memory(n);
4040         }
4041       }
4042     }
4043   } // original reexecute is set back here
4044 
4045   C->set_has_split_ifs(true); // Has chance for split-if optimization
4046   if (!stopped()) {
4047     set_result(newcopy);
4048   }
4049   return true;
4050 }
4051 
4052 
4053 //----------------------generate_virtual_guard---------------------------
4054 // Helper for hashCode and clone.  Peeks inside the vtable to avoid a call.
4055 Node* LibraryCallKit::generate_virtual_guard(Node* obj_klass,
4056                                              RegionNode* slow_region) {
4057   ciMethod* method = callee();
4058   int vtable_index = method->vtable_index();
4059   assert(vtable_index >= 0 || vtable_index == Method::nonvirtual_vtable_index,
4060          "bad index %d", vtable_index);
4061   // Get the Method* out of the appropriate vtable entry.
4062   int entry_offset  = (InstanceKlass::vtable_start_offset() +
4063                      vtable_index*vtableEntry::size()) * wordSize +
4064                      vtableEntry::method_offset_in_bytes();
4065   Node* entry_addr  = basic_plus_adr(obj_klass, entry_offset);
4066   Node* target_call = make_load(NULL, entry_addr, TypePtr::NOTNULL, T_ADDRESS, MemNode::unordered);
4067 
4068   // Compare the target method with the expected method (e.g., Object.hashCode).
4069   const TypePtr* native_call_addr = TypeMetadataPtr::make(method);
4070 
4071   Node* native_call = makecon(native_call_addr);
4072   Node* chk_native  = _gvn.transform(new CmpPNode(target_call, native_call));
4073   Node* test_native = _gvn.transform(new BoolNode(chk_native, BoolTest::ne));
4074 
4075   return generate_slow_guard(test_native, slow_region);
4076 }
4077 
4078 //-----------------------generate_method_call----------------------------
4079 // Use generate_method_call to make a slow-call to the real
4080 // method if the fast path fails.  An alternative would be to
4081 // use a stub like OptoRuntime::slow_arraycopy_Java.
4082 // This only works for expanding the current library call,
4083 // not another intrinsic.  (E.g., don't use this for making an
4084 // arraycopy call inside of the copyOf intrinsic.)
4085 CallJavaNode*
4086 LibraryCallKit::generate_method_call(vmIntrinsics::ID method_id, bool is_virtual, bool is_static) {
4087   // When compiling the intrinsic method itself, do not use this technique.
4088   guarantee(callee() != C->method(), "cannot make slow-call to self");
4089 
4090   ciMethod* method = callee();
4091   // ensure the JVMS we have will be correct for this call
4092   guarantee(method_id == method->intrinsic_id(), "must match");
4093 
4094   const TypeFunc* tf = TypeFunc::make(method);
4095   CallJavaNode* slow_call;
4096   if (is_static) {
4097     assert(!is_virtual, "");
4098     slow_call = new CallStaticJavaNode(C, tf,
4099                            SharedRuntime::get_resolve_static_call_stub(),
4100                            method, bci());
4101   } else if (is_virtual) {
4102     null_check_receiver();
4103     int vtable_index = Method::invalid_vtable_index;
4104     if (UseInlineCaches) {
4105       // Suppress the vtable call
4106     } else {
4107       // hashCode and clone are not a miranda methods,
4108       // so the vtable index is fixed.
4109       // No need to use the linkResolver to get it.
4110        vtable_index = method->vtable_index();
4111        assert(vtable_index >= 0 || vtable_index == Method::nonvirtual_vtable_index,
4112               "bad index %d", vtable_index);
4113     }
4114     slow_call = new CallDynamicJavaNode(tf,
4115                           SharedRuntime::get_resolve_virtual_call_stub(),
4116                           method, vtable_index, bci());
4117   } else {  // neither virtual nor static:  opt_virtual
4118     null_check_receiver();
4119     slow_call = new CallStaticJavaNode(C, tf,
4120                                 SharedRuntime::get_resolve_opt_virtual_call_stub(),
4121                                 method, bci());
4122     slow_call->set_optimized_virtual(true);
4123   }
4124   set_arguments_for_java_call(slow_call);
4125   set_edges_for_java_call(slow_call);
4126   return slow_call;
4127 }
4128 
4129 
4130 /**
4131  * Build special case code for calls to hashCode on an object. This call may
4132  * be virtual (invokevirtual) or bound (invokespecial). For each case we generate
4133  * slightly different code.
4134  */
4135 bool LibraryCallKit::inline_native_hashcode(bool is_virtual, bool is_static) {
4136   assert(is_static == callee()->is_static(), "correct intrinsic selection");
4137   assert(!(is_virtual && is_static), "either virtual, special, or static");
4138 
4139   enum { _slow_path = 1, _fast_path, _null_path, PATH_LIMIT };
4140 
4141   RegionNode* result_reg = new RegionNode(PATH_LIMIT);
4142   PhiNode*    result_val = new PhiNode(result_reg, TypeInt::INT);
4143   PhiNode*    result_io  = new PhiNode(result_reg, Type::ABIO);
4144   PhiNode*    result_mem = new PhiNode(result_reg, Type::MEMORY, TypePtr::BOTTOM);
4145   Node* obj = NULL;
4146   if (!is_static) {
4147     // Check for hashing null object
4148     obj = null_check_receiver();
4149     if (stopped())  return true;        // unconditionally null
4150     result_reg->init_req(_null_path, top());
4151     result_val->init_req(_null_path, top());
4152   } else {
4153     // Do a null check, and return zero if null.
4154     // System.identityHashCode(null) == 0
4155     obj = argument(0);
4156     Node* null_ctl = top();
4157     obj = null_check_oop(obj, &null_ctl);
4158     result_reg->init_req(_null_path, null_ctl);
4159     result_val->init_req(_null_path, _gvn.intcon(0));
4160   }
4161 
4162   // Unconditionally null?  Then return right away.
4163   if (stopped()) {
4164     set_control( result_reg->in(_null_path));
4165     if (!stopped())
4166       set_result(result_val->in(_null_path));
4167     return true;
4168   }
4169 
4170   // We only go to the fast case code if we pass a number of guards.  The
4171   // paths which do not pass are accumulated in the slow_region.
4172   RegionNode* slow_region = new RegionNode(1);
4173   record_for_igvn(slow_region);
4174 
4175   // If this is a virtual call, we generate a funny guard.  We pull out
4176   // the vtable entry corresponding to hashCode() from the target object.
4177   // If the target method which we are calling happens to be the native
4178   // Object hashCode() method, we pass the guard.  We do not need this
4179   // guard for non-virtual calls -- the caller is known to be the native
4180   // Object hashCode().
4181   if (is_virtual) {
4182     // After null check, get the object's klass.
4183     Node* obj_klass = load_object_klass(obj);
4184     generate_virtual_guard(obj_klass, slow_region);
4185   }
4186 
4187   // Get the header out of the object, use LoadMarkNode when available
4188   Node* header_addr = basic_plus_adr(obj, oopDesc::mark_offset_in_bytes());
4189   // The control of the load must be NULL. Otherwise, the load can move before
4190   // the null check after castPP removal.
4191   Node* no_ctrl = NULL;
4192   Node* header = make_load(no_ctrl, header_addr, TypeX_X, TypeX_X->basic_type(), MemNode::unordered);
4193 
4194   // Test the header to see if it is unlocked.
4195   Node *lock_mask      = _gvn.MakeConX(markOopDesc::biased_lock_mask_in_place);
4196   Node *lmasked_header = _gvn.transform(new AndXNode(header, lock_mask));
4197   Node *unlocked_val   = _gvn.MakeConX(markOopDesc::unlocked_value);
4198   Node *chk_unlocked   = _gvn.transform(new CmpXNode( lmasked_header, unlocked_val));
4199   Node *test_unlocked  = _gvn.transform(new BoolNode( chk_unlocked, BoolTest::ne));
4200 
4201   generate_slow_guard(test_unlocked, slow_region);
4202 
4203   // Get the hash value and check to see that it has been properly assigned.
4204   // We depend on hash_mask being at most 32 bits and avoid the use of
4205   // hash_mask_in_place because it could be larger than 32 bits in a 64-bit
4206   // vm: see markOop.hpp.
4207   Node *hash_mask      = _gvn.intcon(markOopDesc::hash_mask);
4208   Node *hash_shift     = _gvn.intcon(markOopDesc::hash_shift);
4209   Node *hshifted_header= _gvn.transform(new URShiftXNode(header, hash_shift));
4210   // This hack lets the hash bits live anywhere in the mark object now, as long
4211   // as the shift drops the relevant bits into the low 32 bits.  Note that
4212   // Java spec says that HashCode is an int so there's no point in capturing
4213   // an 'X'-sized hashcode (32 in 32-bit build or 64 in 64-bit build).
4214   hshifted_header      = ConvX2I(hshifted_header);
4215   Node *hash_val       = _gvn.transform(new AndINode(hshifted_header, hash_mask));
4216 
4217   Node *no_hash_val    = _gvn.intcon(markOopDesc::no_hash);
4218   Node *chk_assigned   = _gvn.transform(new CmpINode( hash_val, no_hash_val));
4219   Node *test_assigned  = _gvn.transform(new BoolNode( chk_assigned, BoolTest::eq));
4220 
4221   generate_slow_guard(test_assigned, slow_region);
4222 
4223   Node* init_mem = reset_memory();
4224   // fill in the rest of the null path:
4225   result_io ->init_req(_null_path, i_o());
4226   result_mem->init_req(_null_path, init_mem);
4227 
4228   result_val->init_req(_fast_path, hash_val);
4229   result_reg->init_req(_fast_path, control());
4230   result_io ->init_req(_fast_path, i_o());
4231   result_mem->init_req(_fast_path, init_mem);
4232 
4233   // Generate code for the slow case.  We make a call to hashCode().
4234   set_control(_gvn.transform(slow_region));
4235   if (!stopped()) {
4236     // No need for PreserveJVMState, because we're using up the present state.
4237     set_all_memory(init_mem);
4238     vmIntrinsics::ID hashCode_id = is_static ? vmIntrinsics::_identityHashCode : vmIntrinsics::_hashCode;
4239     CallJavaNode* slow_call = generate_method_call(hashCode_id, is_virtual, is_static);
4240     Node* slow_result = set_results_for_java_call(slow_call);
4241     // this->control() comes from set_results_for_java_call
4242     result_reg->init_req(_slow_path, control());
4243     result_val->init_req(_slow_path, slow_result);
4244     result_io  ->set_req(_slow_path, i_o());
4245     result_mem ->set_req(_slow_path, reset_memory());
4246   }
4247 
4248   // Return the combined state.
4249   set_i_o(        _gvn.transform(result_io)  );
4250   set_all_memory( _gvn.transform(result_mem));
4251 
4252   set_result(result_reg, result_val);
4253   return true;
4254 }
4255 
4256 //---------------------------inline_native_getClass----------------------------
4257 // public final native Class<?> java.lang.Object.getClass();
4258 //
4259 // Build special case code for calls to getClass on an object.
4260 bool LibraryCallKit::inline_native_getClass() {
4261   Node* obj = null_check_receiver();
4262   if (stopped())  return true;
4263   set_result(load_mirror_from_klass(load_object_klass(obj)));
4264   return true;
4265 }
4266 
4267 //-----------------inline_native_Reflection_getCallerClass---------------------
4268 // public static native Class<?> sun.reflect.Reflection.getCallerClass();
4269 //
4270 // In the presence of deep enough inlining, getCallerClass() becomes a no-op.
4271 //
4272 // NOTE: This code must perform the same logic as JVM_GetCallerClass
4273 // in that it must skip particular security frames and checks for
4274 // caller sensitive methods.
4275 bool LibraryCallKit::inline_native_Reflection_getCallerClass() {
4276 #ifndef PRODUCT
4277   if ((C->print_intrinsics() || C->print_inlining()) && Verbose) {
4278     tty->print_cr("Attempting to inline sun.reflect.Reflection.getCallerClass");
4279   }
4280 #endif
4281 
4282   if (!jvms()->has_method()) {
4283 #ifndef PRODUCT
4284     if ((C->print_intrinsics() || C->print_inlining()) && Verbose) {
4285       tty->print_cr("  Bailing out because intrinsic was inlined at top level");
4286     }
4287 #endif
4288     return false;
4289   }
4290 
4291   // Walk back up the JVM state to find the caller at the required
4292   // depth.
4293   JVMState* caller_jvms = jvms();
4294 
4295   // Cf. JVM_GetCallerClass
4296   // NOTE: Start the loop at depth 1 because the current JVM state does
4297   // not include the Reflection.getCallerClass() frame.
4298   for (int n = 1; caller_jvms != NULL; caller_jvms = caller_jvms->caller(), n++) {
4299     ciMethod* m = caller_jvms->method();
4300     switch (n) {
4301     case 0:
4302       fatal("current JVM state does not include the Reflection.getCallerClass frame");
4303       break;
4304     case 1:
4305       // Frame 0 and 1 must be caller sensitive (see JVM_GetCallerClass).
4306       if (!m->caller_sensitive()) {
4307 #ifndef PRODUCT
4308         if ((C->print_intrinsics() || C->print_inlining()) && Verbose) {
4309           tty->print_cr("  Bailing out: CallerSensitive annotation expected at frame %d", n);
4310         }
4311 #endif
4312         return false;  // bail-out; let JVM_GetCallerClass do the work
4313       }
4314       break;
4315     default:
4316       if (!m->is_ignored_by_security_stack_walk()) {
4317         // We have reached the desired frame; return the holder class.
4318         // Acquire method holder as java.lang.Class and push as constant.
4319         ciInstanceKlass* caller_klass = caller_jvms->method()->holder();
4320         ciInstance* caller_mirror = caller_klass->java_mirror();
4321         set_result(makecon(TypeInstPtr::make(caller_mirror)));
4322 
4323 #ifndef PRODUCT
4324         if ((C->print_intrinsics() || C->print_inlining()) && Verbose) {
4325           tty->print_cr("  Succeeded: caller = %d) %s.%s, JVMS depth = %d", n, caller_klass->name()->as_utf8(), caller_jvms->method()->name()->as_utf8(), jvms()->depth());
4326           tty->print_cr("  JVM state at this point:");
4327           for (int i = jvms()->depth(), n = 1; i >= 1; i--, n++) {
4328             ciMethod* m = jvms()->of_depth(i)->method();
4329             tty->print_cr("   %d) %s.%s", n, m->holder()->name()->as_utf8(), m->name()->as_utf8());
4330           }
4331         }
4332 #endif
4333         return true;
4334       }
4335       break;
4336     }
4337   }
4338 
4339 #ifndef PRODUCT
4340   if ((C->print_intrinsics() || C->print_inlining()) && Verbose) {
4341     tty->print_cr("  Bailing out because caller depth exceeded inlining depth = %d", jvms()->depth());
4342     tty->print_cr("  JVM state at this point:");
4343     for (int i = jvms()->depth(), n = 1; i >= 1; i--, n++) {
4344       ciMethod* m = jvms()->of_depth(i)->method();
4345       tty->print_cr("   %d) %s.%s", n, m->holder()->name()->as_utf8(), m->name()->as_utf8());
4346     }
4347   }
4348 #endif
4349 
4350   return false;  // bail-out; let JVM_GetCallerClass do the work
4351 }
4352 
4353 bool LibraryCallKit::inline_fp_conversions(vmIntrinsics::ID id) {
4354   Node* arg = argument(0);
4355   Node* result = NULL;
4356 
4357   switch (id) {
4358   case vmIntrinsics::_floatToRawIntBits:    result = new MoveF2INode(arg);  break;
4359   case vmIntrinsics::_intBitsToFloat:       result = new MoveI2FNode(arg);  break;
4360   case vmIntrinsics::_doubleToRawLongBits:  result = new MoveD2LNode(arg);  break;
4361   case vmIntrinsics::_longBitsToDouble:     result = new MoveL2DNode(arg);  break;
4362 
4363   case vmIntrinsics::_doubleToLongBits: {
4364     // two paths (plus control) merge in a wood
4365     RegionNode *r = new RegionNode(3);
4366     Node *phi = new PhiNode(r, TypeLong::LONG);
4367 
4368     Node *cmpisnan = _gvn.transform(new CmpDNode(arg, arg));
4369     // Build the boolean node
4370     Node *bolisnan = _gvn.transform(new BoolNode(cmpisnan, BoolTest::ne));
4371 
4372     // Branch either way.
4373     // NaN case is less traveled, which makes all the difference.
4374     IfNode *ifisnan = create_and_xform_if(control(), bolisnan, PROB_STATIC_FREQUENT, COUNT_UNKNOWN);
4375     Node *opt_isnan = _gvn.transform(ifisnan);
4376     assert( opt_isnan->is_If(), "Expect an IfNode");
4377     IfNode *opt_ifisnan = (IfNode*)opt_isnan;
4378     Node *iftrue = _gvn.transform(new IfTrueNode(opt_ifisnan));
4379 
4380     set_control(iftrue);
4381 
4382     static const jlong nan_bits = CONST64(0x7ff8000000000000);
4383     Node *slow_result = longcon(nan_bits); // return NaN
4384     phi->init_req(1, _gvn.transform( slow_result ));
4385     r->init_req(1, iftrue);
4386 
4387     // Else fall through
4388     Node *iffalse = _gvn.transform(new IfFalseNode(opt_ifisnan));
4389     set_control(iffalse);
4390 
4391     phi->init_req(2, _gvn.transform(new MoveD2LNode(arg)));
4392     r->init_req(2, iffalse);
4393 
4394     // Post merge
4395     set_control(_gvn.transform(r));
4396     record_for_igvn(r);
4397 
4398     C->set_has_split_ifs(true); // Has chance for split-if optimization
4399     result = phi;
4400     assert(result->bottom_type()->isa_long(), "must be");
4401     break;
4402   }
4403 
4404   case vmIntrinsics::_floatToIntBits: {
4405     // two paths (plus control) merge in a wood
4406     RegionNode *r = new RegionNode(3);
4407     Node *phi = new PhiNode(r, TypeInt::INT);
4408 
4409     Node *cmpisnan = _gvn.transform(new CmpFNode(arg, arg));
4410     // Build the boolean node
4411     Node *bolisnan = _gvn.transform(new BoolNode(cmpisnan, BoolTest::ne));
4412 
4413     // Branch either way.
4414     // NaN case is less traveled, which makes all the difference.
4415     IfNode *ifisnan = create_and_xform_if(control(), bolisnan, PROB_STATIC_FREQUENT, COUNT_UNKNOWN);
4416     Node *opt_isnan = _gvn.transform(ifisnan);
4417     assert( opt_isnan->is_If(), "Expect an IfNode");
4418     IfNode *opt_ifisnan = (IfNode*)opt_isnan;
4419     Node *iftrue = _gvn.transform(new IfTrueNode(opt_ifisnan));
4420 
4421     set_control(iftrue);
4422 
4423     static const jint nan_bits = 0x7fc00000;
4424     Node *slow_result = makecon(TypeInt::make(nan_bits)); // return NaN
4425     phi->init_req(1, _gvn.transform( slow_result ));
4426     r->init_req(1, iftrue);
4427 
4428     // Else fall through
4429     Node *iffalse = _gvn.transform(new IfFalseNode(opt_ifisnan));
4430     set_control(iffalse);
4431 
4432     phi->init_req(2, _gvn.transform(new MoveF2INode(arg)));
4433     r->init_req(2, iffalse);
4434 
4435     // Post merge
4436     set_control(_gvn.transform(r));
4437     record_for_igvn(r);
4438 
4439     C->set_has_split_ifs(true); // Has chance for split-if optimization
4440     result = phi;
4441     assert(result->bottom_type()->isa_int(), "must be");
4442     break;
4443   }
4444 
4445   default:
4446     fatal_unexpected_iid(id);
4447     break;
4448   }
4449   set_result(_gvn.transform(result));
4450   return true;
4451 }
4452 
4453 //----------------------inline_unsafe_copyMemory-------------------------
4454 // public native void Unsafe.copyMemory(Object srcBase, long srcOffset, Object destBase, long destOffset, long bytes);
4455 bool LibraryCallKit::inline_unsafe_copyMemory() {
4456   if (callee()->is_static())  return false;  // caller must have the capability!
4457   null_check_receiver();  // null-check receiver
4458   if (stopped())  return true;
4459 
4460   C->set_has_unsafe_access(true);  // Mark eventual nmethod as "unsafe".
4461 
4462   Node* src_ptr =         argument(1);   // type: oop
4463   Node* src_off = ConvL2X(argument(2));  // type: long
4464   Node* dst_ptr =         argument(4);   // type: oop
4465   Node* dst_off = ConvL2X(argument(5));  // type: long
4466   Node* size    = ConvL2X(argument(7));  // type: long
4467 
4468   assert(Unsafe_field_offset_to_byte_offset(11) == 11,
4469          "fieldOffset must be byte-scaled");
4470 
4471   Node* src = make_unsafe_address(src_ptr, src_off);
4472   Node* dst = make_unsafe_address(dst_ptr, dst_off);
4473 
4474   // Conservatively insert a memory barrier on all memory slices.
4475   // Do not let writes of the copy source or destination float below the copy.
4476   insert_mem_bar(Op_MemBarCPUOrder);
4477 
4478   // Call it.  Note that the length argument is not scaled.
4479   make_runtime_call(RC_LEAF|RC_NO_FP,
4480                     OptoRuntime::fast_arraycopy_Type(),
4481                     StubRoutines::unsafe_arraycopy(),
4482                     "unsafe_arraycopy",
4483                     TypeRawPtr::BOTTOM,
4484                     src, dst, size XTOP);
4485 
4486   // Do not let reads of the copy destination float above the copy.
4487   insert_mem_bar(Op_MemBarCPUOrder);
4488 
4489   return true;
4490 }
4491 
4492 //------------------------clone_coping-----------------------------------
4493 // Helper function for inline_native_clone.
4494 void LibraryCallKit::copy_to_clone(Node* obj, Node* alloc_obj, Node* obj_size, bool is_array, bool card_mark) {
4495   assert(obj_size != NULL, "");
4496   Node* raw_obj = alloc_obj->in(1);
4497   assert(alloc_obj->is_CheckCastPP() && raw_obj->is_Proj() && raw_obj->in(0)->is_Allocate(), "");
4498 
4499   AllocateNode* alloc = NULL;
4500   if (ReduceBulkZeroing) {
4501     // We will be completely responsible for initializing this object -
4502     // mark Initialize node as complete.
4503     alloc = AllocateNode::Ideal_allocation(alloc_obj, &_gvn);
4504     // The object was just allocated - there should be no any stores!
4505     guarantee(alloc != NULL && alloc->maybe_set_complete(&_gvn), "");
4506     // Mark as complete_with_arraycopy so that on AllocateNode
4507     // expansion, we know this AllocateNode is initialized by an array
4508     // copy and a StoreStore barrier exists after the array copy.
4509     alloc->initialization()->set_complete_with_arraycopy();
4510   }
4511 
4512   // Copy the fastest available way.
4513   // TODO: generate fields copies for small objects instead.
4514   Node* src  = obj;
4515   Node* dest = alloc_obj;
4516   Node* size = _gvn.transform(obj_size);
4517 
4518   // Exclude the header but include array length to copy by 8 bytes words.
4519   // Can't use base_offset_in_bytes(bt) since basic type is unknown.
4520   int base_off = is_array ? arrayOopDesc::length_offset_in_bytes() :
4521                             instanceOopDesc::base_offset_in_bytes();
4522   // base_off:
4523   // 8  - 32-bit VM
4524   // 12 - 64-bit VM, compressed klass
4525   // 16 - 64-bit VM, normal klass
4526   if (base_off % BytesPerLong != 0) {
4527     assert(UseCompressedClassPointers, "");
4528     if (is_array) {
4529       // Exclude length to copy by 8 bytes words.
4530       base_off += sizeof(int);
4531     } else {
4532       // Include klass to copy by 8 bytes words.
4533       base_off = instanceOopDesc::klass_offset_in_bytes();
4534     }
4535     assert(base_off % BytesPerLong == 0, "expect 8 bytes alignment");
4536   }
4537   src  = basic_plus_adr(src,  base_off);
4538   dest = basic_plus_adr(dest, base_off);
4539 
4540   // Compute the length also, if needed:
4541   Node* countx = size;
4542   countx = _gvn.transform(new SubXNode(countx, MakeConX(base_off)));
4543   countx = _gvn.transform(new URShiftXNode(countx, intcon(LogBytesPerLong) ));
4544 
4545   const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
4546 
4547   ArrayCopyNode* ac = ArrayCopyNode::make(this, false, src, NULL, dest, NULL, countx, false);
4548   ac->set_clonebasic();
4549   Node* n = _gvn.transform(ac);
4550   if (n == ac) {
4551     set_predefined_output_for_runtime_call(ac, ac->in(TypeFunc::Memory), raw_adr_type);
4552   } else {
4553     set_all_memory(n);
4554   }
4555 
4556   // If necessary, emit some card marks afterwards.  (Non-arrays only.)
4557   if (card_mark) {
4558     assert(!is_array, "");
4559     // Put in store barrier for any and all oops we are sticking
4560     // into this object.  (We could avoid this if we could prove
4561     // that the object type contains no oop fields at all.)
4562     Node* no_particular_value = NULL;
4563     Node* no_particular_field = NULL;
4564     int raw_adr_idx = Compile::AliasIdxRaw;
4565     post_barrier(control(),
4566                  memory(raw_adr_type),
4567                  alloc_obj,
4568                  no_particular_field,
4569                  raw_adr_idx,
4570                  no_particular_value,
4571                  T_OBJECT,
4572                  false);
4573   }
4574 
4575   // Do not let reads from the cloned object float above the arraycopy.
4576   if (alloc != NULL) {
4577     // Do not let stores that initialize this object be reordered with
4578     // a subsequent store that would make this object accessible by
4579     // other threads.
4580     // Record what AllocateNode this StoreStore protects so that
4581     // escape analysis can go from the MemBarStoreStoreNode to the
4582     // AllocateNode and eliminate the MemBarStoreStoreNode if possible
4583     // based on the escape status of the AllocateNode.
4584     insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out(AllocateNode::RawAddress));
4585   } else {
4586     insert_mem_bar(Op_MemBarCPUOrder);
4587   }
4588 }
4589 
4590 //------------------------inline_native_clone----------------------------
4591 // protected native Object java.lang.Object.clone();
4592 //
4593 // Here are the simple edge cases:
4594 //  null receiver => normal trap
4595 //  virtual and clone was overridden => slow path to out-of-line clone
4596 //  not cloneable or finalizer => slow path to out-of-line Object.clone
4597 //
4598 // The general case has two steps, allocation and copying.
4599 // Allocation has two cases, and uses GraphKit::new_instance or new_array.
4600 //
4601 // Copying also has two cases, oop arrays and everything else.
4602 // Oop arrays use arrayof_oop_arraycopy (same as System.arraycopy).
4603 // Everything else uses the tight inline loop supplied by CopyArrayNode.
4604 //
4605 // These steps fold up nicely if and when the cloned object's klass
4606 // can be sharply typed as an object array, a type array, or an instance.
4607 //
4608 bool LibraryCallKit::inline_native_clone(bool is_virtual) {
4609   PhiNode* result_val;
4610 
4611   // Set the reexecute bit for the interpreter to reexecute
4612   // the bytecode that invokes Object.clone if deoptimization happens.
4613   { PreserveReexecuteState preexecs(this);
4614     jvms()->set_should_reexecute(true);
4615 
4616     Node* obj = null_check_receiver();
4617     if (stopped())  return true;
4618 
4619     const TypeOopPtr* obj_type = _gvn.type(obj)->is_oopptr();
4620 
4621     // If we are going to clone an instance, we need its exact type to
4622     // know the number and types of fields to convert the clone to
4623     // loads/stores. Maybe a speculative type can help us.
4624     if (!obj_type->klass_is_exact() &&
4625         obj_type->speculative_type() != NULL &&
4626         obj_type->speculative_type()->is_instance_klass()) {
4627       ciInstanceKlass* spec_ik = obj_type->speculative_type()->as_instance_klass();
4628       if (spec_ik->nof_nonstatic_fields() <= ArrayCopyLoadStoreMaxElem &&
4629           !spec_ik->has_injected_fields()) {
4630         ciKlass* k = obj_type->klass();
4631         if (!k->is_instance_klass() ||
4632             k->as_instance_klass()->is_interface() ||
4633             k->as_instance_klass()->has_subklass()) {
4634           obj = maybe_cast_profiled_obj(obj, obj_type->speculative_type(), false);
4635         }
4636       }
4637     }
4638 
4639     Node* obj_klass = load_object_klass(obj);
4640     const TypeKlassPtr* tklass = _gvn.type(obj_klass)->isa_klassptr();
4641     const TypeOopPtr*   toop   = ((tklass != NULL)
4642                                 ? tklass->as_instance_type()
4643                                 : TypeInstPtr::NOTNULL);
4644 
4645     // Conservatively insert a memory barrier on all memory slices.
4646     // Do not let writes into the original float below the clone.
4647     insert_mem_bar(Op_MemBarCPUOrder);
4648 
4649     // paths into result_reg:
4650     enum {
4651       _slow_path = 1,     // out-of-line call to clone method (virtual or not)
4652       _objArray_path,     // plain array allocation, plus arrayof_oop_arraycopy
4653       _array_path,        // plain array allocation, plus arrayof_long_arraycopy
4654       _instance_path,     // plain instance allocation, plus arrayof_long_arraycopy
4655       PATH_LIMIT
4656     };
4657     RegionNode* result_reg = new RegionNode(PATH_LIMIT);
4658     result_val             = new PhiNode(result_reg, TypeInstPtr::NOTNULL);
4659     PhiNode*    result_i_o = new PhiNode(result_reg, Type::ABIO);
4660     PhiNode*    result_mem = new PhiNode(result_reg, Type::MEMORY, TypePtr::BOTTOM);
4661     record_for_igvn(result_reg);
4662 
4663     const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
4664     int raw_adr_idx = Compile::AliasIdxRaw;
4665 
4666     Node* array_ctl = generate_array_guard(obj_klass, (RegionNode*)NULL);
4667     if (array_ctl != NULL) {
4668       // It's an array.
4669       PreserveJVMState pjvms(this);
4670       set_control(array_ctl);
4671       Node* obj_length = load_array_length(obj);
4672       Node* obj_size  = NULL;
4673       Node* alloc_obj = new_array(obj_klass, obj_length, 0, &obj_size);  // no arguments to push
4674 
4675       if (!use_ReduceInitialCardMarks()) {
4676         // If it is an oop array, it requires very special treatment,
4677         // because card marking is required on each card of the array.
4678         Node* is_obja = generate_objArray_guard(obj_klass, (RegionNode*)NULL);
4679         if (is_obja != NULL) {
4680           PreserveJVMState pjvms2(this);
4681           set_control(is_obja);
4682           // Generate a direct call to the right arraycopy function(s).
4683           Node* alloc = tightly_coupled_allocation(alloc_obj, NULL);
4684           ArrayCopyNode* ac = ArrayCopyNode::make(this, true, obj, intcon(0), alloc_obj, intcon(0), obj_length, alloc != NULL);
4685           ac->set_cloneoop();
4686           Node* n = _gvn.transform(ac);
4687           assert(n == ac, "cannot disappear");
4688           ac->connect_outputs(this);
4689 
4690           result_reg->init_req(_objArray_path, control());
4691           result_val->init_req(_objArray_path, alloc_obj);
4692           result_i_o ->set_req(_objArray_path, i_o());
4693           result_mem ->set_req(_objArray_path, reset_memory());
4694         }
4695       }
4696       // Otherwise, there are no card marks to worry about.
4697       // (We can dispense with card marks if we know the allocation
4698       //  comes out of eden (TLAB)...  In fact, ReduceInitialCardMarks
4699       //  causes the non-eden paths to take compensating steps to
4700       //  simulate a fresh allocation, so that no further
4701       //  card marks are required in compiled code to initialize
4702       //  the object.)
4703 
4704       if (!stopped()) {
4705         copy_to_clone(obj, alloc_obj, obj_size, true, false);
4706 
4707         // Present the results of the copy.
4708         result_reg->init_req(_array_path, control());
4709         result_val->init_req(_array_path, alloc_obj);
4710         result_i_o ->set_req(_array_path, i_o());
4711         result_mem ->set_req(_array_path, reset_memory());
4712       }
4713     }
4714 
4715     // We only go to the instance fast case code if we pass a number of guards.
4716     // The paths which do not pass are accumulated in the slow_region.
4717     RegionNode* slow_region = new RegionNode(1);
4718     record_for_igvn(slow_region);
4719     if (!stopped()) {
4720       // It's an instance (we did array above).  Make the slow-path tests.
4721       // If this is a virtual call, we generate a funny guard.  We grab
4722       // the vtable entry corresponding to clone() from the target object.
4723       // If the target method which we are calling happens to be the
4724       // Object clone() method, we pass the guard.  We do not need this
4725       // guard for non-virtual calls; the caller is known to be the native
4726       // Object clone().
4727       if (is_virtual) {
4728         generate_virtual_guard(obj_klass, slow_region);
4729       }
4730 
4731       // The object must be cloneable and must not have a finalizer.
4732       // Both of these conditions may be checked in a single test.
4733       // We could optimize the cloneable test further, but we don't care.
4734       generate_access_flags_guard(obj_klass,
4735                                   // Test both conditions:
4736                                   JVM_ACC_IS_CLONEABLE | JVM_ACC_HAS_FINALIZER,
4737                                   // Must be cloneable but not finalizer:
4738                                   JVM_ACC_IS_CLONEABLE,
4739                                   slow_region);
4740     }
4741 
4742     if (!stopped()) {
4743       // It's an instance, and it passed the slow-path tests.
4744       PreserveJVMState pjvms(this);
4745       Node* obj_size  = NULL;
4746       // Need to deoptimize on exception from allocation since Object.clone intrinsic
4747       // is reexecuted if deoptimization occurs and there could be problems when merging
4748       // exception state between multiple Object.clone versions (reexecute=true vs reexecute=false).
4749       Node* alloc_obj = new_instance(obj_klass, NULL, &obj_size, /*deoptimize_on_exception=*/true);
4750 
4751       copy_to_clone(obj, alloc_obj, obj_size, false, !use_ReduceInitialCardMarks());
4752 
4753       // Present the results of the slow call.
4754       result_reg->init_req(_instance_path, control());
4755       result_val->init_req(_instance_path, alloc_obj);
4756       result_i_o ->set_req(_instance_path, i_o());
4757       result_mem ->set_req(_instance_path, reset_memory());
4758     }
4759 
4760     // Generate code for the slow case.  We make a call to clone().
4761     set_control(_gvn.transform(slow_region));
4762     if (!stopped()) {
4763       PreserveJVMState pjvms(this);
4764       CallJavaNode* slow_call = generate_method_call(vmIntrinsics::_clone, is_virtual);
4765       Node* slow_result = set_results_for_java_call(slow_call);
4766       // this->control() comes from set_results_for_java_call
4767       result_reg->init_req(_slow_path, control());
4768       result_val->init_req(_slow_path, slow_result);
4769       result_i_o ->set_req(_slow_path, i_o());
4770       result_mem ->set_req(_slow_path, reset_memory());
4771     }
4772 
4773     // Return the combined state.
4774     set_control(    _gvn.transform(result_reg));
4775     set_i_o(        _gvn.transform(result_i_o));
4776     set_all_memory( _gvn.transform(result_mem));
4777   } // original reexecute is set back here
4778 
4779   set_result(_gvn.transform(result_val));
4780   return true;
4781 }
4782 
4783 // If we have a tighly coupled allocation, the arraycopy may take care
4784 // of the array initialization. If one of the guards we insert between
4785 // the allocation and the arraycopy causes a deoptimization, an
4786 // unitialized array will escape the compiled method. To prevent that
4787 // we set the JVM state for uncommon traps between the allocation and
4788 // the arraycopy to the state before the allocation so, in case of
4789 // deoptimization, we'll reexecute the allocation and the
4790 // initialization.
4791 JVMState* LibraryCallKit::arraycopy_restore_alloc_state(AllocateArrayNode* alloc, int& saved_reexecute_sp) {
4792   if (alloc != NULL) {
4793     ciMethod* trap_method = alloc->jvms()->method();
4794     int trap_bci = alloc->jvms()->bci();
4795 
4796     if (!C->too_many_traps(trap_method, trap_bci, Deoptimization::Reason_intrinsic) &
4797           !C->too_many_traps(trap_method, trap_bci, Deoptimization::Reason_null_check)) {
4798       // Make sure there's no store between the allocation and the
4799       // arraycopy otherwise visible side effects could be rexecuted
4800       // in case of deoptimization and cause incorrect execution.
4801       bool no_interfering_store = true;
4802       Node* mem = alloc->in(TypeFunc::Memory);
4803       if (mem->is_MergeMem()) {
4804         for (MergeMemStream mms(merged_memory(), mem->as_MergeMem()); mms.next_non_empty2(); ) {
4805           Node* n = mms.memory();
4806           if (n != mms.memory2() && !(n->is_Proj() && n->in(0) == alloc->initialization())) {
4807             assert(n->is_Store(), "what else?");
4808             no_interfering_store = false;
4809             break;
4810           }
4811         }
4812       } else {
4813         for (MergeMemStream mms(merged_memory()); mms.next_non_empty(); ) {
4814           Node* n = mms.memory();
4815           if (n != mem && !(n->is_Proj() && n->in(0) == alloc->initialization())) {
4816             assert(n->is_Store(), "what else?");
4817             no_interfering_store = false;
4818             break;
4819           }
4820         }
4821       }
4822 
4823       if (no_interfering_store) {
4824         JVMState* old_jvms = alloc->jvms()->clone_shallow(C);
4825         uint size = alloc->req();
4826         SafePointNode* sfpt = new SafePointNode(size, old_jvms);
4827         old_jvms->set_map(sfpt);
4828         for (uint i = 0; i < size; i++) {
4829           sfpt->init_req(i, alloc->in(i));
4830         }
4831         // re-push array length for deoptimization
4832         sfpt->ins_req(old_jvms->stkoff() + old_jvms->sp(), alloc->in(AllocateNode::ALength));
4833         old_jvms->set_sp(old_jvms->sp()+1);
4834         old_jvms->set_monoff(old_jvms->monoff()+1);
4835         old_jvms->set_scloff(old_jvms->scloff()+1);
4836         old_jvms->set_endoff(old_jvms->endoff()+1);
4837         old_jvms->set_should_reexecute(true);
4838 
4839         sfpt->set_i_o(map()->i_o());
4840         sfpt->set_memory(map()->memory());
4841         sfpt->set_control(map()->control());
4842 
4843         JVMState* saved_jvms = jvms();
4844         saved_reexecute_sp = _reexecute_sp;
4845 
4846         set_jvms(sfpt->jvms());
4847         _reexecute_sp = jvms()->sp();
4848 
4849         return saved_jvms;
4850       }
4851     }
4852   }
4853   return NULL;
4854 }
4855 
4856 // In case of a deoptimization, we restart execution at the
4857 // allocation, allocating a new array. We would leave an uninitialized
4858 // array in the heap that GCs wouldn't expect. Move the allocation
4859 // after the traps so we don't allocate the array if we
4860 // deoptimize. This is possible because tightly_coupled_allocation()
4861 // guarantees there's no observer of the allocated array at this point
4862 // and the control flow is simple enough.
4863 void LibraryCallKit::arraycopy_move_allocation_here(AllocateArrayNode* alloc, Node* dest, JVMState* saved_jvms, int saved_reexecute_sp) {
4864   if (saved_jvms != NULL && !stopped()) {
4865     assert(alloc != NULL, "only with a tightly coupled allocation");
4866     // restore JVM state to the state at the arraycopy
4867     saved_jvms->map()->set_control(map()->control());
4868     assert(saved_jvms->map()->memory() == map()->memory(), "memory state changed?");
4869     assert(saved_jvms->map()->i_o() == map()->i_o(), "IO state changed?");
4870     // If we've improved the types of some nodes (null check) while
4871     // emitting the guards, propagate them to the current state
4872     map()->replaced_nodes().apply(saved_jvms->map());
4873     set_jvms(saved_jvms);
4874     _reexecute_sp = saved_reexecute_sp;
4875 
4876     // Remove the allocation from above the guards
4877     CallProjections callprojs;
4878     alloc->extract_projections(&callprojs, true);
4879     InitializeNode* init = alloc->initialization();
4880     Node* alloc_mem = alloc->in(TypeFunc::Memory);
4881     C->gvn_replace_by(callprojs.fallthrough_ioproj, alloc->in(TypeFunc::I_O));
4882     C->gvn_replace_by(init->proj_out(TypeFunc::Memory), alloc_mem);
4883     C->gvn_replace_by(init->proj_out(TypeFunc::Control), alloc->in(0));
4884 
4885     // move the allocation here (after the guards)
4886     _gvn.hash_delete(alloc);
4887     alloc->set_req(TypeFunc::Control, control());
4888     alloc->set_req(TypeFunc::I_O, i_o());
4889     Node *mem = reset_memory();
4890     set_all_memory(mem);
4891     alloc->set_req(TypeFunc::Memory, mem);
4892     set_control(init->proj_out(TypeFunc::Control));
4893     set_i_o(callprojs.fallthrough_ioproj);
4894 
4895     // Update memory as done in GraphKit::set_output_for_allocation()
4896     const TypeInt* length_type = _gvn.find_int_type(alloc->in(AllocateNode::ALength));
4897     const TypeOopPtr* ary_type = _gvn.type(alloc->in(AllocateNode::KlassNode))->is_klassptr()->as_instance_type();
4898     if (ary_type->isa_aryptr() && length_type != NULL) {
4899       ary_type = ary_type->is_aryptr()->cast_to_size(length_type);
4900     }
4901     const TypePtr* telemref = ary_type->add_offset(Type::OffsetBot);
4902     int            elemidx  = C->get_alias_index(telemref);
4903     set_memory(init->proj_out(TypeFunc::Memory), Compile::AliasIdxRaw);
4904     set_memory(init->proj_out(TypeFunc::Memory), elemidx);
4905 
4906     Node* allocx = _gvn.transform(alloc);
4907     assert(allocx == alloc, "where has the allocation gone?");
4908     assert(dest->is_CheckCastPP(), "not an allocation result?");
4909 
4910     _gvn.hash_delete(dest);
4911     dest->set_req(0, control());
4912     Node* destx = _gvn.transform(dest);
4913     assert(destx == dest, "where has the allocation result gone?");
4914   }
4915 }
4916 
4917 
4918 //------------------------------inline_arraycopy-----------------------
4919 // public static native void java.lang.System.arraycopy(Object src,  int  srcPos,
4920 //                                                      Object dest, int destPos,
4921 //                                                      int length);
4922 bool LibraryCallKit::inline_arraycopy() {
4923   // Get the arguments.
4924   Node* src         = argument(0);  // type: oop
4925   Node* src_offset  = argument(1);  // type: int
4926   Node* dest        = argument(2);  // type: oop
4927   Node* dest_offset = argument(3);  // type: int
4928   Node* length      = argument(4);  // type: int
4929 
4930 
4931   // Check for allocation before we add nodes that would confuse
4932   // tightly_coupled_allocation()
4933   AllocateArrayNode* alloc = tightly_coupled_allocation(dest, NULL);
4934 
4935   int saved_reexecute_sp = -1;
4936   JVMState* saved_jvms = arraycopy_restore_alloc_state(alloc, saved_reexecute_sp);
4937   // See arraycopy_restore_alloc_state() comment
4938   // if alloc == NULL we don't have to worry about a tightly coupled allocation so we can emit all needed guards
4939   // if saved_jvms != NULL (then alloc != NULL) then we can handle guards and a tightly coupled allocation
4940   // if saved_jvms == NULL and alloc != NULL, we can’t emit any guards
4941   bool can_emit_guards = (alloc == NULL || saved_jvms != NULL);
4942 
4943   // The following tests must be performed
4944   // (1) src and dest are arrays.
4945   // (2) src and dest arrays must have elements of the same BasicType
4946   // (3) src and dest must not be null.
4947   // (4) src_offset must not be negative.
4948   // (5) dest_offset must not be negative.
4949   // (6) length must not be negative.
4950   // (7) src_offset + length must not exceed length of src.
4951   // (8) dest_offset + length must not exceed length of dest.
4952   // (9) each element of an oop array must be assignable
4953 
4954   // (3) src and dest must not be null.
4955   // always do this here because we need the JVM state for uncommon traps
4956   Node* null_ctl = top();
4957   src  = saved_jvms != NULL ? null_check_oop(src, &null_ctl, true, true) : null_check(src,  T_ARRAY);
4958   assert(null_ctl->is_top(), "no null control here");
4959   dest = null_check(dest, T_ARRAY);
4960 
4961   if (!can_emit_guards) {
4962     // if saved_jvms == NULL and alloc != NULL, we don't emit any
4963     // guards but the arraycopy node could still take advantage of a
4964     // tightly allocated allocation. tightly_coupled_allocation() is
4965     // called again to make sure it takes the null check above into
4966     // account: the null check is mandatory and if it caused an
4967     // uncommon trap to be emitted then the allocation can't be
4968     // considered tightly coupled in this context.
4969     alloc = tightly_coupled_allocation(dest, NULL);
4970   }
4971 
4972   bool validated = false;
4973 
4974   const Type* src_type  = _gvn.type(src);
4975   const Type* dest_type = _gvn.type(dest);
4976   const TypeAryPtr* top_src  = src_type->isa_aryptr();
4977   const TypeAryPtr* top_dest = dest_type->isa_aryptr();
4978 
4979   // Do we have the type of src?
4980   bool has_src = (top_src != NULL && top_src->klass() != NULL);
4981   // Do we have the type of dest?
4982   bool has_dest = (top_dest != NULL && top_dest->klass() != NULL);
4983   // Is the type for src from speculation?
4984   bool src_spec = false;
4985   // Is the type for dest from speculation?
4986   bool dest_spec = false;
4987 
4988   if ((!has_src || !has_dest) && can_emit_guards) {
4989     // We don't have sufficient type information, let's see if
4990     // speculative types can help. We need to have types for both src
4991     // and dest so that it pays off.
4992 
4993     // Do we already have or could we have type information for src
4994     bool could_have_src = has_src;
4995     // Do we already have or could we have type information for dest
4996     bool could_have_dest = has_dest;
4997 
4998     ciKlass* src_k = NULL;
4999     if (!has_src) {
5000       src_k = src_type->speculative_type_not_null();
5001       if (src_k != NULL && src_k->is_array_klass()) {
5002         could_have_src = true;
5003       }
5004     }
5005 
5006     ciKlass* dest_k = NULL;
5007     if (!has_dest) {
5008       dest_k = dest_type->speculative_type_not_null();
5009       if (dest_k != NULL && dest_k->is_array_klass()) {
5010         could_have_dest = true;
5011       }
5012     }
5013 
5014     if (could_have_src && could_have_dest) {
5015       // This is going to pay off so emit the required guards
5016       if (!has_src) {
5017         src = maybe_cast_profiled_obj(src, src_k, true);
5018         src_type  = _gvn.type(src);
5019         top_src  = src_type->isa_aryptr();
5020         has_src = (top_src != NULL && top_src->klass() != NULL);
5021         src_spec = true;
5022       }
5023       if (!has_dest) {
5024         dest = maybe_cast_profiled_obj(dest, dest_k, true);
5025         dest_type  = _gvn.type(dest);
5026         top_dest  = dest_type->isa_aryptr();
5027         has_dest = (top_dest != NULL && top_dest->klass() != NULL);
5028         dest_spec = true;
5029       }
5030     }
5031   }
5032 
5033   if (has_src && has_dest && can_emit_guards) {
5034     BasicType src_elem  = top_src->klass()->as_array_klass()->element_type()->basic_type();
5035     BasicType dest_elem = top_dest->klass()->as_array_klass()->element_type()->basic_type();
5036     if (src_elem  == T_ARRAY)  src_elem  = T_OBJECT;
5037     if (dest_elem == T_ARRAY)  dest_elem = T_OBJECT;
5038 
5039     if (src_elem == dest_elem && src_elem == T_OBJECT) {
5040       // If both arrays are object arrays then having the exact types
5041       // for both will remove the need for a subtype check at runtime
5042       // before the call and may make it possible to pick a faster copy
5043       // routine (without a subtype check on every element)
5044       // Do we have the exact type of src?
5045       bool could_have_src = src_spec;
5046       // Do we have the exact type of dest?
5047       bool could_have_dest = dest_spec;
5048       ciKlass* src_k = top_src->klass();
5049       ciKlass* dest_k = top_dest->klass();
5050       if (!src_spec) {
5051         src_k = src_type->speculative_type_not_null();
5052         if (src_k != NULL && src_k->is_array_klass()) {
5053           could_have_src = true;
5054         }
5055       }
5056       if (!dest_spec) {
5057         dest_k = dest_type->speculative_type_not_null();
5058         if (dest_k != NULL && dest_k->is_array_klass()) {
5059           could_have_dest = true;
5060         }
5061       }
5062       if (could_have_src && could_have_dest) {
5063         // If we can have both exact types, emit the missing guards
5064         if (could_have_src && !src_spec) {
5065           src = maybe_cast_profiled_obj(src, src_k, true);
5066         }
5067         if (could_have_dest && !dest_spec) {
5068           dest = maybe_cast_profiled_obj(dest, dest_k, true);
5069         }
5070       }
5071     }
5072   }
5073 
5074   ciMethod* trap_method = method();
5075   int trap_bci = bci();
5076   if (saved_jvms != NULL) {
5077     trap_method = alloc->jvms()->method();
5078     trap_bci = alloc->jvms()->bci();
5079   }
5080 
5081   if (!C->too_many_traps(trap_method, trap_bci, Deoptimization::Reason_intrinsic) &&
5082       can_emit_guards &&
5083       !src->is_top() && !dest->is_top()) {
5084     // validate arguments: enables transformation the ArrayCopyNode
5085     validated = true;
5086 
5087     RegionNode* slow_region = new RegionNode(1);
5088     record_for_igvn(slow_region);
5089 
5090     // (1) src and dest are arrays.
5091     generate_non_array_guard(load_object_klass(src), slow_region);
5092     generate_non_array_guard(load_object_klass(dest), slow_region);
5093 
5094     // (2) src and dest arrays must have elements of the same BasicType
5095     // done at macro expansion or at Ideal transformation time
5096 
5097     // (4) src_offset must not be negative.
5098     generate_negative_guard(src_offset, slow_region);
5099 
5100     // (5) dest_offset must not be negative.
5101     generate_negative_guard(dest_offset, slow_region);
5102 
5103     // (7) src_offset + length must not exceed length of src.
5104     generate_limit_guard(src_offset, length,
5105                          load_array_length(src),
5106                          slow_region);
5107 
5108     // (8) dest_offset + length must not exceed length of dest.
5109     generate_limit_guard(dest_offset, length,
5110                          load_array_length(dest),
5111                          slow_region);
5112 
5113     // (9) each element of an oop array must be assignable
5114     Node* src_klass  = load_object_klass(src);
5115     Node* dest_klass = load_object_klass(dest);
5116     Node* not_subtype_ctrl = gen_subtype_check(src_klass, dest_klass);
5117 
5118     if (not_subtype_ctrl != top()) {
5119       PreserveJVMState pjvms(this);
5120       set_control(not_subtype_ctrl);
5121       uncommon_trap(Deoptimization::Reason_intrinsic,
5122                     Deoptimization::Action_make_not_entrant);
5123       assert(stopped(), "Should be stopped");
5124     }
5125     {
5126       PreserveJVMState pjvms(this);
5127       set_control(_gvn.transform(slow_region));
5128       uncommon_trap(Deoptimization::Reason_intrinsic,
5129                     Deoptimization::Action_make_not_entrant);
5130       assert(stopped(), "Should be stopped");
5131     }
5132   }
5133 
5134   arraycopy_move_allocation_here(alloc, dest, saved_jvms, saved_reexecute_sp);
5135 
5136   if (stopped()) {
5137     return true;
5138   }
5139 
5140   ArrayCopyNode* ac = ArrayCopyNode::make(this, true, src, src_offset, dest, dest_offset, length, alloc != NULL,
5141                                           // Create LoadRange and LoadKlass nodes for use during macro expansion here
5142                                           // so the compiler has a chance to eliminate them: during macro expansion,
5143                                           // we have to set their control (CastPP nodes are eliminated).
5144                                           load_object_klass(src), load_object_klass(dest),
5145                                           load_array_length(src), load_array_length(dest));
5146 
5147   ac->set_arraycopy(validated);
5148 
5149   Node* n = _gvn.transform(ac);
5150   if (n == ac) {
5151     ac->connect_outputs(this);
5152   } else {
5153     assert(validated, "shouldn't transform if all arguments not validated");
5154     set_all_memory(n);
5155   }
5156 
5157   return true;
5158 }
5159 
5160 
5161 // Helper function which determines if an arraycopy immediately follows
5162 // an allocation, with no intervening tests or other escapes for the object.
5163 AllocateArrayNode*
5164 LibraryCallKit::tightly_coupled_allocation(Node* ptr,
5165                                            RegionNode* slow_region) {
5166   if (stopped())             return NULL;  // no fast path
5167   if (C->AliasLevel() == 0)  return NULL;  // no MergeMems around
5168 
5169   AllocateArrayNode* alloc = AllocateArrayNode::Ideal_array_allocation(ptr, &_gvn);
5170   if (alloc == NULL)  return NULL;
5171 
5172   Node* rawmem = memory(Compile::AliasIdxRaw);
5173   // Is the allocation's memory state untouched?
5174   if (!(rawmem->is_Proj() && rawmem->in(0)->is_Initialize())) {
5175     // Bail out if there have been raw-memory effects since the allocation.
5176     // (Example:  There might have been a call or safepoint.)
5177     return NULL;
5178   }
5179   rawmem = rawmem->in(0)->as_Initialize()->memory(Compile::AliasIdxRaw);
5180   if (!(rawmem->is_Proj() && rawmem->in(0) == alloc)) {
5181     return NULL;
5182   }
5183 
5184   // There must be no unexpected observers of this allocation.
5185   for (DUIterator_Fast imax, i = ptr->fast_outs(imax); i < imax; i++) {
5186     Node* obs = ptr->fast_out(i);
5187     if (obs != this->map()) {
5188       return NULL;
5189     }
5190   }
5191 
5192   // This arraycopy must unconditionally follow the allocation of the ptr.
5193   Node* alloc_ctl = ptr->in(0);
5194   assert(just_allocated_object(alloc_ctl) == ptr, "most recent allo");
5195 
5196   Node* ctl = control();
5197   while (ctl != alloc_ctl) {
5198     // There may be guards which feed into the slow_region.
5199     // Any other control flow means that we might not get a chance
5200     // to finish initializing the allocated object.
5201     if ((ctl->is_IfFalse() || ctl->is_IfTrue()) && ctl->in(0)->is_If()) {
5202       IfNode* iff = ctl->in(0)->as_If();
5203       Node* not_ctl = iff->proj_out(1 - ctl->as_Proj()->_con);
5204       assert(not_ctl != NULL && not_ctl != ctl, "found alternate");
5205       if (slow_region != NULL && slow_region->find_edge(not_ctl) >= 1) {
5206         ctl = iff->in(0);       // This test feeds the known slow_region.
5207         continue;
5208       }
5209       // One more try:  Various low-level checks bottom out in
5210       // uncommon traps.  If the debug-info of the trap omits
5211       // any reference to the allocation, as we've already
5212       // observed, then there can be no objection to the trap.
5213       bool found_trap = false;
5214       for (DUIterator_Fast jmax, j = not_ctl->fast_outs(jmax); j < jmax; j++) {
5215         Node* obs = not_ctl->fast_out(j);
5216         if (obs->in(0) == not_ctl && obs->is_Call() &&
5217             (obs->as_Call()->entry_point() == SharedRuntime::uncommon_trap_blob()->entry_point())) {
5218           found_trap = true; break;
5219         }
5220       }
5221       if (found_trap) {
5222         ctl = iff->in(0);       // This test feeds a harmless uncommon trap.
5223         continue;
5224       }
5225     }
5226     return NULL;
5227   }
5228 
5229   // If we get this far, we have an allocation which immediately
5230   // precedes the arraycopy, and we can take over zeroing the new object.
5231   // The arraycopy will finish the initialization, and provide
5232   // a new control state to which we will anchor the destination pointer.
5233 
5234   return alloc;
5235 }
5236 
5237 //-------------inline_encodeISOArray-----------------------------------
5238 // encode char[] to byte[] in ISO_8859_1
5239 bool LibraryCallKit::inline_encodeISOArray() {
5240   assert(callee()->signature()->size() == 5, "encodeISOArray has 5 parameters");
5241   // no receiver since it is static method
5242   Node *src         = argument(0);
5243   Node *src_offset  = argument(1);
5244   Node *dst         = argument(2);
5245   Node *dst_offset  = argument(3);
5246   Node *length      = argument(4);
5247 
5248   const Type* src_type = src->Value(&_gvn);
5249   const Type* dst_type = dst->Value(&_gvn);
5250   const TypeAryPtr* top_src = src_type->isa_aryptr();
5251   const TypeAryPtr* top_dest = dst_type->isa_aryptr();
5252   if (top_src  == NULL || top_src->klass()  == NULL ||
5253       top_dest == NULL || top_dest->klass() == NULL) {
5254     // failed array check
5255     return false;
5256   }
5257 
5258   // Figure out the size and type of the elements we will be copying.
5259   BasicType src_elem = src_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
5260   BasicType dst_elem = dst_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
5261   if (!((src_elem == T_CHAR) || (src_elem== T_BYTE)) || dst_elem != T_BYTE) {
5262     return false;
5263   }
5264 
5265   Node* src_start = array_element_address(src, src_offset, T_CHAR);
5266   Node* dst_start = array_element_address(dst, dst_offset, dst_elem);
5267   // 'src_start' points to src array + scaled offset
5268   // 'dst_start' points to dst array + scaled offset
5269 
5270   const TypeAryPtr* mtype = TypeAryPtr::BYTES;
5271   Node* enc = new EncodeISOArrayNode(control(), memory(mtype), src_start, dst_start, length);
5272   enc = _gvn.transform(enc);
5273   Node* res_mem = _gvn.transform(new SCMemProjNode(enc));
5274   set_memory(res_mem, mtype);
5275   set_result(enc);
5276   return true;
5277 }
5278 
5279 //-------------inline_multiplyToLen-----------------------------------
5280 bool LibraryCallKit::inline_multiplyToLen() {
5281   assert(UseMultiplyToLenIntrinsic, "not implemented on this platform");
5282 
5283   address stubAddr = StubRoutines::multiplyToLen();
5284   if (stubAddr == NULL) {
5285     return false; // Intrinsic's stub is not implemented on this platform
5286   }
5287   const char* stubName = "multiplyToLen";
5288 
5289   assert(callee()->signature()->size() == 5, "multiplyToLen has 5 parameters");
5290 
5291   // no receiver because it is a static method
5292   Node* x    = argument(0);
5293   Node* xlen = argument(1);
5294   Node* y    = argument(2);
5295   Node* ylen = argument(3);
5296   Node* z    = argument(4);
5297 
5298   const Type* x_type = x->Value(&_gvn);
5299   const Type* y_type = y->Value(&_gvn);
5300   const TypeAryPtr* top_x = x_type->isa_aryptr();
5301   const TypeAryPtr* top_y = y_type->isa_aryptr();
5302   if (top_x  == NULL || top_x->klass()  == NULL ||
5303       top_y == NULL || top_y->klass() == NULL) {
5304     // failed array check
5305     return false;
5306   }
5307 
5308   BasicType x_elem = x_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
5309   BasicType y_elem = y_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
5310   if (x_elem != T_INT || y_elem != T_INT) {
5311     return false;
5312   }
5313 
5314   // Set the original stack and the reexecute bit for the interpreter to reexecute
5315   // the bytecode that invokes BigInteger.multiplyToLen() if deoptimization happens
5316   // on the return from z array allocation in runtime.
5317   { PreserveReexecuteState preexecs(this);
5318     jvms()->set_should_reexecute(true);
5319 
5320     Node* x_start = array_element_address(x, intcon(0), x_elem);
5321     Node* y_start = array_element_address(y, intcon(0), y_elem);
5322     // 'x_start' points to x array + scaled xlen
5323     // 'y_start' points to y array + scaled ylen
5324 
5325     // Allocate the result array
5326     Node* zlen = _gvn.transform(new AddINode(xlen, ylen));
5327     ciKlass* klass = ciTypeArrayKlass::make(T_INT);
5328     Node* klass_node = makecon(TypeKlassPtr::make(klass));
5329 
5330     IdealKit ideal(this);
5331 
5332 #define __ ideal.
5333      Node* one = __ ConI(1);
5334      Node* zero = __ ConI(0);
5335      IdealVariable need_alloc(ideal), z_alloc(ideal);  __ declarations_done();
5336      __ set(need_alloc, zero);
5337      __ set(z_alloc, z);
5338      __ if_then(z, BoolTest::eq, null()); {
5339        __ increment (need_alloc, one);
5340      } __ else_(); {
5341        // Update graphKit memory and control from IdealKit.
5342        sync_kit(ideal);
5343        Node* zlen_arg = load_array_length(z);
5344        // Update IdealKit memory and control from graphKit.
5345        __ sync_kit(this);
5346        __ if_then(zlen_arg, BoolTest::lt, zlen); {
5347          __ increment (need_alloc, one);
5348        } __ end_if();
5349      } __ end_if();
5350 
5351      __ if_then(__ value(need_alloc), BoolTest::ne, zero); {
5352        // Update graphKit memory and control from IdealKit.
5353        sync_kit(ideal);
5354        Node * narr = new_array(klass_node, zlen, 1);
5355        // Update IdealKit memory and control from graphKit.
5356        __ sync_kit(this);
5357        __ set(z_alloc, narr);
5358      } __ end_if();
5359 
5360      sync_kit(ideal);
5361      z = __ value(z_alloc);
5362      // Can't use TypeAryPtr::INTS which uses Bottom offset.
5363      _gvn.set_type(z, TypeOopPtr::make_from_klass(klass));
5364      // Final sync IdealKit and GraphKit.
5365      final_sync(ideal);
5366 #undef __
5367 
5368     Node* z_start = array_element_address(z, intcon(0), T_INT);
5369 
5370     Node* call = make_runtime_call(RC_LEAF|RC_NO_FP,
5371                                    OptoRuntime::multiplyToLen_Type(),
5372                                    stubAddr, stubName, TypePtr::BOTTOM,
5373                                    x_start, xlen, y_start, ylen, z_start, zlen);
5374   } // original reexecute is set back here
5375 
5376   C->set_has_split_ifs(true); // Has chance for split-if optimization
5377   set_result(z);
5378   return true;
5379 }
5380 
5381 //-------------inline_squareToLen------------------------------------
5382 bool LibraryCallKit::inline_squareToLen() {
5383   assert(UseSquareToLenIntrinsic, "not implemented on this platform");
5384 
5385   address stubAddr = StubRoutines::squareToLen();
5386   if (stubAddr == NULL) {
5387     return false; // Intrinsic's stub is not implemented on this platform
5388   }
5389   const char* stubName = "squareToLen";
5390 
5391   assert(callee()->signature()->size() == 4, "implSquareToLen has 4 parameters");
5392 
5393   Node* x    = argument(0);
5394   Node* len  = argument(1);
5395   Node* z    = argument(2);
5396   Node* zlen = argument(3);
5397 
5398   const Type* x_type = x->Value(&_gvn);
5399   const Type* z_type = z->Value(&_gvn);
5400   const TypeAryPtr* top_x = x_type->isa_aryptr();
5401   const TypeAryPtr* top_z = z_type->isa_aryptr();
5402   if (top_x  == NULL || top_x->klass()  == NULL ||
5403       top_z  == NULL || top_z->klass()  == NULL) {
5404     // failed array check
5405     return false;
5406   }
5407 
5408   BasicType x_elem = x_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
5409   BasicType z_elem = z_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
5410   if (x_elem != T_INT || z_elem != T_INT) {
5411     return false;
5412   }
5413 
5414 
5415   Node* x_start = array_element_address(x, intcon(0), x_elem);
5416   Node* z_start = array_element_address(z, intcon(0), z_elem);
5417 
5418   Node*  call = make_runtime_call(RC_LEAF|RC_NO_FP,
5419                                   OptoRuntime::squareToLen_Type(),
5420                                   stubAddr, stubName, TypePtr::BOTTOM,
5421                                   x_start, len, z_start, zlen);
5422 
5423   set_result(z);
5424   return true;
5425 }
5426 
5427 //-------------inline_mulAdd------------------------------------------
5428 bool LibraryCallKit::inline_mulAdd() {
5429   assert(UseMulAddIntrinsic, "not implemented on this platform");
5430 
5431   address stubAddr = StubRoutines::mulAdd();
5432   if (stubAddr == NULL) {
5433     return false; // Intrinsic's stub is not implemented on this platform
5434   }
5435   const char* stubName = "mulAdd";
5436 
5437   assert(callee()->signature()->size() == 5, "mulAdd has 5 parameters");
5438 
5439   Node* out      = argument(0);
5440   Node* in       = argument(1);
5441   Node* offset   = argument(2);
5442   Node* len      = argument(3);
5443   Node* k        = argument(4);
5444 
5445   const Type* out_type = out->Value(&_gvn);
5446   const Type* in_type = in->Value(&_gvn);
5447   const TypeAryPtr* top_out = out_type->isa_aryptr();
5448   const TypeAryPtr* top_in = in_type->isa_aryptr();
5449   if (top_out  == NULL || top_out->klass()  == NULL ||
5450       top_in == NULL || top_in->klass() == NULL) {
5451     // failed array check
5452     return false;
5453   }
5454 
5455   BasicType out_elem = out_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
5456   BasicType in_elem = in_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
5457   if (out_elem != T_INT || in_elem != T_INT) {
5458     return false;
5459   }
5460 
5461   Node* outlen = load_array_length(out);
5462   Node* new_offset = _gvn.transform(new SubINode(outlen, offset));
5463   Node* out_start = array_element_address(out, intcon(0), out_elem);
5464   Node* in_start = array_element_address(in, intcon(0), in_elem);
5465 
5466   Node*  call = make_runtime_call(RC_LEAF|RC_NO_FP,
5467                                   OptoRuntime::mulAdd_Type(),
5468                                   stubAddr, stubName, TypePtr::BOTTOM,
5469                                   out_start,in_start, new_offset, len, k);
5470   Node* result = _gvn.transform(new ProjNode(call, TypeFunc::Parms));
5471   set_result(result);
5472   return true;
5473 }
5474 
5475 //-------------inline_montgomeryMultiply-----------------------------------
5476 bool LibraryCallKit::inline_montgomeryMultiply() {
5477   address stubAddr = StubRoutines::montgomeryMultiply();
5478   if (stubAddr == NULL) {
5479     return false; // Intrinsic's stub is not implemented on this platform
5480   }
5481 
5482   assert(UseMontgomeryMultiplyIntrinsic, "not implemented on this platform");
5483   const char* stubName = "montgomery_square";
5484 
5485   assert(callee()->signature()->size() == 7, "montgomeryMultiply has 7 parameters");
5486 
5487   Node* a    = argument(0);
5488   Node* b    = argument(1);
5489   Node* n    = argument(2);
5490   Node* len  = argument(3);
5491   Node* inv  = argument(4);
5492   Node* m    = argument(6);
5493 
5494   const Type* a_type = a->Value(&_gvn);
5495   const TypeAryPtr* top_a = a_type->isa_aryptr();
5496   const Type* b_type = b->Value(&_gvn);
5497   const TypeAryPtr* top_b = b_type->isa_aryptr();
5498   const Type* n_type = a->Value(&_gvn);
5499   const TypeAryPtr* top_n = n_type->isa_aryptr();
5500   const Type* m_type = a->Value(&_gvn);
5501   const TypeAryPtr* top_m = m_type->isa_aryptr();
5502   if (top_a  == NULL || top_a->klass()  == NULL ||
5503       top_b == NULL || top_b->klass()  == NULL ||
5504       top_n == NULL || top_n->klass()  == NULL ||
5505       top_m == NULL || top_m->klass()  == NULL) {
5506     // failed array check
5507     return false;
5508   }
5509 
5510   BasicType a_elem = a_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
5511   BasicType b_elem = b_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
5512   BasicType n_elem = n_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
5513   BasicType m_elem = m_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
5514   if (a_elem != T_INT || b_elem != T_INT || n_elem != T_INT || m_elem != T_INT) {
5515     return false;
5516   }
5517 
5518   // Make the call
5519   {
5520     Node* a_start = array_element_address(a, intcon(0), a_elem);
5521     Node* b_start = array_element_address(b, intcon(0), b_elem);
5522     Node* n_start = array_element_address(n, intcon(0), n_elem);
5523     Node* m_start = array_element_address(m, intcon(0), m_elem);
5524 
5525     Node* call = make_runtime_call(RC_LEAF,
5526                                    OptoRuntime::montgomeryMultiply_Type(),
5527                                    stubAddr, stubName, TypePtr::BOTTOM,
5528                                    a_start, b_start, n_start, len, inv, top(),
5529                                    m_start);
5530     set_result(m);
5531   }
5532 
5533   return true;
5534 }
5535 
5536 bool LibraryCallKit::inline_montgomerySquare() {
5537   address stubAddr = StubRoutines::montgomerySquare();
5538   if (stubAddr == NULL) {
5539     return false; // Intrinsic's stub is not implemented on this platform
5540   }
5541 
5542   assert(UseMontgomerySquareIntrinsic, "not implemented on this platform");
5543   const char* stubName = "montgomery_square";
5544 
5545   assert(callee()->signature()->size() == 6, "montgomerySquare has 6 parameters");
5546 
5547   Node* a    = argument(0);
5548   Node* n    = argument(1);
5549   Node* len  = argument(2);
5550   Node* inv  = argument(3);
5551   Node* m    = argument(5);
5552 
5553   const Type* a_type = a->Value(&_gvn);
5554   const TypeAryPtr* top_a = a_type->isa_aryptr();
5555   const Type* n_type = a->Value(&_gvn);
5556   const TypeAryPtr* top_n = n_type->isa_aryptr();
5557   const Type* m_type = a->Value(&_gvn);
5558   const TypeAryPtr* top_m = m_type->isa_aryptr();
5559   if (top_a  == NULL || top_a->klass()  == NULL ||
5560       top_n == NULL || top_n->klass()  == NULL ||
5561       top_m == NULL || top_m->klass()  == NULL) {
5562     // failed array check
5563     return false;
5564   }
5565 
5566   BasicType a_elem = a_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
5567   BasicType n_elem = n_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
5568   BasicType m_elem = m_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
5569   if (a_elem != T_INT || n_elem != T_INT || m_elem != T_INT) {
5570     return false;
5571   }
5572 
5573   // Make the call
5574   {
5575     Node* a_start = array_element_address(a, intcon(0), a_elem);
5576     Node* n_start = array_element_address(n, intcon(0), n_elem);
5577     Node* m_start = array_element_address(m, intcon(0), m_elem);
5578 
5579     Node* call = make_runtime_call(RC_LEAF,
5580                                    OptoRuntime::montgomerySquare_Type(),
5581                                    stubAddr, stubName, TypePtr::BOTTOM,
5582                                    a_start, n_start, len, inv, top(),
5583                                    m_start);
5584     set_result(m);
5585   }
5586 
5587   return true;
5588 }
5589 
5590 
5591 /**
5592  * Calculate CRC32 for byte.
5593  * int java.util.zip.CRC32.update(int crc, int b)
5594  */
5595 bool LibraryCallKit::inline_updateCRC32() {
5596   assert(UseCRC32Intrinsics, "need AVX and LCMUL instructions support");
5597   assert(callee()->signature()->size() == 2, "update has 2 parameters");
5598   // no receiver since it is static method
5599   Node* crc  = argument(0); // type: int
5600   Node* b    = argument(1); // type: int
5601 
5602   /*
5603    *    int c = ~ crc;
5604    *    b = timesXtoThe32[(b ^ c) & 0xFF];
5605    *    b = b ^ (c >>> 8);
5606    *    crc = ~b;
5607    */
5608 
5609   Node* M1 = intcon(-1);
5610   crc = _gvn.transform(new XorINode(crc, M1));
5611   Node* result = _gvn.transform(new XorINode(crc, b));
5612   result = _gvn.transform(new AndINode(result, intcon(0xFF)));
5613 
5614   Node* base = makecon(TypeRawPtr::make(StubRoutines::crc_table_addr()));
5615   Node* offset = _gvn.transform(new LShiftINode(result, intcon(0x2)));
5616   Node* adr = basic_plus_adr(top(), base, ConvI2X(offset));
5617   result = make_load(control(), adr, TypeInt::INT, T_INT, MemNode::unordered);
5618 
5619   crc = _gvn.transform(new URShiftINode(crc, intcon(8)));
5620   result = _gvn.transform(new XorINode(crc, result));
5621   result = _gvn.transform(new XorINode(result, M1));
5622   set_result(result);
5623   return true;
5624 }
5625 
5626 /**
5627  * Calculate CRC32 for byte[] array.
5628  * int java.util.zip.CRC32.updateBytes(int crc, byte[] buf, int off, int len)
5629  */
5630 bool LibraryCallKit::inline_updateBytesCRC32() {
5631   assert(UseCRC32Intrinsics, "need AVX and LCMUL instructions support");
5632   assert(callee()->signature()->size() == 4, "updateBytes has 4 parameters");
5633   // no receiver since it is static method
5634   Node* crc     = argument(0); // type: int
5635   Node* src     = argument(1); // type: oop
5636   Node* offset  = argument(2); // type: int
5637   Node* length  = argument(3); // type: int
5638 
5639   const Type* src_type = src->Value(&_gvn);
5640   const TypeAryPtr* top_src = src_type->isa_aryptr();
5641   if (top_src  == NULL || top_src->klass()  == NULL) {
5642     // failed array check
5643     return false;
5644   }
5645 
5646   // Figure out the size and type of the elements we will be copying.
5647   BasicType src_elem = src_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
5648   if (src_elem != T_BYTE) {
5649     return false;
5650   }
5651 
5652   // 'src_start' points to src array + scaled offset
5653   Node* src_start = array_element_address(src, offset, src_elem);
5654 
5655   // We assume that range check is done by caller.
5656   // TODO: generate range check (offset+length < src.length) in debug VM.
5657 
5658   // Call the stub.
5659   address stubAddr = StubRoutines::updateBytesCRC32();
5660   const char *stubName = "updateBytesCRC32";
5661 
5662   Node* call = make_runtime_call(RC_LEAF|RC_NO_FP, OptoRuntime::updateBytesCRC32_Type(),
5663                                  stubAddr, stubName, TypePtr::BOTTOM,
5664                                  crc, src_start, length);
5665   Node* result = _gvn.transform(new ProjNode(call, TypeFunc::Parms));
5666   set_result(result);
5667   return true;
5668 }
5669 
5670 /**
5671  * Calculate CRC32 for ByteBuffer.
5672  * int java.util.zip.CRC32.updateByteBuffer(int crc, long buf, int off, int len)
5673  */
5674 bool LibraryCallKit::inline_updateByteBufferCRC32() {
5675   assert(UseCRC32Intrinsics, "need AVX and LCMUL instructions support");
5676   assert(callee()->signature()->size() == 5, "updateByteBuffer has 4 parameters and one is long");
5677   // no receiver since it is static method
5678   Node* crc     = argument(0); // type: int
5679   Node* src     = argument(1); // type: long
5680   Node* offset  = argument(3); // type: int
5681   Node* length  = argument(4); // type: int
5682 
5683   src = ConvL2X(src);  // adjust Java long to machine word
5684   Node* base = _gvn.transform(new CastX2PNode(src));
5685   offset = ConvI2X(offset);
5686 
5687   // 'src_start' points to src array + scaled offset
5688   Node* src_start = basic_plus_adr(top(), base, offset);
5689 
5690   // Call the stub.
5691   address stubAddr = StubRoutines::updateBytesCRC32();
5692   const char *stubName = "updateBytesCRC32";
5693 
5694   Node* call = make_runtime_call(RC_LEAF|RC_NO_FP, OptoRuntime::updateBytesCRC32_Type(),
5695                                  stubAddr, stubName, TypePtr::BOTTOM,
5696                                  crc, src_start, length);
5697   Node* result = _gvn.transform(new ProjNode(call, TypeFunc::Parms));
5698   set_result(result);
5699   return true;
5700 }
5701 
5702 //------------------------------get_table_from_crc32c_class-----------------------
5703 Node * LibraryCallKit::get_table_from_crc32c_class(ciInstanceKlass *crc32c_class) {
5704   Node* table = load_field_from_object(NULL, "byteTable", "[I", /*is_exact*/ false, /*is_static*/ true, crc32c_class);
5705   assert (table != NULL, "wrong version of java.util.zip.CRC32C");
5706 
5707   return table;
5708 }
5709 
5710 //------------------------------inline_updateBytesCRC32C-----------------------
5711 //
5712 // Calculate CRC32C for byte[] array.
5713 // int java.util.zip.CRC32C.updateBytes(int crc, byte[] buf, int off, int end)
5714 //
5715 bool LibraryCallKit::inline_updateBytesCRC32C() {
5716   assert(UseCRC32CIntrinsics, "need CRC32C instruction support");
5717   assert(callee()->signature()->size() == 4, "updateBytes has 4 parameters");
5718   assert(callee()->holder()->is_loaded(), "CRC32C class must be loaded");
5719   // no receiver since it is a static method
5720   Node* crc     = argument(0); // type: int
5721   Node* src     = argument(1); // type: oop
5722   Node* offset  = argument(2); // type: int
5723   Node* end     = argument(3); // type: int
5724 
5725   Node* length = _gvn.transform(new SubINode(end, offset));
5726 
5727   const Type* src_type = src->Value(&_gvn);
5728   const TypeAryPtr* top_src = src_type->isa_aryptr();
5729   if (top_src  == NULL || top_src->klass()  == NULL) {
5730     // failed array check
5731     return false;
5732   }
5733 
5734   // Figure out the size and type of the elements we will be copying.
5735   BasicType src_elem = src_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
5736   if (src_elem != T_BYTE) {
5737     return false;
5738   }
5739 
5740   // 'src_start' points to src array + scaled offset
5741   Node* src_start = array_element_address(src, offset, src_elem);
5742 
5743   // static final int[] byteTable in class CRC32C
5744   Node* table = get_table_from_crc32c_class(callee()->holder());
5745   Node* table_start = array_element_address(table, intcon(0), T_INT);
5746 
5747   // We assume that range check is done by caller.
5748   // TODO: generate range check (offset+length < src.length) in debug VM.
5749 
5750   // Call the stub.
5751   address stubAddr = StubRoutines::updateBytesCRC32C();
5752   const char *stubName = "updateBytesCRC32C";
5753 
5754   Node* call = make_runtime_call(RC_LEAF, OptoRuntime::updateBytesCRC32C_Type(),
5755                                  stubAddr, stubName, TypePtr::BOTTOM,
5756                                  crc, src_start, length, table_start);
5757   Node* result = _gvn.transform(new ProjNode(call, TypeFunc::Parms));
5758   set_result(result);
5759   return true;
5760 }
5761 
5762 //------------------------------inline_updateDirectByteBufferCRC32C-----------------------
5763 //
5764 // Calculate CRC32C for DirectByteBuffer.
5765 // int java.util.zip.CRC32C.updateDirectByteBuffer(int crc, long buf, int off, int end)
5766 //
5767 bool LibraryCallKit::inline_updateDirectByteBufferCRC32C() {
5768   assert(UseCRC32CIntrinsics, "need CRC32C instruction support");
5769   assert(callee()->signature()->size() == 5, "updateDirectByteBuffer has 4 parameters and one is long");
5770   assert(callee()->holder()->is_loaded(), "CRC32C class must be loaded");
5771   // no receiver since it is a static method
5772   Node* crc     = argument(0); // type: int
5773   Node* src     = argument(1); // type: long
5774   Node* offset  = argument(3); // type: int
5775   Node* end     = argument(4); // type: int
5776 
5777   Node* length = _gvn.transform(new SubINode(end, offset));
5778 
5779   src = ConvL2X(src);  // adjust Java long to machine word
5780   Node* base = _gvn.transform(new CastX2PNode(src));
5781   offset = ConvI2X(offset);
5782 
5783   // 'src_start' points to src array + scaled offset
5784   Node* src_start = basic_plus_adr(top(), base, offset);
5785 
5786   // static final int[] byteTable in class CRC32C
5787   Node* table = get_table_from_crc32c_class(callee()->holder());
5788   Node* table_start = array_element_address(table, intcon(0), T_INT);
5789 
5790   // Call the stub.
5791   address stubAddr = StubRoutines::updateBytesCRC32C();
5792   const char *stubName = "updateBytesCRC32C";
5793 
5794   Node* call = make_runtime_call(RC_LEAF, OptoRuntime::updateBytesCRC32C_Type(),
5795                                  stubAddr, stubName, TypePtr::BOTTOM,
5796                                  crc, src_start, length, table_start);
5797   Node* result = _gvn.transform(new ProjNode(call, TypeFunc::Parms));
5798   set_result(result);
5799   return true;
5800 }
5801 
5802 //------------------------------inline_updateBytesAdler32----------------------
5803 //
5804 // Calculate Adler32 checksum for byte[] array.
5805 // int java.util.zip.Adler32.updateBytes(int crc, byte[] buf, int off, int len)
5806 //
5807 bool LibraryCallKit::inline_updateBytesAdler32() {
5808   assert(UseAdler32Intrinsics, "Adler32 Instrinsic support need"); // check if we actually need to check this flag or check a different one
5809   assert(callee()->signature()->size() == 4, "updateBytes has 4 parameters");
5810   assert(callee()->holder()->is_loaded(), "Adler32 class must be loaded");
5811   // no receiver since it is static method
5812   Node* crc     = argument(0); // type: int
5813   Node* src     = argument(1); // type: oop
5814   Node* offset  = argument(2); // type: int
5815   Node* length  = argument(3); // type: int
5816 
5817   const Type* src_type = src->Value(&_gvn);
5818   const TypeAryPtr* top_src = src_type->isa_aryptr();
5819   if (top_src  == NULL || top_src->klass()  == NULL) {
5820     // failed array check
5821     return false;
5822   }
5823 
5824   // Figure out the size and type of the elements we will be copying.
5825   BasicType src_elem = src_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
5826   if (src_elem != T_BYTE) {
5827     return false;
5828   }
5829 
5830   // 'src_start' points to src array + scaled offset
5831   Node* src_start = array_element_address(src, offset, src_elem);
5832 
5833   // We assume that range check is done by caller.
5834   // TODO: generate range check (offset+length < src.length) in debug VM.
5835 
5836   // Call the stub.
5837   address stubAddr = StubRoutines::updateBytesAdler32();
5838   const char *stubName = "updateBytesAdler32";
5839 
5840   Node* call = make_runtime_call(RC_LEAF, OptoRuntime::updateBytesAdler32_Type(),
5841                                  stubAddr, stubName, TypePtr::BOTTOM,
5842                                  crc, src_start, length);
5843   Node* result = _gvn.transform(new ProjNode(call, TypeFunc::Parms));
5844   set_result(result);
5845   return true;
5846 }
5847 
5848 //------------------------------inline_updateByteBufferAdler32---------------
5849 //
5850 // Calculate Adler32 checksum for DirectByteBuffer.
5851 // int java.util.zip.Adler32.updateByteBuffer(int crc, long buf, int off, int len)
5852 //
5853 bool LibraryCallKit::inline_updateByteBufferAdler32() {
5854   assert(UseAdler32Intrinsics, "Adler32 Instrinsic support need"); // check if we actually need to check this flag or check a different one
5855   assert(callee()->signature()->size() == 5, "updateByteBuffer has 4 parameters and one is long");
5856   assert(callee()->holder()->is_loaded(), "Adler32 class must be loaded");
5857   // no receiver since it is static method
5858   Node* crc     = argument(0); // type: int
5859   Node* src     = argument(1); // type: long
5860   Node* offset  = argument(3); // type: int
5861   Node* length  = argument(4); // type: int
5862 
5863   src = ConvL2X(src);  // adjust Java long to machine word
5864   Node* base = _gvn.transform(new CastX2PNode(src));
5865   offset = ConvI2X(offset);
5866 
5867   // 'src_start' points to src array + scaled offset
5868   Node* src_start = basic_plus_adr(top(), base, offset);
5869 
5870   // Call the stub.
5871   address stubAddr = StubRoutines::updateBytesAdler32();
5872   const char *stubName = "updateBytesAdler32";
5873 
5874   Node* call = make_runtime_call(RC_LEAF, OptoRuntime::updateBytesAdler32_Type(),
5875                                  stubAddr, stubName, TypePtr::BOTTOM,
5876                                  crc, src_start, length);
5877 
5878   Node* result = _gvn.transform(new ProjNode(call, TypeFunc::Parms));
5879   set_result(result);
5880   return true;
5881 }
5882 
5883 //----------------------------inline_reference_get----------------------------
5884 // public T java.lang.ref.Reference.get();
5885 bool LibraryCallKit::inline_reference_get() {
5886   const int referent_offset = java_lang_ref_Reference::referent_offset;
5887   guarantee(referent_offset > 0, "should have already been set");
5888 
5889   // Get the argument:
5890   Node* reference_obj = null_check_receiver();
5891   if (stopped()) return true;
5892 
5893   Node* adr = basic_plus_adr(reference_obj, reference_obj, referent_offset);
5894 
5895   ciInstanceKlass* klass = env()->Object_klass();
5896   const TypeOopPtr* object_type = TypeOopPtr::make_from_klass(klass);
5897 
5898   Node* no_ctrl = NULL;
5899   Node* result = make_load(no_ctrl, adr, object_type, T_OBJECT, MemNode::unordered);
5900 
5901   // Use the pre-barrier to record the value in the referent field
5902   pre_barrier(false /* do_load */,
5903               control(),
5904               NULL /* obj */, NULL /* adr */, max_juint /* alias_idx */, NULL /* val */, NULL /* val_type */,
5905               result /* pre_val */,
5906               T_OBJECT);
5907 
5908   // Add memory barrier to prevent commoning reads from this field
5909   // across safepoint since GC can change its value.
5910   insert_mem_bar(Op_MemBarCPUOrder);
5911 
5912   set_result(result);
5913   return true;
5914 }
5915 
5916 
5917 Node * LibraryCallKit::load_field_from_object(Node * fromObj, const char * fieldName, const char * fieldTypeString,
5918                                               bool is_exact=true, bool is_static=false,
5919                                               ciInstanceKlass * fromKls=NULL) {
5920   if (fromKls == NULL) {
5921     const TypeInstPtr* tinst = _gvn.type(fromObj)->isa_instptr();
5922     assert(tinst != NULL, "obj is null");
5923     assert(tinst->klass()->is_loaded(), "obj is not loaded");
5924     assert(!is_exact || tinst->klass_is_exact(), "klass not exact");
5925     fromKls = tinst->klass()->as_instance_klass();
5926   } else {
5927     assert(is_static, "only for static field access");
5928   }
5929   ciField* field = fromKls->get_field_by_name(ciSymbol::make(fieldName),
5930                                               ciSymbol::make(fieldTypeString),
5931                                               is_static);
5932 
5933   assert (field != NULL, "undefined field");
5934   if (field == NULL) return (Node *) NULL;
5935 
5936   if (is_static) {
5937     const TypeInstPtr* tip = TypeInstPtr::make(fromKls->java_mirror());
5938     fromObj = makecon(tip);
5939   }
5940 
5941   // Next code  copied from Parse::do_get_xxx():
5942 
5943   // Compute address and memory type.
5944   int offset  = field->offset_in_bytes();
5945   bool is_vol = field->is_volatile();
5946   ciType* field_klass = field->type();
5947   assert(field_klass->is_loaded(), "should be loaded");
5948   const TypePtr* adr_type = C->alias_type(field)->adr_type();
5949   Node *adr = basic_plus_adr(fromObj, fromObj, offset);
5950   BasicType bt = field->layout_type();
5951 
5952   // Build the resultant type of the load
5953   const Type *type;
5954   if (bt == T_OBJECT) {
5955     type = TypeOopPtr::make_from_klass(field_klass->as_klass());
5956   } else {
5957     type = Type::get_const_basic_type(bt);
5958   }
5959 
5960   if (support_IRIW_for_not_multiple_copy_atomic_cpu && is_vol) {
5961     insert_mem_bar(Op_MemBarVolatile);   // StoreLoad barrier
5962   }
5963   // Build the load.
5964   MemNode::MemOrd mo = is_vol ? MemNode::acquire : MemNode::unordered;
5965   Node* loadedField = make_load(NULL, adr, type, bt, adr_type, mo, LoadNode::DependsOnlyOnTest, is_vol);
5966   // If reference is volatile, prevent following memory ops from
5967   // floating up past the volatile read.  Also prevents commoning
5968   // another volatile read.
5969   if (is_vol) {
5970     // Memory barrier includes bogus read of value to force load BEFORE membar
5971     insert_mem_bar(Op_MemBarAcquire, loadedField);
5972   }
5973   return loadedField;
5974 }
5975 
5976 
5977 //------------------------------inline_aescrypt_Block-----------------------
5978 bool LibraryCallKit::inline_aescrypt_Block(vmIntrinsics::ID id) {
5979   address stubAddr = NULL;
5980   const char *stubName;
5981   assert(UseAES, "need AES instruction support");
5982 
5983   switch(id) {
5984   case vmIntrinsics::_aescrypt_encryptBlock:
5985     stubAddr = StubRoutines::aescrypt_encryptBlock();
5986     stubName = "aescrypt_encryptBlock";
5987     break;
5988   case vmIntrinsics::_aescrypt_decryptBlock:
5989     stubAddr = StubRoutines::aescrypt_decryptBlock();
5990     stubName = "aescrypt_decryptBlock";
5991     break;
5992   }
5993   if (stubAddr == NULL) return false;
5994 
5995   Node* aescrypt_object = argument(0);
5996   Node* src             = argument(1);
5997   Node* src_offset      = argument(2);
5998   Node* dest            = argument(3);
5999   Node* dest_offset     = argument(4);
6000 
6001   // (1) src and dest are arrays.
6002   const Type* src_type = src->Value(&_gvn);
6003   const Type* dest_type = dest->Value(&_gvn);
6004   const TypeAryPtr* top_src = src_type->isa_aryptr();
6005   const TypeAryPtr* top_dest = dest_type->isa_aryptr();
6006   assert (top_src  != NULL && top_src->klass()  != NULL &&  top_dest != NULL && top_dest->klass() != NULL, "args are strange");
6007 
6008   // for the quick and dirty code we will skip all the checks.
6009   // we are just trying to get the call to be generated.
6010   Node* src_start  = src;
6011   Node* dest_start = dest;
6012   if (src_offset != NULL || dest_offset != NULL) {
6013     assert(src_offset != NULL && dest_offset != NULL, "");
6014     src_start  = array_element_address(src,  src_offset,  T_BYTE);
6015     dest_start = array_element_address(dest, dest_offset, T_BYTE);
6016   }
6017 
6018   // now need to get the start of its expanded key array
6019   // this requires a newer class file that has this array as littleEndian ints, otherwise we revert to java
6020   Node* k_start = get_key_start_from_aescrypt_object(aescrypt_object);
6021   if (k_start == NULL) return false;
6022 
6023   if (Matcher::pass_original_key_for_aes()) {
6024     // on SPARC we need to pass the original key since key expansion needs to happen in intrinsics due to
6025     // compatibility issues between Java key expansion and SPARC crypto instructions
6026     Node* original_k_start = get_original_key_start_from_aescrypt_object(aescrypt_object);
6027     if (original_k_start == NULL) return false;
6028 
6029     // Call the stub.
6030     make_runtime_call(RC_LEAF|RC_NO_FP, OptoRuntime::aescrypt_block_Type(),
6031                       stubAddr, stubName, TypePtr::BOTTOM,
6032                       src_start, dest_start, k_start, original_k_start);
6033   } else {
6034     // Call the stub.
6035     make_runtime_call(RC_LEAF|RC_NO_FP, OptoRuntime::aescrypt_block_Type(),
6036                       stubAddr, stubName, TypePtr::BOTTOM,
6037                       src_start, dest_start, k_start);
6038   }
6039 
6040   return true;
6041 }
6042 
6043 //------------------------------inline_cipherBlockChaining_AESCrypt-----------------------
6044 bool LibraryCallKit::inline_cipherBlockChaining_AESCrypt(vmIntrinsics::ID id) {
6045   address stubAddr = NULL;
6046   const char *stubName = NULL;
6047 
6048   assert(UseAES, "need AES instruction support");
6049 
6050   switch(id) {
6051   case vmIntrinsics::_cipherBlockChaining_encryptAESCrypt:
6052     stubAddr = StubRoutines::cipherBlockChaining_encryptAESCrypt();
6053     stubName = "cipherBlockChaining_encryptAESCrypt";
6054     break;
6055   case vmIntrinsics::_cipherBlockChaining_decryptAESCrypt:
6056     stubAddr = StubRoutines::cipherBlockChaining_decryptAESCrypt();
6057     stubName = "cipherBlockChaining_decryptAESCrypt";
6058     break;
6059   }
6060   if (stubAddr == NULL) return false;
6061 
6062   Node* cipherBlockChaining_object = argument(0);
6063   Node* src                        = argument(1);
6064   Node* src_offset                 = argument(2);
6065   Node* len                        = argument(3);
6066   Node* dest                       = argument(4);
6067   Node* dest_offset                = argument(5);
6068 
6069   // (1) src and dest are arrays.
6070   const Type* src_type = src->Value(&_gvn);
6071   const Type* dest_type = dest->Value(&_gvn);
6072   const TypeAryPtr* top_src = src_type->isa_aryptr();
6073   const TypeAryPtr* top_dest = dest_type->isa_aryptr();
6074   assert (top_src  != NULL && top_src->klass()  != NULL
6075           &&  top_dest != NULL && top_dest->klass() != NULL, "args are strange");
6076 
6077   // checks are the responsibility of the caller
6078   Node* src_start  = src;
6079   Node* dest_start = dest;
6080   if (src_offset != NULL || dest_offset != NULL) {
6081     assert(src_offset != NULL && dest_offset != NULL, "");
6082     src_start  = array_element_address(src,  src_offset,  T_BYTE);
6083     dest_start = array_element_address(dest, dest_offset, T_BYTE);
6084   }
6085 
6086   // if we are in this set of code, we "know" the embeddedCipher is an AESCrypt object
6087   // (because of the predicated logic executed earlier).
6088   // so we cast it here safely.
6089   // this requires a newer class file that has this array as littleEndian ints, otherwise we revert to java
6090 
6091   Node* embeddedCipherObj = load_field_from_object(cipherBlockChaining_object, "embeddedCipher", "Lcom/sun/crypto/provider/SymmetricCipher;", /*is_exact*/ false);
6092   if (embeddedCipherObj == NULL) return false;
6093 
6094   // cast it to what we know it will be at runtime
6095   const TypeInstPtr* tinst = _gvn.type(cipherBlockChaining_object)->isa_instptr();
6096   assert(tinst != NULL, "CBC obj is null");
6097   assert(tinst->klass()->is_loaded(), "CBC obj is not loaded");
6098   ciKlass* klass_AESCrypt = tinst->klass()->as_instance_klass()->find_klass(ciSymbol::make("com/sun/crypto/provider/AESCrypt"));
6099   assert(klass_AESCrypt->is_loaded(), "predicate checks that this class is loaded");
6100 
6101   ciInstanceKlass* instklass_AESCrypt = klass_AESCrypt->as_instance_klass();
6102   const TypeKlassPtr* aklass = TypeKlassPtr::make(instklass_AESCrypt);
6103   const TypeOopPtr* xtype = aklass->as_instance_type();
6104   Node* aescrypt_object = new CheckCastPPNode(control(), embeddedCipherObj, xtype);
6105   aescrypt_object = _gvn.transform(aescrypt_object);
6106 
6107   // we need to get the start of the aescrypt_object's expanded key array
6108   Node* k_start = get_key_start_from_aescrypt_object(aescrypt_object);
6109   if (k_start == NULL) return false;
6110 
6111   // similarly, get the start address of the r vector
6112   Node* objRvec = load_field_from_object(cipherBlockChaining_object, "r", "[B", /*is_exact*/ false);
6113   if (objRvec == NULL) return false;
6114   Node* r_start = array_element_address(objRvec, intcon(0), T_BYTE);
6115 
6116   Node* cbcCrypt;
6117   if (Matcher::pass_original_key_for_aes()) {
6118     // on SPARC we need to pass the original key since key expansion needs to happen in intrinsics due to
6119     // compatibility issues between Java key expansion and SPARC crypto instructions
6120     Node* original_k_start = get_original_key_start_from_aescrypt_object(aescrypt_object);
6121     if (original_k_start == NULL) return false;
6122 
6123     // Call the stub, passing src_start, dest_start, k_start, r_start, src_len and original_k_start
6124     cbcCrypt = make_runtime_call(RC_LEAF|RC_NO_FP,
6125                                  OptoRuntime::cipherBlockChaining_aescrypt_Type(),
6126                                  stubAddr, stubName, TypePtr::BOTTOM,
6127                                  src_start, dest_start, k_start, r_start, len, original_k_start);
6128   } else {
6129     // Call the stub, passing src_start, dest_start, k_start, r_start and src_len
6130     cbcCrypt = make_runtime_call(RC_LEAF|RC_NO_FP,
6131                                  OptoRuntime::cipherBlockChaining_aescrypt_Type(),
6132                                  stubAddr, stubName, TypePtr::BOTTOM,
6133                                  src_start, dest_start, k_start, r_start, len);
6134   }
6135 
6136   // return cipher length (int)
6137   Node* retvalue = _gvn.transform(new ProjNode(cbcCrypt, TypeFunc::Parms));
6138   set_result(retvalue);
6139   return true;
6140 }
6141 
6142 //------------------------------get_key_start_from_aescrypt_object-----------------------
6143 Node * LibraryCallKit::get_key_start_from_aescrypt_object(Node *aescrypt_object) {
6144   Node* objAESCryptKey = load_field_from_object(aescrypt_object, "K", "[I", /*is_exact*/ false);
6145   assert (objAESCryptKey != NULL, "wrong version of com.sun.crypto.provider.AESCrypt");
6146   if (objAESCryptKey == NULL) return (Node *) NULL;
6147 
6148   // now have the array, need to get the start address of the K array
6149   Node* k_start = array_element_address(objAESCryptKey, intcon(0), T_INT);
6150   return k_start;
6151 }
6152 
6153 //------------------------------get_original_key_start_from_aescrypt_object-----------------------
6154 Node * LibraryCallKit::get_original_key_start_from_aescrypt_object(Node *aescrypt_object) {
6155   Node* objAESCryptKey = load_field_from_object(aescrypt_object, "lastKey", "[B", /*is_exact*/ false);
6156   assert (objAESCryptKey != NULL, "wrong version of com.sun.crypto.provider.AESCrypt");
6157   if (objAESCryptKey == NULL) return (Node *) NULL;
6158 
6159   // now have the array, need to get the start address of the lastKey array
6160   Node* original_k_start = array_element_address(objAESCryptKey, intcon(0), T_BYTE);
6161   return original_k_start;
6162 }
6163 
6164 //----------------------------inline_cipherBlockChaining_AESCrypt_predicate----------------------------
6165 // Return node representing slow path of predicate check.
6166 // the pseudo code we want to emulate with this predicate is:
6167 // for encryption:
6168 //    if (embeddedCipherObj instanceof AESCrypt) do_intrinsic, else do_javapath
6169 // for decryption:
6170 //    if ((embeddedCipherObj instanceof AESCrypt) && (cipher!=plain)) do_intrinsic, else do_javapath
6171 //    note cipher==plain is more conservative than the original java code but that's OK
6172 //
6173 Node* LibraryCallKit::inline_cipherBlockChaining_AESCrypt_predicate(bool decrypting) {
6174   // The receiver was checked for NULL already.
6175   Node* objCBC = argument(0);
6176 
6177   // Load embeddedCipher field of CipherBlockChaining object.
6178   Node* embeddedCipherObj = load_field_from_object(objCBC, "embeddedCipher", "Lcom/sun/crypto/provider/SymmetricCipher;", /*is_exact*/ false);
6179 
6180   // get AESCrypt klass for instanceOf check
6181   // AESCrypt might not be loaded yet if some other SymmetricCipher got us to this compile point
6182   // will have same classloader as CipherBlockChaining object
6183   const TypeInstPtr* tinst = _gvn.type(objCBC)->isa_instptr();
6184   assert(tinst != NULL, "CBCobj is null");
6185   assert(tinst->klass()->is_loaded(), "CBCobj is not loaded");
6186 
6187   // we want to do an instanceof comparison against the AESCrypt class
6188   ciKlass* klass_AESCrypt = tinst->klass()->as_instance_klass()->find_klass(ciSymbol::make("com/sun/crypto/provider/AESCrypt"));
6189   if (!klass_AESCrypt->is_loaded()) {
6190     // if AESCrypt is not even loaded, we never take the intrinsic fast path
6191     Node* ctrl = control();
6192     set_control(top()); // no regular fast path
6193     return ctrl;
6194   }
6195   ciInstanceKlass* instklass_AESCrypt = klass_AESCrypt->as_instance_klass();
6196 
6197   Node* instof = gen_instanceof(embeddedCipherObj, makecon(TypeKlassPtr::make(instklass_AESCrypt)));
6198   Node* cmp_instof  = _gvn.transform(new CmpINode(instof, intcon(1)));
6199   Node* bool_instof  = _gvn.transform(new BoolNode(cmp_instof, BoolTest::ne));
6200 
6201   Node* instof_false = generate_guard(bool_instof, NULL, PROB_MIN);
6202 
6203   // for encryption, we are done
6204   if (!decrypting)
6205     return instof_false;  // even if it is NULL
6206 
6207   // for decryption, we need to add a further check to avoid
6208   // taking the intrinsic path when cipher and plain are the same
6209   // see the original java code for why.
6210   RegionNode* region = new RegionNode(3);
6211   region->init_req(1, instof_false);
6212   Node* src = argument(1);
6213   Node* dest = argument(4);
6214   Node* cmp_src_dest = _gvn.transform(new CmpPNode(src, dest));
6215   Node* bool_src_dest = _gvn.transform(new BoolNode(cmp_src_dest, BoolTest::eq));
6216   Node* src_dest_conjoint = generate_guard(bool_src_dest, NULL, PROB_MIN);
6217   region->init_req(2, src_dest_conjoint);
6218 
6219   record_for_igvn(region);
6220   return _gvn.transform(region);
6221 }
6222 
6223 //------------------------------inline_ghash_processBlocks
6224 bool LibraryCallKit::inline_ghash_processBlocks() {
6225   address stubAddr;
6226   const char *stubName;
6227   assert(UseGHASHIntrinsics, "need GHASH intrinsics support");
6228 
6229   stubAddr = StubRoutines::ghash_processBlocks();
6230   stubName = "ghash_processBlocks";
6231 
6232   Node* data           = argument(0);
6233   Node* offset         = argument(1);
6234   Node* len            = argument(2);
6235   Node* state          = argument(3);
6236   Node* subkeyH        = argument(4);
6237 
6238   Node* state_start  = array_element_address(state, intcon(0), T_LONG);
6239   assert(state_start, "state is NULL");
6240   Node* subkeyH_start  = array_element_address(subkeyH, intcon(0), T_LONG);
6241   assert(subkeyH_start, "subkeyH is NULL");
6242   Node* data_start  = array_element_address(data, offset, T_BYTE);
6243   assert(data_start, "data is NULL");
6244 
6245   Node* ghash = make_runtime_call(RC_LEAF|RC_NO_FP,
6246                                   OptoRuntime::ghash_processBlocks_Type(),
6247                                   stubAddr, stubName, TypePtr::BOTTOM,
6248                                   state_start, subkeyH_start, data_start, len);
6249   return true;
6250 }
6251 
6252 //------------------------------inline_sha_implCompress-----------------------
6253 //
6254 // Calculate SHA (i.e., SHA-1) for single-block byte[] array.
6255 // void com.sun.security.provider.SHA.implCompress(byte[] buf, int ofs)
6256 //
6257 // Calculate SHA2 (i.e., SHA-244 or SHA-256) for single-block byte[] array.
6258 // void com.sun.security.provider.SHA2.implCompress(byte[] buf, int ofs)
6259 //
6260 // Calculate SHA5 (i.e., SHA-384 or SHA-512) for single-block byte[] array.
6261 // void com.sun.security.provider.SHA5.implCompress(byte[] buf, int ofs)
6262 //
6263 bool LibraryCallKit::inline_sha_implCompress(vmIntrinsics::ID id) {
6264   assert(callee()->signature()->size() == 2, "sha_implCompress has 2 parameters");
6265 
6266   Node* sha_obj = argument(0);
6267   Node* src     = argument(1); // type oop
6268   Node* ofs     = argument(2); // type int
6269 
6270   const Type* src_type = src->Value(&_gvn);
6271   const TypeAryPtr* top_src = src_type->isa_aryptr();
6272   if (top_src  == NULL || top_src->klass()  == NULL) {
6273     // failed array check
6274     return false;
6275   }
6276   // Figure out the size and type of the elements we will be copying.
6277   BasicType src_elem = src_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
6278   if (src_elem != T_BYTE) {
6279     return false;
6280   }
6281   // 'src_start' points to src array + offset
6282   Node* src_start = array_element_address(src, ofs, src_elem);
6283   Node* state = NULL;
6284   address stubAddr;
6285   const char *stubName;
6286 
6287   switch(id) {
6288   case vmIntrinsics::_sha_implCompress:
6289     assert(UseSHA1Intrinsics, "need SHA1 instruction support");
6290     state = get_state_from_sha_object(sha_obj);
6291     stubAddr = StubRoutines::sha1_implCompress();
6292     stubName = "sha1_implCompress";
6293     break;
6294   case vmIntrinsics::_sha2_implCompress:
6295     assert(UseSHA256Intrinsics, "need SHA256 instruction support");
6296     state = get_state_from_sha_object(sha_obj);
6297     stubAddr = StubRoutines::sha256_implCompress();
6298     stubName = "sha256_implCompress";
6299     break;
6300   case vmIntrinsics::_sha5_implCompress:
6301     assert(UseSHA512Intrinsics, "need SHA512 instruction support");
6302     state = get_state_from_sha5_object(sha_obj);
6303     stubAddr = StubRoutines::sha512_implCompress();
6304     stubName = "sha512_implCompress";
6305     break;
6306   default:
6307     fatal_unexpected_iid(id);
6308     return false;
6309   }
6310   if (state == NULL) return false;
6311 
6312   // Call the stub.
6313   Node* call = make_runtime_call(RC_LEAF|RC_NO_FP, OptoRuntime::sha_implCompress_Type(),
6314                                  stubAddr, stubName, TypePtr::BOTTOM,
6315                                  src_start, state);
6316 
6317   return true;
6318 }
6319 
6320 //------------------------------inline_digestBase_implCompressMB-----------------------
6321 //
6322 // Calculate SHA/SHA2/SHA5 for multi-block byte[] array.
6323 // int com.sun.security.provider.DigestBase.implCompressMultiBlock(byte[] b, int ofs, int limit)
6324 //
6325 bool LibraryCallKit::inline_digestBase_implCompressMB(int predicate) {
6326   assert(UseSHA1Intrinsics || UseSHA256Intrinsics || UseSHA512Intrinsics,
6327          "need SHA1/SHA256/SHA512 instruction support");
6328   assert((uint)predicate < 3, "sanity");
6329   assert(callee()->signature()->size() == 3, "digestBase_implCompressMB has 3 parameters");
6330 
6331   Node* digestBase_obj = argument(0); // The receiver was checked for NULL already.
6332   Node* src            = argument(1); // byte[] array
6333   Node* ofs            = argument(2); // type int
6334   Node* limit          = argument(3); // type int
6335 
6336   const Type* src_type = src->Value(&_gvn);
6337   const TypeAryPtr* top_src = src_type->isa_aryptr();
6338   if (top_src  == NULL || top_src->klass()  == NULL) {
6339     // failed array check
6340     return false;
6341   }
6342   // Figure out the size and type of the elements we will be copying.
6343   BasicType src_elem = src_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
6344   if (src_elem != T_BYTE) {
6345     return false;
6346   }
6347   // 'src_start' points to src array + offset
6348   Node* src_start = array_element_address(src, ofs, src_elem);
6349 
6350   const char* klass_SHA_name = NULL;
6351   const char* stub_name = NULL;
6352   address     stub_addr = NULL;
6353   bool        long_state = false;
6354 
6355   switch (predicate) {
6356   case 0:
6357     if (UseSHA1Intrinsics) {
6358       klass_SHA_name = "sun/security/provider/SHA";
6359       stub_name = "sha1_implCompressMB";
6360       stub_addr = StubRoutines::sha1_implCompressMB();
6361     }
6362     break;
6363   case 1:
6364     if (UseSHA256Intrinsics) {
6365       klass_SHA_name = "sun/security/provider/SHA2";
6366       stub_name = "sha256_implCompressMB";
6367       stub_addr = StubRoutines::sha256_implCompressMB();
6368     }
6369     break;
6370   case 2:
6371     if (UseSHA512Intrinsics) {
6372       klass_SHA_name = "sun/security/provider/SHA5";
6373       stub_name = "sha512_implCompressMB";
6374       stub_addr = StubRoutines::sha512_implCompressMB();
6375       long_state = true;
6376     }
6377     break;
6378   default:
6379     fatal("unknown SHA intrinsic predicate: %d", predicate);
6380   }
6381   if (klass_SHA_name != NULL) {
6382     // get DigestBase klass to lookup for SHA klass
6383     const TypeInstPtr* tinst = _gvn.type(digestBase_obj)->isa_instptr();
6384     assert(tinst != NULL, "digestBase_obj is not instance???");
6385     assert(tinst->klass()->is_loaded(), "DigestBase is not loaded");
6386 
6387     ciKlass* klass_SHA = tinst->klass()->as_instance_klass()->find_klass(ciSymbol::make(klass_SHA_name));
6388     assert(klass_SHA->is_loaded(), "predicate checks that this class is loaded");
6389     ciInstanceKlass* instklass_SHA = klass_SHA->as_instance_klass();
6390     return inline_sha_implCompressMB(digestBase_obj, instklass_SHA, long_state, stub_addr, stub_name, src_start, ofs, limit);
6391   }
6392   return false;
6393 }
6394 //------------------------------inline_sha_implCompressMB-----------------------
6395 bool LibraryCallKit::inline_sha_implCompressMB(Node* digestBase_obj, ciInstanceKlass* instklass_SHA,
6396                                                bool long_state, address stubAddr, const char *stubName,
6397                                                Node* src_start, Node* ofs, Node* limit) {
6398   const TypeKlassPtr* aklass = TypeKlassPtr::make(instklass_SHA);
6399   const TypeOopPtr* xtype = aklass->as_instance_type();
6400   Node* sha_obj = new CheckCastPPNode(control(), digestBase_obj, xtype);
6401   sha_obj = _gvn.transform(sha_obj);
6402 
6403   Node* state;
6404   if (long_state) {
6405     state = get_state_from_sha5_object(sha_obj);
6406   } else {
6407     state = get_state_from_sha_object(sha_obj);
6408   }
6409   if (state == NULL) return false;
6410 
6411   // Call the stub.
6412   Node* call = make_runtime_call(RC_LEAF|RC_NO_FP,
6413                                  OptoRuntime::digestBase_implCompressMB_Type(),
6414                                  stubAddr, stubName, TypePtr::BOTTOM,
6415                                  src_start, state, ofs, limit);
6416   // return ofs (int)
6417   Node* result = _gvn.transform(new ProjNode(call, TypeFunc::Parms));
6418   set_result(result);
6419 
6420   return true;
6421 }
6422 
6423 //------------------------------get_state_from_sha_object-----------------------
6424 Node * LibraryCallKit::get_state_from_sha_object(Node *sha_object) {
6425   Node* sha_state = load_field_from_object(sha_object, "state", "[I", /*is_exact*/ false);
6426   assert (sha_state != NULL, "wrong version of sun.security.provider.SHA/SHA2");
6427   if (sha_state == NULL) return (Node *) NULL;
6428 
6429   // now have the array, need to get the start address of the state array
6430   Node* state = array_element_address(sha_state, intcon(0), T_INT);
6431   return state;
6432 }
6433 
6434 //------------------------------get_state_from_sha5_object-----------------------
6435 Node * LibraryCallKit::get_state_from_sha5_object(Node *sha_object) {
6436   Node* sha_state = load_field_from_object(sha_object, "state", "[J", /*is_exact*/ false);
6437   assert (sha_state != NULL, "wrong version of sun.security.provider.SHA5");
6438   if (sha_state == NULL) return (Node *) NULL;
6439 
6440   // now have the array, need to get the start address of the state array
6441   Node* state = array_element_address(sha_state, intcon(0), T_LONG);
6442   return state;
6443 }
6444 
6445 //----------------------------inline_digestBase_implCompressMB_predicate----------------------------
6446 // Return node representing slow path of predicate check.
6447 // the pseudo code we want to emulate with this predicate is:
6448 //    if (digestBaseObj instanceof SHA/SHA2/SHA5) do_intrinsic, else do_javapath
6449 //
6450 Node* LibraryCallKit::inline_digestBase_implCompressMB_predicate(int predicate) {
6451   assert(UseSHA1Intrinsics || UseSHA256Intrinsics || UseSHA512Intrinsics,
6452          "need SHA1/SHA256/SHA512 instruction support");
6453   assert((uint)predicate < 3, "sanity");
6454 
6455   // The receiver was checked for NULL already.
6456   Node* digestBaseObj = argument(0);
6457 
6458   // get DigestBase klass for instanceOf check
6459   const TypeInstPtr* tinst = _gvn.type(digestBaseObj)->isa_instptr();
6460   assert(tinst != NULL, "digestBaseObj is null");
6461   assert(tinst->klass()->is_loaded(), "DigestBase is not loaded");
6462 
6463   const char* klass_SHA_name = NULL;
6464   switch (predicate) {
6465   case 0:
6466     if (UseSHA1Intrinsics) {
6467       // we want to do an instanceof comparison against the SHA class
6468       klass_SHA_name = "sun/security/provider/SHA";
6469     }
6470     break;
6471   case 1:
6472     if (UseSHA256Intrinsics) {
6473       // we want to do an instanceof comparison against the SHA2 class
6474       klass_SHA_name = "sun/security/provider/SHA2";
6475     }
6476     break;
6477   case 2:
6478     if (UseSHA512Intrinsics) {
6479       // we want to do an instanceof comparison against the SHA5 class
6480       klass_SHA_name = "sun/security/provider/SHA5";
6481     }
6482     break;
6483   default:
6484     fatal("unknown SHA intrinsic predicate: %d", predicate);
6485   }
6486 
6487   ciKlass* klass_SHA = NULL;
6488   if (klass_SHA_name != NULL) {
6489     klass_SHA = tinst->klass()->as_instance_klass()->find_klass(ciSymbol::make(klass_SHA_name));
6490   }
6491   if ((klass_SHA == NULL) || !klass_SHA->is_loaded()) {
6492     // if none of SHA/SHA2/SHA5 is loaded, we never take the intrinsic fast path
6493     Node* ctrl = control();
6494     set_control(top()); // no intrinsic path
6495     return ctrl;
6496   }
6497   ciInstanceKlass* instklass_SHA = klass_SHA->as_instance_klass();
6498 
6499   Node* instofSHA = gen_instanceof(digestBaseObj, makecon(TypeKlassPtr::make(instklass_SHA)));
6500   Node* cmp_instof = _gvn.transform(new CmpINode(instofSHA, intcon(1)));
6501   Node* bool_instof = _gvn.transform(new BoolNode(cmp_instof, BoolTest::ne));
6502   Node* instof_false = generate_guard(bool_instof, NULL, PROB_MIN);
6503 
6504   return instof_false;  // even if it is NULL
6505 }
6506 
6507 bool LibraryCallKit::inline_profileBoolean() {
6508   Node* counts = argument(1);
6509   const TypeAryPtr* ary = NULL;
6510   ciArray* aobj = NULL;
6511   if (counts->is_Con()
6512       && (ary = counts->bottom_type()->isa_aryptr()) != NULL
6513       && (aobj = ary->const_oop()->as_array()) != NULL
6514       && (aobj->length() == 2)) {
6515     // Profile is int[2] where [0] and [1] correspond to false and true value occurrences respectively.
6516     jint false_cnt = aobj->element_value(0).as_int();
6517     jint  true_cnt = aobj->element_value(1).as_int();
6518 
6519     if (C->log() != NULL) {
6520       C->log()->elem("observe source='profileBoolean' false='%d' true='%d'",
6521                      false_cnt, true_cnt);
6522     }
6523 
6524     if (false_cnt + true_cnt == 0) {
6525       // According to profile, never executed.
6526       uncommon_trap_exact(Deoptimization::Reason_intrinsic,
6527                           Deoptimization::Action_reinterpret);
6528       return true;
6529     }
6530 
6531     // result is a boolean (0 or 1) and its profile (false_cnt & true_cnt)
6532     // is a number of each value occurrences.
6533     Node* result = argument(0);
6534     if (false_cnt == 0 || true_cnt == 0) {
6535       // According to profile, one value has been never seen.
6536       int expected_val = (false_cnt == 0) ? 1 : 0;
6537 
6538       Node* cmp  = _gvn.transform(new CmpINode(result, intcon(expected_val)));
6539       Node* test = _gvn.transform(new BoolNode(cmp, BoolTest::eq));
6540 
6541       IfNode* check = create_and_map_if(control(), test, PROB_ALWAYS, COUNT_UNKNOWN);
6542       Node* fast_path = _gvn.transform(new IfTrueNode(check));
6543       Node* slow_path = _gvn.transform(new IfFalseNode(check));
6544 
6545       { // Slow path: uncommon trap for never seen value and then reexecute
6546         // MethodHandleImpl::profileBoolean() to bump the count, so JIT knows
6547         // the value has been seen at least once.
6548         PreserveJVMState pjvms(this);
6549         PreserveReexecuteState preexecs(this);
6550         jvms()->set_should_reexecute(true);
6551 
6552         set_control(slow_path);
6553         set_i_o(i_o());
6554 
6555         uncommon_trap_exact(Deoptimization::Reason_intrinsic,
6556                             Deoptimization::Action_reinterpret);
6557       }
6558       // The guard for never seen value enables sharpening of the result and
6559       // returning a constant. It allows to eliminate branches on the same value
6560       // later on.
6561       set_control(fast_path);
6562       result = intcon(expected_val);
6563     }
6564     // Stop profiling.
6565     // MethodHandleImpl::profileBoolean() has profiling logic in its bytecode.
6566     // By replacing method body with profile data (represented as ProfileBooleanNode
6567     // on IR level) we effectively disable profiling.
6568     // It enables full speed execution once optimized code is generated.
6569     Node* profile = _gvn.transform(new ProfileBooleanNode(result, false_cnt, true_cnt));
6570     C->record_for_igvn(profile);
6571     set_result(profile);
6572     return true;
6573   } else {
6574     // Continue profiling.
6575     // Profile data isn't available at the moment. So, execute method's bytecode version.
6576     // Usually, when GWT LambdaForms are profiled it means that a stand-alone nmethod
6577     // is compiled and counters aren't available since corresponding MethodHandle
6578     // isn't a compile-time constant.
6579     return false;
6580   }
6581 }
6582 
6583 bool LibraryCallKit::inline_isCompileConstant() {
6584   Node* n = argument(0);
6585   set_result(n->is_Con() ? intcon(1) : intcon(0));
6586   return true;
6587 }