src/share/vm/opto/library_call.cpp
Index Unified diffs Context diffs Sdiffs Patch New Old Previous File Next File g1-bulk-zeroing-reduction Sdiff src/share/vm/opto

src/share/vm/opto/library_call.cpp

Print this page


   1 /*
   2  * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *


  80   int               arg_size()  const    { return callee()->arg_size(); }
  81 
  82   bool try_to_inline();
  83 
  84   // Helper functions to inline natives
  85   void push_result(RegionNode* region, PhiNode* value);
  86   Node* generate_guard(Node* test, RegionNode* region, float true_prob);
  87   Node* generate_slow_guard(Node* test, RegionNode* region);
  88   Node* generate_fair_guard(Node* test, RegionNode* region);
  89   Node* generate_negative_guard(Node* index, RegionNode* region,
  90                                 // resulting CastII of index:
  91                                 Node* *pos_index = NULL);
  92   Node* generate_nonpositive_guard(Node* index, bool never_negative,
  93                                    // resulting CastII of index:
  94                                    Node* *pos_index = NULL);
  95   Node* generate_limit_guard(Node* offset, Node* subseq_length,
  96                              Node* array_length,
  97                              RegionNode* region);
  98   Node* generate_current_thread(Node* &tls_output);
  99   address basictype2arraycopy(BasicType t, Node *src_offset, Node *dest_offset,
 100                               bool disjoint_bases, const char* &name);
 101   Node* load_mirror_from_klass(Node* klass);
 102   Node* load_klass_from_mirror_common(Node* mirror, bool never_see_null,
 103                                       int nargs,
 104                                       RegionNode* region, int null_path,
 105                                       int offset);
 106   Node* load_klass_from_mirror(Node* mirror, bool never_see_null, int nargs,
 107                                RegionNode* region, int null_path) {
 108     int offset = java_lang_Class::klass_offset_in_bytes();
 109     return load_klass_from_mirror_common(mirror, never_see_null, nargs,
 110                                          region, null_path,
 111                                          offset);
 112   }
 113   Node* load_array_klass_from_mirror(Node* mirror, bool never_see_null,
 114                                      int nargs,
 115                                      RegionNode* region, int null_path) {
 116     int offset = java_lang_Class::array_klass_offset_in_bytes();
 117     return load_klass_from_mirror_common(mirror, never_see_null, nargs,
 118                                          region, null_path,
 119                                          offset);
 120   }


 204   void generate_clear_array(const TypePtr* adr_type,
 205                             Node* dest,
 206                             BasicType basic_elem_type,
 207                             Node* slice_off,
 208                             Node* slice_len,
 209                             Node* slice_end);
 210   bool generate_block_arraycopy(const TypePtr* adr_type,
 211                                 BasicType basic_elem_type,
 212                                 AllocateNode* alloc,
 213                                 Node* src,  Node* src_offset,
 214                                 Node* dest, Node* dest_offset,
 215                                 Node* dest_size);
 216   void generate_slow_arraycopy(const TypePtr* adr_type,
 217                                Node* src,  Node* src_offset,
 218                                Node* dest, Node* dest_offset,
 219                                Node* copy_length);
 220   Node* generate_checkcast_arraycopy(const TypePtr* adr_type,
 221                                      Node* dest_elem_klass,
 222                                      Node* src,  Node* src_offset,
 223                                      Node* dest, Node* dest_offset,
 224                                      Node* copy_length);
 225   Node* generate_generic_arraycopy(const TypePtr* adr_type,
 226                                    Node* src,  Node* src_offset,
 227                                    Node* dest, Node* dest_offset,
 228                                    Node* copy_length);
 229   void generate_unchecked_arraycopy(const TypePtr* adr_type,
 230                                     BasicType basic_elem_type,
 231                                     bool disjoint_bases,
 232                                     Node* src,  Node* src_offset,
 233                                     Node* dest, Node* dest_offset,
 234                                     Node* copy_length);
 235   bool inline_unsafe_CAS(BasicType type);
 236   bool inline_unsafe_ordered_store(BasicType type);
 237   bool inline_fp_conversions(vmIntrinsics::ID id);
 238   bool inline_numberOfLeadingZeros(vmIntrinsics::ID id);
 239   bool inline_numberOfTrailingZeros(vmIntrinsics::ID id);
 240   bool inline_bitCount(vmIntrinsics::ID id);
 241   bool inline_reverseBytes(vmIntrinsics::ID id);
 242 };
 243 
 244 
 245 //---------------------------make_vm_intrinsic----------------------------
 246 CallGenerator* Compile::make_vm_intrinsic(ciMethod* m, bool is_virtual) {
 247   vmIntrinsics::ID id = m->intrinsic_id();
 248   assert(id != vmIntrinsics::_none, "must be a VM intrinsic");
 249 
 250   if (DisableIntrinsic[0] != '\0'
 251       && strstr(DisableIntrinsic, vmIntrinsics::name_at(id)) != NULL) {
 252     // disabled by a user request on the command line:
 253     // example: -XX:DisableIntrinsic=_hashCode,_getClass
 254     return NULL;


4278     set_all_memory( _gvn.transform(result_mem) );
4279   } //original reexecute and sp are set back here
4280 
4281   push(_gvn.transform(result_val));
4282 
4283   return true;
4284 }
4285 
4286 
4287 // constants for computing the copy function
4288 enum {
4289   COPYFUNC_UNALIGNED = 0,
4290   COPYFUNC_ALIGNED = 1,                 // src, dest aligned to HeapWordSize
4291   COPYFUNC_CONJOINT = 0,
4292   COPYFUNC_DISJOINT = 2                 // src != dest, or transfer can descend
4293 };
4294 
4295 // Note:  The condition "disjoint" applies also for overlapping copies
4296 // where an descending copy is permitted (i.e., dest_offset <= src_offset).
4297 static address
4298 select_arraycopy_function(BasicType t, bool aligned, bool disjoint, const char* &name) {
4299   int selector =
4300     (aligned  ? COPYFUNC_ALIGNED  : COPYFUNC_UNALIGNED) +
4301     (disjoint ? COPYFUNC_DISJOINT : COPYFUNC_CONJOINT);
4302 
4303 #define RETURN_STUB(xxx_arraycopy) { \
4304   name = #xxx_arraycopy; \
4305   return StubRoutines::xxx_arraycopy(); }
4306 




4307   switch (t) {
4308   case T_BYTE:
4309   case T_BOOLEAN:
4310     switch (selector) {
4311     case COPYFUNC_CONJOINT | COPYFUNC_UNALIGNED:  RETURN_STUB(jbyte_arraycopy);
4312     case COPYFUNC_CONJOINT | COPYFUNC_ALIGNED:    RETURN_STUB(arrayof_jbyte_arraycopy);
4313     case COPYFUNC_DISJOINT | COPYFUNC_UNALIGNED:  RETURN_STUB(jbyte_disjoint_arraycopy);
4314     case COPYFUNC_DISJOINT | COPYFUNC_ALIGNED:    RETURN_STUB(arrayof_jbyte_disjoint_arraycopy);
4315     }
4316   case T_CHAR:
4317   case T_SHORT:
4318     switch (selector) {
4319     case COPYFUNC_CONJOINT | COPYFUNC_UNALIGNED:  RETURN_STUB(jshort_arraycopy);
4320     case COPYFUNC_CONJOINT | COPYFUNC_ALIGNED:    RETURN_STUB(arrayof_jshort_arraycopy);
4321     case COPYFUNC_DISJOINT | COPYFUNC_UNALIGNED:  RETURN_STUB(jshort_disjoint_arraycopy);
4322     case COPYFUNC_DISJOINT | COPYFUNC_ALIGNED:    RETURN_STUB(arrayof_jshort_disjoint_arraycopy);
4323     }
4324   case T_INT:
4325   case T_FLOAT:
4326     switch (selector) {
4327     case COPYFUNC_CONJOINT | COPYFUNC_UNALIGNED:  RETURN_STUB(jint_arraycopy);
4328     case COPYFUNC_CONJOINT | COPYFUNC_ALIGNED:    RETURN_STUB(arrayof_jint_arraycopy);
4329     case COPYFUNC_DISJOINT | COPYFUNC_UNALIGNED:  RETURN_STUB(jint_disjoint_arraycopy);
4330     case COPYFUNC_DISJOINT | COPYFUNC_ALIGNED:    RETURN_STUB(arrayof_jint_disjoint_arraycopy);
4331     }
4332   case T_DOUBLE:
4333   case T_LONG:
4334     switch (selector) {
4335     case COPYFUNC_CONJOINT | COPYFUNC_UNALIGNED:  RETURN_STUB(jlong_arraycopy);
4336     case COPYFUNC_CONJOINT | COPYFUNC_ALIGNED:    RETURN_STUB(arrayof_jlong_arraycopy);
4337     case COPYFUNC_DISJOINT | COPYFUNC_UNALIGNED:  RETURN_STUB(jlong_disjoint_arraycopy);
4338     case COPYFUNC_DISJOINT | COPYFUNC_ALIGNED:    RETURN_STUB(arrayof_jlong_disjoint_arraycopy);
4339     }
4340   case T_ARRAY:
4341   case T_OBJECT:
4342     switch (selector) {
4343     case COPYFUNC_CONJOINT | COPYFUNC_UNALIGNED:  RETURN_STUB(oop_arraycopy);
4344     case COPYFUNC_CONJOINT | COPYFUNC_ALIGNED:    RETURN_STUB(arrayof_oop_arraycopy);
4345     case COPYFUNC_DISJOINT | COPYFUNC_UNALIGNED:  RETURN_STUB(oop_disjoint_arraycopy);
4346     case COPYFUNC_DISJOINT | COPYFUNC_ALIGNED:    RETURN_STUB(arrayof_oop_disjoint_arraycopy);
4347     }
4348   default:
4349     ShouldNotReachHere();
4350     return NULL;
4351   }
4352 
4353 #undef RETURN_STUB

4354 }
4355 
4356 //------------------------------basictype2arraycopy----------------------------
4357 address LibraryCallKit::basictype2arraycopy(BasicType t,
4358                                             Node* src_offset,
4359                                             Node* dest_offset,
4360                                             bool disjoint_bases,
4361                                             const char* &name) {

4362   const TypeInt* src_offset_inttype  = gvn().find_int_type(src_offset);;
4363   const TypeInt* dest_offset_inttype = gvn().find_int_type(dest_offset);;
4364 
4365   bool aligned = false;
4366   bool disjoint = disjoint_bases;
4367 
4368   // if the offsets are the same, we can treat the memory regions as
4369   // disjoint, because either the memory regions are in different arrays,
4370   // or they are identical (which we can treat as disjoint.)  We can also
4371   // treat a copy with a destination index  less that the source index
4372   // as disjoint since a low->high copy will work correctly in this case.
4373   if (src_offset_inttype != NULL && src_offset_inttype->is_con() &&
4374       dest_offset_inttype != NULL && dest_offset_inttype->is_con()) {
4375     // both indices are constants
4376     int s_offs = src_offset_inttype->get_con();
4377     int d_offs = dest_offset_inttype->get_con();
4378     int element_size = type2aelembytes(t);
4379     aligned = ((arrayOopDesc::base_offset_in_bytes(t) + s_offs * element_size) % HeapWordSize == 0) &&
4380               ((arrayOopDesc::base_offset_in_bytes(t) + d_offs * element_size) % HeapWordSize == 0);
4381     if (s_offs >= d_offs)  disjoint = true;
4382   } else if (src_offset == dest_offset && src_offset != NULL) {
4383     // This can occur if the offsets are identical non-constants.
4384     disjoint = true;
4385   }
4386 
4387   return select_arraycopy_function(t, aligned, disjoint, name);
4388 }
4389 
4390 
4391 //------------------------------inline_arraycopy-----------------------
4392 bool LibraryCallKit::inline_arraycopy() {
4393   // Restore the stack and pop off the arguments.
4394   int nargs = 5;  // 2 oops, 3 ints, no size_t or long
4395   assert(callee()->signature()->size() == nargs, "copy has 5 arguments");
4396 
4397   Node *src         = argument(0);
4398   Node *src_offset  = argument(1);
4399   Node *dest        = argument(2);
4400   Node *dest_offset = argument(3);
4401   Node *length      = argument(4);
4402 
4403   // Compile time checks.  If any of these checks cannot be verified at compile time,
4404   // we do not make a fast path for this call.  Instead, we let the call remain as it
4405   // is.  The checks we choose to mandate at compile time are:
4406   //
4407   // (1) src and dest are arrays.


4757     // The backing store for a List<String> is always an Object[],
4758     // but its elements are always type String, if the generic types
4759     // are correct at the source level.
4760     //
4761     // Test S[] against D[], not S against D, because (probably)
4762     // the secondary supertype cache is less busy for S[] than S.
4763     // This usually only matters when D is an interface.
4764     Node* not_subtype_ctrl = gen_subtype_check(src_klass, dest_klass);
4765     // Plug failing path into checked_oop_disjoint_arraycopy
4766     if (not_subtype_ctrl != top()) {
4767       PreserveJVMState pjvms(this);
4768       set_control(not_subtype_ctrl);
4769       // (At this point we can assume disjoint_bases, since types differ.)
4770       int ek_offset = objArrayKlass::element_klass_offset_in_bytes() + sizeof(oopDesc);
4771       Node* p1 = basic_plus_adr(dest_klass, ek_offset);
4772       Node* n1 = LoadKlassNode::make(_gvn, immutable_memory(), p1, TypeRawPtr::BOTTOM);
4773       Node* dest_elem_klass = _gvn.transform(n1);
4774       Node* cv = generate_checkcast_arraycopy(adr_type,
4775                                               dest_elem_klass,
4776                                               src, src_offset, dest, dest_offset,
4777                                               ConvI2X(copy_length));
4778       if (cv == NULL)  cv = intcon(-1);  // failure (no stub available)
4779       checked_control = control();
4780       checked_i_o     = i_o();
4781       checked_mem     = memory(adr_type);
4782       checked_value   = cv;
4783     }
4784     // At this point we know we do not need type checks on oop stores.
4785 
4786     // Let's see if we need card marks:
4787     if (alloc != NULL && use_ReduceInitialCardMarks()) {
4788       // If we do not need card marks, copy using the jint or jlong stub.
4789       copy_type = LP64_ONLY(UseCompressedOops ? T_INT : T_LONG) NOT_LP64(T_INT);
4790       assert(type2aelembytes(basic_elem_type) == type2aelembytes(copy_type),
4791              "sizes agree");
4792     }
4793   }
4794 
4795   if (!stopped()) {
4796     // Generate the fast path, if possible.
4797     PreserveJVMState pjvms(this);
4798     generate_unchecked_arraycopy(adr_type, copy_type, disjoint_bases,
4799                                  src, src_offset, dest, dest_offset,
4800                                  ConvI2X(copy_length));
4801 
4802     // Present the results of the fast call.
4803     result_region->init_req(fast_path, control());
4804     result_i_o   ->init_req(fast_path, i_o());
4805     result_memory->init_req(fast_path, memory(adr_type));
4806   }
4807 
4808   // Here are all the slow paths up to this point, in one bundle:
4809   slow_control = top();
4810   if (slow_region != NULL)
4811     slow_control = _gvn.transform(slow_region);
4812   debug_only(slow_region = (RegionNode*)badAddress);
4813 
4814   set_control(checked_control);
4815   if (!stopped()) {
4816     // Clean up after the checked call.
4817     // The returned value is either 0 or -1^K,
4818     // where K = number of partially transferred array elements.
4819     Node* cmp = _gvn.transform( new(C, 3) CmpINode(checked_value, intcon(0)) );
4820     Node* bol = _gvn.transform( new(C, 2) BoolNode(cmp, BoolTest::eq) );


5187                                         Node* src,  Node* src_offset,
5188                                         Node* dest, Node* dest_offset,
5189                                         Node* copy_length) {
5190   Node* call = make_runtime_call(RC_NO_LEAF | RC_UNCOMMON,
5191                                  OptoRuntime::slow_arraycopy_Type(),
5192                                  OptoRuntime::slow_arraycopy_Java(),
5193                                  "slow_arraycopy", adr_type,
5194                                  src, src_offset, dest, dest_offset,
5195                                  copy_length);
5196 
5197   // Handle exceptions thrown by this fellow:
5198   make_slow_call_ex(call, env()->Throwable_klass(), false);
5199 }
5200 
5201 // Helper function; generates code for cases requiring runtime checks.
5202 Node*
5203 LibraryCallKit::generate_checkcast_arraycopy(const TypePtr* adr_type,
5204                                              Node* dest_elem_klass,
5205                                              Node* src,  Node* src_offset,
5206                                              Node* dest, Node* dest_offset,
5207                                              Node* copy_length) {
5208   if (stopped())  return NULL;
5209 
5210   address copyfunc_addr = StubRoutines::checkcast_arraycopy();
5211   if (copyfunc_addr == NULL) { // Stub was not generated, go slow path.
5212     return NULL;
5213   }
5214 
5215   // Pick out the parameters required to perform a store-check
5216   // for the target array.  This is an optimistic check.  It will
5217   // look in each non-null element's class, at the desired klass's
5218   // super_check_offset, for the desired klass.
5219   int sco_offset = Klass::super_check_offset_offset_in_bytes() + sizeof(oopDesc);
5220   Node* p3 = basic_plus_adr(dest_elem_klass, sco_offset);
5221   Node* n3 = new(C, 3) LoadINode(NULL, memory(p3), p3, _gvn.type(p3)->is_ptr());
5222   Node* check_offset = ConvI2X(_gvn.transform(n3));
5223   Node* check_value  = dest_elem_klass;
5224 
5225   Node* src_start  = array_element_address(src,  src_offset,  T_OBJECT);
5226   Node* dest_start = array_element_address(dest, dest_offset, T_OBJECT);
5227 
5228   // (We know the arrays are never conjoint, because their types differ.)
5229   Node* call = make_runtime_call(RC_LEAF|RC_NO_FP,
5230                                  OptoRuntime::checkcast_arraycopy_Type(),


5251   address copyfunc_addr = StubRoutines::generic_arraycopy();
5252   if (copyfunc_addr == NULL) { // Stub was not generated, go slow path.
5253     return NULL;
5254   }
5255 
5256   Node* call = make_runtime_call(RC_LEAF|RC_NO_FP,
5257                     OptoRuntime::generic_arraycopy_Type(),
5258                     copyfunc_addr, "generic_arraycopy", adr_type,
5259                     src, src_offset, dest, dest_offset, copy_length);
5260 
5261   return _gvn.transform(new (C, 1) ProjNode(call, TypeFunc::Parms));
5262 }
5263 
5264 // Helper function; generates the fast out-of-line call to an arraycopy stub.
5265 void
5266 LibraryCallKit::generate_unchecked_arraycopy(const TypePtr* adr_type,
5267                                              BasicType basic_elem_type,
5268                                              bool disjoint_bases,
5269                                              Node* src,  Node* src_offset,
5270                                              Node* dest, Node* dest_offset,
5271                                              Node* copy_length) {
5272   if (stopped())  return;               // nothing to do
5273 
5274   Node* src_start  = src;
5275   Node* dest_start = dest;
5276   if (src_offset != NULL || dest_offset != NULL) {
5277     assert(src_offset != NULL && dest_offset != NULL, "");
5278     src_start  = array_element_address(src,  src_offset,  basic_elem_type);
5279     dest_start = array_element_address(dest, dest_offset, basic_elem_type);
5280   }
5281 
5282   // Figure out which arraycopy runtime method to call.
5283   const char* copyfunc_name = "arraycopy";
5284   address     copyfunc_addr =
5285       basictype2arraycopy(basic_elem_type, src_offset, dest_offset,
5286                           disjoint_bases, copyfunc_name);
5287 
5288   // Call it.  Note that the count_ix value is not scaled to a byte-size.
5289   make_runtime_call(RC_LEAF|RC_NO_FP,
5290                     OptoRuntime::fast_arraycopy_Type(),
5291                     copyfunc_addr, copyfunc_name, adr_type,
5292                     src_start, dest_start, copy_length XTOP);
5293 }
   1 /*
   2  * Copyright (c) 1999, 2011, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *


  80   int               arg_size()  const    { return callee()->arg_size(); }
  81 
  82   bool try_to_inline();
  83 
  84   // Helper functions to inline natives
  85   void push_result(RegionNode* region, PhiNode* value);
  86   Node* generate_guard(Node* test, RegionNode* region, float true_prob);
  87   Node* generate_slow_guard(Node* test, RegionNode* region);
  88   Node* generate_fair_guard(Node* test, RegionNode* region);
  89   Node* generate_negative_guard(Node* index, RegionNode* region,
  90                                 // resulting CastII of index:
  91                                 Node* *pos_index = NULL);
  92   Node* generate_nonpositive_guard(Node* index, bool never_negative,
  93                                    // resulting CastII of index:
  94                                    Node* *pos_index = NULL);
  95   Node* generate_limit_guard(Node* offset, Node* subseq_length,
  96                              Node* array_length,
  97                              RegionNode* region);
  98   Node* generate_current_thread(Node* &tls_output);
  99   address basictype2arraycopy(BasicType t, Node *src_offset, Node *dest_offset,
 100                               bool disjoint_bases, const char* &name, bool need_pre_barrier = true);
 101   Node* load_mirror_from_klass(Node* klass);
 102   Node* load_klass_from_mirror_common(Node* mirror, bool never_see_null,
 103                                       int nargs,
 104                                       RegionNode* region, int null_path,
 105                                       int offset);
 106   Node* load_klass_from_mirror(Node* mirror, bool never_see_null, int nargs,
 107                                RegionNode* region, int null_path) {
 108     int offset = java_lang_Class::klass_offset_in_bytes();
 109     return load_klass_from_mirror_common(mirror, never_see_null, nargs,
 110                                          region, null_path,
 111                                          offset);
 112   }
 113   Node* load_array_klass_from_mirror(Node* mirror, bool never_see_null,
 114                                      int nargs,
 115                                      RegionNode* region, int null_path) {
 116     int offset = java_lang_Class::array_klass_offset_in_bytes();
 117     return load_klass_from_mirror_common(mirror, never_see_null, nargs,
 118                                          region, null_path,
 119                                          offset);
 120   }


 204   void generate_clear_array(const TypePtr* adr_type,
 205                             Node* dest,
 206                             BasicType basic_elem_type,
 207                             Node* slice_off,
 208                             Node* slice_len,
 209                             Node* slice_end);
 210   bool generate_block_arraycopy(const TypePtr* adr_type,
 211                                 BasicType basic_elem_type,
 212                                 AllocateNode* alloc,
 213                                 Node* src,  Node* src_offset,
 214                                 Node* dest, Node* dest_offset,
 215                                 Node* dest_size);
 216   void generate_slow_arraycopy(const TypePtr* adr_type,
 217                                Node* src,  Node* src_offset,
 218                                Node* dest, Node* dest_offset,
 219                                Node* copy_length);
 220   Node* generate_checkcast_arraycopy(const TypePtr* adr_type,
 221                                      Node* dest_elem_klass,
 222                                      Node* src,  Node* src_offset,
 223                                      Node* dest, Node* dest_offset,
 224                                      Node* copy_length, bool need_pre_barrier = true);
 225   Node* generate_generic_arraycopy(const TypePtr* adr_type,
 226                                    Node* src,  Node* src_offset,
 227                                    Node* dest, Node* dest_offset,
 228                                    Node* copy_length);
 229   void generate_unchecked_arraycopy(const TypePtr* adr_type,
 230                                     BasicType basic_elem_type,
 231                                     bool disjoint_bases,
 232                                     Node* src,  Node* src_offset,
 233                                     Node* dest, Node* dest_offset,
 234                                     Node* copy_length, bool need_pre_barrier = true);
 235   bool inline_unsafe_CAS(BasicType type);
 236   bool inline_unsafe_ordered_store(BasicType type);
 237   bool inline_fp_conversions(vmIntrinsics::ID id);
 238   bool inline_numberOfLeadingZeros(vmIntrinsics::ID id);
 239   bool inline_numberOfTrailingZeros(vmIntrinsics::ID id);
 240   bool inline_bitCount(vmIntrinsics::ID id);
 241   bool inline_reverseBytes(vmIntrinsics::ID id);
 242 };
 243 
 244 
 245 //---------------------------make_vm_intrinsic----------------------------
 246 CallGenerator* Compile::make_vm_intrinsic(ciMethod* m, bool is_virtual) {
 247   vmIntrinsics::ID id = m->intrinsic_id();
 248   assert(id != vmIntrinsics::_none, "must be a VM intrinsic");
 249 
 250   if (DisableIntrinsic[0] != '\0'
 251       && strstr(DisableIntrinsic, vmIntrinsics::name_at(id)) != NULL) {
 252     // disabled by a user request on the command line:
 253     // example: -XX:DisableIntrinsic=_hashCode,_getClass
 254     return NULL;


4278     set_all_memory( _gvn.transform(result_mem) );
4279   } //original reexecute and sp are set back here
4280 
4281   push(_gvn.transform(result_val));
4282 
4283   return true;
4284 }
4285 
4286 
4287 // constants for computing the copy function
4288 enum {
4289   COPYFUNC_UNALIGNED = 0,
4290   COPYFUNC_ALIGNED = 1,                 // src, dest aligned to HeapWordSize
4291   COPYFUNC_CONJOINT = 0,
4292   COPYFUNC_DISJOINT = 2                 // src != dest, or transfer can descend
4293 };
4294 
4295 // Note:  The condition "disjoint" applies also for overlapping copies
4296 // where an descending copy is permitted (i.e., dest_offset <= src_offset).
4297 static address
4298 select_arraycopy_function(BasicType t, bool aligned, bool disjoint, const char* &name, bool need_pre_barrier) {
4299   int selector =
4300     (aligned  ? COPYFUNC_ALIGNED  : COPYFUNC_UNALIGNED) +
4301     (disjoint ? COPYFUNC_DISJOINT : COPYFUNC_CONJOINT);
4302 
4303 #define RETURN_STUB(xxx_arraycopy) { \
4304   name = #xxx_arraycopy; \
4305   return StubRoutines::xxx_arraycopy(); }
4306 
4307 #define RETURN_STUB_PARM(xxx_arraycopy, parm) {           \
4308   name = #xxx_arraycopy; \
4309   return StubRoutines::xxx_arraycopy(parm); }
4310 
4311   switch (t) {
4312   case T_BYTE:
4313   case T_BOOLEAN:
4314     switch (selector) {
4315     case COPYFUNC_CONJOINT | COPYFUNC_UNALIGNED:  RETURN_STUB(jbyte_arraycopy);
4316     case COPYFUNC_CONJOINT | COPYFUNC_ALIGNED:    RETURN_STUB(arrayof_jbyte_arraycopy);
4317     case COPYFUNC_DISJOINT | COPYFUNC_UNALIGNED:  RETURN_STUB(jbyte_disjoint_arraycopy);
4318     case COPYFUNC_DISJOINT | COPYFUNC_ALIGNED:    RETURN_STUB(arrayof_jbyte_disjoint_arraycopy);
4319     }
4320   case T_CHAR:
4321   case T_SHORT:
4322     switch (selector) {
4323     case COPYFUNC_CONJOINT | COPYFUNC_UNALIGNED:  RETURN_STUB(jshort_arraycopy);
4324     case COPYFUNC_CONJOINT | COPYFUNC_ALIGNED:    RETURN_STUB(arrayof_jshort_arraycopy);
4325     case COPYFUNC_DISJOINT | COPYFUNC_UNALIGNED:  RETURN_STUB(jshort_disjoint_arraycopy);
4326     case COPYFUNC_DISJOINT | COPYFUNC_ALIGNED:    RETURN_STUB(arrayof_jshort_disjoint_arraycopy);
4327     }
4328   case T_INT:
4329   case T_FLOAT:
4330     switch (selector) {
4331     case COPYFUNC_CONJOINT | COPYFUNC_UNALIGNED:  RETURN_STUB(jint_arraycopy);
4332     case COPYFUNC_CONJOINT | COPYFUNC_ALIGNED:    RETURN_STUB(arrayof_jint_arraycopy);
4333     case COPYFUNC_DISJOINT | COPYFUNC_UNALIGNED:  RETURN_STUB(jint_disjoint_arraycopy);
4334     case COPYFUNC_DISJOINT | COPYFUNC_ALIGNED:    RETURN_STUB(arrayof_jint_disjoint_arraycopy);
4335     }
4336   case T_DOUBLE:
4337   case T_LONG:
4338     switch (selector) {
4339     case COPYFUNC_CONJOINT | COPYFUNC_UNALIGNED:  RETURN_STUB(jlong_arraycopy);
4340     case COPYFUNC_CONJOINT | COPYFUNC_ALIGNED:    RETURN_STUB(arrayof_jlong_arraycopy);
4341     case COPYFUNC_DISJOINT | COPYFUNC_UNALIGNED:  RETURN_STUB(jlong_disjoint_arraycopy);
4342     case COPYFUNC_DISJOINT | COPYFUNC_ALIGNED:    RETURN_STUB(arrayof_jlong_disjoint_arraycopy);
4343     }
4344   case T_ARRAY:
4345   case T_OBJECT:
4346     switch (selector) {
4347     case COPYFUNC_CONJOINT | COPYFUNC_UNALIGNED:  RETURN_STUB_PARM(oop_arraycopy, need_pre_barrier);
4348     case COPYFUNC_CONJOINT | COPYFUNC_ALIGNED:    RETURN_STUB_PARM(arrayof_oop_arraycopy, need_pre_barrier);
4349     case COPYFUNC_DISJOINT | COPYFUNC_UNALIGNED:  RETURN_STUB_PARM(oop_disjoint_arraycopy, need_pre_barrier);
4350     case COPYFUNC_DISJOINT | COPYFUNC_ALIGNED:    RETURN_STUB_PARM(arrayof_oop_disjoint_arraycopy, need_pre_barrier);
4351     }
4352   default:
4353     ShouldNotReachHere();
4354     return NULL;
4355   }
4356 
4357 #undef RETURN_STUB
4358 #undef RETURN_STUB_PARM
4359 }
4360 
4361 //------------------------------basictype2arraycopy----------------------------
4362 address LibraryCallKit::basictype2arraycopy(BasicType t,
4363                                             Node* src_offset,
4364                                             Node* dest_offset,
4365                                             bool disjoint_bases,
4366                                             const char* &name,
4367                                             bool need_pre_barrier) {
4368   const TypeInt* src_offset_inttype  = gvn().find_int_type(src_offset);;
4369   const TypeInt* dest_offset_inttype = gvn().find_int_type(dest_offset);;
4370 
4371   bool aligned = false;
4372   bool disjoint = disjoint_bases;
4373 
4374   // if the offsets are the same, we can treat the memory regions as
4375   // disjoint, because either the memory regions are in different arrays,
4376   // or they are identical (which we can treat as disjoint.)  We can also
4377   // treat a copy with a destination index  less that the source index
4378   // as disjoint since a low->high copy will work correctly in this case.
4379   if (src_offset_inttype != NULL && src_offset_inttype->is_con() &&
4380       dest_offset_inttype != NULL && dest_offset_inttype->is_con()) {
4381     // both indices are constants
4382     int s_offs = src_offset_inttype->get_con();
4383     int d_offs = dest_offset_inttype->get_con();
4384     int element_size = type2aelembytes(t);
4385     aligned = ((arrayOopDesc::base_offset_in_bytes(t) + s_offs * element_size) % HeapWordSize == 0) &&
4386               ((arrayOopDesc::base_offset_in_bytes(t) + d_offs * element_size) % HeapWordSize == 0);
4387     if (s_offs >= d_offs)  disjoint = true;
4388   } else if (src_offset == dest_offset && src_offset != NULL) {
4389     // This can occur if the offsets are identical non-constants.
4390     disjoint = true;
4391   }
4392 
4393   return select_arraycopy_function(t, aligned, disjoint, name, need_pre_barrier);
4394 }
4395 
4396 
4397 //------------------------------inline_arraycopy-----------------------
4398 bool LibraryCallKit::inline_arraycopy() {
4399   // Restore the stack and pop off the arguments.
4400   int nargs = 5;  // 2 oops, 3 ints, no size_t or long
4401   assert(callee()->signature()->size() == nargs, "copy has 5 arguments");
4402 
4403   Node *src         = argument(0);
4404   Node *src_offset  = argument(1);
4405   Node *dest        = argument(2);
4406   Node *dest_offset = argument(3);
4407   Node *length      = argument(4);
4408 
4409   // Compile time checks.  If any of these checks cannot be verified at compile time,
4410   // we do not make a fast path for this call.  Instead, we let the call remain as it
4411   // is.  The checks we choose to mandate at compile time are:
4412   //
4413   // (1) src and dest are arrays.


4763     // The backing store for a List<String> is always an Object[],
4764     // but its elements are always type String, if the generic types
4765     // are correct at the source level.
4766     //
4767     // Test S[] against D[], not S against D, because (probably)
4768     // the secondary supertype cache is less busy for S[] than S.
4769     // This usually only matters when D is an interface.
4770     Node* not_subtype_ctrl = gen_subtype_check(src_klass, dest_klass);
4771     // Plug failing path into checked_oop_disjoint_arraycopy
4772     if (not_subtype_ctrl != top()) {
4773       PreserveJVMState pjvms(this);
4774       set_control(not_subtype_ctrl);
4775       // (At this point we can assume disjoint_bases, since types differ.)
4776       int ek_offset = objArrayKlass::element_klass_offset_in_bytes() + sizeof(oopDesc);
4777       Node* p1 = basic_plus_adr(dest_klass, ek_offset);
4778       Node* n1 = LoadKlassNode::make(_gvn, immutable_memory(), p1, TypeRawPtr::BOTTOM);
4779       Node* dest_elem_klass = _gvn.transform(n1);
4780       Node* cv = generate_checkcast_arraycopy(adr_type,
4781                                               dest_elem_klass,
4782                                               src, src_offset, dest, dest_offset,
4783                                               ConvI2X(copy_length), !must_clear_dest);
4784       if (cv == NULL)  cv = intcon(-1);  // failure (no stub available)
4785       checked_control = control();
4786       checked_i_o     = i_o();
4787       checked_mem     = memory(adr_type);
4788       checked_value   = cv;
4789     }
4790     // At this point we know we do not need type checks on oop stores.
4791 
4792     // Let's see if we need card marks:
4793     if (alloc != NULL && use_ReduceInitialCardMarks()) {
4794       // If we do not need card marks, copy using the jint or jlong stub.
4795       copy_type = LP64_ONLY(UseCompressedOops ? T_INT : T_LONG) NOT_LP64(T_INT);
4796       assert(type2aelembytes(basic_elem_type) == type2aelembytes(copy_type),
4797              "sizes agree");
4798     }
4799   }
4800 
4801   if (!stopped()) {
4802     // Generate the fast path, if possible.
4803     PreserveJVMState pjvms(this);
4804     generate_unchecked_arraycopy(adr_type, copy_type, disjoint_bases,
4805                                  src, src_offset, dest, dest_offset,
4806                                  ConvI2X(copy_length), !must_clear_dest);
4807 
4808     // Present the results of the fast call.
4809     result_region->init_req(fast_path, control());
4810     result_i_o   ->init_req(fast_path, i_o());
4811     result_memory->init_req(fast_path, memory(adr_type));
4812   }
4813 
4814   // Here are all the slow paths up to this point, in one bundle:
4815   slow_control = top();
4816   if (slow_region != NULL)
4817     slow_control = _gvn.transform(slow_region);
4818   debug_only(slow_region = (RegionNode*)badAddress);
4819 
4820   set_control(checked_control);
4821   if (!stopped()) {
4822     // Clean up after the checked call.
4823     // The returned value is either 0 or -1^K,
4824     // where K = number of partially transferred array elements.
4825     Node* cmp = _gvn.transform( new(C, 3) CmpINode(checked_value, intcon(0)) );
4826     Node* bol = _gvn.transform( new(C, 2) BoolNode(cmp, BoolTest::eq) );


5193                                         Node* src,  Node* src_offset,
5194                                         Node* dest, Node* dest_offset,
5195                                         Node* copy_length) {
5196   Node* call = make_runtime_call(RC_NO_LEAF | RC_UNCOMMON,
5197                                  OptoRuntime::slow_arraycopy_Type(),
5198                                  OptoRuntime::slow_arraycopy_Java(),
5199                                  "slow_arraycopy", adr_type,
5200                                  src, src_offset, dest, dest_offset,
5201                                  copy_length);
5202 
5203   // Handle exceptions thrown by this fellow:
5204   make_slow_call_ex(call, env()->Throwable_klass(), false);
5205 }
5206 
5207 // Helper function; generates code for cases requiring runtime checks.
5208 Node*
5209 LibraryCallKit::generate_checkcast_arraycopy(const TypePtr* adr_type,
5210                                              Node* dest_elem_klass,
5211                                              Node* src,  Node* src_offset,
5212                                              Node* dest, Node* dest_offset,
5213                                              Node* copy_length, bool need_pre_barrier) {
5214   if (stopped())  return NULL;
5215 
5216   address copyfunc_addr = StubRoutines::checkcast_arraycopy(need_pre_barrier);
5217   if (copyfunc_addr == NULL) { // Stub was not generated, go slow path.
5218     return NULL;
5219   }
5220 
5221   // Pick out the parameters required to perform a store-check
5222   // for the target array.  This is an optimistic check.  It will
5223   // look in each non-null element's class, at the desired klass's
5224   // super_check_offset, for the desired klass.
5225   int sco_offset = Klass::super_check_offset_offset_in_bytes() + sizeof(oopDesc);
5226   Node* p3 = basic_plus_adr(dest_elem_klass, sco_offset);
5227   Node* n3 = new(C, 3) LoadINode(NULL, memory(p3), p3, _gvn.type(p3)->is_ptr());
5228   Node* check_offset = ConvI2X(_gvn.transform(n3));
5229   Node* check_value  = dest_elem_klass;
5230 
5231   Node* src_start  = array_element_address(src,  src_offset,  T_OBJECT);
5232   Node* dest_start = array_element_address(dest, dest_offset, T_OBJECT);
5233 
5234   // (We know the arrays are never conjoint, because their types differ.)
5235   Node* call = make_runtime_call(RC_LEAF|RC_NO_FP,
5236                                  OptoRuntime::checkcast_arraycopy_Type(),


5257   address copyfunc_addr = StubRoutines::generic_arraycopy();
5258   if (copyfunc_addr == NULL) { // Stub was not generated, go slow path.
5259     return NULL;
5260   }
5261 
5262   Node* call = make_runtime_call(RC_LEAF|RC_NO_FP,
5263                     OptoRuntime::generic_arraycopy_Type(),
5264                     copyfunc_addr, "generic_arraycopy", adr_type,
5265                     src, src_offset, dest, dest_offset, copy_length);
5266 
5267   return _gvn.transform(new (C, 1) ProjNode(call, TypeFunc::Parms));
5268 }
5269 
5270 // Helper function; generates the fast out-of-line call to an arraycopy stub.
5271 void
5272 LibraryCallKit::generate_unchecked_arraycopy(const TypePtr* adr_type,
5273                                              BasicType basic_elem_type,
5274                                              bool disjoint_bases,
5275                                              Node* src,  Node* src_offset,
5276                                              Node* dest, Node* dest_offset,
5277                                              Node* copy_length, bool need_pre_barrier) {
5278   if (stopped())  return;               // nothing to do
5279 
5280   Node* src_start  = src;
5281   Node* dest_start = dest;
5282   if (src_offset != NULL || dest_offset != NULL) {
5283     assert(src_offset != NULL && dest_offset != NULL, "");
5284     src_start  = array_element_address(src,  src_offset,  basic_elem_type);
5285     dest_start = array_element_address(dest, dest_offset, basic_elem_type);
5286   }
5287 
5288   // Figure out which arraycopy runtime method to call.
5289   const char* copyfunc_name = "arraycopy";
5290   address     copyfunc_addr =
5291       basictype2arraycopy(basic_elem_type, src_offset, dest_offset,
5292                           disjoint_bases, copyfunc_name, need_pre_barrier);
5293 
5294   // Call it.  Note that the count_ix value is not scaled to a byte-size.
5295   make_runtime_call(RC_LEAF|RC_NO_FP,
5296                     OptoRuntime::fast_arraycopy_Type(),
5297                     copyfunc_addr, copyfunc_name, adr_type,
5298                     src_start, dest_start, copy_length XTOP);
5299 }
src/share/vm/opto/library_call.cpp
Index Unified diffs Context diffs Sdiffs Patch New Old Previous File Next File