128 }
129
130 private:
131 void fatal_unexpected_iid(vmIntrinsics::ID iid) {
132 fatal(err_msg_res("unexpected intrinsic %d: %s", iid, vmIntrinsics::name_at(iid)));
133 }
134
135 void set_result(Node* n) { assert(_result == NULL, "only set once"); _result = n; }
136 void set_result(RegionNode* region, PhiNode* value);
137 Node* result() { return _result; }
138
139 virtual int reexecute_sp() { return _reexecute_sp; }
140
141 // Helper functions to inline natives
142 Node* generate_guard(Node* test, RegionNode* region, float true_prob);
143 Node* generate_slow_guard(Node* test, RegionNode* region);
144 Node* generate_fair_guard(Node* test, RegionNode* region);
145 Node* generate_negative_guard(Node* index, RegionNode* region,
146 // resulting CastII of index:
147 Node* *pos_index = NULL);
148 Node* generate_nonpositive_guard(Node* index, bool never_negative,
149 // resulting CastII of index:
150 Node* *pos_index = NULL);
151 Node* generate_limit_guard(Node* offset, Node* subseq_length,
152 Node* array_length,
153 RegionNode* region);
154 Node* generate_current_thread(Node* &tls_output);
155 address basictype2arraycopy(BasicType t, Node *src_offset, Node *dest_offset,
156 bool disjoint_bases, const char* &name, bool dest_uninitialized);
157 Node* load_mirror_from_klass(Node* klass);
158 Node* load_klass_from_mirror_common(Node* mirror, bool never_see_null,
159 RegionNode* region, int null_path,
160 int offset);
161 Node* load_klass_from_mirror(Node* mirror, bool never_see_null,
162 RegionNode* region, int null_path) {
163 int offset = java_lang_Class::klass_offset_in_bytes();
164 return load_klass_from_mirror_common(mirror, never_see_null,
165 region, null_path,
166 offset);
167 }
168 Node* load_array_klass_from_mirror(Node* mirror, bool never_see_null,
169 RegionNode* region, int null_path) {
170 int offset = java_lang_Class::array_klass_offset_in_bytes();
171 return load_klass_from_mirror_common(mirror, never_see_null,
172 region, null_path,
173 offset);
174 }
175 Node* generate_access_flags_guard(Node* kls,
176 int modifier_mask, int modifier_bits,
246 bool inline_native_threadID();
247 #endif
248 bool inline_native_time_funcs(address method, const char* funcName);
249 bool inline_native_isInterrupted();
250 bool inline_native_Class_query(vmIntrinsics::ID id);
251 bool inline_native_subtype_check();
252
253 bool inline_native_newArray();
254 bool inline_native_getLength();
255 bool inline_array_copyOf(bool is_copyOfRange);
256 bool inline_array_equals();
257 void copy_to_clone(Node* obj, Node* alloc_obj, Node* obj_size, bool is_array, bool card_mark);
258 bool inline_native_clone(bool is_virtual);
259 bool inline_native_Reflection_getCallerClass();
260 // Helper function for inlining native object hash method
261 bool inline_native_hashcode(bool is_virtual, bool is_static);
262 bool inline_native_getClass();
263
264 // Helper functions for inlining arraycopy
265 bool inline_arraycopy();
266 void generate_arraycopy(const TypePtr* adr_type,
267 BasicType basic_elem_type,
268 Node* src, Node* src_offset,
269 Node* dest, Node* dest_offset,
270 Node* copy_length,
271 bool disjoint_bases = false,
272 bool length_never_negative = false,
273 RegionNode* slow_region = NULL);
274 AllocateArrayNode* tightly_coupled_allocation(Node* ptr,
275 RegionNode* slow_region);
276 void generate_clear_array(const TypePtr* adr_type,
277 Node* dest,
278 BasicType basic_elem_type,
279 Node* slice_off,
280 Node* slice_len,
281 Node* slice_end);
282 bool generate_block_arraycopy(const TypePtr* adr_type,
283 BasicType basic_elem_type,
284 AllocateNode* alloc,
285 Node* src, Node* src_offset,
286 Node* dest, Node* dest_offset,
287 Node* dest_size, bool dest_uninitialized);
288 void generate_slow_arraycopy(const TypePtr* adr_type,
289 Node* src, Node* src_offset,
290 Node* dest, Node* dest_offset,
291 Node* copy_length, bool dest_uninitialized);
292 Node* generate_checkcast_arraycopy(const TypePtr* adr_type,
293 Node* dest_elem_klass,
294 Node* src, Node* src_offset,
295 Node* dest, Node* dest_offset,
296 Node* copy_length, bool dest_uninitialized);
297 Node* generate_generic_arraycopy(const TypePtr* adr_type,
298 Node* src, Node* src_offset,
299 Node* dest, Node* dest_offset,
300 Node* copy_length, bool dest_uninitialized);
301 void generate_unchecked_arraycopy(const TypePtr* adr_type,
302 BasicType basic_elem_type,
303 bool disjoint_bases,
304 Node* src, Node* src_offset,
305 Node* dest, Node* dest_offset,
306 Node* copy_length, bool dest_uninitialized);
307 typedef enum { LS_xadd, LS_xchg, LS_cmpxchg } LoadStoreKind;
308 bool inline_unsafe_load_store(BasicType type, LoadStoreKind kind);
309 bool inline_unsafe_ordered_store(BasicType type);
310 bool inline_unsafe_fence(vmIntrinsics::ID id);
311 bool inline_fp_conversions(vmIntrinsics::ID id);
312 bool inline_number_methods(vmIntrinsics::ID id);
313 bool inline_reference_get();
314 bool inline_aescrypt_Block(vmIntrinsics::ID id);
315 bool inline_cipherBlockChaining_AESCrypt(vmIntrinsics::ID id);
316 Node* inline_cipherBlockChaining_AESCrypt_predicate(bool decrypting);
317 Node* get_key_start_from_aescrypt_object(Node* aescrypt_object);
318 Node* get_original_key_start_from_aescrypt_object(Node* aescrypt_object);
319 bool inline_sha_implCompress(vmIntrinsics::ID id);
320 bool inline_digestBase_implCompressMB(int predicate);
321 bool inline_sha_implCompressMB(Node* digestBaseObj, ciInstanceKlass* instklass_SHA,
322 bool long_state, address stubAddr, const char *stubName,
323 Node* src_start, Node* ofs, Node* limit);
324 Node* get_state_from_sha_object(Node *sha_object);
325 Node* get_state_from_sha5_object(Node *sha_object);
326 Node* inline_digestBase_implCompressMB_predicate(int predicate);
1031 }
1032
1033 inline Node* LibraryCallKit::generate_negative_guard(Node* index, RegionNode* region,
1034 Node* *pos_index) {
1035 if (stopped())
1036 return NULL; // already stopped
1037 if (_gvn.type(index)->higher_equal(TypeInt::POS)) // [0,maxint]
1038 return NULL; // index is already adequately typed
1039 Node* cmp_lt = _gvn.transform(new CmpINode(index, intcon(0)));
1040 Node* bol_lt = _gvn.transform(new BoolNode(cmp_lt, BoolTest::lt));
1041 Node* is_neg = generate_guard(bol_lt, region, PROB_MIN);
1042 if (is_neg != NULL && pos_index != NULL) {
1043 // Emulate effect of Parse::adjust_map_after_if.
1044 Node* ccast = new CastIINode(index, TypeInt::POS);
1045 ccast->set_req(0, control());
1046 (*pos_index) = _gvn.transform(ccast);
1047 }
1048 return is_neg;
1049 }
1050
1051 inline Node* LibraryCallKit::generate_nonpositive_guard(Node* index, bool never_negative,
1052 Node* *pos_index) {
1053 if (stopped())
1054 return NULL; // already stopped
1055 if (_gvn.type(index)->higher_equal(TypeInt::POS1)) // [1,maxint]
1056 return NULL; // index is already adequately typed
1057 Node* cmp_le = _gvn.transform(new CmpINode(index, intcon(0)));
1058 BoolTest::mask le_or_eq = (never_negative ? BoolTest::eq : BoolTest::le);
1059 Node* bol_le = _gvn.transform(new BoolNode(cmp_le, le_or_eq));
1060 Node* is_notp = generate_guard(bol_le, NULL, PROB_MIN);
1061 if (is_notp != NULL && pos_index != NULL) {
1062 // Emulate effect of Parse::adjust_map_after_if.
1063 Node* ccast = new CastIINode(index, TypeInt::POS1);
1064 ccast->set_req(0, control());
1065 (*pos_index) = _gvn.transform(ccast);
1066 }
1067 return is_notp;
1068 }
1069
1070 // Make sure that 'position' is a valid limit index, in [0..length].
1071 // There are two equivalent plans for checking this:
1072 // A. (offset + copyLength) unsigned<= arrayLength
1073 // B. offset <= (arrayLength - copyLength)
1074 // We require that all of the values above, except for the sum and
1075 // difference, are already known to be non-negative.
1076 // Plan A is robust in the face of overflow, if offset and copyLength
1077 // are both hugely positive.
1078 //
1079 // Plan B is less direct and intuitive, but it does not overflow at
1080 // all, since the difference of two non-negatives is always
1081 // representable. Whenever Java methods must perform the equivalent
1082 // check they generally use Plan B instead of Plan A.
1083 // For the moment we use Plan A.
1084 inline Node* LibraryCallKit::generate_limit_guard(Node* offset,
1085 Node* subseq_length,
1086 Node* array_length,
1087 RegionNode* region) {
1088 if (stopped())
1089 return NULL; // already stopped
3910 if (bailout->req() > 1) {
3911 PreserveJVMState pjvms(this);
3912 set_control(_gvn.transform(bailout));
3913 uncommon_trap(Deoptimization::Reason_intrinsic,
3914 Deoptimization::Action_maybe_recompile);
3915 }
3916
3917 if (!stopped()) {
3918 // How many elements will we copy from the original?
3919 // The answer is MinI(orig_length - start, length).
3920 Node* orig_tail = _gvn.transform(new SubINode(orig_length, start));
3921 Node* moved = generate_min_max(vmIntrinsics::_min, orig_tail, length);
3922
3923 newcopy = new_array(klass_node, length, 0); // no argments to push
3924
3925 // Generate a direct call to the right arraycopy function(s).
3926 // We know the copy is disjoint but we might not know if the
3927 // oop stores need checking.
3928 // Extreme case: Arrays.copyOf((Integer[])x, 10, String[].class).
3929 // This will fail a store-check if x contains any non-nulls.
3930 bool disjoint_bases = true;
3931 // if start > orig_length then the length of the copy may be
3932 // negative.
3933 bool length_never_negative = !is_copyOfRange;
3934 generate_arraycopy(TypeAryPtr::OOPS, T_OBJECT,
3935 original, start, newcopy, intcon(0), moved,
3936 disjoint_bases, length_never_negative);
3937 }
3938 } // original reexecute is set back here
3939
3940 C->set_has_split_ifs(true); // Has chance for split-if optimization
3941 if (!stopped()) {
3942 set_result(newcopy);
3943 }
3944 return true;
3945 }
3946
3947
3948 //----------------------generate_virtual_guard---------------------------
3949 // Helper for hashCode and clone. Peeks inside the vtable to avoid a call.
3950 Node* LibraryCallKit::generate_virtual_guard(Node* obj_klass,
3951 RegionNode* slow_region) {
3952 ciMethod* method = callee();
3953 int vtable_index = method->vtable_index();
3954 assert(vtable_index >= 0 || vtable_index == Method::nonvirtual_vtable_index,
3955 err_msg_res("bad index %d", vtable_index));
3956 // Get the Method* out of the appropriate vtable entry.
4427 if (base_off % BytesPerLong != 0) {
4428 assert(UseCompressedClassPointers, "");
4429 if (is_array) {
4430 // Exclude length to copy by 8 bytes words.
4431 base_off += sizeof(int);
4432 } else {
4433 // Include klass to copy by 8 bytes words.
4434 base_off = instanceOopDesc::klass_offset_in_bytes();
4435 }
4436 assert(base_off % BytesPerLong == 0, "expect 8 bytes alignment");
4437 }
4438 src = basic_plus_adr(src, base_off);
4439 dest = basic_plus_adr(dest, base_off);
4440
4441 // Compute the length also, if needed:
4442 Node* countx = size;
4443 countx = _gvn.transform(new SubXNode(countx, MakeConX(base_off)));
4444 countx = _gvn.transform(new URShiftXNode(countx, intcon(LogBytesPerLong) ));
4445
4446 const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
4447 bool disjoint_bases = true;
4448 generate_unchecked_arraycopy(raw_adr_type, T_LONG, disjoint_bases,
4449 src, NULL, dest, NULL, countx,
4450 /*dest_uninitialized*/true);
4451
4452 // If necessary, emit some card marks afterwards. (Non-arrays only.)
4453 if (card_mark) {
4454 assert(!is_array, "");
4455 // Put in store barrier for any and all oops we are sticking
4456 // into this object. (We could avoid this if we could prove
4457 // that the object type contains no oop fields at all.)
4458 Node* no_particular_value = NULL;
4459 Node* no_particular_field = NULL;
4460 int raw_adr_idx = Compile::AliasIdxRaw;
4461 post_barrier(control(),
4462 memory(raw_adr_type),
4463 alloc_obj,
4464 no_particular_field,
4465 raw_adr_idx,
4466 no_particular_value,
4467 T_OBJECT,
4468 false);
4469 }
4470
4539 const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
4540 int raw_adr_idx = Compile::AliasIdxRaw;
4541
4542 Node* array_ctl = generate_array_guard(obj_klass, (RegionNode*)NULL);
4543 if (array_ctl != NULL) {
4544 // It's an array.
4545 PreserveJVMState pjvms(this);
4546 set_control(array_ctl);
4547 Node* obj_length = load_array_length(obj);
4548 Node* obj_size = NULL;
4549 Node* alloc_obj = new_array(obj_klass, obj_length, 0, &obj_size); // no arguments to push
4550
4551 if (!use_ReduceInitialCardMarks()) {
4552 // If it is an oop array, it requires very special treatment,
4553 // because card marking is required on each card of the array.
4554 Node* is_obja = generate_objArray_guard(obj_klass, (RegionNode*)NULL);
4555 if (is_obja != NULL) {
4556 PreserveJVMState pjvms2(this);
4557 set_control(is_obja);
4558 // Generate a direct call to the right arraycopy function(s).
4559 bool disjoint_bases = true;
4560 bool length_never_negative = true;
4561 generate_arraycopy(TypeAryPtr::OOPS, T_OBJECT,
4562 obj, intcon(0), alloc_obj, intcon(0),
4563 obj_length,
4564 disjoint_bases, length_never_negative);
4565 result_reg->init_req(_objArray_path, control());
4566 result_val->init_req(_objArray_path, alloc_obj);
4567 result_i_o ->set_req(_objArray_path, i_o());
4568 result_mem ->set_req(_objArray_path, reset_memory());
4569 }
4570 }
4571 // Otherwise, there are no card marks to worry about.
4572 // (We can dispense with card marks if we know the allocation
4573 // comes out of eden (TLAB)... In fact, ReduceInitialCardMarks
4574 // causes the non-eden paths to take compensating steps to
4575 // simulate a fresh allocation, so that no further
4576 // card marks are required in compiled code to initialize
4577 // the object.)
4578
4579 if (!stopped()) {
4580 copy_to_clone(obj, alloc_obj, obj_size, true, false);
4581
4582 // Present the results of the copy.
4583 result_reg->init_req(_array_path, control());
4584 result_val->init_req(_array_path, alloc_obj);
4638 PreserveJVMState pjvms(this);
4639 CallJavaNode* slow_call = generate_method_call(vmIntrinsics::_clone, is_virtual);
4640 Node* slow_result = set_results_for_java_call(slow_call);
4641 // this->control() comes from set_results_for_java_call
4642 result_reg->init_req(_slow_path, control());
4643 result_val->init_req(_slow_path, slow_result);
4644 result_i_o ->set_req(_slow_path, i_o());
4645 result_mem ->set_req(_slow_path, reset_memory());
4646 }
4647
4648 // Return the combined state.
4649 set_control( _gvn.transform(result_reg));
4650 set_i_o( _gvn.transform(result_i_o));
4651 set_all_memory( _gvn.transform(result_mem));
4652 } // original reexecute is set back here
4653
4654 set_result(_gvn.transform(result_val));
4655 return true;
4656 }
4657
4658 //------------------------------basictype2arraycopy----------------------------
4659 address LibraryCallKit::basictype2arraycopy(BasicType t,
4660 Node* src_offset,
4661 Node* dest_offset,
4662 bool disjoint_bases,
4663 const char* &name,
4664 bool dest_uninitialized) {
4665 const TypeInt* src_offset_inttype = gvn().find_int_type(src_offset);;
4666 const TypeInt* dest_offset_inttype = gvn().find_int_type(dest_offset);;
4667
4668 bool aligned = false;
4669 bool disjoint = disjoint_bases;
4670
4671 // if the offsets are the same, we can treat the memory regions as
4672 // disjoint, because either the memory regions are in different arrays,
4673 // or they are identical (which we can treat as disjoint.) We can also
4674 // treat a copy with a destination index less that the source index
4675 // as disjoint since a low->high copy will work correctly in this case.
4676 if (src_offset_inttype != NULL && src_offset_inttype->is_con() &&
4677 dest_offset_inttype != NULL && dest_offset_inttype->is_con()) {
4678 // both indices are constants
4679 int s_offs = src_offset_inttype->get_con();
4680 int d_offs = dest_offset_inttype->get_con();
4681 int element_size = type2aelembytes(t);
4682 aligned = ((arrayOopDesc::base_offset_in_bytes(t) + s_offs * element_size) % HeapWordSize == 0) &&
4683 ((arrayOopDesc::base_offset_in_bytes(t) + d_offs * element_size) % HeapWordSize == 0);
4684 if (s_offs >= d_offs) disjoint = true;
4685 } else if (src_offset == dest_offset && src_offset != NULL) {
4686 // This can occur if the offsets are identical non-constants.
4687 disjoint = true;
4688 }
4689
4690 return StubRoutines::select_arraycopy_function(t, aligned, disjoint, name, dest_uninitialized);
4691 }
4692
4693
4694 //------------------------------inline_arraycopy-----------------------
4695 // public static native void java.lang.System.arraycopy(Object src, int srcPos,
4696 // Object dest, int destPos,
4697 // int length);
4698 bool LibraryCallKit::inline_arraycopy() {
4699 // Get the arguments.
4700 Node* src = argument(0); // type: oop
4701 Node* src_offset = argument(1); // type: int
4702 Node* dest = argument(2); // type: oop
4703 Node* dest_offset = argument(3); // type: int
4704 Node* length = argument(4); // type: int
4705
4706 // Compile time checks. If any of these checks cannot be verified at compile time,
4707 // we do not make a fast path for this call. Instead, we let the call remain as it
4708 // is. The checks we choose to mandate at compile time are:
4709 //
4710 // (1) src and dest are arrays.
4711 const Type* src_type = src->Value(&_gvn);
4712 const Type* dest_type = dest->Value(&_gvn);
4713 const TypeAryPtr* top_src = src_type->isa_aryptr();
4714 const TypeAryPtr* top_dest = dest_type->isa_aryptr();
4715
4716 // Do we have the type of src?
4717 bool has_src = (top_src != NULL && top_src->klass() != NULL);
4718 // Do we have the type of dest?
4719 bool has_dest = (top_dest != NULL && top_dest->klass() != NULL);
4720 // Is the type for src from speculation?
4721 bool src_spec = false;
4722 // Is the type for dest from speculation?
4723 bool dest_spec = false;
4724
4725 if (!has_src || !has_dest) {
4726 // We don't have sufficient type information, let's see if
4727 // speculative types can help. We need to have types for both src
4728 // and dest so that it pays off.
4729
4730 // Do we already have or could we have type information for src
4731 bool could_have_src = has_src;
4732 // Do we already have or could we have type information for dest
4750
4751 if (could_have_src && could_have_dest) {
4752 // This is going to pay off so emit the required guards
4753 if (!has_src) {
4754 src = maybe_cast_profiled_obj(src, src_k);
4755 src_type = _gvn.type(src);
4756 top_src = src_type->isa_aryptr();
4757 has_src = (top_src != NULL && top_src->klass() != NULL);
4758 src_spec = true;
4759 }
4760 if (!has_dest) {
4761 dest = maybe_cast_profiled_obj(dest, dest_k);
4762 dest_type = _gvn.type(dest);
4763 top_dest = dest_type->isa_aryptr();
4764 has_dest = (top_dest != NULL && top_dest->klass() != NULL);
4765 dest_spec = true;
4766 }
4767 }
4768 }
4769
4770 if (!has_src || !has_dest) {
4771 // Conservatively insert a memory barrier on all memory slices.
4772 // Do not let writes into the source float below the arraycopy.
4773 insert_mem_bar(Op_MemBarCPUOrder);
4774
4775 // Call StubRoutines::generic_arraycopy stub.
4776 generate_arraycopy(TypeRawPtr::BOTTOM, T_CONFLICT,
4777 src, src_offset, dest, dest_offset, length);
4778
4779 // Do not let reads from the destination float above the arraycopy.
4780 // Since we cannot type the arrays, we don't know which slices
4781 // might be affected. We could restrict this barrier only to those
4782 // memory slices which pertain to array elements--but don't bother.
4783 if (!InsertMemBarAfterArraycopy)
4784 // (If InsertMemBarAfterArraycopy, there is already one in place.)
4785 insert_mem_bar(Op_MemBarCPUOrder);
4786 return true;
4787 }
4788
4789 // (2) src and dest arrays must have elements of the same BasicType
4790 // Figure out the size and type of the elements we will be copying.
4791 BasicType src_elem = top_src->klass()->as_array_klass()->element_type()->basic_type();
4792 BasicType dest_elem = top_dest->klass()->as_array_klass()->element_type()->basic_type();
4793 if (src_elem == T_ARRAY) src_elem = T_OBJECT;
4794 if (dest_elem == T_ARRAY) dest_elem = T_OBJECT;
4795
4796 if (src_elem != dest_elem || dest_elem == T_VOID) {
4797 // The component types are not the same or are not recognized. Punt.
4798 // (But, avoid the native method wrapper to JVM_ArrayCopy.)
4799 generate_slow_arraycopy(TypePtr::BOTTOM,
4800 src, src_offset, dest, dest_offset, length,
4801 /*dest_uninitialized*/false);
4802 return true;
4803 }
4804
4805 if (src_elem == T_OBJECT) {
4806 // If both arrays are object arrays then having the exact types
4807 // for both will remove the need for a subtype check at runtime
4808 // before the call and may make it possible to pick a faster copy
4809 // routine (without a subtype check on every element)
4810 // Do we have the exact type of src?
4811 bool could_have_src = src_spec;
4812 // Do we have the exact type of dest?
4813 bool could_have_dest = dest_spec;
4814 ciKlass* src_k = top_src->klass();
4815 ciKlass* dest_k = top_dest->klass();
4816 if (!src_spec) {
4817 src_k = src_type->speculative_type_not_null();
4818 if (src_k != NULL && src_k->is_array_klass()) {
4819 could_have_src = true;
4820 }
4821 }
4822 if (!dest_spec) {
4823 dest_k = dest_type->speculative_type_not_null();
4824 if (dest_k != NULL && dest_k->is_array_klass()) {
4825 could_have_dest = true;
4826 }
4827 }
4828 if (could_have_src && could_have_dest) {
4829 // If we can have both exact types, emit the missing guards
4830 if (could_have_src && !src_spec) {
4831 src = maybe_cast_profiled_obj(src, src_k);
4832 }
4833 if (could_have_dest && !dest_spec) {
4834 dest = maybe_cast_profiled_obj(dest, dest_k);
4835 }
4836 }
4837 }
4838
4839 //---------------------------------------------------------------------------
4840 // We will make a fast path for this call to arraycopy.
4841
4842 // We have the following tests left to perform:
4843 //
4844 // (3) src and dest must not be null.
4845 // (4) src_offset must not be negative.
4846 // (5) dest_offset must not be negative.
4847 // (6) length must not be negative.
4848 // (7) src_offset + length must not exceed length of src.
4849 // (8) dest_offset + length must not exceed length of dest.
4850 // (9) each element of an oop array must be assignable
4851
4852 RegionNode* slow_region = new RegionNode(1);
4853 record_for_igvn(slow_region);
4854
4855 // (3) operands must not be null
4856 // We currently perform our null checks with the null_check routine.
4857 // This means that the null exceptions will be reported in the caller
4858 // rather than (correctly) reported inside of the native arraycopy call.
4859 // This should be corrected, given time. We do our null check with the
4860 // stack pointer restored.
4861 src = null_check(src, T_ARRAY);
4862 dest = null_check(dest, T_ARRAY);
4863
4864 // (4) src_offset must not be negative.
4865 generate_negative_guard(src_offset, slow_region);
4866
4867 // (5) dest_offset must not be negative.
4868 generate_negative_guard(dest_offset, slow_region);
4869
4870 // (6) length must not be negative (moved to generate_arraycopy()).
4871 // generate_negative_guard(length, slow_region);
4872
4873 // (7) src_offset + length must not exceed length of src.
4874 generate_limit_guard(src_offset, length,
4875 load_array_length(src),
4876 slow_region);
4877
4878 // (8) dest_offset + length must not exceed length of dest.
4879 generate_limit_guard(dest_offset, length,
4880 load_array_length(dest),
4881 slow_region);
4882
4883 // (9) each element of an oop array must be assignable
4884 // The generate_arraycopy subroutine checks this.
4885
4886 // This is where the memory effects are placed:
4887 const TypePtr* adr_type = TypeAryPtr::get_array_body_type(dest_elem);
4888 generate_arraycopy(adr_type, dest_elem,
4889 src, src_offset, dest, dest_offset, length,
4890 false, false, slow_region);
4891
4892 return true;
4893 }
4894
4895 //-----------------------------generate_arraycopy----------------------
4896 // Generate an optimized call to arraycopy.
4897 // Caller must guard against non-arrays.
4898 // Caller must determine a common array basic-type for both arrays.
4899 // Caller must validate offsets against array bounds.
4900 // The slow_region has already collected guard failure paths
4901 // (such as out of bounds length or non-conformable array types).
4902 // The generated code has this shape, in general:
4903 //
4904 // if (length == 0) return // via zero_path
4905 // slowval = -1
4906 // if (types unknown) {
4907 // slowval = call generic copy loop
4908 // if (slowval == 0) return // via checked_path
4909 // } else if (indexes in bounds) {
4910 // if ((is object array) && !(array type check)) {
4911 // slowval = call checked copy loop
4912 // if (slowval == 0) return // via checked_path
4913 // } else {
4914 // call bulk copy loop
4915 // return // via fast_path
4916 // }
4917 // }
4918 // // adjust params for remaining work:
4919 // if (slowval != -1) {
4920 // n = -1^slowval; src_offset += n; dest_offset += n; length -= n
4921 // }
4922 // slow_region:
4923 // call slow arraycopy(src, src_offset, dest, dest_offset, length)
4924 // return // via slow_call_path
4925 //
4926 // This routine is used from several intrinsics: System.arraycopy,
4927 // Object.clone (the array subcase), and Arrays.copyOf[Range].
4928 //
4929 void
4930 LibraryCallKit::generate_arraycopy(const TypePtr* adr_type,
4931 BasicType basic_elem_type,
4932 Node* src, Node* src_offset,
4933 Node* dest, Node* dest_offset,
4934 Node* copy_length,
4935 bool disjoint_bases,
4936 bool length_never_negative,
4937 RegionNode* slow_region) {
4938
4939 if (slow_region == NULL) {
4940 slow_region = new RegionNode(1);
4941 record_for_igvn(slow_region);
4942 }
4943
4944 Node* original_dest = dest;
4945 AllocateArrayNode* alloc = NULL; // used for zeroing, if needed
4946 bool dest_uninitialized = false;
4947
4948 // See if this is the initialization of a newly-allocated array.
4949 // If so, we will take responsibility here for initializing it to zero.
4950 // (Note: Because tightly_coupled_allocation performs checks on the
4951 // out-edges of the dest, we need to avoid making derived pointers
4952 // from it until we have checked its uses.)
4953 if (ReduceBulkZeroing
4954 && !ZeroTLAB // pointless if already zeroed
4955 && basic_elem_type != T_CONFLICT // avoid corner case
4956 && !src->eqv_uncast(dest)
4957 && ((alloc = tightly_coupled_allocation(dest, slow_region))
4958 != NULL)
4959 && _gvn.find_int_con(alloc->in(AllocateNode::ALength), 1) > 0
4960 && alloc->maybe_set_complete(&_gvn)) {
4961 // "You break it, you buy it."
4962 InitializeNode* init = alloc->initialization();
4963 assert(init->is_complete(), "we just did this");
4964 init->set_complete_with_arraycopy();
4965 assert(dest->is_CheckCastPP(), "sanity");
4966 assert(dest->in(0)->in(0) == init, "dest pinned");
4967 adr_type = TypeRawPtr::BOTTOM; // all initializations are into raw memory
4968 // From this point on, every exit path is responsible for
4969 // initializing any non-copied parts of the object to zero.
4970 // Also, if this flag is set we make sure that arraycopy interacts properly
4971 // with G1, eliding pre-barriers. See CR 6627983.
4972 dest_uninitialized = true;
4973 } else {
4974 // No zeroing elimination here.
4975 alloc = NULL;
4976 //original_dest = dest;
4977 //dest_uninitialized = false;
4978 }
4979
4980 // Results are placed here:
4981 enum { fast_path = 1, // normal void-returning assembly stub
4982 checked_path = 2, // special assembly stub with cleanup
4983 slow_call_path = 3, // something went wrong; call the VM
4984 zero_path = 4, // bypass when length of copy is zero
4985 bcopy_path = 5, // copy primitive array by 64-bit blocks
4986 PATH_LIMIT = 6
4987 };
4988 RegionNode* result_region = new RegionNode(PATH_LIMIT);
4989 PhiNode* result_i_o = new PhiNode(result_region, Type::ABIO);
4990 PhiNode* result_memory = new PhiNode(result_region, Type::MEMORY, adr_type);
4991 record_for_igvn(result_region);
4992 _gvn.set_type_bottom(result_i_o);
4993 _gvn.set_type_bottom(result_memory);
4994 assert(adr_type != TypePtr::BOTTOM, "must be RawMem or a T[] slice");
4995
4996 // The slow_control path:
4997 Node* slow_control;
4998 Node* slow_i_o = i_o();
4999 Node* slow_mem = memory(adr_type);
5000 debug_only(slow_control = (Node*) badAddress);
5001
5002 // Checked control path:
5003 Node* checked_control = top();
5004 Node* checked_mem = NULL;
5005 Node* checked_i_o = NULL;
5006 Node* checked_value = NULL;
5007
5008 if (basic_elem_type == T_CONFLICT) {
5009 assert(!dest_uninitialized, "");
5010 Node* cv = generate_generic_arraycopy(adr_type,
5011 src, src_offset, dest, dest_offset,
5012 copy_length, dest_uninitialized);
5013 if (cv == NULL) cv = intcon(-1); // failure (no stub available)
5014 checked_control = control();
5015 checked_i_o = i_o();
5016 checked_mem = memory(adr_type);
5017 checked_value = cv;
5018 set_control(top()); // no fast path
5019 }
5020
5021 Node* not_pos = generate_nonpositive_guard(copy_length, length_never_negative);
5022 if (not_pos != NULL) {
5023 PreserveJVMState pjvms(this);
5024 set_control(not_pos);
5025
5026 // (6) length must not be negative.
5027 if (!length_never_negative) {
5028 generate_negative_guard(copy_length, slow_region);
5029 }
5030
5031 // copy_length is 0.
5032 if (!stopped() && dest_uninitialized) {
5033 Node* dest_length = alloc->in(AllocateNode::ALength);
5034 if (copy_length->eqv_uncast(dest_length)
5035 || _gvn.find_int_con(dest_length, 1) <= 0) {
5036 // There is no zeroing to do. No need for a secondary raw memory barrier.
5037 } else {
5038 // Clear the whole thing since there are no source elements to copy.
5039 generate_clear_array(adr_type, dest, basic_elem_type,
5040 intcon(0), NULL,
5041 alloc->in(AllocateNode::AllocSize));
5042 // Use a secondary InitializeNode as raw memory barrier.
5043 // Currently it is needed only on this path since other
5044 // paths have stub or runtime calls as raw memory barriers.
5045 InitializeNode* init = insert_mem_bar_volatile(Op_Initialize,
5046 Compile::AliasIdxRaw,
5047 top())->as_Initialize();
5048 init->set_complete(&_gvn); // (there is no corresponding AllocateNode)
5049 }
5050 }
5051
5052 // Present the results of the fast call.
5053 result_region->init_req(zero_path, control());
5054 result_i_o ->init_req(zero_path, i_o());
5055 result_memory->init_req(zero_path, memory(adr_type));
5056 }
5057
5058 if (!stopped() && dest_uninitialized) {
5059 // We have to initialize the *uncopied* part of the array to zero.
5060 // The copy destination is the slice dest[off..off+len]. The other slices
5061 // are dest_head = dest[0..off] and dest_tail = dest[off+len..dest.length].
5062 Node* dest_size = alloc->in(AllocateNode::AllocSize);
5063 Node* dest_length = alloc->in(AllocateNode::ALength);
5064 Node* dest_tail = _gvn.transform(new AddINode(dest_offset, copy_length));
5065
5066 // If there is a head section that needs zeroing, do it now.
5067 if (find_int_con(dest_offset, -1) != 0) {
5068 generate_clear_array(adr_type, dest, basic_elem_type,
5069 intcon(0), dest_offset,
5070 NULL);
5071 }
5072
5073 // Next, perform a dynamic check on the tail length.
5074 // It is often zero, and we can win big if we prove this.
5075 // There are two wins: Avoid generating the ClearArray
5076 // with its attendant messy index arithmetic, and upgrade
5077 // the copy to a more hardware-friendly word size of 64 bits.
5078 Node* tail_ctl = NULL;
5079 if (!stopped() && !dest_tail->eqv_uncast(dest_length)) {
5080 Node* cmp_lt = _gvn.transform(new CmpINode(dest_tail, dest_length));
5081 Node* bol_lt = _gvn.transform(new BoolNode(cmp_lt, BoolTest::lt));
5082 tail_ctl = generate_slow_guard(bol_lt, NULL);
5083 assert(tail_ctl != NULL || !stopped(), "must be an outcome");
5084 }
5085
5086 // At this point, let's assume there is no tail.
5087 if (!stopped() && alloc != NULL && basic_elem_type != T_OBJECT) {
5088 // There is no tail. Try an upgrade to a 64-bit copy.
5089 bool didit = false;
5090 { PreserveJVMState pjvms(this);
5091 didit = generate_block_arraycopy(adr_type, basic_elem_type, alloc,
5092 src, src_offset, dest, dest_offset,
5093 dest_size, dest_uninitialized);
5094 if (didit) {
5095 // Present the results of the block-copying fast call.
5096 result_region->init_req(bcopy_path, control());
5097 result_i_o ->init_req(bcopy_path, i_o());
5098 result_memory->init_req(bcopy_path, memory(adr_type));
5099 }
5100 }
5101 if (didit)
5102 set_control(top()); // no regular fast path
5103 }
5104
5105 // Clear the tail, if any.
5106 if (tail_ctl != NULL) {
5107 Node* notail_ctl = stopped() ? NULL : control();
5108 set_control(tail_ctl);
5109 if (notail_ctl == NULL) {
5110 generate_clear_array(adr_type, dest, basic_elem_type,
5111 dest_tail, NULL,
5112 dest_size);
5113 } else {
5114 // Make a local merge.
5115 Node* done_ctl = new RegionNode(3);
5116 Node* done_mem = new PhiNode(done_ctl, Type::MEMORY, adr_type);
5117 done_ctl->init_req(1, notail_ctl);
5118 done_mem->init_req(1, memory(adr_type));
5119 generate_clear_array(adr_type, dest, basic_elem_type,
5120 dest_tail, NULL,
5121 dest_size);
5122 done_ctl->init_req(2, control());
5123 done_mem->init_req(2, memory(adr_type));
5124 set_control( _gvn.transform(done_ctl));
5125 set_memory( _gvn.transform(done_mem), adr_type );
5126 }
5127 }
5128 }
5129
5130 BasicType copy_type = basic_elem_type;
5131 assert(basic_elem_type != T_ARRAY, "caller must fix this");
5132 if (!stopped() && copy_type == T_OBJECT) {
5133 // If src and dest have compatible element types, we can copy bits.
5134 // Types S[] and D[] are compatible if D is a supertype of S.
5135 //
5136 // If they are not, we will use checked_oop_disjoint_arraycopy,
5137 // which performs a fast optimistic per-oop check, and backs off
5138 // further to JVM_ArrayCopy on the first per-oop check that fails.
5139 // (Actually, we don't move raw bits only; the GC requires card marks.)
5140
5141 // Get the Klass* for both src and dest
5142 Node* src_klass = load_object_klass(src);
5143 Node* dest_klass = load_object_klass(dest);
5144
5145 // Generate the subtype check.
5146 // This might fold up statically, or then again it might not.
5147 //
5148 // Non-static example: Copying List<String>.elements to a new String[].
5149 // The backing store for a List<String> is always an Object[],
5150 // but its elements are always type String, if the generic types
5151 // are correct at the source level.
5152 //
5153 // Test S[] against D[], not S against D, because (probably)
5154 // the secondary supertype cache is less busy for S[] than S.
5155 // This usually only matters when D is an interface.
5156 Node* not_subtype_ctrl = gen_subtype_check(src_klass, dest_klass);
5157 // Plug failing path into checked_oop_disjoint_arraycopy
5158 if (not_subtype_ctrl != top()) {
5159 PreserveJVMState pjvms(this);
5160 set_control(not_subtype_ctrl);
5161 // (At this point we can assume disjoint_bases, since types differ.)
5162 int ek_offset = in_bytes(ObjArrayKlass::element_klass_offset());
5163 Node* p1 = basic_plus_adr(dest_klass, ek_offset);
5164 Node* n1 = LoadKlassNode::make(_gvn, immutable_memory(), p1, TypeRawPtr::BOTTOM);
5165 Node* dest_elem_klass = _gvn.transform(n1);
5166 Node* cv = generate_checkcast_arraycopy(adr_type,
5167 dest_elem_klass,
5168 src, src_offset, dest, dest_offset,
5169 ConvI2X(copy_length), dest_uninitialized);
5170 if (cv == NULL) cv = intcon(-1); // failure (no stub available)
5171 checked_control = control();
5172 checked_i_o = i_o();
5173 checked_mem = memory(adr_type);
5174 checked_value = cv;
5175 }
5176 // At this point we know we do not need type checks on oop stores.
5177
5178 // Let's see if we need card marks:
5179 if (alloc != NULL && use_ReduceInitialCardMarks()) {
5180 // If we do not need card marks, copy using the jint or jlong stub.
5181 copy_type = LP64_ONLY(UseCompressedOops ? T_INT : T_LONG) NOT_LP64(T_INT);
5182 assert(type2aelembytes(basic_elem_type) == type2aelembytes(copy_type),
5183 "sizes agree");
5184 }
5185 }
5186
5187 if (!stopped()) {
5188 // Generate the fast path, if possible.
5189 PreserveJVMState pjvms(this);
5190 generate_unchecked_arraycopy(adr_type, copy_type, disjoint_bases,
5191 src, src_offset, dest, dest_offset,
5192 ConvI2X(copy_length), dest_uninitialized);
5193
5194 // Present the results of the fast call.
5195 result_region->init_req(fast_path, control());
5196 result_i_o ->init_req(fast_path, i_o());
5197 result_memory->init_req(fast_path, memory(adr_type));
5198 }
5199
5200 // Here are all the slow paths up to this point, in one bundle:
5201 slow_control = top();
5202 if (slow_region != NULL)
5203 slow_control = _gvn.transform(slow_region);
5204 DEBUG_ONLY(slow_region = (RegionNode*)badAddress);
5205
5206 set_control(checked_control);
5207 if (!stopped()) {
5208 // Clean up after the checked call.
5209 // The returned value is either 0 or -1^K,
5210 // where K = number of partially transferred array elements.
5211 Node* cmp = _gvn.transform(new CmpINode(checked_value, intcon(0)));
5212 Node* bol = _gvn.transform(new BoolNode(cmp, BoolTest::eq));
5213 IfNode* iff = create_and_map_if(control(), bol, PROB_MAX, COUNT_UNKNOWN);
5214
5215 // If it is 0, we are done, so transfer to the end.
5216 Node* checks_done = _gvn.transform(new IfTrueNode(iff));
5217 result_region->init_req(checked_path, checks_done);
5218 result_i_o ->init_req(checked_path, checked_i_o);
5219 result_memory->init_req(checked_path, checked_mem);
5220
5221 // If it is not zero, merge into the slow call.
5222 set_control( _gvn.transform(new IfFalseNode(iff) ));
5223 RegionNode* slow_reg2 = new RegionNode(3);
5224 PhiNode* slow_i_o2 = new PhiNode(slow_reg2, Type::ABIO);
5225 PhiNode* slow_mem2 = new PhiNode(slow_reg2, Type::MEMORY, adr_type);
5226 record_for_igvn(slow_reg2);
5227 slow_reg2 ->init_req(1, slow_control);
5228 slow_i_o2 ->init_req(1, slow_i_o);
5229 slow_mem2 ->init_req(1, slow_mem);
5230 slow_reg2 ->init_req(2, control());
5231 slow_i_o2 ->init_req(2, checked_i_o);
5232 slow_mem2 ->init_req(2, checked_mem);
5233
5234 slow_control = _gvn.transform(slow_reg2);
5235 slow_i_o = _gvn.transform(slow_i_o2);
5236 slow_mem = _gvn.transform(slow_mem2);
5237
5238 if (alloc != NULL) {
5239 // We'll restart from the very beginning, after zeroing the whole thing.
5240 // This can cause double writes, but that's OK since dest is brand new.
5241 // So we ignore the low 31 bits of the value returned from the stub.
5242 } else {
5243 // We must continue the copy exactly where it failed, or else
5244 // another thread might see the wrong number of writes to dest.
5245 Node* checked_offset = _gvn.transform(new XorINode(checked_value, intcon(-1)));
5246 Node* slow_offset = new PhiNode(slow_reg2, TypeInt::INT);
5247 slow_offset->init_req(1, intcon(0));
5248 slow_offset->init_req(2, checked_offset);
5249 slow_offset = _gvn.transform(slow_offset);
5250
5251 // Adjust the arguments by the conditionally incoming offset.
5252 Node* src_off_plus = _gvn.transform(new AddINode(src_offset, slow_offset));
5253 Node* dest_off_plus = _gvn.transform(new AddINode(dest_offset, slow_offset));
5254 Node* length_minus = _gvn.transform(new SubINode(copy_length, slow_offset));
5255
5256 // Tweak the node variables to adjust the code produced below:
5257 src_offset = src_off_plus;
5258 dest_offset = dest_off_plus;
5259 copy_length = length_minus;
5260 }
5261 }
5262
5263 set_control(slow_control);
5264 if (!stopped()) {
5265 // Generate the slow path, if needed.
5266 PreserveJVMState pjvms(this); // replace_in_map may trash the map
5267
5268 set_memory(slow_mem, adr_type);
5269 set_i_o(slow_i_o);
5270
5271 if (dest_uninitialized) {
5272 generate_clear_array(adr_type, dest, basic_elem_type,
5273 intcon(0), NULL,
5274 alloc->in(AllocateNode::AllocSize));
5275 }
5276
5277 generate_slow_arraycopy(adr_type,
5278 src, src_offset, dest, dest_offset,
5279 copy_length, /*dest_uninitialized*/false);
5280
5281 result_region->init_req(slow_call_path, control());
5282 result_i_o ->init_req(slow_call_path, i_o());
5283 result_memory->init_req(slow_call_path, memory(adr_type));
5284 }
5285
5286 // Remove unused edges.
5287 for (uint i = 1; i < result_region->req(); i++) {
5288 if (result_region->in(i) == NULL)
5289 result_region->init_req(i, top());
5290 }
5291
5292 // Finished; return the combined state.
5293 set_control( _gvn.transform(result_region));
5294 set_i_o( _gvn.transform(result_i_o) );
5295 set_memory( _gvn.transform(result_memory), adr_type );
5296
5297 // The memory edges above are precise in order to model effects around
5298 // array copies accurately to allow value numbering of field loads around
5299 // arraycopy. Such field loads, both before and after, are common in Java
5300 // collections and similar classes involving header/array data structures.
5301 //
5302 // But with low number of register or when some registers are used or killed
5303 // by arraycopy calls it causes registers spilling on stack. See 6544710.
5304 // The next memory barrier is added to avoid it. If the arraycopy can be
5305 // optimized away (which it can, sometimes) then we can manually remove
5306 // the membar also.
5307 //
5308 // Do not let reads from the cloned object float above the arraycopy.
5309 if (alloc != NULL) {
5310 // Do not let stores that initialize this object be reordered with
5311 // a subsequent store that would make this object accessible by
5312 // other threads.
5313 // Record what AllocateNode this StoreStore protects so that
5314 // escape analysis can go from the MemBarStoreStoreNode to the
5315 // AllocateNode and eliminate the MemBarStoreStoreNode if possible
5316 // based on the escape status of the AllocateNode.
5317 insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out(AllocateNode::RawAddress));
5318 } else if (InsertMemBarAfterArraycopy)
5319 insert_mem_bar(Op_MemBarCPUOrder);
5320 }
5321
5322
5323 // Helper function which determines if an arraycopy immediately follows
5324 // an allocation, with no intervening tests or other escapes for the object.
5325 AllocateArrayNode*
5326 LibraryCallKit::tightly_coupled_allocation(Node* ptr,
5327 RegionNode* slow_region) {
5328 if (stopped()) return NULL; // no fast path
5329 if (C->AliasLevel() == 0) return NULL; // no MergeMems around
5330
5331 AllocateArrayNode* alloc = AllocateArrayNode::Ideal_array_allocation(ptr, &_gvn);
5332 if (alloc == NULL) return NULL;
5333
5334 Node* rawmem = memory(Compile::AliasIdxRaw);
5335 // Is the allocation's memory state untouched?
5336 if (!(rawmem->is_Proj() && rawmem->in(0)->is_Initialize())) {
5337 // Bail out if there have been raw-memory effects since the allocation.
5338 // (Example: There might have been a call or safepoint.)
5339 return NULL;
5377 Node* obs = not_ctl->fast_out(j);
5378 if (obs->in(0) == not_ctl && obs->is_Call() &&
5379 (obs->as_Call()->entry_point() == SharedRuntime::uncommon_trap_blob()->entry_point())) {
5380 found_trap = true; break;
5381 }
5382 }
5383 if (found_trap) {
5384 ctl = iff->in(0); // This test feeds a harmless uncommon trap.
5385 continue;
5386 }
5387 }
5388 return NULL;
5389 }
5390
5391 // If we get this far, we have an allocation which immediately
5392 // precedes the arraycopy, and we can take over zeroing the new object.
5393 // The arraycopy will finish the initialization, and provide
5394 // a new control state to which we will anchor the destination pointer.
5395
5396 return alloc;
5397 }
5398
5399 // Helper for initialization of arrays, creating a ClearArray.
5400 // It writes zero bits in [start..end), within the body of an array object.
5401 // The memory effects are all chained onto the 'adr_type' alias category.
5402 //
5403 // Since the object is otherwise uninitialized, we are free
5404 // to put a little "slop" around the edges of the cleared area,
5405 // as long as it does not go back into the array's header,
5406 // or beyond the array end within the heap.
5407 //
5408 // The lower edge can be rounded down to the nearest jint and the
5409 // upper edge can be rounded up to the nearest MinObjAlignmentInBytes.
5410 //
5411 // Arguments:
5412 // adr_type memory slice where writes are generated
5413 // dest oop of the destination array
5414 // basic_elem_type element type of the destination
5415 // slice_idx array index of first element to store
5416 // slice_len number of elements to store (or NULL)
5417 // dest_size total size in bytes of the array object
5418 //
5419 // Exactly one of slice_len or dest_size must be non-NULL.
5420 // If dest_size is non-NULL, zeroing extends to the end of the object.
5421 // If slice_len is non-NULL, the slice_idx value must be a constant.
5422 void
5423 LibraryCallKit::generate_clear_array(const TypePtr* adr_type,
5424 Node* dest,
5425 BasicType basic_elem_type,
5426 Node* slice_idx,
5427 Node* slice_len,
5428 Node* dest_size) {
5429 // one or the other but not both of slice_len and dest_size:
5430 assert((slice_len != NULL? 1: 0) + (dest_size != NULL? 1: 0) == 1, "");
5431 if (slice_len == NULL) slice_len = top();
5432 if (dest_size == NULL) dest_size = top();
5433
5434 // operate on this memory slice:
5435 Node* mem = memory(adr_type); // memory slice to operate on
5436
5437 // scaling and rounding of indexes:
5438 int scale = exact_log2(type2aelembytes(basic_elem_type));
5439 int abase = arrayOopDesc::base_offset_in_bytes(basic_elem_type);
5440 int clear_low = (-1 << scale) & (BytesPerInt - 1);
5441 int bump_bit = (-1 << scale) & BytesPerInt;
5442
5443 // determine constant starts and ends
5444 const intptr_t BIG_NEG = -128;
5445 assert(BIG_NEG + 2*abase < 0, "neg enough");
5446 intptr_t slice_idx_con = (intptr_t) find_int_con(slice_idx, BIG_NEG);
5447 intptr_t slice_len_con = (intptr_t) find_int_con(slice_len, BIG_NEG);
5448 if (slice_len_con == 0) {
5449 return; // nothing to do here
5450 }
5451 intptr_t start_con = (abase + (slice_idx_con << scale)) & ~clear_low;
5452 intptr_t end_con = find_intptr_t_con(dest_size, -1);
5453 if (slice_idx_con >= 0 && slice_len_con >= 0) {
5454 assert(end_con < 0, "not two cons");
5455 end_con = round_to(abase + ((slice_idx_con + slice_len_con) << scale),
5456 BytesPerLong);
5457 }
5458
5459 if (start_con >= 0 && end_con >= 0) {
5460 // Constant start and end. Simple.
5461 mem = ClearArrayNode::clear_memory(control(), mem, dest,
5462 start_con, end_con, &_gvn);
5463 } else if (start_con >= 0 && dest_size != top()) {
5464 // Constant start, pre-rounded end after the tail of the array.
5465 Node* end = dest_size;
5466 mem = ClearArrayNode::clear_memory(control(), mem, dest,
5467 start_con, end, &_gvn);
5468 } else if (start_con >= 0 && slice_len != top()) {
5469 // Constant start, non-constant end. End needs rounding up.
5470 // End offset = round_up(abase + ((slice_idx_con + slice_len) << scale), 8)
5471 intptr_t end_base = abase + (slice_idx_con << scale);
5472 int end_round = (-1 << scale) & (BytesPerLong - 1);
5473 Node* end = ConvI2X(slice_len);
5474 if (scale != 0)
5475 end = _gvn.transform(new LShiftXNode(end, intcon(scale) ));
5476 end_base += end_round;
5477 end = _gvn.transform(new AddXNode(end, MakeConX(end_base)));
5478 end = _gvn.transform(new AndXNode(end, MakeConX(~end_round)));
5479 mem = ClearArrayNode::clear_memory(control(), mem, dest,
5480 start_con, end, &_gvn);
5481 } else if (start_con < 0 && dest_size != top()) {
5482 // Non-constant start, pre-rounded end after the tail of the array.
5483 // This is almost certainly a "round-to-end" operation.
5484 Node* start = slice_idx;
5485 start = ConvI2X(start);
5486 if (scale != 0)
5487 start = _gvn.transform(new LShiftXNode( start, intcon(scale) ));
5488 start = _gvn.transform(new AddXNode(start, MakeConX(abase)));
5489 if ((bump_bit | clear_low) != 0) {
5490 int to_clear = (bump_bit | clear_low);
5491 // Align up mod 8, then store a jint zero unconditionally
5492 // just before the mod-8 boundary.
5493 if (((abase + bump_bit) & ~to_clear) - bump_bit
5494 < arrayOopDesc::length_offset_in_bytes() + BytesPerInt) {
5495 bump_bit = 0;
5496 assert((abase & to_clear) == 0, "array base must be long-aligned");
5497 } else {
5498 // Bump 'start' up to (or past) the next jint boundary:
5499 start = _gvn.transform(new AddXNode(start, MakeConX(bump_bit)));
5500 assert((abase & clear_low) == 0, "array base must be int-aligned");
5501 }
5502 // Round bumped 'start' down to jlong boundary in body of array.
5503 start = _gvn.transform(new AndXNode(start, MakeConX(~to_clear)));
5504 if (bump_bit != 0) {
5505 // Store a zero to the immediately preceding jint:
5506 Node* x1 = _gvn.transform(new AddXNode(start, MakeConX(-bump_bit)));
5507 Node* p1 = basic_plus_adr(dest, x1);
5508 mem = StoreNode::make(_gvn, control(), mem, p1, adr_type, intcon(0), T_INT, MemNode::unordered);
5509 mem = _gvn.transform(mem);
5510 }
5511 }
5512 Node* end = dest_size; // pre-rounded
5513 mem = ClearArrayNode::clear_memory(control(), mem, dest,
5514 start, end, &_gvn);
5515 } else {
5516 // Non-constant start, unrounded non-constant end.
5517 // (Nobody zeroes a random midsection of an array using this routine.)
5518 ShouldNotReachHere(); // fix caller
5519 }
5520
5521 // Done.
5522 set_memory(mem, adr_type);
5523 }
5524
5525
5526 bool
5527 LibraryCallKit::generate_block_arraycopy(const TypePtr* adr_type,
5528 BasicType basic_elem_type,
5529 AllocateNode* alloc,
5530 Node* src, Node* src_offset,
5531 Node* dest, Node* dest_offset,
5532 Node* dest_size, bool dest_uninitialized) {
5533 // See if there is an advantage from block transfer.
5534 int scale = exact_log2(type2aelembytes(basic_elem_type));
5535 if (scale >= LogBytesPerLong)
5536 return false; // it is already a block transfer
5537
5538 // Look at the alignment of the starting offsets.
5539 int abase = arrayOopDesc::base_offset_in_bytes(basic_elem_type);
5540
5541 intptr_t src_off_con = (intptr_t) find_int_con(src_offset, -1);
5542 intptr_t dest_off_con = (intptr_t) find_int_con(dest_offset, -1);
5543 if (src_off_con < 0 || dest_off_con < 0)
5544 // At present, we can only understand constants.
5545 return false;
5546
5547 intptr_t src_off = abase + (src_off_con << scale);
5548 intptr_t dest_off = abase + (dest_off_con << scale);
5549
5550 if (((src_off | dest_off) & (BytesPerLong-1)) != 0) {
5551 // Non-aligned; too bad.
5552 // One more chance: Pick off an initial 32-bit word.
5553 // This is a common case, since abase can be odd mod 8.
5554 if (((src_off | dest_off) & (BytesPerLong-1)) == BytesPerInt &&
5555 ((src_off ^ dest_off) & (BytesPerLong-1)) == 0) {
5556 Node* sptr = basic_plus_adr(src, src_off);
5557 Node* dptr = basic_plus_adr(dest, dest_off);
5558 Node* sval = make_load(control(), sptr, TypeInt::INT, T_INT, adr_type, MemNode::unordered);
5559 store_to_memory(control(), dptr, sval, T_INT, adr_type, MemNode::unordered);
5560 src_off += BytesPerInt;
5561 dest_off += BytesPerInt;
5562 } else {
5563 return false;
5564 }
5565 }
5566 assert(src_off % BytesPerLong == 0, "");
5567 assert(dest_off % BytesPerLong == 0, "");
5568
5569 // Do this copy by giant steps.
5570 Node* sptr = basic_plus_adr(src, src_off);
5571 Node* dptr = basic_plus_adr(dest, dest_off);
5572 Node* countx = dest_size;
5573 countx = _gvn.transform(new SubXNode(countx, MakeConX(dest_off)));
5574 countx = _gvn.transform(new URShiftXNode(countx, intcon(LogBytesPerLong)));
5575
5576 bool disjoint_bases = true; // since alloc != NULL
5577 generate_unchecked_arraycopy(adr_type, T_LONG, disjoint_bases,
5578 sptr, NULL, dptr, NULL, countx, dest_uninitialized);
5579
5580 return true;
5581 }
5582
5583
5584 // Helper function; generates code for the slow case.
5585 // We make a call to a runtime method which emulates the native method,
5586 // but without the native wrapper overhead.
5587 void
5588 LibraryCallKit::generate_slow_arraycopy(const TypePtr* adr_type,
5589 Node* src, Node* src_offset,
5590 Node* dest, Node* dest_offset,
5591 Node* copy_length, bool dest_uninitialized) {
5592 assert(!dest_uninitialized, "Invariant");
5593 Node* call = make_runtime_call(RC_NO_LEAF | RC_UNCOMMON,
5594 OptoRuntime::slow_arraycopy_Type(),
5595 OptoRuntime::slow_arraycopy_Java(),
5596 "slow_arraycopy", adr_type,
5597 src, src_offset, dest, dest_offset,
5598 copy_length);
5599
5600 // Handle exceptions thrown by this fellow:
5601 make_slow_call_ex(call, env()->Throwable_klass(), false);
5602 }
5603
5604 // Helper function; generates code for cases requiring runtime checks.
5605 Node*
5606 LibraryCallKit::generate_checkcast_arraycopy(const TypePtr* adr_type,
5607 Node* dest_elem_klass,
5608 Node* src, Node* src_offset,
5609 Node* dest, Node* dest_offset,
5610 Node* copy_length, bool dest_uninitialized) {
5611 if (stopped()) return NULL;
5612
5613 address copyfunc_addr = StubRoutines::checkcast_arraycopy(dest_uninitialized);
5614 if (copyfunc_addr == NULL) { // Stub was not generated, go slow path.
5615 return NULL;
5616 }
5617
5618 // Pick out the parameters required to perform a store-check
5619 // for the target array. This is an optimistic check. It will
5620 // look in each non-null element's class, at the desired klass's
5621 // super_check_offset, for the desired klass.
5622 int sco_offset = in_bytes(Klass::super_check_offset_offset());
5623 Node* p3 = basic_plus_adr(dest_elem_klass, sco_offset);
5624 Node* n3 = new LoadINode(NULL, memory(p3), p3, _gvn.type(p3)->is_ptr(), TypeInt::INT, MemNode::unordered);
5625 Node* check_offset = ConvI2X(_gvn.transform(n3));
5626 Node* check_value = dest_elem_klass;
5627
5628 Node* src_start = array_element_address(src, src_offset, T_OBJECT);
5629 Node* dest_start = array_element_address(dest, dest_offset, T_OBJECT);
5630
5631 // (We know the arrays are never conjoint, because their types differ.)
5632 Node* call = make_runtime_call(RC_LEAF|RC_NO_FP,
5633 OptoRuntime::checkcast_arraycopy_Type(),
5634 copyfunc_addr, "checkcast_arraycopy", adr_type,
5635 // five arguments, of which two are
5636 // intptr_t (jlong in LP64)
5637 src_start, dest_start,
5638 copy_length XTOP,
5639 check_offset XTOP,
5640 check_value);
5641
5642 return _gvn.transform(new ProjNode(call, TypeFunc::Parms));
5643 }
5644
5645
5646 // Helper function; generates code for cases requiring runtime checks.
5647 Node*
5648 LibraryCallKit::generate_generic_arraycopy(const TypePtr* adr_type,
5649 Node* src, Node* src_offset,
5650 Node* dest, Node* dest_offset,
5651 Node* copy_length, bool dest_uninitialized) {
5652 assert(!dest_uninitialized, "Invariant");
5653 if (stopped()) return NULL;
5654 address copyfunc_addr = StubRoutines::generic_arraycopy();
5655 if (copyfunc_addr == NULL) { // Stub was not generated, go slow path.
5656 return NULL;
5657 }
5658
5659 Node* call = make_runtime_call(RC_LEAF|RC_NO_FP,
5660 OptoRuntime::generic_arraycopy_Type(),
5661 copyfunc_addr, "generic_arraycopy", adr_type,
5662 src, src_offset, dest, dest_offset, copy_length);
5663
5664 return _gvn.transform(new ProjNode(call, TypeFunc::Parms));
5665 }
5666
5667 // Helper function; generates the fast out-of-line call to an arraycopy stub.
5668 void
5669 LibraryCallKit::generate_unchecked_arraycopy(const TypePtr* adr_type,
5670 BasicType basic_elem_type,
5671 bool disjoint_bases,
5672 Node* src, Node* src_offset,
5673 Node* dest, Node* dest_offset,
5674 Node* copy_length, bool dest_uninitialized) {
5675 if (stopped()) return; // nothing to do
5676
5677 Node* src_start = src;
5678 Node* dest_start = dest;
5679 if (src_offset != NULL || dest_offset != NULL) {
5680 assert(src_offset != NULL && dest_offset != NULL, "");
5681 src_start = array_element_address(src, src_offset, basic_elem_type);
5682 dest_start = array_element_address(dest, dest_offset, basic_elem_type);
5683 }
5684
5685 // Figure out which arraycopy runtime method to call.
5686 const char* copyfunc_name = "arraycopy";
5687 address copyfunc_addr =
5688 basictype2arraycopy(basic_elem_type, src_offset, dest_offset,
5689 disjoint_bases, copyfunc_name, dest_uninitialized);
5690
5691 // Call it. Note that the count_ix value is not scaled to a byte-size.
5692 make_runtime_call(RC_LEAF|RC_NO_FP,
5693 OptoRuntime::fast_arraycopy_Type(),
5694 copyfunc_addr, copyfunc_name, adr_type,
5695 src_start, dest_start, copy_length XTOP);
5696 }
5697
5698 //-------------inline_encodeISOArray-----------------------------------
5699 // encode char[] to byte[] in ISO_8859_1
5700 bool LibraryCallKit::inline_encodeISOArray() {
5701 assert(callee()->signature()->size() == 5, "encodeISOArray has 5 parameters");
5702 // no receiver since it is static method
5703 Node *src = argument(0);
5704 Node *src_offset = argument(1);
5705 Node *dst = argument(2);
5706 Node *dst_offset = argument(3);
5707 Node *length = argument(4);
5708
5709 const Type* src_type = src->Value(&_gvn);
5710 const Type* dst_type = dst->Value(&_gvn);
5711 const TypeAryPtr* top_src = src_type->isa_aryptr();
5712 const TypeAryPtr* top_dest = dst_type->isa_aryptr();
5713 if (top_src == NULL || top_src->klass() == NULL ||
5714 top_dest == NULL || top_dest->klass() == NULL) {
5715 // failed array check
|
128 }
129
130 private:
131 void fatal_unexpected_iid(vmIntrinsics::ID iid) {
132 fatal(err_msg_res("unexpected intrinsic %d: %s", iid, vmIntrinsics::name_at(iid)));
133 }
134
135 void set_result(Node* n) { assert(_result == NULL, "only set once"); _result = n; }
136 void set_result(RegionNode* region, PhiNode* value);
137 Node* result() { return _result; }
138
139 virtual int reexecute_sp() { return _reexecute_sp; }
140
141 // Helper functions to inline natives
142 Node* generate_guard(Node* test, RegionNode* region, float true_prob);
143 Node* generate_slow_guard(Node* test, RegionNode* region);
144 Node* generate_fair_guard(Node* test, RegionNode* region);
145 Node* generate_negative_guard(Node* index, RegionNode* region,
146 // resulting CastII of index:
147 Node* *pos_index = NULL);
148 Node* generate_limit_guard(Node* offset, Node* subseq_length,
149 Node* array_length,
150 RegionNode* region);
151 Node* generate_current_thread(Node* &tls_output);
152 Node* load_mirror_from_klass(Node* klass);
153 Node* load_klass_from_mirror_common(Node* mirror, bool never_see_null,
154 RegionNode* region, int null_path,
155 int offset);
156 Node* load_klass_from_mirror(Node* mirror, bool never_see_null,
157 RegionNode* region, int null_path) {
158 int offset = java_lang_Class::klass_offset_in_bytes();
159 return load_klass_from_mirror_common(mirror, never_see_null,
160 region, null_path,
161 offset);
162 }
163 Node* load_array_klass_from_mirror(Node* mirror, bool never_see_null,
164 RegionNode* region, int null_path) {
165 int offset = java_lang_Class::array_klass_offset_in_bytes();
166 return load_klass_from_mirror_common(mirror, never_see_null,
167 region, null_path,
168 offset);
169 }
170 Node* generate_access_flags_guard(Node* kls,
171 int modifier_mask, int modifier_bits,
241 bool inline_native_threadID();
242 #endif
243 bool inline_native_time_funcs(address method, const char* funcName);
244 bool inline_native_isInterrupted();
245 bool inline_native_Class_query(vmIntrinsics::ID id);
246 bool inline_native_subtype_check();
247
248 bool inline_native_newArray();
249 bool inline_native_getLength();
250 bool inline_array_copyOf(bool is_copyOfRange);
251 bool inline_array_equals();
252 void copy_to_clone(Node* obj, Node* alloc_obj, Node* obj_size, bool is_array, bool card_mark);
253 bool inline_native_clone(bool is_virtual);
254 bool inline_native_Reflection_getCallerClass();
255 // Helper function for inlining native object hash method
256 bool inline_native_hashcode(bool is_virtual, bool is_static);
257 bool inline_native_getClass();
258
259 // Helper functions for inlining arraycopy
260 bool inline_arraycopy();
261 AllocateArrayNode* tightly_coupled_allocation(Node* ptr,
262 RegionNode* slow_region);
263 typedef enum { LS_xadd, LS_xchg, LS_cmpxchg } LoadStoreKind;
264 bool inline_unsafe_load_store(BasicType type, LoadStoreKind kind);
265 bool inline_unsafe_ordered_store(BasicType type);
266 bool inline_unsafe_fence(vmIntrinsics::ID id);
267 bool inline_fp_conversions(vmIntrinsics::ID id);
268 bool inline_number_methods(vmIntrinsics::ID id);
269 bool inline_reference_get();
270 bool inline_aescrypt_Block(vmIntrinsics::ID id);
271 bool inline_cipherBlockChaining_AESCrypt(vmIntrinsics::ID id);
272 Node* inline_cipherBlockChaining_AESCrypt_predicate(bool decrypting);
273 Node* get_key_start_from_aescrypt_object(Node* aescrypt_object);
274 Node* get_original_key_start_from_aescrypt_object(Node* aescrypt_object);
275 bool inline_sha_implCompress(vmIntrinsics::ID id);
276 bool inline_digestBase_implCompressMB(int predicate);
277 bool inline_sha_implCompressMB(Node* digestBaseObj, ciInstanceKlass* instklass_SHA,
278 bool long_state, address stubAddr, const char *stubName,
279 Node* src_start, Node* ofs, Node* limit);
280 Node* get_state_from_sha_object(Node *sha_object);
281 Node* get_state_from_sha5_object(Node *sha_object);
282 Node* inline_digestBase_implCompressMB_predicate(int predicate);
987 }
988
989 inline Node* LibraryCallKit::generate_negative_guard(Node* index, RegionNode* region,
990 Node* *pos_index) {
991 if (stopped())
992 return NULL; // already stopped
993 if (_gvn.type(index)->higher_equal(TypeInt::POS)) // [0,maxint]
994 return NULL; // index is already adequately typed
995 Node* cmp_lt = _gvn.transform(new CmpINode(index, intcon(0)));
996 Node* bol_lt = _gvn.transform(new BoolNode(cmp_lt, BoolTest::lt));
997 Node* is_neg = generate_guard(bol_lt, region, PROB_MIN);
998 if (is_neg != NULL && pos_index != NULL) {
999 // Emulate effect of Parse::adjust_map_after_if.
1000 Node* ccast = new CastIINode(index, TypeInt::POS);
1001 ccast->set_req(0, control());
1002 (*pos_index) = _gvn.transform(ccast);
1003 }
1004 return is_neg;
1005 }
1006
1007 // Make sure that 'position' is a valid limit index, in [0..length].
1008 // There are two equivalent plans for checking this:
1009 // A. (offset + copyLength) unsigned<= arrayLength
1010 // B. offset <= (arrayLength - copyLength)
1011 // We require that all of the values above, except for the sum and
1012 // difference, are already known to be non-negative.
1013 // Plan A is robust in the face of overflow, if offset and copyLength
1014 // are both hugely positive.
1015 //
1016 // Plan B is less direct and intuitive, but it does not overflow at
1017 // all, since the difference of two non-negatives is always
1018 // representable. Whenever Java methods must perform the equivalent
1019 // check they generally use Plan B instead of Plan A.
1020 // For the moment we use Plan A.
1021 inline Node* LibraryCallKit::generate_limit_guard(Node* offset,
1022 Node* subseq_length,
1023 Node* array_length,
1024 RegionNode* region) {
1025 if (stopped())
1026 return NULL; // already stopped
3847 if (bailout->req() > 1) {
3848 PreserveJVMState pjvms(this);
3849 set_control(_gvn.transform(bailout));
3850 uncommon_trap(Deoptimization::Reason_intrinsic,
3851 Deoptimization::Action_maybe_recompile);
3852 }
3853
3854 if (!stopped()) {
3855 // How many elements will we copy from the original?
3856 // The answer is MinI(orig_length - start, length).
3857 Node* orig_tail = _gvn.transform(new SubINode(orig_length, start));
3858 Node* moved = generate_min_max(vmIntrinsics::_min, orig_tail, length);
3859
3860 newcopy = new_array(klass_node, length, 0); // no argments to push
3861
3862 // Generate a direct call to the right arraycopy function(s).
3863 // We know the copy is disjoint but we might not know if the
3864 // oop stores need checking.
3865 // Extreme case: Arrays.copyOf((Integer[])x, 10, String[].class).
3866 // This will fail a store-check if x contains any non-nulls.
3867
3868 Node* alloc = tightly_coupled_allocation(newcopy, NULL);
3869
3870 ArrayCopyNode* ac = ArrayCopyNode::make(this, true, original, start, newcopy, intcon(0), moved, alloc != NULL);
3871 if (!is_copyOfRange) {
3872 ac->set_copyof();
3873 } else {
3874 ac->set_copyofrange();
3875 }
3876 Node* n = _gvn.transform(ac);
3877 assert(n == ac, "cannot disappear");
3878 ac->connect_outputs(this);
3879 }
3880 } // original reexecute is set back here
3881
3882 C->set_has_split_ifs(true); // Has chance for split-if optimization
3883 if (!stopped()) {
3884 set_result(newcopy);
3885 }
3886 return true;
3887 }
3888
3889
3890 //----------------------generate_virtual_guard---------------------------
3891 // Helper for hashCode and clone. Peeks inside the vtable to avoid a call.
3892 Node* LibraryCallKit::generate_virtual_guard(Node* obj_klass,
3893 RegionNode* slow_region) {
3894 ciMethod* method = callee();
3895 int vtable_index = method->vtable_index();
3896 assert(vtable_index >= 0 || vtable_index == Method::nonvirtual_vtable_index,
3897 err_msg_res("bad index %d", vtable_index));
3898 // Get the Method* out of the appropriate vtable entry.
4369 if (base_off % BytesPerLong != 0) {
4370 assert(UseCompressedClassPointers, "");
4371 if (is_array) {
4372 // Exclude length to copy by 8 bytes words.
4373 base_off += sizeof(int);
4374 } else {
4375 // Include klass to copy by 8 bytes words.
4376 base_off = instanceOopDesc::klass_offset_in_bytes();
4377 }
4378 assert(base_off % BytesPerLong == 0, "expect 8 bytes alignment");
4379 }
4380 src = basic_plus_adr(src, base_off);
4381 dest = basic_plus_adr(dest, base_off);
4382
4383 // Compute the length also, if needed:
4384 Node* countx = size;
4385 countx = _gvn.transform(new SubXNode(countx, MakeConX(base_off)));
4386 countx = _gvn.transform(new URShiftXNode(countx, intcon(LogBytesPerLong) ));
4387
4388 const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
4389
4390 ArrayCopyNode* ac = ArrayCopyNode::make(this, false, src, NULL, dest, NULL, countx, false);
4391 ac->set_clonebasic();
4392 Node* n = _gvn.transform(ac);
4393 assert(n == ac, "cannot disappear");
4394 set_predefined_output_for_runtime_call(ac, ac->in(TypeFunc::Memory), raw_adr_type);
4395
4396 // If necessary, emit some card marks afterwards. (Non-arrays only.)
4397 if (card_mark) {
4398 assert(!is_array, "");
4399 // Put in store barrier for any and all oops we are sticking
4400 // into this object. (We could avoid this if we could prove
4401 // that the object type contains no oop fields at all.)
4402 Node* no_particular_value = NULL;
4403 Node* no_particular_field = NULL;
4404 int raw_adr_idx = Compile::AliasIdxRaw;
4405 post_barrier(control(),
4406 memory(raw_adr_type),
4407 alloc_obj,
4408 no_particular_field,
4409 raw_adr_idx,
4410 no_particular_value,
4411 T_OBJECT,
4412 false);
4413 }
4414
4483 const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
4484 int raw_adr_idx = Compile::AliasIdxRaw;
4485
4486 Node* array_ctl = generate_array_guard(obj_klass, (RegionNode*)NULL);
4487 if (array_ctl != NULL) {
4488 // It's an array.
4489 PreserveJVMState pjvms(this);
4490 set_control(array_ctl);
4491 Node* obj_length = load_array_length(obj);
4492 Node* obj_size = NULL;
4493 Node* alloc_obj = new_array(obj_klass, obj_length, 0, &obj_size); // no arguments to push
4494
4495 if (!use_ReduceInitialCardMarks()) {
4496 // If it is an oop array, it requires very special treatment,
4497 // because card marking is required on each card of the array.
4498 Node* is_obja = generate_objArray_guard(obj_klass, (RegionNode*)NULL);
4499 if (is_obja != NULL) {
4500 PreserveJVMState pjvms2(this);
4501 set_control(is_obja);
4502 // Generate a direct call to the right arraycopy function(s).
4503 Node* alloc = tightly_coupled_allocation(alloc_obj, NULL);
4504 ArrayCopyNode* ac = ArrayCopyNode::make(this, true, obj, intcon(0), alloc_obj, intcon(0), obj_length, alloc != NULL);
4505 ac->set_cloneoop();
4506 Node* n = _gvn.transform(ac);
4507 assert(n == ac, "cannot disappear");
4508 ac->connect_outputs(this);
4509
4510 result_reg->init_req(_objArray_path, control());
4511 result_val->init_req(_objArray_path, alloc_obj);
4512 result_i_o ->set_req(_objArray_path, i_o());
4513 result_mem ->set_req(_objArray_path, reset_memory());
4514 }
4515 }
4516 // Otherwise, there are no card marks to worry about.
4517 // (We can dispense with card marks if we know the allocation
4518 // comes out of eden (TLAB)... In fact, ReduceInitialCardMarks
4519 // causes the non-eden paths to take compensating steps to
4520 // simulate a fresh allocation, so that no further
4521 // card marks are required in compiled code to initialize
4522 // the object.)
4523
4524 if (!stopped()) {
4525 copy_to_clone(obj, alloc_obj, obj_size, true, false);
4526
4527 // Present the results of the copy.
4528 result_reg->init_req(_array_path, control());
4529 result_val->init_req(_array_path, alloc_obj);
4583 PreserveJVMState pjvms(this);
4584 CallJavaNode* slow_call = generate_method_call(vmIntrinsics::_clone, is_virtual);
4585 Node* slow_result = set_results_for_java_call(slow_call);
4586 // this->control() comes from set_results_for_java_call
4587 result_reg->init_req(_slow_path, control());
4588 result_val->init_req(_slow_path, slow_result);
4589 result_i_o ->set_req(_slow_path, i_o());
4590 result_mem ->set_req(_slow_path, reset_memory());
4591 }
4592
4593 // Return the combined state.
4594 set_control( _gvn.transform(result_reg));
4595 set_i_o( _gvn.transform(result_i_o));
4596 set_all_memory( _gvn.transform(result_mem));
4597 } // original reexecute is set back here
4598
4599 set_result(_gvn.transform(result_val));
4600 return true;
4601 }
4602
4603 //------------------------------inline_arraycopy-----------------------
4604 // public static native void java.lang.System.arraycopy(Object src, int srcPos,
4605 // Object dest, int destPos,
4606 // int length);
4607 bool LibraryCallKit::inline_arraycopy() {
4608 // Get the arguments.
4609 Node* src = argument(0); // type: oop
4610 Node* src_offset = argument(1); // type: int
4611 Node* dest = argument(2); // type: oop
4612 Node* dest_offset = argument(3); // type: int
4613 Node* length = argument(4); // type: int
4614
4615 // The following tests must be performed
4616 // (1) src and dest are arrays.
4617 // (2) src and dest arrays must have elements of the same BasicType
4618 // (3) src and dest must not be null.
4619 // (4) src_offset must not be negative.
4620 // (5) dest_offset must not be negative.
4621 // (6) length must not be negative.
4622 // (7) src_offset + length must not exceed length of src.
4623 // (8) dest_offset + length must not exceed length of dest.
4624 // (9) each element of an oop array must be assignable
4625
4626 // (3) src and dest must not be null.
4627 // always do this here because we need the JVM state for uncommon traps
4628 src = null_check(src, T_ARRAY);
4629 dest = null_check(dest, T_ARRAY);
4630
4631 bool notest = false;
4632
4633 const Type* src_type = _gvn.type(src);
4634 const Type* dest_type = _gvn.type(dest);
4635 const TypeAryPtr* top_src = src_type->isa_aryptr();
4636 const TypeAryPtr* top_dest = dest_type->isa_aryptr();
4637
4638 // Do we have the type of src?
4639 bool has_src = (top_src != NULL && top_src->klass() != NULL);
4640 // Do we have the type of dest?
4641 bool has_dest = (top_dest != NULL && top_dest->klass() != NULL);
4642 // Is the type for src from speculation?
4643 bool src_spec = false;
4644 // Is the type for dest from speculation?
4645 bool dest_spec = false;
4646
4647 if (!has_src || !has_dest) {
4648 // We don't have sufficient type information, let's see if
4649 // speculative types can help. We need to have types for both src
4650 // and dest so that it pays off.
4651
4652 // Do we already have or could we have type information for src
4653 bool could_have_src = has_src;
4654 // Do we already have or could we have type information for dest
4672
4673 if (could_have_src && could_have_dest) {
4674 // This is going to pay off so emit the required guards
4675 if (!has_src) {
4676 src = maybe_cast_profiled_obj(src, src_k);
4677 src_type = _gvn.type(src);
4678 top_src = src_type->isa_aryptr();
4679 has_src = (top_src != NULL && top_src->klass() != NULL);
4680 src_spec = true;
4681 }
4682 if (!has_dest) {
4683 dest = maybe_cast_profiled_obj(dest, dest_k);
4684 dest_type = _gvn.type(dest);
4685 top_dest = dest_type->isa_aryptr();
4686 has_dest = (top_dest != NULL && top_dest->klass() != NULL);
4687 dest_spec = true;
4688 }
4689 }
4690 }
4691
4692 if (has_src && has_dest) {
4693 BasicType src_elem = top_src->klass()->as_array_klass()->element_type()->basic_type();
4694 BasicType dest_elem = top_dest->klass()->as_array_klass()->element_type()->basic_type();
4695 if (src_elem == T_ARRAY) src_elem = T_OBJECT;
4696 if (dest_elem == T_ARRAY) dest_elem = T_OBJECT;
4697
4698 if (src_elem == dest_elem && src_elem == T_OBJECT) {
4699 // If both arrays are object arrays then having the exact types
4700 // for both will remove the need for a subtype check at runtime
4701 // before the call and may make it possible to pick a faster copy
4702 // routine (without a subtype check on every element)
4703 // Do we have the exact type of src?
4704 bool could_have_src = src_spec;
4705 // Do we have the exact type of dest?
4706 bool could_have_dest = dest_spec;
4707 ciKlass* src_k = top_src->klass();
4708 ciKlass* dest_k = top_dest->klass();
4709 if (!src_spec) {
4710 src_k = src_type->speculative_type_not_null();
4711 if (src_k != NULL && src_k->is_array_klass()) {
4712 could_have_src = true;
4713 }
4714 }
4715 if (!dest_spec) {
4716 dest_k = dest_type->speculative_type_not_null();
4717 if (dest_k != NULL && dest_k->is_array_klass()) {
4718 could_have_dest = true;
4719 }
4720 }
4721 if (could_have_src && could_have_dest) {
4722 // If we can have both exact types, emit the missing guards
4723 if (could_have_src && !src_spec) {
4724 src = maybe_cast_profiled_obj(src, src_k);
4725 }
4726 if (could_have_dest && !dest_spec) {
4727 dest = maybe_cast_profiled_obj(dest, dest_k);
4728 }
4729 }
4730 }
4731 }
4732
4733 if (!too_many_traps(Deoptimization::Reason_intrinsic) && !src->is_top() && !dest->is_top()) {
4734 // validate arguments: enables transformation the ArrayCopyNode
4735 notest = true;
4736
4737 RegionNode* slow_region = new RegionNode(1);
4738 record_for_igvn(slow_region);
4739
4740 // (1) src and dest are arrays.
4741 generate_non_array_guard(load_object_klass(src), slow_region);
4742 generate_non_array_guard(load_object_klass(dest), slow_region);
4743
4744 // (2) src and dest arrays must have elements of the same BasicType
4745 // done at macro expansion or at Ideal transformation time
4746
4747 // (4) src_offset must not be negative.
4748 generate_negative_guard(src_offset, slow_region);
4749
4750 // (5) dest_offset must not be negative.
4751 generate_negative_guard(dest_offset, slow_region);
4752
4753 // (7) src_offset + length must not exceed length of src.
4754 generate_limit_guard(src_offset, length,
4755 load_array_length(src),
4756 slow_region);
4757
4758 // (8) dest_offset + length must not exceed length of dest.
4759 generate_limit_guard(dest_offset, length,
4760 load_array_length(dest),
4761 slow_region);
4762
4763 // (9) each element of an oop array must be assignable
4764 Node* src_klass = load_object_klass(src);
4765 Node* dest_klass = load_object_klass(dest);
4766 Node* not_subtype_ctrl = gen_subtype_check(src_klass, dest_klass);
4767
4768 if (not_subtype_ctrl != top()) {
4769 PreserveJVMState pjvms(this);
4770 set_control(not_subtype_ctrl);
4771 uncommon_trap(Deoptimization::Reason_intrinsic,
4772 Deoptimization::Action_make_not_entrant);
4773 assert(stopped(), "Should be stopped");
4774 }
4775 {
4776 PreserveJVMState pjvms(this);
4777 set_control(_gvn.transform(slow_region));
4778 uncommon_trap(Deoptimization::Reason_intrinsic,
4779 Deoptimization::Action_make_not_entrant);
4780 assert(stopped(), "Should be stopped");
4781 }
4782 }
4783
4784 if (stopped()) {
4785 return true;
4786 }
4787
4788 AllocateArrayNode* alloc = tightly_coupled_allocation(dest, NULL);
4789 ArrayCopyNode* ac = ArrayCopyNode::make(this, true, src, src_offset, dest, dest_offset, length, alloc != NULL);
4790
4791 if (notest) {
4792 ac->set_arraycopy_notest();
4793 }
4794
4795 Node* n = _gvn.transform(ac);
4796 assert(n == ac, "cannot disappear");
4797 ac->connect_outputs(this);
4798
4799 return true;
4800 }
4801
4802
4803 // Helper function which determines if an arraycopy immediately follows
4804 // an allocation, with no intervening tests or other escapes for the object.
4805 AllocateArrayNode*
4806 LibraryCallKit::tightly_coupled_allocation(Node* ptr,
4807 RegionNode* slow_region) {
4808 if (stopped()) return NULL; // no fast path
4809 if (C->AliasLevel() == 0) return NULL; // no MergeMems around
4810
4811 AllocateArrayNode* alloc = AllocateArrayNode::Ideal_array_allocation(ptr, &_gvn);
4812 if (alloc == NULL) return NULL;
4813
4814 Node* rawmem = memory(Compile::AliasIdxRaw);
4815 // Is the allocation's memory state untouched?
4816 if (!(rawmem->is_Proj() && rawmem->in(0)->is_Initialize())) {
4817 // Bail out if there have been raw-memory effects since the allocation.
4818 // (Example: There might have been a call or safepoint.)
4819 return NULL;
4857 Node* obs = not_ctl->fast_out(j);
4858 if (obs->in(0) == not_ctl && obs->is_Call() &&
4859 (obs->as_Call()->entry_point() == SharedRuntime::uncommon_trap_blob()->entry_point())) {
4860 found_trap = true; break;
4861 }
4862 }
4863 if (found_trap) {
4864 ctl = iff->in(0); // This test feeds a harmless uncommon trap.
4865 continue;
4866 }
4867 }
4868 return NULL;
4869 }
4870
4871 // If we get this far, we have an allocation which immediately
4872 // precedes the arraycopy, and we can take over zeroing the new object.
4873 // The arraycopy will finish the initialization, and provide
4874 // a new control state to which we will anchor the destination pointer.
4875
4876 return alloc;
4877 }
4878
4879 //-------------inline_encodeISOArray-----------------------------------
4880 // encode char[] to byte[] in ISO_8859_1
4881 bool LibraryCallKit::inline_encodeISOArray() {
4882 assert(callee()->signature()->size() == 5, "encodeISOArray has 5 parameters");
4883 // no receiver since it is static method
4884 Node *src = argument(0);
4885 Node *src_offset = argument(1);
4886 Node *dst = argument(2);
4887 Node *dst_offset = argument(3);
4888 Node *length = argument(4);
4889
4890 const Type* src_type = src->Value(&_gvn);
4891 const Type* dst_type = dst->Value(&_gvn);
4892 const TypeAryPtr* top_src = src_type->isa_aryptr();
4893 const TypeAryPtr* top_dest = dst_type->isa_aryptr();
4894 if (top_src == NULL || top_src->klass() == NULL ||
4895 top_dest == NULL || top_dest->klass() == NULL) {
4896 // failed array check
|