1 /*
2 * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
80 int arg_size() const { return callee()->arg_size(); }
81
82 bool try_to_inline();
83
84 // Helper functions to inline natives
85 void push_result(RegionNode* region, PhiNode* value);
86 Node* generate_guard(Node* test, RegionNode* region, float true_prob);
87 Node* generate_slow_guard(Node* test, RegionNode* region);
88 Node* generate_fair_guard(Node* test, RegionNode* region);
89 Node* generate_negative_guard(Node* index, RegionNode* region,
90 // resulting CastII of index:
91 Node* *pos_index = NULL);
92 Node* generate_nonpositive_guard(Node* index, bool never_negative,
93 // resulting CastII of index:
94 Node* *pos_index = NULL);
95 Node* generate_limit_guard(Node* offset, Node* subseq_length,
96 Node* array_length,
97 RegionNode* region);
98 Node* generate_current_thread(Node* &tls_output);
99 address basictype2arraycopy(BasicType t, Node *src_offset, Node *dest_offset,
100 bool disjoint_bases, const char* &name);
101 Node* load_mirror_from_klass(Node* klass);
102 Node* load_klass_from_mirror_common(Node* mirror, bool never_see_null,
103 int nargs,
104 RegionNode* region, int null_path,
105 int offset);
106 Node* load_klass_from_mirror(Node* mirror, bool never_see_null, int nargs,
107 RegionNode* region, int null_path) {
108 int offset = java_lang_Class::klass_offset_in_bytes();
109 return load_klass_from_mirror_common(mirror, never_see_null, nargs,
110 region, null_path,
111 offset);
112 }
113 Node* load_array_klass_from_mirror(Node* mirror, bool never_see_null,
114 int nargs,
115 RegionNode* region, int null_path) {
116 int offset = java_lang_Class::array_klass_offset_in_bytes();
117 return load_klass_from_mirror_common(mirror, never_see_null, nargs,
118 region, null_path,
119 offset);
120 }
195 BasicType basic_elem_type,
196 Node* src, Node* src_offset,
197 Node* dest, Node* dest_offset,
198 Node* copy_length,
199 bool disjoint_bases = false,
200 bool length_never_negative = false,
201 RegionNode* slow_region = NULL);
202 AllocateArrayNode* tightly_coupled_allocation(Node* ptr,
203 RegionNode* slow_region);
204 void generate_clear_array(const TypePtr* adr_type,
205 Node* dest,
206 BasicType basic_elem_type,
207 Node* slice_off,
208 Node* slice_len,
209 Node* slice_end);
210 bool generate_block_arraycopy(const TypePtr* adr_type,
211 BasicType basic_elem_type,
212 AllocateNode* alloc,
213 Node* src, Node* src_offset,
214 Node* dest, Node* dest_offset,
215 Node* dest_size);
216 void generate_slow_arraycopy(const TypePtr* adr_type,
217 Node* src, Node* src_offset,
218 Node* dest, Node* dest_offset,
219 Node* copy_length);
220 Node* generate_checkcast_arraycopy(const TypePtr* adr_type,
221 Node* dest_elem_klass,
222 Node* src, Node* src_offset,
223 Node* dest, Node* dest_offset,
224 Node* copy_length);
225 Node* generate_generic_arraycopy(const TypePtr* adr_type,
226 Node* src, Node* src_offset,
227 Node* dest, Node* dest_offset,
228 Node* copy_length);
229 void generate_unchecked_arraycopy(const TypePtr* adr_type,
230 BasicType basic_elem_type,
231 bool disjoint_bases,
232 Node* src, Node* src_offset,
233 Node* dest, Node* dest_offset,
234 Node* copy_length);
235 bool inline_unsafe_CAS(BasicType type);
236 bool inline_unsafe_ordered_store(BasicType type);
237 bool inline_fp_conversions(vmIntrinsics::ID id);
238 bool inline_numberOfLeadingZeros(vmIntrinsics::ID id);
239 bool inline_numberOfTrailingZeros(vmIntrinsics::ID id);
240 bool inline_bitCount(vmIntrinsics::ID id);
241 bool inline_reverseBytes(vmIntrinsics::ID id);
242 };
243
244
245 //---------------------------make_vm_intrinsic----------------------------
246 CallGenerator* Compile::make_vm_intrinsic(ciMethod* m, bool is_virtual) {
247 vmIntrinsics::ID id = m->intrinsic_id();
248 assert(id != vmIntrinsics::_none, "must be a VM intrinsic");
249
250 if (DisableIntrinsic[0] != '\0'
251 && strstr(DisableIntrinsic, vmIntrinsics::name_at(id)) != NULL) {
252 // disabled by a user request on the command line:
253 // example: -XX:DisableIntrinsic=_hashCode,_getClass
254 return NULL;
4064 if (is_array) {
4065 // Exclude length to copy by 8 bytes words.
4066 base_off += sizeof(int);
4067 } else {
4068 // Include klass to copy by 8 bytes words.
4069 base_off = instanceOopDesc::klass_offset_in_bytes();
4070 }
4071 assert(base_off % BytesPerLong == 0, "expect 8 bytes alignment");
4072 }
4073 src = basic_plus_adr(src, base_off);
4074 dest = basic_plus_adr(dest, base_off);
4075
4076 // Compute the length also, if needed:
4077 Node* countx = size;
4078 countx = _gvn.transform( new (C, 3) SubXNode(countx, MakeConX(base_off)) );
4079 countx = _gvn.transform( new (C, 3) URShiftXNode(countx, intcon(LogBytesPerLong) ));
4080
4081 const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
4082 bool disjoint_bases = true;
4083 generate_unchecked_arraycopy(raw_adr_type, T_LONG, disjoint_bases,
4084 src, NULL, dest, NULL, countx);
4085
4086 // If necessary, emit some card marks afterwards. (Non-arrays only.)
4087 if (card_mark) {
4088 assert(!is_array, "");
4089 // Put in store barrier for any and all oops we are sticking
4090 // into this object. (We could avoid this if we could prove
4091 // that the object type contains no oop fields at all.)
4092 Node* no_particular_value = NULL;
4093 Node* no_particular_field = NULL;
4094 int raw_adr_idx = Compile::AliasIdxRaw;
4095 post_barrier(control(),
4096 memory(raw_adr_type),
4097 alloc_obj,
4098 no_particular_field,
4099 raw_adr_idx,
4100 no_particular_value,
4101 T_OBJECT,
4102 false);
4103 }
4104
4278 set_all_memory( _gvn.transform(result_mem) );
4279 } //original reexecute and sp are set back here
4280
4281 push(_gvn.transform(result_val));
4282
4283 return true;
4284 }
4285
4286
4287 // constants for computing the copy function
4288 enum {
4289 COPYFUNC_UNALIGNED = 0,
4290 COPYFUNC_ALIGNED = 1, // src, dest aligned to HeapWordSize
4291 COPYFUNC_CONJOINT = 0,
4292 COPYFUNC_DISJOINT = 2 // src != dest, or transfer can descend
4293 };
4294
4295 // Note: The condition "disjoint" applies also for overlapping copies
4296 // where an descending copy is permitted (i.e., dest_offset <= src_offset).
4297 static address
4298 select_arraycopy_function(BasicType t, bool aligned, bool disjoint, const char* &name) {
4299 int selector =
4300 (aligned ? COPYFUNC_ALIGNED : COPYFUNC_UNALIGNED) +
4301 (disjoint ? COPYFUNC_DISJOINT : COPYFUNC_CONJOINT);
4302
4303 #define RETURN_STUB(xxx_arraycopy) { \
4304 name = #xxx_arraycopy; \
4305 return StubRoutines::xxx_arraycopy(); }
4306
4307 switch (t) {
4308 case T_BYTE:
4309 case T_BOOLEAN:
4310 switch (selector) {
4311 case COPYFUNC_CONJOINT | COPYFUNC_UNALIGNED: RETURN_STUB(jbyte_arraycopy);
4312 case COPYFUNC_CONJOINT | COPYFUNC_ALIGNED: RETURN_STUB(arrayof_jbyte_arraycopy);
4313 case COPYFUNC_DISJOINT | COPYFUNC_UNALIGNED: RETURN_STUB(jbyte_disjoint_arraycopy);
4314 case COPYFUNC_DISJOINT | COPYFUNC_ALIGNED: RETURN_STUB(arrayof_jbyte_disjoint_arraycopy);
4315 }
4316 case T_CHAR:
4317 case T_SHORT:
4318 switch (selector) {
4319 case COPYFUNC_CONJOINT | COPYFUNC_UNALIGNED: RETURN_STUB(jshort_arraycopy);
4320 case COPYFUNC_CONJOINT | COPYFUNC_ALIGNED: RETURN_STUB(arrayof_jshort_arraycopy);
4321 case COPYFUNC_DISJOINT | COPYFUNC_UNALIGNED: RETURN_STUB(jshort_disjoint_arraycopy);
4322 case COPYFUNC_DISJOINT | COPYFUNC_ALIGNED: RETURN_STUB(arrayof_jshort_disjoint_arraycopy);
4323 }
4324 case T_INT:
4325 case T_FLOAT:
4326 switch (selector) {
4327 case COPYFUNC_CONJOINT | COPYFUNC_UNALIGNED: RETURN_STUB(jint_arraycopy);
4328 case COPYFUNC_CONJOINT | COPYFUNC_ALIGNED: RETURN_STUB(arrayof_jint_arraycopy);
4329 case COPYFUNC_DISJOINT | COPYFUNC_UNALIGNED: RETURN_STUB(jint_disjoint_arraycopy);
4330 case COPYFUNC_DISJOINT | COPYFUNC_ALIGNED: RETURN_STUB(arrayof_jint_disjoint_arraycopy);
4331 }
4332 case T_DOUBLE:
4333 case T_LONG:
4334 switch (selector) {
4335 case COPYFUNC_CONJOINT | COPYFUNC_UNALIGNED: RETURN_STUB(jlong_arraycopy);
4336 case COPYFUNC_CONJOINT | COPYFUNC_ALIGNED: RETURN_STUB(arrayof_jlong_arraycopy);
4337 case COPYFUNC_DISJOINT | COPYFUNC_UNALIGNED: RETURN_STUB(jlong_disjoint_arraycopy);
4338 case COPYFUNC_DISJOINT | COPYFUNC_ALIGNED: RETURN_STUB(arrayof_jlong_disjoint_arraycopy);
4339 }
4340 case T_ARRAY:
4341 case T_OBJECT:
4342 switch (selector) {
4343 case COPYFUNC_CONJOINT | COPYFUNC_UNALIGNED: RETURN_STUB(oop_arraycopy);
4344 case COPYFUNC_CONJOINT | COPYFUNC_ALIGNED: RETURN_STUB(arrayof_oop_arraycopy);
4345 case COPYFUNC_DISJOINT | COPYFUNC_UNALIGNED: RETURN_STUB(oop_disjoint_arraycopy);
4346 case COPYFUNC_DISJOINT | COPYFUNC_ALIGNED: RETURN_STUB(arrayof_oop_disjoint_arraycopy);
4347 }
4348 default:
4349 ShouldNotReachHere();
4350 return NULL;
4351 }
4352
4353 #undef RETURN_STUB
4354 }
4355
4356 //------------------------------basictype2arraycopy----------------------------
4357 address LibraryCallKit::basictype2arraycopy(BasicType t,
4358 Node* src_offset,
4359 Node* dest_offset,
4360 bool disjoint_bases,
4361 const char* &name) {
4362 const TypeInt* src_offset_inttype = gvn().find_int_type(src_offset);;
4363 const TypeInt* dest_offset_inttype = gvn().find_int_type(dest_offset);;
4364
4365 bool aligned = false;
4366 bool disjoint = disjoint_bases;
4367
4368 // if the offsets are the same, we can treat the memory regions as
4369 // disjoint, because either the memory regions are in different arrays,
4370 // or they are identical (which we can treat as disjoint.) We can also
4371 // treat a copy with a destination index less that the source index
4372 // as disjoint since a low->high copy will work correctly in this case.
4373 if (src_offset_inttype != NULL && src_offset_inttype->is_con() &&
4374 dest_offset_inttype != NULL && dest_offset_inttype->is_con()) {
4375 // both indices are constants
4376 int s_offs = src_offset_inttype->get_con();
4377 int d_offs = dest_offset_inttype->get_con();
4378 int element_size = type2aelembytes(t);
4379 aligned = ((arrayOopDesc::base_offset_in_bytes(t) + s_offs * element_size) % HeapWordSize == 0) &&
4380 ((arrayOopDesc::base_offset_in_bytes(t) + d_offs * element_size) % HeapWordSize == 0);
4381 if (s_offs >= d_offs) disjoint = true;
4382 } else if (src_offset == dest_offset && src_offset != NULL) {
4383 // This can occur if the offsets are identical non-constants.
4384 disjoint = true;
4385 }
4386
4387 return select_arraycopy_function(t, aligned, disjoint, name);
4388 }
4389
4390
4391 //------------------------------inline_arraycopy-----------------------
4392 bool LibraryCallKit::inline_arraycopy() {
4393 // Restore the stack and pop off the arguments.
4394 int nargs = 5; // 2 oops, 3 ints, no size_t or long
4395 assert(callee()->signature()->size() == nargs, "copy has 5 arguments");
4396
4397 Node *src = argument(0);
4398 Node *src_offset = argument(1);
4399 Node *dest = argument(2);
4400 Node *dest_offset = argument(3);
4401 Node *length = argument(4);
4402
4403 // Compile time checks. If any of these checks cannot be verified at compile time,
4404 // we do not make a fast path for this call. Instead, we let the call remain as it
4405 // is. The checks we choose to mandate at compile time are:
4406 //
4407 // (1) src and dest are arrays.
4423 // Since we cannot type the arrays, we don't know which slices
4424 // might be affected. We could restrict this barrier only to those
4425 // memory slices which pertain to array elements--but don't bother.
4426 if (!InsertMemBarAfterArraycopy)
4427 // (If InsertMemBarAfterArraycopy, there is already one in place.)
4428 insert_mem_bar(Op_MemBarCPUOrder);
4429 return true;
4430 }
4431
4432 // (2) src and dest arrays must have elements of the same BasicType
4433 // Figure out the size and type of the elements we will be copying.
4434 BasicType src_elem = top_src->klass()->as_array_klass()->element_type()->basic_type();
4435 BasicType dest_elem = top_dest->klass()->as_array_klass()->element_type()->basic_type();
4436 if (src_elem == T_ARRAY) src_elem = T_OBJECT;
4437 if (dest_elem == T_ARRAY) dest_elem = T_OBJECT;
4438
4439 if (src_elem != dest_elem || dest_elem == T_VOID) {
4440 // The component types are not the same or are not recognized. Punt.
4441 // (But, avoid the native method wrapper to JVM_ArrayCopy.)
4442 generate_slow_arraycopy(TypePtr::BOTTOM,
4443 src, src_offset, dest, dest_offset, length);
4444 return true;
4445 }
4446
4447 //---------------------------------------------------------------------------
4448 // We will make a fast path for this call to arraycopy.
4449
4450 // We have the following tests left to perform:
4451 //
4452 // (3) src and dest must not be null.
4453 // (4) src_offset must not be negative.
4454 // (5) dest_offset must not be negative.
4455 // (6) length must not be negative.
4456 // (7) src_offset + length must not exceed length of src.
4457 // (8) dest_offset + length must not exceed length of dest.
4458 // (9) each element of an oop array must be assignable
4459
4460 RegionNode* slow_region = new (C, 1) RegionNode(1);
4461 record_for_igvn(slow_region);
4462
4463 // (3) operands must not be null
4567 && ((alloc = tightly_coupled_allocation(dest, slow_region))
4568 != NULL)
4569 && _gvn.find_int_con(alloc->in(AllocateNode::ALength), 1) > 0
4570 && alloc->maybe_set_complete(&_gvn)) {
4571 // "You break it, you buy it."
4572 InitializeNode* init = alloc->initialization();
4573 assert(init->is_complete(), "we just did this");
4574 assert(dest->is_CheckCastPP(), "sanity");
4575 assert(dest->in(0)->in(0) == init, "dest pinned");
4576 adr_type = TypeRawPtr::BOTTOM; // all initializations are into raw memory
4577 // From this point on, every exit path is responsible for
4578 // initializing any non-copied parts of the object to zero.
4579 must_clear_dest = true;
4580 } else {
4581 // No zeroing elimination here.
4582 alloc = NULL;
4583 //original_dest = dest;
4584 //must_clear_dest = false;
4585 }
4586
4587 // Results are placed here:
4588 enum { fast_path = 1, // normal void-returning assembly stub
4589 checked_path = 2, // special assembly stub with cleanup
4590 slow_call_path = 3, // something went wrong; call the VM
4591 zero_path = 4, // bypass when length of copy is zero
4592 bcopy_path = 5, // copy primitive array by 64-bit blocks
4593 PATH_LIMIT = 6
4594 };
4595 RegionNode* result_region = new(C, PATH_LIMIT) RegionNode(PATH_LIMIT);
4596 PhiNode* result_i_o = new(C, PATH_LIMIT) PhiNode(result_region, Type::ABIO);
4597 PhiNode* result_memory = new(C, PATH_LIMIT) PhiNode(result_region, Type::MEMORY, adr_type);
4598 record_for_igvn(result_region);
4599 _gvn.set_type_bottom(result_i_o);
4600 _gvn.set_type_bottom(result_memory);
4601 assert(adr_type != TypePtr::BOTTOM, "must be RawMem or a T[] slice");
4602
4603 // The slow_control path:
4604 Node* slow_control;
4605 Node* slow_i_o = i_o();
4606 Node* slow_mem = memory(adr_type);
4607 debug_only(slow_control = (Node*) badAddress);
4608
4609 // Checked control path:
4610 Node* checked_control = top();
4611 Node* checked_mem = NULL;
4612 Node* checked_i_o = NULL;
4613 Node* checked_value = NULL;
4614
4615 if (basic_elem_type == T_CONFLICT) {
4616 assert(!must_clear_dest, "");
4617 Node* cv = generate_generic_arraycopy(adr_type,
4618 src, src_offset, dest, dest_offset,
4619 copy_length);
4620 if (cv == NULL) cv = intcon(-1); // failure (no stub available)
4621 checked_control = control();
4622 checked_i_o = i_o();
4623 checked_mem = memory(adr_type);
4624 checked_value = cv;
4625 set_control(top()); // no fast path
4626 }
4627
4628 Node* not_pos = generate_nonpositive_guard(copy_length, length_never_negative);
4629 if (not_pos != NULL) {
4630 PreserveJVMState pjvms(this);
4631 set_control(not_pos);
4632
4633 // (6) length must not be negative.
4634 if (!length_never_negative) {
4635 generate_negative_guard(copy_length, slow_region);
4636 }
4637
4638 // copy_length is 0.
4639 if (!stopped() && must_clear_dest) {
4681 // Next, perform a dynamic check on the tail length.
4682 // It is often zero, and we can win big if we prove this.
4683 // There are two wins: Avoid generating the ClearArray
4684 // with its attendant messy index arithmetic, and upgrade
4685 // the copy to a more hardware-friendly word size of 64 bits.
4686 Node* tail_ctl = NULL;
4687 if (!stopped() && !_gvn.eqv_uncast(dest_tail, dest_length)) {
4688 Node* cmp_lt = _gvn.transform( new(C,3) CmpINode(dest_tail, dest_length) );
4689 Node* bol_lt = _gvn.transform( new(C,2) BoolNode(cmp_lt, BoolTest::lt) );
4690 tail_ctl = generate_slow_guard(bol_lt, NULL);
4691 assert(tail_ctl != NULL || !stopped(), "must be an outcome");
4692 }
4693
4694 // At this point, let's assume there is no tail.
4695 if (!stopped() && alloc != NULL && basic_elem_type != T_OBJECT) {
4696 // There is no tail. Try an upgrade to a 64-bit copy.
4697 bool didit = false;
4698 { PreserveJVMState pjvms(this);
4699 didit = generate_block_arraycopy(adr_type, basic_elem_type, alloc,
4700 src, src_offset, dest, dest_offset,
4701 dest_size);
4702 if (didit) {
4703 // Present the results of the block-copying fast call.
4704 result_region->init_req(bcopy_path, control());
4705 result_i_o ->init_req(bcopy_path, i_o());
4706 result_memory->init_req(bcopy_path, memory(adr_type));
4707 }
4708 }
4709 if (didit)
4710 set_control(top()); // no regular fast path
4711 }
4712
4713 // Clear the tail, if any.
4714 if (tail_ctl != NULL) {
4715 Node* notail_ctl = stopped() ? NULL : control();
4716 set_control(tail_ctl);
4717 if (notail_ctl == NULL) {
4718 generate_clear_array(adr_type, dest, basic_elem_type,
4719 dest_tail, NULL,
4720 dest_size);
4721 } else {
4757 // The backing store for a List<String> is always an Object[],
4758 // but its elements are always type String, if the generic types
4759 // are correct at the source level.
4760 //
4761 // Test S[] against D[], not S against D, because (probably)
4762 // the secondary supertype cache is less busy for S[] than S.
4763 // This usually only matters when D is an interface.
4764 Node* not_subtype_ctrl = gen_subtype_check(src_klass, dest_klass);
4765 // Plug failing path into checked_oop_disjoint_arraycopy
4766 if (not_subtype_ctrl != top()) {
4767 PreserveJVMState pjvms(this);
4768 set_control(not_subtype_ctrl);
4769 // (At this point we can assume disjoint_bases, since types differ.)
4770 int ek_offset = objArrayKlass::element_klass_offset_in_bytes() + sizeof(oopDesc);
4771 Node* p1 = basic_plus_adr(dest_klass, ek_offset);
4772 Node* n1 = LoadKlassNode::make(_gvn, immutable_memory(), p1, TypeRawPtr::BOTTOM);
4773 Node* dest_elem_klass = _gvn.transform(n1);
4774 Node* cv = generate_checkcast_arraycopy(adr_type,
4775 dest_elem_klass,
4776 src, src_offset, dest, dest_offset,
4777 ConvI2X(copy_length));
4778 if (cv == NULL) cv = intcon(-1); // failure (no stub available)
4779 checked_control = control();
4780 checked_i_o = i_o();
4781 checked_mem = memory(adr_type);
4782 checked_value = cv;
4783 }
4784 // At this point we know we do not need type checks on oop stores.
4785
4786 // Let's see if we need card marks:
4787 if (alloc != NULL && use_ReduceInitialCardMarks()) {
4788 // If we do not need card marks, copy using the jint or jlong stub.
4789 copy_type = LP64_ONLY(UseCompressedOops ? T_INT : T_LONG) NOT_LP64(T_INT);
4790 assert(type2aelembytes(basic_elem_type) == type2aelembytes(copy_type),
4791 "sizes agree");
4792 }
4793 }
4794
4795 if (!stopped()) {
4796 // Generate the fast path, if possible.
4797 PreserveJVMState pjvms(this);
4798 generate_unchecked_arraycopy(adr_type, copy_type, disjoint_bases,
4799 src, src_offset, dest, dest_offset,
4800 ConvI2X(copy_length));
4801
4802 // Present the results of the fast call.
4803 result_region->init_req(fast_path, control());
4804 result_i_o ->init_req(fast_path, i_o());
4805 result_memory->init_req(fast_path, memory(adr_type));
4806 }
4807
4808 // Here are all the slow paths up to this point, in one bundle:
4809 slow_control = top();
4810 if (slow_region != NULL)
4811 slow_control = _gvn.transform(slow_region);
4812 debug_only(slow_region = (RegionNode*)badAddress);
4813
4814 set_control(checked_control);
4815 if (!stopped()) {
4816 // Clean up after the checked call.
4817 // The returned value is either 0 or -1^K,
4818 // where K = number of partially transferred array elements.
4819 Node* cmp = _gvn.transform( new(C, 3) CmpINode(checked_value, intcon(0)) );
4820 Node* bol = _gvn.transform( new(C, 2) BoolNode(cmp, BoolTest::eq) );
4867 copy_length = length_minus;
4868 }
4869 }
4870
4871 set_control(slow_control);
4872 if (!stopped()) {
4873 // Generate the slow path, if needed.
4874 PreserveJVMState pjvms(this); // replace_in_map may trash the map
4875
4876 set_memory(slow_mem, adr_type);
4877 set_i_o(slow_i_o);
4878
4879 if (must_clear_dest) {
4880 generate_clear_array(adr_type, dest, basic_elem_type,
4881 intcon(0), NULL,
4882 alloc->in(AllocateNode::AllocSize));
4883 }
4884
4885 generate_slow_arraycopy(adr_type,
4886 src, src_offset, dest, dest_offset,
4887 copy_length);
4888
4889 result_region->init_req(slow_call_path, control());
4890 result_i_o ->init_req(slow_call_path, i_o());
4891 result_memory->init_req(slow_call_path, memory(adr_type));
4892 }
4893
4894 // Remove unused edges.
4895 for (uint i = 1; i < result_region->req(); i++) {
4896 if (result_region->in(i) == NULL)
4897 result_region->init_req(i, top());
4898 }
4899
4900 // Finished; return the combined state.
4901 set_control( _gvn.transform(result_region) );
4902 set_i_o( _gvn.transform(result_i_o) );
4903 set_memory( _gvn.transform(result_memory), adr_type );
4904
4905 // The memory edges above are precise in order to model effects around
4906 // array copies accurately to allow value numbering of field loads around
4907 // arraycopy. Such field loads, both before and after, are common in Java
5111 Node* end = dest_size; // pre-rounded
5112 mem = ClearArrayNode::clear_memory(control(), mem, dest,
5113 start, end, &_gvn);
5114 } else {
5115 // Non-constant start, unrounded non-constant end.
5116 // (Nobody zeroes a random midsection of an array using this routine.)
5117 ShouldNotReachHere(); // fix caller
5118 }
5119
5120 // Done.
5121 set_memory(mem, adr_type);
5122 }
5123
5124
5125 bool
5126 LibraryCallKit::generate_block_arraycopy(const TypePtr* adr_type,
5127 BasicType basic_elem_type,
5128 AllocateNode* alloc,
5129 Node* src, Node* src_offset,
5130 Node* dest, Node* dest_offset,
5131 Node* dest_size) {
5132 // See if there is an advantage from block transfer.
5133 int scale = exact_log2(type2aelembytes(basic_elem_type));
5134 if (scale >= LogBytesPerLong)
5135 return false; // it is already a block transfer
5136
5137 // Look at the alignment of the starting offsets.
5138 int abase = arrayOopDesc::base_offset_in_bytes(basic_elem_type);
5139 const intptr_t BIG_NEG = -128;
5140 assert(BIG_NEG + 2*abase < 0, "neg enough");
5141
5142 intptr_t src_off = abase + ((intptr_t) find_int_con(src_offset, -1) << scale);
5143 intptr_t dest_off = abase + ((intptr_t) find_int_con(dest_offset, -1) << scale);
5144 if (src_off < 0 || dest_off < 0)
5145 // At present, we can only understand constants.
5146 return false;
5147
5148 if (((src_off | dest_off) & (BytesPerLong-1)) != 0) {
5149 // Non-aligned; too bad.
5150 // One more chance: Pick off an initial 32-bit word.
5151 // This is a common case, since abase can be odd mod 8.
5156 Node* sval = make_load(control(), sptr, TypeInt::INT, T_INT, adr_type);
5157 store_to_memory(control(), dptr, sval, T_INT, adr_type);
5158 src_off += BytesPerInt;
5159 dest_off += BytesPerInt;
5160 } else {
5161 return false;
5162 }
5163 }
5164 assert(src_off % BytesPerLong == 0, "");
5165 assert(dest_off % BytesPerLong == 0, "");
5166
5167 // Do this copy by giant steps.
5168 Node* sptr = basic_plus_adr(src, src_off);
5169 Node* dptr = basic_plus_adr(dest, dest_off);
5170 Node* countx = dest_size;
5171 countx = _gvn.transform( new (C, 3) SubXNode(countx, MakeConX(dest_off)) );
5172 countx = _gvn.transform( new (C, 3) URShiftXNode(countx, intcon(LogBytesPerLong)) );
5173
5174 bool disjoint_bases = true; // since alloc != NULL
5175 generate_unchecked_arraycopy(adr_type, T_LONG, disjoint_bases,
5176 sptr, NULL, dptr, NULL, countx);
5177
5178 return true;
5179 }
5180
5181
5182 // Helper function; generates code for the slow case.
5183 // We make a call to a runtime method which emulates the native method,
5184 // but without the native wrapper overhead.
5185 void
5186 LibraryCallKit::generate_slow_arraycopy(const TypePtr* adr_type,
5187 Node* src, Node* src_offset,
5188 Node* dest, Node* dest_offset,
5189 Node* copy_length) {
5190 Node* call = make_runtime_call(RC_NO_LEAF | RC_UNCOMMON,
5191 OptoRuntime::slow_arraycopy_Type(),
5192 OptoRuntime::slow_arraycopy_Java(),
5193 "slow_arraycopy", adr_type,
5194 src, src_offset, dest, dest_offset,
5195 copy_length);
5196
5197 // Handle exceptions thrown by this fellow:
5198 make_slow_call_ex(call, env()->Throwable_klass(), false);
5199 }
5200
5201 // Helper function; generates code for cases requiring runtime checks.
5202 Node*
5203 LibraryCallKit::generate_checkcast_arraycopy(const TypePtr* adr_type,
5204 Node* dest_elem_klass,
5205 Node* src, Node* src_offset,
5206 Node* dest, Node* dest_offset,
5207 Node* copy_length) {
5208 if (stopped()) return NULL;
5209
5210 address copyfunc_addr = StubRoutines::checkcast_arraycopy();
5211 if (copyfunc_addr == NULL) { // Stub was not generated, go slow path.
5212 return NULL;
5213 }
5214
5215 // Pick out the parameters required to perform a store-check
5216 // for the target array. This is an optimistic check. It will
5217 // look in each non-null element's class, at the desired klass's
5218 // super_check_offset, for the desired klass.
5219 int sco_offset = Klass::super_check_offset_offset_in_bytes() + sizeof(oopDesc);
5220 Node* p3 = basic_plus_adr(dest_elem_klass, sco_offset);
5221 Node* n3 = new(C, 3) LoadINode(NULL, memory(p3), p3, _gvn.type(p3)->is_ptr());
5222 Node* check_offset = ConvI2X(_gvn.transform(n3));
5223 Node* check_value = dest_elem_klass;
5224
5225 Node* src_start = array_element_address(src, src_offset, T_OBJECT);
5226 Node* dest_start = array_element_address(dest, dest_offset, T_OBJECT);
5227
5228 // (We know the arrays are never conjoint, because their types differ.)
5229 Node* call = make_runtime_call(RC_LEAF|RC_NO_FP,
5230 OptoRuntime::checkcast_arraycopy_Type(),
5231 copyfunc_addr, "checkcast_arraycopy", adr_type,
5232 // five arguments, of which two are
5233 // intptr_t (jlong in LP64)
5234 src_start, dest_start,
5235 copy_length XTOP,
5236 check_offset XTOP,
5237 check_value);
5238
5239 return _gvn.transform(new (C, 1) ProjNode(call, TypeFunc::Parms));
5240 }
5241
5242
5243 // Helper function; generates code for cases requiring runtime checks.
5244 Node*
5245 LibraryCallKit::generate_generic_arraycopy(const TypePtr* adr_type,
5246 Node* src, Node* src_offset,
5247 Node* dest, Node* dest_offset,
5248 Node* copy_length) {
5249 if (stopped()) return NULL;
5250
5251 address copyfunc_addr = StubRoutines::generic_arraycopy();
5252 if (copyfunc_addr == NULL) { // Stub was not generated, go slow path.
5253 return NULL;
5254 }
5255
5256 Node* call = make_runtime_call(RC_LEAF|RC_NO_FP,
5257 OptoRuntime::generic_arraycopy_Type(),
5258 copyfunc_addr, "generic_arraycopy", adr_type,
5259 src, src_offset, dest, dest_offset, copy_length);
5260
5261 return _gvn.transform(new (C, 1) ProjNode(call, TypeFunc::Parms));
5262 }
5263
5264 // Helper function; generates the fast out-of-line call to an arraycopy stub.
5265 void
5266 LibraryCallKit::generate_unchecked_arraycopy(const TypePtr* adr_type,
5267 BasicType basic_elem_type,
5268 bool disjoint_bases,
5269 Node* src, Node* src_offset,
5270 Node* dest, Node* dest_offset,
5271 Node* copy_length) {
5272 if (stopped()) return; // nothing to do
5273
5274 Node* src_start = src;
5275 Node* dest_start = dest;
5276 if (src_offset != NULL || dest_offset != NULL) {
5277 assert(src_offset != NULL && dest_offset != NULL, "");
5278 src_start = array_element_address(src, src_offset, basic_elem_type);
5279 dest_start = array_element_address(dest, dest_offset, basic_elem_type);
5280 }
5281
5282 // Figure out which arraycopy runtime method to call.
5283 const char* copyfunc_name = "arraycopy";
5284 address copyfunc_addr =
5285 basictype2arraycopy(basic_elem_type, src_offset, dest_offset,
5286 disjoint_bases, copyfunc_name);
5287
5288 // Call it. Note that the count_ix value is not scaled to a byte-size.
5289 make_runtime_call(RC_LEAF|RC_NO_FP,
5290 OptoRuntime::fast_arraycopy_Type(),
5291 copyfunc_addr, copyfunc_name, adr_type,
5292 src_start, dest_start, copy_length XTOP);
5293 }
|
1 /*
2 * Copyright (c) 1999, 2011, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
80 int arg_size() const { return callee()->arg_size(); }
81
82 bool try_to_inline();
83
84 // Helper functions to inline natives
85 void push_result(RegionNode* region, PhiNode* value);
86 Node* generate_guard(Node* test, RegionNode* region, float true_prob);
87 Node* generate_slow_guard(Node* test, RegionNode* region);
88 Node* generate_fair_guard(Node* test, RegionNode* region);
89 Node* generate_negative_guard(Node* index, RegionNode* region,
90 // resulting CastII of index:
91 Node* *pos_index = NULL);
92 Node* generate_nonpositive_guard(Node* index, bool never_negative,
93 // resulting CastII of index:
94 Node* *pos_index = NULL);
95 Node* generate_limit_guard(Node* offset, Node* subseq_length,
96 Node* array_length,
97 RegionNode* region);
98 Node* generate_current_thread(Node* &tls_output);
99 address basictype2arraycopy(BasicType t, Node *src_offset, Node *dest_offset,
100 bool disjoint_bases, const char* &name, bool dest_uninitialized);
101 Node* load_mirror_from_klass(Node* klass);
102 Node* load_klass_from_mirror_common(Node* mirror, bool never_see_null,
103 int nargs,
104 RegionNode* region, int null_path,
105 int offset);
106 Node* load_klass_from_mirror(Node* mirror, bool never_see_null, int nargs,
107 RegionNode* region, int null_path) {
108 int offset = java_lang_Class::klass_offset_in_bytes();
109 return load_klass_from_mirror_common(mirror, never_see_null, nargs,
110 region, null_path,
111 offset);
112 }
113 Node* load_array_klass_from_mirror(Node* mirror, bool never_see_null,
114 int nargs,
115 RegionNode* region, int null_path) {
116 int offset = java_lang_Class::array_klass_offset_in_bytes();
117 return load_klass_from_mirror_common(mirror, never_see_null, nargs,
118 region, null_path,
119 offset);
120 }
195 BasicType basic_elem_type,
196 Node* src, Node* src_offset,
197 Node* dest, Node* dest_offset,
198 Node* copy_length,
199 bool disjoint_bases = false,
200 bool length_never_negative = false,
201 RegionNode* slow_region = NULL);
202 AllocateArrayNode* tightly_coupled_allocation(Node* ptr,
203 RegionNode* slow_region);
204 void generate_clear_array(const TypePtr* adr_type,
205 Node* dest,
206 BasicType basic_elem_type,
207 Node* slice_off,
208 Node* slice_len,
209 Node* slice_end);
210 bool generate_block_arraycopy(const TypePtr* adr_type,
211 BasicType basic_elem_type,
212 AllocateNode* alloc,
213 Node* src, Node* src_offset,
214 Node* dest, Node* dest_offset,
215 Node* dest_size, bool dest_uninitialized);
216 void generate_slow_arraycopy(const TypePtr* adr_type,
217 Node* src, Node* src_offset,
218 Node* dest, Node* dest_offset,
219 Node* copy_length, bool dest_uninitialized);
220 Node* generate_checkcast_arraycopy(const TypePtr* adr_type,
221 Node* dest_elem_klass,
222 Node* src, Node* src_offset,
223 Node* dest, Node* dest_offset,
224 Node* copy_length, bool dest_uninitialized);
225 Node* generate_generic_arraycopy(const TypePtr* adr_type,
226 Node* src, Node* src_offset,
227 Node* dest, Node* dest_offset,
228 Node* copy_length, bool dest_uninitialized);
229 void generate_unchecked_arraycopy(const TypePtr* adr_type,
230 BasicType basic_elem_type,
231 bool disjoint_bases,
232 Node* src, Node* src_offset,
233 Node* dest, Node* dest_offset,
234 Node* copy_length, bool dest_uninitialized);
235 bool inline_unsafe_CAS(BasicType type);
236 bool inline_unsafe_ordered_store(BasicType type);
237 bool inline_fp_conversions(vmIntrinsics::ID id);
238 bool inline_numberOfLeadingZeros(vmIntrinsics::ID id);
239 bool inline_numberOfTrailingZeros(vmIntrinsics::ID id);
240 bool inline_bitCount(vmIntrinsics::ID id);
241 bool inline_reverseBytes(vmIntrinsics::ID id);
242 };
243
244
245 //---------------------------make_vm_intrinsic----------------------------
246 CallGenerator* Compile::make_vm_intrinsic(ciMethod* m, bool is_virtual) {
247 vmIntrinsics::ID id = m->intrinsic_id();
248 assert(id != vmIntrinsics::_none, "must be a VM intrinsic");
249
250 if (DisableIntrinsic[0] != '\0'
251 && strstr(DisableIntrinsic, vmIntrinsics::name_at(id)) != NULL) {
252 // disabled by a user request on the command line:
253 // example: -XX:DisableIntrinsic=_hashCode,_getClass
254 return NULL;
4064 if (is_array) {
4065 // Exclude length to copy by 8 bytes words.
4066 base_off += sizeof(int);
4067 } else {
4068 // Include klass to copy by 8 bytes words.
4069 base_off = instanceOopDesc::klass_offset_in_bytes();
4070 }
4071 assert(base_off % BytesPerLong == 0, "expect 8 bytes alignment");
4072 }
4073 src = basic_plus_adr(src, base_off);
4074 dest = basic_plus_adr(dest, base_off);
4075
4076 // Compute the length also, if needed:
4077 Node* countx = size;
4078 countx = _gvn.transform( new (C, 3) SubXNode(countx, MakeConX(base_off)) );
4079 countx = _gvn.transform( new (C, 3) URShiftXNode(countx, intcon(LogBytesPerLong) ));
4080
4081 const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
4082 bool disjoint_bases = true;
4083 generate_unchecked_arraycopy(raw_adr_type, T_LONG, disjoint_bases,
4084 src, NULL, dest, NULL, countx,
4085 /*dest_uninitialized*/false);
4086
4087 // If necessary, emit some card marks afterwards. (Non-arrays only.)
4088 if (card_mark) {
4089 assert(!is_array, "");
4090 // Put in store barrier for any and all oops we are sticking
4091 // into this object. (We could avoid this if we could prove
4092 // that the object type contains no oop fields at all.)
4093 Node* no_particular_value = NULL;
4094 Node* no_particular_field = NULL;
4095 int raw_adr_idx = Compile::AliasIdxRaw;
4096 post_barrier(control(),
4097 memory(raw_adr_type),
4098 alloc_obj,
4099 no_particular_field,
4100 raw_adr_idx,
4101 no_particular_value,
4102 T_OBJECT,
4103 false);
4104 }
4105
4279 set_all_memory( _gvn.transform(result_mem) );
4280 } //original reexecute and sp are set back here
4281
4282 push(_gvn.transform(result_val));
4283
4284 return true;
4285 }
4286
4287
4288 // constants for computing the copy function
4289 enum {
4290 COPYFUNC_UNALIGNED = 0,
4291 COPYFUNC_ALIGNED = 1, // src, dest aligned to HeapWordSize
4292 COPYFUNC_CONJOINT = 0,
4293 COPYFUNC_DISJOINT = 2 // src != dest, or transfer can descend
4294 };
4295
4296 // Note: The condition "disjoint" applies also for overlapping copies
4297 // where an descending copy is permitted (i.e., dest_offset <= src_offset).
4298 static address
4299 select_arraycopy_function(BasicType t, bool aligned, bool disjoint, const char* &name, bool dest_uninitialized) {
4300 int selector =
4301 (aligned ? COPYFUNC_ALIGNED : COPYFUNC_UNALIGNED) +
4302 (disjoint ? COPYFUNC_DISJOINT : COPYFUNC_CONJOINT);
4303
4304 #define RETURN_STUB(xxx_arraycopy) { \
4305 name = #xxx_arraycopy; \
4306 return StubRoutines::xxx_arraycopy(); }
4307
4308 #define RETURN_STUB_PARM(xxx_arraycopy, parm) { \
4309 name = #xxx_arraycopy; \
4310 return StubRoutines::xxx_arraycopy(parm); }
4311
4312 switch (t) {
4313 case T_BYTE:
4314 case T_BOOLEAN:
4315 switch (selector) {
4316 case COPYFUNC_CONJOINT | COPYFUNC_UNALIGNED: RETURN_STUB(jbyte_arraycopy);
4317 case COPYFUNC_CONJOINT | COPYFUNC_ALIGNED: RETURN_STUB(arrayof_jbyte_arraycopy);
4318 case COPYFUNC_DISJOINT | COPYFUNC_UNALIGNED: RETURN_STUB(jbyte_disjoint_arraycopy);
4319 case COPYFUNC_DISJOINT | COPYFUNC_ALIGNED: RETURN_STUB(arrayof_jbyte_disjoint_arraycopy);
4320 }
4321 case T_CHAR:
4322 case T_SHORT:
4323 switch (selector) {
4324 case COPYFUNC_CONJOINT | COPYFUNC_UNALIGNED: RETURN_STUB(jshort_arraycopy);
4325 case COPYFUNC_CONJOINT | COPYFUNC_ALIGNED: RETURN_STUB(arrayof_jshort_arraycopy);
4326 case COPYFUNC_DISJOINT | COPYFUNC_UNALIGNED: RETURN_STUB(jshort_disjoint_arraycopy);
4327 case COPYFUNC_DISJOINT | COPYFUNC_ALIGNED: RETURN_STUB(arrayof_jshort_disjoint_arraycopy);
4328 }
4329 case T_INT:
4330 case T_FLOAT:
4331 switch (selector) {
4332 case COPYFUNC_CONJOINT | COPYFUNC_UNALIGNED: RETURN_STUB(jint_arraycopy);
4333 case COPYFUNC_CONJOINT | COPYFUNC_ALIGNED: RETURN_STUB(arrayof_jint_arraycopy);
4334 case COPYFUNC_DISJOINT | COPYFUNC_UNALIGNED: RETURN_STUB(jint_disjoint_arraycopy);
4335 case COPYFUNC_DISJOINT | COPYFUNC_ALIGNED: RETURN_STUB(arrayof_jint_disjoint_arraycopy);
4336 }
4337 case T_DOUBLE:
4338 case T_LONG:
4339 switch (selector) {
4340 case COPYFUNC_CONJOINT | COPYFUNC_UNALIGNED: RETURN_STUB(jlong_arraycopy);
4341 case COPYFUNC_CONJOINT | COPYFUNC_ALIGNED: RETURN_STUB(arrayof_jlong_arraycopy);
4342 case COPYFUNC_DISJOINT | COPYFUNC_UNALIGNED: RETURN_STUB(jlong_disjoint_arraycopy);
4343 case COPYFUNC_DISJOINT | COPYFUNC_ALIGNED: RETURN_STUB(arrayof_jlong_disjoint_arraycopy);
4344 }
4345 case T_ARRAY:
4346 case T_OBJECT:
4347 switch (selector) {
4348 case COPYFUNC_CONJOINT | COPYFUNC_UNALIGNED: RETURN_STUB_PARM(oop_arraycopy, dest_uninitialized);
4349 case COPYFUNC_CONJOINT | COPYFUNC_ALIGNED: RETURN_STUB_PARM(arrayof_oop_arraycopy, dest_uninitialized);
4350 case COPYFUNC_DISJOINT | COPYFUNC_UNALIGNED: RETURN_STUB_PARM(oop_disjoint_arraycopy, dest_uninitialized);
4351 case COPYFUNC_DISJOINT | COPYFUNC_ALIGNED: RETURN_STUB_PARM(arrayof_oop_disjoint_arraycopy, dest_uninitialized);
4352 }
4353 default:
4354 ShouldNotReachHere();
4355 return NULL;
4356 }
4357
4358 #undef RETURN_STUB
4359 #undef RETURN_STUB_PARM
4360 }
4361
4362 //------------------------------basictype2arraycopy----------------------------
4363 address LibraryCallKit::basictype2arraycopy(BasicType t,
4364 Node* src_offset,
4365 Node* dest_offset,
4366 bool disjoint_bases,
4367 const char* &name,
4368 bool dest_uninitialized) {
4369 const TypeInt* src_offset_inttype = gvn().find_int_type(src_offset);;
4370 const TypeInt* dest_offset_inttype = gvn().find_int_type(dest_offset);;
4371
4372 bool aligned = false;
4373 bool disjoint = disjoint_bases;
4374
4375 // if the offsets are the same, we can treat the memory regions as
4376 // disjoint, because either the memory regions are in different arrays,
4377 // or they are identical (which we can treat as disjoint.) We can also
4378 // treat a copy with a destination index less that the source index
4379 // as disjoint since a low->high copy will work correctly in this case.
4380 if (src_offset_inttype != NULL && src_offset_inttype->is_con() &&
4381 dest_offset_inttype != NULL && dest_offset_inttype->is_con()) {
4382 // both indices are constants
4383 int s_offs = src_offset_inttype->get_con();
4384 int d_offs = dest_offset_inttype->get_con();
4385 int element_size = type2aelembytes(t);
4386 aligned = ((arrayOopDesc::base_offset_in_bytes(t) + s_offs * element_size) % HeapWordSize == 0) &&
4387 ((arrayOopDesc::base_offset_in_bytes(t) + d_offs * element_size) % HeapWordSize == 0);
4388 if (s_offs >= d_offs) disjoint = true;
4389 } else if (src_offset == dest_offset && src_offset != NULL) {
4390 // This can occur if the offsets are identical non-constants.
4391 disjoint = true;
4392 }
4393
4394 return select_arraycopy_function(t, aligned, disjoint, name, dest_uninitialized);
4395 }
4396
4397
4398 //------------------------------inline_arraycopy-----------------------
4399 bool LibraryCallKit::inline_arraycopy() {
4400 // Restore the stack and pop off the arguments.
4401 int nargs = 5; // 2 oops, 3 ints, no size_t or long
4402 assert(callee()->signature()->size() == nargs, "copy has 5 arguments");
4403
4404 Node *src = argument(0);
4405 Node *src_offset = argument(1);
4406 Node *dest = argument(2);
4407 Node *dest_offset = argument(3);
4408 Node *length = argument(4);
4409
4410 // Compile time checks. If any of these checks cannot be verified at compile time,
4411 // we do not make a fast path for this call. Instead, we let the call remain as it
4412 // is. The checks we choose to mandate at compile time are:
4413 //
4414 // (1) src and dest are arrays.
4430 // Since we cannot type the arrays, we don't know which slices
4431 // might be affected. We could restrict this barrier only to those
4432 // memory slices which pertain to array elements--but don't bother.
4433 if (!InsertMemBarAfterArraycopy)
4434 // (If InsertMemBarAfterArraycopy, there is already one in place.)
4435 insert_mem_bar(Op_MemBarCPUOrder);
4436 return true;
4437 }
4438
4439 // (2) src and dest arrays must have elements of the same BasicType
4440 // Figure out the size and type of the elements we will be copying.
4441 BasicType src_elem = top_src->klass()->as_array_klass()->element_type()->basic_type();
4442 BasicType dest_elem = top_dest->klass()->as_array_klass()->element_type()->basic_type();
4443 if (src_elem == T_ARRAY) src_elem = T_OBJECT;
4444 if (dest_elem == T_ARRAY) dest_elem = T_OBJECT;
4445
4446 if (src_elem != dest_elem || dest_elem == T_VOID) {
4447 // The component types are not the same or are not recognized. Punt.
4448 // (But, avoid the native method wrapper to JVM_ArrayCopy.)
4449 generate_slow_arraycopy(TypePtr::BOTTOM,
4450 src, src_offset, dest, dest_offset, length,
4451 /*uninitialized_target*/false);
4452 return true;
4453 }
4454
4455 //---------------------------------------------------------------------------
4456 // We will make a fast path for this call to arraycopy.
4457
4458 // We have the following tests left to perform:
4459 //
4460 // (3) src and dest must not be null.
4461 // (4) src_offset must not be negative.
4462 // (5) dest_offset must not be negative.
4463 // (6) length must not be negative.
4464 // (7) src_offset + length must not exceed length of src.
4465 // (8) dest_offset + length must not exceed length of dest.
4466 // (9) each element of an oop array must be assignable
4467
4468 RegionNode* slow_region = new (C, 1) RegionNode(1);
4469 record_for_igvn(slow_region);
4470
4471 // (3) operands must not be null
4575 && ((alloc = tightly_coupled_allocation(dest, slow_region))
4576 != NULL)
4577 && _gvn.find_int_con(alloc->in(AllocateNode::ALength), 1) > 0
4578 && alloc->maybe_set_complete(&_gvn)) {
4579 // "You break it, you buy it."
4580 InitializeNode* init = alloc->initialization();
4581 assert(init->is_complete(), "we just did this");
4582 assert(dest->is_CheckCastPP(), "sanity");
4583 assert(dest->in(0)->in(0) == init, "dest pinned");
4584 adr_type = TypeRawPtr::BOTTOM; // all initializations are into raw memory
4585 // From this point on, every exit path is responsible for
4586 // initializing any non-copied parts of the object to zero.
4587 must_clear_dest = true;
4588 } else {
4589 // No zeroing elimination here.
4590 alloc = NULL;
4591 //original_dest = dest;
4592 //must_clear_dest = false;
4593 }
4594
4595 // We must choose different arraycopy stubs if the target array are not initialized,
4596 // so the the pre-barriers wouldn't peek into the old values. See CR 6627983.
4597 const bool& dest_uninitialized = must_clear_dest;
4598
4599 // Results are placed here:
4600 enum { fast_path = 1, // normal void-returning assembly stub
4601 checked_path = 2, // special assembly stub with cleanup
4602 slow_call_path = 3, // something went wrong; call the VM
4603 zero_path = 4, // bypass when length of copy is zero
4604 bcopy_path = 5, // copy primitive array by 64-bit blocks
4605 PATH_LIMIT = 6
4606 };
4607 RegionNode* result_region = new(C, PATH_LIMIT) RegionNode(PATH_LIMIT);
4608 PhiNode* result_i_o = new(C, PATH_LIMIT) PhiNode(result_region, Type::ABIO);
4609 PhiNode* result_memory = new(C, PATH_LIMIT) PhiNode(result_region, Type::MEMORY, adr_type);
4610 record_for_igvn(result_region);
4611 _gvn.set_type_bottom(result_i_o);
4612 _gvn.set_type_bottom(result_memory);
4613 assert(adr_type != TypePtr::BOTTOM, "must be RawMem or a T[] slice");
4614
4615 // The slow_control path:
4616 Node* slow_control;
4617 Node* slow_i_o = i_o();
4618 Node* slow_mem = memory(adr_type);
4619 debug_only(slow_control = (Node*) badAddress);
4620
4621 // Checked control path:
4622 Node* checked_control = top();
4623 Node* checked_mem = NULL;
4624 Node* checked_i_o = NULL;
4625 Node* checked_value = NULL;
4626
4627 if (basic_elem_type == T_CONFLICT) {
4628 assert(!must_clear_dest, "");
4629 Node* cv = generate_generic_arraycopy(adr_type,
4630 src, src_offset, dest, dest_offset,
4631 copy_length, dest_uninitialized);
4632 if (cv == NULL) cv = intcon(-1); // failure (no stub available)
4633 checked_control = control();
4634 checked_i_o = i_o();
4635 checked_mem = memory(adr_type);
4636 checked_value = cv;
4637 set_control(top()); // no fast path
4638 }
4639
4640 Node* not_pos = generate_nonpositive_guard(copy_length, length_never_negative);
4641 if (not_pos != NULL) {
4642 PreserveJVMState pjvms(this);
4643 set_control(not_pos);
4644
4645 // (6) length must not be negative.
4646 if (!length_never_negative) {
4647 generate_negative_guard(copy_length, slow_region);
4648 }
4649
4650 // copy_length is 0.
4651 if (!stopped() && must_clear_dest) {
4693 // Next, perform a dynamic check on the tail length.
4694 // It is often zero, and we can win big if we prove this.
4695 // There are two wins: Avoid generating the ClearArray
4696 // with its attendant messy index arithmetic, and upgrade
4697 // the copy to a more hardware-friendly word size of 64 bits.
4698 Node* tail_ctl = NULL;
4699 if (!stopped() && !_gvn.eqv_uncast(dest_tail, dest_length)) {
4700 Node* cmp_lt = _gvn.transform( new(C,3) CmpINode(dest_tail, dest_length) );
4701 Node* bol_lt = _gvn.transform( new(C,2) BoolNode(cmp_lt, BoolTest::lt) );
4702 tail_ctl = generate_slow_guard(bol_lt, NULL);
4703 assert(tail_ctl != NULL || !stopped(), "must be an outcome");
4704 }
4705
4706 // At this point, let's assume there is no tail.
4707 if (!stopped() && alloc != NULL && basic_elem_type != T_OBJECT) {
4708 // There is no tail. Try an upgrade to a 64-bit copy.
4709 bool didit = false;
4710 { PreserveJVMState pjvms(this);
4711 didit = generate_block_arraycopy(adr_type, basic_elem_type, alloc,
4712 src, src_offset, dest, dest_offset,
4713 dest_size, dest_uninitialized);
4714 if (didit) {
4715 // Present the results of the block-copying fast call.
4716 result_region->init_req(bcopy_path, control());
4717 result_i_o ->init_req(bcopy_path, i_o());
4718 result_memory->init_req(bcopy_path, memory(adr_type));
4719 }
4720 }
4721 if (didit)
4722 set_control(top()); // no regular fast path
4723 }
4724
4725 // Clear the tail, if any.
4726 if (tail_ctl != NULL) {
4727 Node* notail_ctl = stopped() ? NULL : control();
4728 set_control(tail_ctl);
4729 if (notail_ctl == NULL) {
4730 generate_clear_array(adr_type, dest, basic_elem_type,
4731 dest_tail, NULL,
4732 dest_size);
4733 } else {
4769 // The backing store for a List<String> is always an Object[],
4770 // but its elements are always type String, if the generic types
4771 // are correct at the source level.
4772 //
4773 // Test S[] against D[], not S against D, because (probably)
4774 // the secondary supertype cache is less busy for S[] than S.
4775 // This usually only matters when D is an interface.
4776 Node* not_subtype_ctrl = gen_subtype_check(src_klass, dest_klass);
4777 // Plug failing path into checked_oop_disjoint_arraycopy
4778 if (not_subtype_ctrl != top()) {
4779 PreserveJVMState pjvms(this);
4780 set_control(not_subtype_ctrl);
4781 // (At this point we can assume disjoint_bases, since types differ.)
4782 int ek_offset = objArrayKlass::element_klass_offset_in_bytes() + sizeof(oopDesc);
4783 Node* p1 = basic_plus_adr(dest_klass, ek_offset);
4784 Node* n1 = LoadKlassNode::make(_gvn, immutable_memory(), p1, TypeRawPtr::BOTTOM);
4785 Node* dest_elem_klass = _gvn.transform(n1);
4786 Node* cv = generate_checkcast_arraycopy(adr_type,
4787 dest_elem_klass,
4788 src, src_offset, dest, dest_offset,
4789 ConvI2X(copy_length), dest_uninitialized);
4790 if (cv == NULL) cv = intcon(-1); // failure (no stub available)
4791 checked_control = control();
4792 checked_i_o = i_o();
4793 checked_mem = memory(adr_type);
4794 checked_value = cv;
4795 }
4796 // At this point we know we do not need type checks on oop stores.
4797
4798 // Let's see if we need card marks:
4799 if (alloc != NULL && use_ReduceInitialCardMarks()) {
4800 // If we do not need card marks, copy using the jint or jlong stub.
4801 copy_type = LP64_ONLY(UseCompressedOops ? T_INT : T_LONG) NOT_LP64(T_INT);
4802 assert(type2aelembytes(basic_elem_type) == type2aelembytes(copy_type),
4803 "sizes agree");
4804 }
4805 }
4806
4807 if (!stopped()) {
4808 // Generate the fast path, if possible.
4809 PreserveJVMState pjvms(this);
4810 generate_unchecked_arraycopy(adr_type, copy_type, disjoint_bases,
4811 src, src_offset, dest, dest_offset,
4812 ConvI2X(copy_length), dest_uninitialized);
4813
4814 // Present the results of the fast call.
4815 result_region->init_req(fast_path, control());
4816 result_i_o ->init_req(fast_path, i_o());
4817 result_memory->init_req(fast_path, memory(adr_type));
4818 }
4819
4820 // Here are all the slow paths up to this point, in one bundle:
4821 slow_control = top();
4822 if (slow_region != NULL)
4823 slow_control = _gvn.transform(slow_region);
4824 debug_only(slow_region = (RegionNode*)badAddress);
4825
4826 set_control(checked_control);
4827 if (!stopped()) {
4828 // Clean up after the checked call.
4829 // The returned value is either 0 or -1^K,
4830 // where K = number of partially transferred array elements.
4831 Node* cmp = _gvn.transform( new(C, 3) CmpINode(checked_value, intcon(0)) );
4832 Node* bol = _gvn.transform( new(C, 2) BoolNode(cmp, BoolTest::eq) );
4879 copy_length = length_minus;
4880 }
4881 }
4882
4883 set_control(slow_control);
4884 if (!stopped()) {
4885 // Generate the slow path, if needed.
4886 PreserveJVMState pjvms(this); // replace_in_map may trash the map
4887
4888 set_memory(slow_mem, adr_type);
4889 set_i_o(slow_i_o);
4890
4891 if (must_clear_dest) {
4892 generate_clear_array(adr_type, dest, basic_elem_type,
4893 intcon(0), NULL,
4894 alloc->in(AllocateNode::AllocSize));
4895 }
4896
4897 generate_slow_arraycopy(adr_type,
4898 src, src_offset, dest, dest_offset,
4899 copy_length, /*dest_uninitialized*/false);
4900
4901 result_region->init_req(slow_call_path, control());
4902 result_i_o ->init_req(slow_call_path, i_o());
4903 result_memory->init_req(slow_call_path, memory(adr_type));
4904 }
4905
4906 // Remove unused edges.
4907 for (uint i = 1; i < result_region->req(); i++) {
4908 if (result_region->in(i) == NULL)
4909 result_region->init_req(i, top());
4910 }
4911
4912 // Finished; return the combined state.
4913 set_control( _gvn.transform(result_region) );
4914 set_i_o( _gvn.transform(result_i_o) );
4915 set_memory( _gvn.transform(result_memory), adr_type );
4916
4917 // The memory edges above are precise in order to model effects around
4918 // array copies accurately to allow value numbering of field loads around
4919 // arraycopy. Such field loads, both before and after, are common in Java
5123 Node* end = dest_size; // pre-rounded
5124 mem = ClearArrayNode::clear_memory(control(), mem, dest,
5125 start, end, &_gvn);
5126 } else {
5127 // Non-constant start, unrounded non-constant end.
5128 // (Nobody zeroes a random midsection of an array using this routine.)
5129 ShouldNotReachHere(); // fix caller
5130 }
5131
5132 // Done.
5133 set_memory(mem, adr_type);
5134 }
5135
5136
5137 bool
5138 LibraryCallKit::generate_block_arraycopy(const TypePtr* adr_type,
5139 BasicType basic_elem_type,
5140 AllocateNode* alloc,
5141 Node* src, Node* src_offset,
5142 Node* dest, Node* dest_offset,
5143 Node* dest_size, bool dest_uninitialized) {
5144 // See if there is an advantage from block transfer.
5145 int scale = exact_log2(type2aelembytes(basic_elem_type));
5146 if (scale >= LogBytesPerLong)
5147 return false; // it is already a block transfer
5148
5149 // Look at the alignment of the starting offsets.
5150 int abase = arrayOopDesc::base_offset_in_bytes(basic_elem_type);
5151 const intptr_t BIG_NEG = -128;
5152 assert(BIG_NEG + 2*abase < 0, "neg enough");
5153
5154 intptr_t src_off = abase + ((intptr_t) find_int_con(src_offset, -1) << scale);
5155 intptr_t dest_off = abase + ((intptr_t) find_int_con(dest_offset, -1) << scale);
5156 if (src_off < 0 || dest_off < 0)
5157 // At present, we can only understand constants.
5158 return false;
5159
5160 if (((src_off | dest_off) & (BytesPerLong-1)) != 0) {
5161 // Non-aligned; too bad.
5162 // One more chance: Pick off an initial 32-bit word.
5163 // This is a common case, since abase can be odd mod 8.
5168 Node* sval = make_load(control(), sptr, TypeInt::INT, T_INT, adr_type);
5169 store_to_memory(control(), dptr, sval, T_INT, adr_type);
5170 src_off += BytesPerInt;
5171 dest_off += BytesPerInt;
5172 } else {
5173 return false;
5174 }
5175 }
5176 assert(src_off % BytesPerLong == 0, "");
5177 assert(dest_off % BytesPerLong == 0, "");
5178
5179 // Do this copy by giant steps.
5180 Node* sptr = basic_plus_adr(src, src_off);
5181 Node* dptr = basic_plus_adr(dest, dest_off);
5182 Node* countx = dest_size;
5183 countx = _gvn.transform( new (C, 3) SubXNode(countx, MakeConX(dest_off)) );
5184 countx = _gvn.transform( new (C, 3) URShiftXNode(countx, intcon(LogBytesPerLong)) );
5185
5186 bool disjoint_bases = true; // since alloc != NULL
5187 generate_unchecked_arraycopy(adr_type, T_LONG, disjoint_bases,
5188 sptr, NULL, dptr, NULL, countx, dest_uninitialized);
5189
5190 return true;
5191 }
5192
5193
5194 // Helper function; generates code for the slow case.
5195 // We make a call to a runtime method which emulates the native method,
5196 // but without the native wrapper overhead.
5197 void
5198 LibraryCallKit::generate_slow_arraycopy(const TypePtr* adr_type,
5199 Node* src, Node* src_offset,
5200 Node* dest, Node* dest_offset,
5201 Node* copy_length, bool dest_uninitialized) {
5202 assert(!dest_uninitialized, "Invariant");
5203 Node* call = make_runtime_call(RC_NO_LEAF | RC_UNCOMMON,
5204 OptoRuntime::slow_arraycopy_Type(),
5205 OptoRuntime::slow_arraycopy_Java(),
5206 "slow_arraycopy", adr_type,
5207 src, src_offset, dest, dest_offset,
5208 copy_length);
5209
5210 // Handle exceptions thrown by this fellow:
5211 make_slow_call_ex(call, env()->Throwable_klass(), false);
5212 }
5213
5214 // Helper function; generates code for cases requiring runtime checks.
5215 Node*
5216 LibraryCallKit::generate_checkcast_arraycopy(const TypePtr* adr_type,
5217 Node* dest_elem_klass,
5218 Node* src, Node* src_offset,
5219 Node* dest, Node* dest_offset,
5220 Node* copy_length, bool dest_uninitialized) {
5221 if (stopped()) return NULL;
5222
5223 address copyfunc_addr = StubRoutines::checkcast_arraycopy(dest_uninitialized);
5224 if (copyfunc_addr == NULL) { // Stub was not generated, go slow path.
5225 return NULL;
5226 }
5227
5228 // Pick out the parameters required to perform a store-check
5229 // for the target array. This is an optimistic check. It will
5230 // look in each non-null element's class, at the desired klass's
5231 // super_check_offset, for the desired klass.
5232 int sco_offset = Klass::super_check_offset_offset_in_bytes() + sizeof(oopDesc);
5233 Node* p3 = basic_plus_adr(dest_elem_klass, sco_offset);
5234 Node* n3 = new(C, 3) LoadINode(NULL, memory(p3), p3, _gvn.type(p3)->is_ptr());
5235 Node* check_offset = ConvI2X(_gvn.transform(n3));
5236 Node* check_value = dest_elem_klass;
5237
5238 Node* src_start = array_element_address(src, src_offset, T_OBJECT);
5239 Node* dest_start = array_element_address(dest, dest_offset, T_OBJECT);
5240
5241 // (We know the arrays are never conjoint, because their types differ.)
5242 Node* call = make_runtime_call(RC_LEAF|RC_NO_FP,
5243 OptoRuntime::checkcast_arraycopy_Type(),
5244 copyfunc_addr, "checkcast_arraycopy", adr_type,
5245 // five arguments, of which two are
5246 // intptr_t (jlong in LP64)
5247 src_start, dest_start,
5248 copy_length XTOP,
5249 check_offset XTOP,
5250 check_value);
5251
5252 return _gvn.transform(new (C, 1) ProjNode(call, TypeFunc::Parms));
5253 }
5254
5255
5256 // Helper function; generates code for cases requiring runtime checks.
5257 Node*
5258 LibraryCallKit::generate_generic_arraycopy(const TypePtr* adr_type,
5259 Node* src, Node* src_offset,
5260 Node* dest, Node* dest_offset,
5261 Node* copy_length, bool dest_uninitialized) {
5262 assert(!dest_uninitialized, "Invariant");
5263 if (stopped()) return NULL;
5264 address copyfunc_addr = StubRoutines::generic_arraycopy();
5265 if (copyfunc_addr == NULL) { // Stub was not generated, go slow path.
5266 return NULL;
5267 }
5268
5269 Node* call = make_runtime_call(RC_LEAF|RC_NO_FP,
5270 OptoRuntime::generic_arraycopy_Type(),
5271 copyfunc_addr, "generic_arraycopy", adr_type,
5272 src, src_offset, dest, dest_offset, copy_length);
5273
5274 return _gvn.transform(new (C, 1) ProjNode(call, TypeFunc::Parms));
5275 }
5276
5277 // Helper function; generates the fast out-of-line call to an arraycopy stub.
5278 void
5279 LibraryCallKit::generate_unchecked_arraycopy(const TypePtr* adr_type,
5280 BasicType basic_elem_type,
5281 bool disjoint_bases,
5282 Node* src, Node* src_offset,
5283 Node* dest, Node* dest_offset,
5284 Node* copy_length, bool dest_uninitialized) {
5285 if (stopped()) return; // nothing to do
5286
5287 Node* src_start = src;
5288 Node* dest_start = dest;
5289 if (src_offset != NULL || dest_offset != NULL) {
5290 assert(src_offset != NULL && dest_offset != NULL, "");
5291 src_start = array_element_address(src, src_offset, basic_elem_type);
5292 dest_start = array_element_address(dest, dest_offset, basic_elem_type);
5293 }
5294
5295 // Figure out which arraycopy runtime method to call.
5296 const char* copyfunc_name = "arraycopy";
5297 address copyfunc_addr =
5298 basictype2arraycopy(basic_elem_type, src_offset, dest_offset,
5299 disjoint_bases, copyfunc_name, dest_uninitialized);
5300
5301 // Call it. Note that the count_ix value is not scaled to a byte-size.
5302 make_runtime_call(RC_LEAF|RC_NO_FP,
5303 OptoRuntime::fast_arraycopy_Type(),
5304 copyfunc_addr, copyfunc_name, adr_type,
5305 src_start, dest_start, copy_length XTOP);
5306 }
|