73 }
74
75 ciMethod* caller() const { return jvms()->method(); }
76 int bci() const { return jvms()->bci(); }
77 LibraryIntrinsic* intrinsic() const { return _intrinsic; }
78 vmIntrinsics::ID intrinsic_id() const { return _intrinsic->intrinsic_id(); }
79 ciMethod* callee() const { return _intrinsic->method(); }
80 ciSignature* signature() const { return callee()->signature(); }
81 int arg_size() const { return callee()->arg_size(); }
82
83 bool try_to_inline();
84
85 // Helper functions to inline natives
86 void push_result(RegionNode* region, PhiNode* value);
87 Node* generate_guard(Node* test, RegionNode* region, float true_prob);
88 Node* generate_slow_guard(Node* test, RegionNode* region);
89 Node* generate_fair_guard(Node* test, RegionNode* region);
90 Node* generate_negative_guard(Node* index, RegionNode* region,
91 // resulting CastII of index:
92 Node* *pos_index = NULL);
93 Node* generate_nonpositive_guard(Node* index, bool never_negative,
94 // resulting CastII of index:
95 Node* *pos_index = NULL);
96 Node* generate_limit_guard(Node* offset, Node* subseq_length,
97 Node* array_length,
98 RegionNode* region);
99 Node* generate_current_thread(Node* &tls_output);
100 address basictype2arraycopy(BasicType t, Node *src_offset, Node *dest_offset,
101 bool disjoint_bases, const char* &name, bool dest_uninitialized);
102 Node* load_mirror_from_klass(Node* klass);
103 Node* load_klass_from_mirror_common(Node* mirror, bool never_see_null,
104 int nargs,
105 RegionNode* region, int null_path,
106 int offset);
107 Node* load_klass_from_mirror(Node* mirror, bool never_see_null, int nargs,
108 RegionNode* region, int null_path) {
109 int offset = java_lang_Class::klass_offset_in_bytes();
110 return load_klass_from_mirror_common(mirror, never_see_null, nargs,
111 region, null_path,
112 offset);
113 }
114 Node* load_array_klass_from_mirror(Node* mirror, bool never_see_null,
115 int nargs,
184 bool inline_native_isInterrupted();
185 bool inline_native_Class_query(vmIntrinsics::ID id);
186 bool inline_native_subtype_check();
187
188 bool inline_native_newArray();
189 bool inline_native_getLength();
190 bool inline_array_copyOf(bool is_copyOfRange);
191 bool inline_array_equals();
192 void copy_to_clone(Node* obj, Node* alloc_obj, Node* obj_size, bool is_array, bool card_mark);
193 bool inline_native_clone(bool is_virtual);
194 bool inline_native_Reflection_getCallerClass();
195 bool inline_native_AtomicLong_get();
196 bool inline_native_AtomicLong_attemptUpdate();
197 bool is_method_invoke_or_aux_frame(JVMState* jvms);
198 // Helper function for inlining native object hash method
199 bool inline_native_hashcode(bool is_virtual, bool is_static);
200 bool inline_native_getClass();
201
202 // Helper functions for inlining arraycopy
203 bool inline_arraycopy();
204 void generate_arraycopy(const TypePtr* adr_type,
205 BasicType basic_elem_type,
206 Node* src, Node* src_offset,
207 Node* dest, Node* dest_offset,
208 Node* copy_length,
209 bool disjoint_bases = false,
210 bool length_never_negative = false,
211 RegionNode* slow_region = NULL);
212 AllocateArrayNode* tightly_coupled_allocation(Node* ptr,
213 RegionNode* slow_region);
214 void generate_clear_array(const TypePtr* adr_type,
215 Node* dest,
216 BasicType basic_elem_type,
217 Node* slice_off,
218 Node* slice_len,
219 Node* slice_end);
220 bool generate_block_arraycopy(const TypePtr* adr_type,
221 BasicType basic_elem_type,
222 AllocateNode* alloc,
223 Node* src, Node* src_offset,
224 Node* dest, Node* dest_offset,
225 Node* dest_size, bool dest_uninitialized);
226 void generate_slow_arraycopy(const TypePtr* adr_type,
227 Node* src, Node* src_offset,
228 Node* dest, Node* dest_offset,
229 Node* copy_length, bool dest_uninitialized);
230 Node* generate_checkcast_arraycopy(const TypePtr* adr_type,
231 Node* dest_elem_klass,
232 Node* src, Node* src_offset,
233 Node* dest, Node* dest_offset,
234 Node* copy_length, bool dest_uninitialized);
235 Node* generate_generic_arraycopy(const TypePtr* adr_type,
236 Node* src, Node* src_offset,
237 Node* dest, Node* dest_offset,
238 Node* copy_length, bool dest_uninitialized);
239 void generate_unchecked_arraycopy(const TypePtr* adr_type,
240 BasicType basic_elem_type,
241 bool disjoint_bases,
242 Node* src, Node* src_offset,
243 Node* dest, Node* dest_offset,
244 Node* copy_length, bool dest_uninitialized);
245 bool inline_unsafe_CAS(BasicType type);
246 bool inline_unsafe_ordered_store(BasicType type);
247 bool inline_fp_conversions(vmIntrinsics::ID id);
248 bool inline_numberOfLeadingZeros(vmIntrinsics::ID id);
249 bool inline_numberOfTrailingZeros(vmIntrinsics::ID id);
250 bool inline_bitCount(vmIntrinsics::ID id);
251 bool inline_reverseBytes(vmIntrinsics::ID id);
252
253 bool inline_reference_get();
254 };
255
256
257 //---------------------------make_vm_intrinsic----------------------------
258 CallGenerator* Compile::make_vm_intrinsic(ciMethod* m, bool is_virtual) {
259 vmIntrinsics::ID id = m->intrinsic_id();
260 assert(id != vmIntrinsics::_none, "must be a VM intrinsic");
261
262 if (DisableIntrinsic[0] != '\0'
263 && strstr(DisableIntrinsic, vmIntrinsics::name_at(id)) != NULL) {
264 // disabled by a user request on the command line:
792 }
793
794 inline Node* LibraryCallKit::generate_negative_guard(Node* index, RegionNode* region,
795 Node* *pos_index) {
796 if (stopped())
797 return NULL; // already stopped
798 if (_gvn.type(index)->higher_equal(TypeInt::POS)) // [0,maxint]
799 return NULL; // index is already adequately typed
800 Node* cmp_lt = _gvn.transform( new (C, 3) CmpINode(index, intcon(0)) );
801 Node* bol_lt = _gvn.transform( new (C, 2) BoolNode(cmp_lt, BoolTest::lt) );
802 Node* is_neg = generate_guard(bol_lt, region, PROB_MIN);
803 if (is_neg != NULL && pos_index != NULL) {
804 // Emulate effect of Parse::adjust_map_after_if.
805 Node* ccast = new (C, 2) CastIINode(index, TypeInt::POS);
806 ccast->set_req(0, control());
807 (*pos_index) = _gvn.transform(ccast);
808 }
809 return is_neg;
810 }
811
812 inline Node* LibraryCallKit::generate_nonpositive_guard(Node* index, bool never_negative,
813 Node* *pos_index) {
814 if (stopped())
815 return NULL; // already stopped
816 if (_gvn.type(index)->higher_equal(TypeInt::POS1)) // [1,maxint]
817 return NULL; // index is already adequately typed
818 Node* cmp_le = _gvn.transform( new (C, 3) CmpINode(index, intcon(0)) );
819 BoolTest::mask le_or_eq = (never_negative ? BoolTest::eq : BoolTest::le);
820 Node* bol_le = _gvn.transform( new (C, 2) BoolNode(cmp_le, le_or_eq) );
821 Node* is_notp = generate_guard(bol_le, NULL, PROB_MIN);
822 if (is_notp != NULL && pos_index != NULL) {
823 // Emulate effect of Parse::adjust_map_after_if.
824 Node* ccast = new (C, 2) CastIINode(index, TypeInt::POS1);
825 ccast->set_req(0, control());
826 (*pos_index) = _gvn.transform(ccast);
827 }
828 return is_notp;
829 }
830
831 // Make sure that 'position' is a valid limit index, in [0..length].
832 // There are two equivalent plans for checking this:
833 // A. (offset + copyLength) unsigned<= arrayLength
834 // B. offset <= (arrayLength - copyLength)
835 // We require that all of the values above, except for the sum and
836 // difference, are already known to be non-negative.
837 // Plan A is robust in the face of overflow, if offset and copyLength
838 // are both hugely positive.
839 //
840 // Plan B is less direct and intuitive, but it does not overflow at
841 // all, since the difference of two non-negatives is always
842 // representable. Whenever Java methods must perform the equivalent
843 // check they generally use Plan B instead of Plan A.
844 // For the moment we use Plan A.
845 inline Node* LibraryCallKit::generate_limit_guard(Node* offset,
846 Node* subseq_length,
847 Node* array_length,
848 RegionNode* region) {
849 if (stopped())
850 return NULL; // already stopped
3611 PreserveJVMState pjvms(this);
3612 set_control( _gvn.transform(bailout) );
3613 uncommon_trap(Deoptimization::Reason_intrinsic,
3614 Deoptimization::Action_maybe_recompile);
3615 }
3616
3617 if (!stopped()) {
3618
3619 // How many elements will we copy from the original?
3620 // The answer is MinI(orig_length - start, length).
3621 Node* orig_tail = _gvn.transform( new(C, 3) SubINode(orig_length, start) );
3622 Node* moved = generate_min_max(vmIntrinsics::_min, orig_tail, length);
3623
3624 newcopy = new_array(klass_node, length, 0);
3625
3626 // Generate a direct call to the right arraycopy function(s).
3627 // We know the copy is disjoint but we might not know if the
3628 // oop stores need checking.
3629 // Extreme case: Arrays.copyOf((Integer[])x, 10, String[].class).
3630 // This will fail a store-check if x contains any non-nulls.
3631 bool disjoint_bases = true;
3632 bool length_never_negative = true;
3633 generate_arraycopy(TypeAryPtr::OOPS, T_OBJECT,
3634 original, start, newcopy, intcon(0), moved,
3635 disjoint_bases, length_never_negative);
3636 }
3637 } //original reexecute and sp are set back here
3638
3639 if(!stopped()) {
3640 push(newcopy);
3641 }
3642
3643 C->set_has_split_ifs(true); // Has chance for split-if optimization
3644
3645 return true;
3646 }
3647
3648
3649 //----------------------generate_virtual_guard---------------------------
3650 // Helper for hashCode and clone. Peeks inside the vtable to avoid a call.
3651 Node* LibraryCallKit::generate_virtual_guard(Node* obj_klass,
3652 RegionNode* slow_region) {
3653 ciMethod* method = callee();
3654 int vtable_index = method->vtable_index();
3655 // Get the methodOop out of the appropriate vtable entry.
4319 if (base_off % BytesPerLong != 0) {
4320 assert(UseCompressedOops, "");
4321 if (is_array) {
4322 // Exclude length to copy by 8 bytes words.
4323 base_off += sizeof(int);
4324 } else {
4325 // Include klass to copy by 8 bytes words.
4326 base_off = instanceOopDesc::klass_offset_in_bytes();
4327 }
4328 assert(base_off % BytesPerLong == 0, "expect 8 bytes alignment");
4329 }
4330 src = basic_plus_adr(src, base_off);
4331 dest = basic_plus_adr(dest, base_off);
4332
4333 // Compute the length also, if needed:
4334 Node* countx = size;
4335 countx = _gvn.transform( new (C, 3) SubXNode(countx, MakeConX(base_off)) );
4336 countx = _gvn.transform( new (C, 3) URShiftXNode(countx, intcon(LogBytesPerLong) ));
4337
4338 const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
4339 bool disjoint_bases = true;
4340 generate_unchecked_arraycopy(raw_adr_type, T_LONG, disjoint_bases,
4341 src, NULL, dest, NULL, countx,
4342 /*dest_uninitialized*/true);
4343
4344 // If necessary, emit some card marks afterwards. (Non-arrays only.)
4345 if (card_mark) {
4346 assert(!is_array, "");
4347 // Put in store barrier for any and all oops we are sticking
4348 // into this object. (We could avoid this if we could prove
4349 // that the object type contains no oop fields at all.)
4350 Node* no_particular_value = NULL;
4351 Node* no_particular_field = NULL;
4352 int raw_adr_idx = Compile::AliasIdxRaw;
4353 post_barrier(control(),
4354 memory(raw_adr_type),
4355 alloc_obj,
4356 no_particular_field,
4357 raw_adr_idx,
4358 no_particular_value,
4359 T_OBJECT,
4360 false);
4361 }
4362
4435 const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
4436 int raw_adr_idx = Compile::AliasIdxRaw;
4437
4438 Node* array_ctl = generate_array_guard(obj_klass, (RegionNode*)NULL);
4439 if (array_ctl != NULL) {
4440 // It's an array.
4441 PreserveJVMState pjvms(this);
4442 set_control(array_ctl);
4443 Node* obj_length = load_array_length(obj);
4444 Node* obj_size = NULL;
4445 Node* alloc_obj = new_array(obj_klass, obj_length, 0, &obj_size);
4446
4447 if (!use_ReduceInitialCardMarks()) {
4448 // If it is an oop array, it requires very special treatment,
4449 // because card marking is required on each card of the array.
4450 Node* is_obja = generate_objArray_guard(obj_klass, (RegionNode*)NULL);
4451 if (is_obja != NULL) {
4452 PreserveJVMState pjvms2(this);
4453 set_control(is_obja);
4454 // Generate a direct call to the right arraycopy function(s).
4455 bool disjoint_bases = true;
4456 bool length_never_negative = true;
4457 generate_arraycopy(TypeAryPtr::OOPS, T_OBJECT,
4458 obj, intcon(0), alloc_obj, intcon(0),
4459 obj_length,
4460 disjoint_bases, length_never_negative);
4461 result_reg->init_req(_objArray_path, control());
4462 result_val->init_req(_objArray_path, alloc_obj);
4463 result_i_o ->set_req(_objArray_path, i_o());
4464 result_mem ->set_req(_objArray_path, reset_memory());
4465 }
4466 }
4467 // Otherwise, there are no card marks to worry about.
4468 // (We can dispense with card marks if we know the allocation
4469 // comes out of eden (TLAB)... In fact, ReduceInitialCardMarks
4470 // causes the non-eden paths to take compensating steps to
4471 // simulate a fresh allocation, so that no further
4472 // card marks are required in compiled code to initialize
4473 // the object.)
4474
4475 if (!stopped()) {
4476 copy_to_clone(obj, alloc_obj, obj_size, true, false);
4477
4478 // Present the results of the copy.
4479 result_reg->init_req(_array_path, control());
4480 result_val->init_req(_array_path, alloc_obj);
4580 // This can occur if the offsets are identical non-constants.
4581 disjoint = true;
4582 }
4583
4584 return StubRoutines::select_arraycopy_function(t, aligned, disjoint, name, dest_uninitialized);
4585 }
4586
4587
4588 //------------------------------inline_arraycopy-----------------------
4589 bool LibraryCallKit::inline_arraycopy() {
4590 // Restore the stack and pop off the arguments.
4591 int nargs = 5; // 2 oops, 3 ints, no size_t or long
4592 assert(callee()->signature()->size() == nargs, "copy has 5 arguments");
4593
4594 Node *src = argument(0);
4595 Node *src_offset = argument(1);
4596 Node *dest = argument(2);
4597 Node *dest_offset = argument(3);
4598 Node *length = argument(4);
4599
4600 // Compile time checks. If any of these checks cannot be verified at compile time,
4601 // we do not make a fast path for this call. Instead, we let the call remain as it
4602 // is. The checks we choose to mandate at compile time are:
4603 //
4604 // (1) src and dest are arrays.
4605 const Type* src_type = src->Value(&_gvn);
4606 const Type* dest_type = dest->Value(&_gvn);
4607 const TypeAryPtr* top_src = src_type->isa_aryptr();
4608 const TypeAryPtr* top_dest = dest_type->isa_aryptr();
4609 if (top_src == NULL || top_src->klass() == NULL ||
4610 top_dest == NULL || top_dest->klass() == NULL) {
4611 // Conservatively insert a memory barrier on all memory slices.
4612 // Do not let writes into the source float below the arraycopy.
4613 insert_mem_bar(Op_MemBarCPUOrder);
4614
4615 // Call StubRoutines::generic_arraycopy stub.
4616 generate_arraycopy(TypeRawPtr::BOTTOM, T_CONFLICT,
4617 src, src_offset, dest, dest_offset, length);
4618
4619 // Do not let reads from the destination float above the arraycopy.
4620 // Since we cannot type the arrays, we don't know which slices
4621 // might be affected. We could restrict this barrier only to those
4622 // memory slices which pertain to array elements--but don't bother.
4623 if (!InsertMemBarAfterArraycopy)
4624 // (If InsertMemBarAfterArraycopy, there is already one in place.)
4625 insert_mem_bar(Op_MemBarCPUOrder);
4626 return true;
4627 }
4628
4629 // (2) src and dest arrays must have elements of the same BasicType
4630 // Figure out the size and type of the elements we will be copying.
4631 BasicType src_elem = top_src->klass()->as_array_klass()->element_type()->basic_type();
4632 BasicType dest_elem = top_dest->klass()->as_array_klass()->element_type()->basic_type();
4633 if (src_elem == T_ARRAY) src_elem = T_OBJECT;
4634 if (dest_elem == T_ARRAY) dest_elem = T_OBJECT;
4635
4636 if (src_elem != dest_elem || dest_elem == T_VOID) {
4637 // The component types are not the same or are not recognized. Punt.
4638 // (But, avoid the native method wrapper to JVM_ArrayCopy.)
4639 generate_slow_arraycopy(TypePtr::BOTTOM,
4640 src, src_offset, dest, dest_offset, length,
4641 /*dest_uninitialized*/false);
4642 return true;
4643 }
4644
4645 //---------------------------------------------------------------------------
4646 // We will make a fast path for this call to arraycopy.
4647
4648 // We have the following tests left to perform:
4649 //
4650 // (3) src and dest must not be null.
4651 // (4) src_offset must not be negative.
4652 // (5) dest_offset must not be negative.
4653 // (6) length must not be negative.
4654 // (7) src_offset + length must not exceed length of src.
4655 // (8) dest_offset + length must not exceed length of dest.
4656 // (9) each element of an oop array must be assignable
4657
4658 RegionNode* slow_region = new (C, 1) RegionNode(1);
4659 record_for_igvn(slow_region);
4660
4661 // (3) operands must not be null
4662 // We currently perform our null checks with the do_null_check routine.
4663 // This means that the null exceptions will be reported in the caller
4664 // rather than (correctly) reported inside of the native arraycopy call.
4665 // This should be corrected, given time. We do our null check with the
4666 // stack pointer restored.
4667 _sp += nargs;
4668 src = do_null_check(src, T_ARRAY);
4669 dest = do_null_check(dest, T_ARRAY);
4670 _sp -= nargs;
4671
4672 // (4) src_offset must not be negative.
4673 generate_negative_guard(src_offset, slow_region);
4674
4675 // (5) dest_offset must not be negative.
4676 generate_negative_guard(dest_offset, slow_region);
4677
4678 // (6) length must not be negative (moved to generate_arraycopy()).
4679 // generate_negative_guard(length, slow_region);
4680
4681 // (7) src_offset + length must not exceed length of src.
4682 generate_limit_guard(src_offset, length,
4683 load_array_length(src),
4684 slow_region);
4685
4686 // (8) dest_offset + length must not exceed length of dest.
4687 generate_limit_guard(dest_offset, length,
4688 load_array_length(dest),
4689 slow_region);
4690
4691 // (9) each element of an oop array must be assignable
4692 // The generate_arraycopy subroutine checks this.
4693
4694 // This is where the memory effects are placed:
4695 const TypePtr* adr_type = TypeAryPtr::get_array_body_type(dest_elem);
4696 generate_arraycopy(adr_type, dest_elem,
4697 src, src_offset, dest, dest_offset, length,
4698 false, false, slow_region);
4699
4700 return true;
4701 }
4702
4703 //-----------------------------generate_arraycopy----------------------
4704 // Generate an optimized call to arraycopy.
4705 // Caller must guard against non-arrays.
4706 // Caller must determine a common array basic-type for both arrays.
4707 // Caller must validate offsets against array bounds.
4708 // The slow_region has already collected guard failure paths
4709 // (such as out of bounds length or non-conformable array types).
4710 // The generated code has this shape, in general:
4711 //
4712 // if (length == 0) return // via zero_path
4713 // slowval = -1
4714 // if (types unknown) {
4715 // slowval = call generic copy loop
4716 // if (slowval == 0) return // via checked_path
4717 // } else if (indexes in bounds) {
4718 // if ((is object array) && !(array type check)) {
4719 // slowval = call checked copy loop
4720 // if (slowval == 0) return // via checked_path
4721 // } else {
4722 // call bulk copy loop
4723 // return // via fast_path
4724 // }
4725 // }
4726 // // adjust params for remaining work:
4727 // if (slowval != -1) {
4728 // n = -1^slowval; src_offset += n; dest_offset += n; length -= n
4729 // }
4730 // slow_region:
4731 // call slow arraycopy(src, src_offset, dest, dest_offset, length)
4732 // return // via slow_call_path
4733 //
4734 // This routine is used from several intrinsics: System.arraycopy,
4735 // Object.clone (the array subcase), and Arrays.copyOf[Range].
4736 //
4737 void
4738 LibraryCallKit::generate_arraycopy(const TypePtr* adr_type,
4739 BasicType basic_elem_type,
4740 Node* src, Node* src_offset,
4741 Node* dest, Node* dest_offset,
4742 Node* copy_length,
4743 bool disjoint_bases,
4744 bool length_never_negative,
4745 RegionNode* slow_region) {
4746
4747 if (slow_region == NULL) {
4748 slow_region = new(C,1) RegionNode(1);
4749 record_for_igvn(slow_region);
4750 }
4751
4752 Node* original_dest = dest;
4753 AllocateArrayNode* alloc = NULL; // used for zeroing, if needed
4754 bool dest_uninitialized = false;
4755
4756 // See if this is the initialization of a newly-allocated array.
4757 // If so, we will take responsibility here for initializing it to zero.
4758 // (Note: Because tightly_coupled_allocation performs checks on the
4759 // out-edges of the dest, we need to avoid making derived pointers
4760 // from it until we have checked its uses.)
4761 if (ReduceBulkZeroing
4762 && !ZeroTLAB // pointless if already zeroed
4763 && basic_elem_type != T_CONFLICT // avoid corner case
4764 && !src->eqv_uncast(dest)
4765 && ((alloc = tightly_coupled_allocation(dest, slow_region))
4766 != NULL)
4767 && _gvn.find_int_con(alloc->in(AllocateNode::ALength), 1) > 0
4768 && alloc->maybe_set_complete(&_gvn)) {
4769 // "You break it, you buy it."
4770 InitializeNode* init = alloc->initialization();
4771 assert(init->is_complete(), "we just did this");
4772 init->set_complete_with_arraycopy();
4773 assert(dest->is_CheckCastPP(), "sanity");
4774 assert(dest->in(0)->in(0) == init, "dest pinned");
4775 adr_type = TypeRawPtr::BOTTOM; // all initializations are into raw memory
4776 // From this point on, every exit path is responsible for
4777 // initializing any non-copied parts of the object to zero.
4778 // Also, if this flag is set we make sure that arraycopy interacts properly
4779 // with G1, eliding pre-barriers. See CR 6627983.
4780 dest_uninitialized = true;
4781 } else {
4782 // No zeroing elimination here.
4783 alloc = NULL;
4784 //original_dest = dest;
4785 //dest_uninitialized = false;
4786 }
4787
4788 // Results are placed here:
4789 enum { fast_path = 1, // normal void-returning assembly stub
4790 checked_path = 2, // special assembly stub with cleanup
4791 slow_call_path = 3, // something went wrong; call the VM
4792 zero_path = 4, // bypass when length of copy is zero
4793 bcopy_path = 5, // copy primitive array by 64-bit blocks
4794 PATH_LIMIT = 6
4795 };
4796 RegionNode* result_region = new(C, PATH_LIMIT) RegionNode(PATH_LIMIT);
4797 PhiNode* result_i_o = new(C, PATH_LIMIT) PhiNode(result_region, Type::ABIO);
4798 PhiNode* result_memory = new(C, PATH_LIMIT) PhiNode(result_region, Type::MEMORY, adr_type);
4799 record_for_igvn(result_region);
4800 _gvn.set_type_bottom(result_i_o);
4801 _gvn.set_type_bottom(result_memory);
4802 assert(adr_type != TypePtr::BOTTOM, "must be RawMem or a T[] slice");
4803
4804 // The slow_control path:
4805 Node* slow_control;
4806 Node* slow_i_o = i_o();
4807 Node* slow_mem = memory(adr_type);
4808 debug_only(slow_control = (Node*) badAddress);
4809
4810 // Checked control path:
4811 Node* checked_control = top();
4812 Node* checked_mem = NULL;
4813 Node* checked_i_o = NULL;
4814 Node* checked_value = NULL;
4815
4816 if (basic_elem_type == T_CONFLICT) {
4817 assert(!dest_uninitialized, "");
4818 Node* cv = generate_generic_arraycopy(adr_type,
4819 src, src_offset, dest, dest_offset,
4820 copy_length, dest_uninitialized);
4821 if (cv == NULL) cv = intcon(-1); // failure (no stub available)
4822 checked_control = control();
4823 checked_i_o = i_o();
4824 checked_mem = memory(adr_type);
4825 checked_value = cv;
4826 set_control(top()); // no fast path
4827 }
4828
4829 Node* not_pos = generate_nonpositive_guard(copy_length, length_never_negative);
4830 if (not_pos != NULL) {
4831 PreserveJVMState pjvms(this);
4832 set_control(not_pos);
4833
4834 // (6) length must not be negative.
4835 if (!length_never_negative) {
4836 generate_negative_guard(copy_length, slow_region);
4837 }
4838
4839 // copy_length is 0.
4840 if (!stopped() && dest_uninitialized) {
4841 Node* dest_length = alloc->in(AllocateNode::ALength);
4842 if (copy_length->eqv_uncast(dest_length)
4843 || _gvn.find_int_con(dest_length, 1) <= 0) {
4844 // There is no zeroing to do. No need for a secondary raw memory barrier.
4845 } else {
4846 // Clear the whole thing since there are no source elements to copy.
4847 generate_clear_array(adr_type, dest, basic_elem_type,
4848 intcon(0), NULL,
4849 alloc->in(AllocateNode::AllocSize));
4850 // Use a secondary InitializeNode as raw memory barrier.
4851 // Currently it is needed only on this path since other
4852 // paths have stub or runtime calls as raw memory barriers.
4853 InitializeNode* init = insert_mem_bar_volatile(Op_Initialize,
4854 Compile::AliasIdxRaw,
4855 top())->as_Initialize();
4856 init->set_complete(&_gvn); // (there is no corresponding AllocateNode)
4857 }
4858 }
4859
4860 // Present the results of the fast call.
4861 result_region->init_req(zero_path, control());
4862 result_i_o ->init_req(zero_path, i_o());
4863 result_memory->init_req(zero_path, memory(adr_type));
4864 }
4865
4866 if (!stopped() && dest_uninitialized) {
4867 // We have to initialize the *uncopied* part of the array to zero.
4868 // The copy destination is the slice dest[off..off+len]. The other slices
4869 // are dest_head = dest[0..off] and dest_tail = dest[off+len..dest.length].
4870 Node* dest_size = alloc->in(AllocateNode::AllocSize);
4871 Node* dest_length = alloc->in(AllocateNode::ALength);
4872 Node* dest_tail = _gvn.transform( new(C,3) AddINode(dest_offset,
4873 copy_length) );
4874
4875 // If there is a head section that needs zeroing, do it now.
4876 if (find_int_con(dest_offset, -1) != 0) {
4877 generate_clear_array(adr_type, dest, basic_elem_type,
4878 intcon(0), dest_offset,
4879 NULL);
4880 }
4881
4882 // Next, perform a dynamic check on the tail length.
4883 // It is often zero, and we can win big if we prove this.
4884 // There are two wins: Avoid generating the ClearArray
4885 // with its attendant messy index arithmetic, and upgrade
4886 // the copy to a more hardware-friendly word size of 64 bits.
4887 Node* tail_ctl = NULL;
4888 if (!stopped() && !dest_tail->eqv_uncast(dest_length)) {
4889 Node* cmp_lt = _gvn.transform( new(C,3) CmpINode(dest_tail, dest_length) );
4890 Node* bol_lt = _gvn.transform( new(C,2) BoolNode(cmp_lt, BoolTest::lt) );
4891 tail_ctl = generate_slow_guard(bol_lt, NULL);
4892 assert(tail_ctl != NULL || !stopped(), "must be an outcome");
4893 }
4894
4895 // At this point, let's assume there is no tail.
4896 if (!stopped() && alloc != NULL && basic_elem_type != T_OBJECT) {
4897 // There is no tail. Try an upgrade to a 64-bit copy.
4898 bool didit = false;
4899 { PreserveJVMState pjvms(this);
4900 didit = generate_block_arraycopy(adr_type, basic_elem_type, alloc,
4901 src, src_offset, dest, dest_offset,
4902 dest_size, dest_uninitialized);
4903 if (didit) {
4904 // Present the results of the block-copying fast call.
4905 result_region->init_req(bcopy_path, control());
4906 result_i_o ->init_req(bcopy_path, i_o());
4907 result_memory->init_req(bcopy_path, memory(adr_type));
4908 }
4909 }
4910 if (didit)
4911 set_control(top()); // no regular fast path
4912 }
4913
4914 // Clear the tail, if any.
4915 if (tail_ctl != NULL) {
4916 Node* notail_ctl = stopped() ? NULL : control();
4917 set_control(tail_ctl);
4918 if (notail_ctl == NULL) {
4919 generate_clear_array(adr_type, dest, basic_elem_type,
4920 dest_tail, NULL,
4921 dest_size);
4922 } else {
4923 // Make a local merge.
4924 Node* done_ctl = new(C,3) RegionNode(3);
4925 Node* done_mem = new(C,3) PhiNode(done_ctl, Type::MEMORY, adr_type);
4926 done_ctl->init_req(1, notail_ctl);
4927 done_mem->init_req(1, memory(adr_type));
4928 generate_clear_array(adr_type, dest, basic_elem_type,
4929 dest_tail, NULL,
4930 dest_size);
4931 done_ctl->init_req(2, control());
4932 done_mem->init_req(2, memory(adr_type));
4933 set_control( _gvn.transform(done_ctl) );
4934 set_memory( _gvn.transform(done_mem), adr_type );
4935 }
4936 }
4937 }
4938
4939 BasicType copy_type = basic_elem_type;
4940 assert(basic_elem_type != T_ARRAY, "caller must fix this");
4941 if (!stopped() && copy_type == T_OBJECT) {
4942 // If src and dest have compatible element types, we can copy bits.
4943 // Types S[] and D[] are compatible if D is a supertype of S.
4944 //
4945 // If they are not, we will use checked_oop_disjoint_arraycopy,
4946 // which performs a fast optimistic per-oop check, and backs off
4947 // further to JVM_ArrayCopy on the first per-oop check that fails.
4948 // (Actually, we don't move raw bits only; the GC requires card marks.)
4949
4950 // Get the klassOop for both src and dest
4951 Node* src_klass = load_object_klass(src);
4952 Node* dest_klass = load_object_klass(dest);
4953
4954 // Generate the subtype check.
4955 // This might fold up statically, or then again it might not.
4956 //
4957 // Non-static example: Copying List<String>.elements to a new String[].
4958 // The backing store for a List<String> is always an Object[],
4959 // but its elements are always type String, if the generic types
4960 // are correct at the source level.
4961 //
4962 // Test S[] against D[], not S against D, because (probably)
4963 // the secondary supertype cache is less busy for S[] than S.
4964 // This usually only matters when D is an interface.
4965 Node* not_subtype_ctrl = gen_subtype_check(src_klass, dest_klass);
4966 // Plug failing path into checked_oop_disjoint_arraycopy
4967 if (not_subtype_ctrl != top()) {
4968 PreserveJVMState pjvms(this);
4969 set_control(not_subtype_ctrl);
4970 // (At this point we can assume disjoint_bases, since types differ.)
4971 int ek_offset = in_bytes(objArrayKlass::element_klass_offset());
4972 Node* p1 = basic_plus_adr(dest_klass, ek_offset);
4973 Node* n1 = LoadKlassNode::make(_gvn, immutable_memory(), p1, TypeRawPtr::BOTTOM);
4974 Node* dest_elem_klass = _gvn.transform(n1);
4975 Node* cv = generate_checkcast_arraycopy(adr_type,
4976 dest_elem_klass,
4977 src, src_offset, dest, dest_offset,
4978 ConvI2X(copy_length), dest_uninitialized);
4979 if (cv == NULL) cv = intcon(-1); // failure (no stub available)
4980 checked_control = control();
4981 checked_i_o = i_o();
4982 checked_mem = memory(adr_type);
4983 checked_value = cv;
4984 }
4985 // At this point we know we do not need type checks on oop stores.
4986
4987 // Let's see if we need card marks:
4988 if (alloc != NULL && use_ReduceInitialCardMarks()) {
4989 // If we do not need card marks, copy using the jint or jlong stub.
4990 copy_type = LP64_ONLY(UseCompressedOops ? T_INT : T_LONG) NOT_LP64(T_INT);
4991 assert(type2aelembytes(basic_elem_type) == type2aelembytes(copy_type),
4992 "sizes agree");
4993 }
4994 }
4995
4996 if (!stopped()) {
4997 // Generate the fast path, if possible.
4998 PreserveJVMState pjvms(this);
4999 generate_unchecked_arraycopy(adr_type, copy_type, disjoint_bases,
5000 src, src_offset, dest, dest_offset,
5001 ConvI2X(copy_length), dest_uninitialized);
5002
5003 // Present the results of the fast call.
5004 result_region->init_req(fast_path, control());
5005 result_i_o ->init_req(fast_path, i_o());
5006 result_memory->init_req(fast_path, memory(adr_type));
5007 }
5008
5009 // Here are all the slow paths up to this point, in one bundle:
5010 slow_control = top();
5011 if (slow_region != NULL)
5012 slow_control = _gvn.transform(slow_region);
5013 debug_only(slow_region = (RegionNode*)badAddress);
5014
5015 set_control(checked_control);
5016 if (!stopped()) {
5017 // Clean up after the checked call.
5018 // The returned value is either 0 or -1^K,
5019 // where K = number of partially transferred array elements.
5020 Node* cmp = _gvn.transform( new(C, 3) CmpINode(checked_value, intcon(0)) );
5021 Node* bol = _gvn.transform( new(C, 2) BoolNode(cmp, BoolTest::eq) );
5022 IfNode* iff = create_and_map_if(control(), bol, PROB_MAX, COUNT_UNKNOWN);
5023
5024 // If it is 0, we are done, so transfer to the end.
5025 Node* checks_done = _gvn.transform( new(C, 1) IfTrueNode(iff) );
5026 result_region->init_req(checked_path, checks_done);
5027 result_i_o ->init_req(checked_path, checked_i_o);
5028 result_memory->init_req(checked_path, checked_mem);
5029
5030 // If it is not zero, merge into the slow call.
5031 set_control( _gvn.transform( new(C, 1) IfFalseNode(iff) ));
5032 RegionNode* slow_reg2 = new(C, 3) RegionNode(3);
5033 PhiNode* slow_i_o2 = new(C, 3) PhiNode(slow_reg2, Type::ABIO);
5034 PhiNode* slow_mem2 = new(C, 3) PhiNode(slow_reg2, Type::MEMORY, adr_type);
5035 record_for_igvn(slow_reg2);
5036 slow_reg2 ->init_req(1, slow_control);
5037 slow_i_o2 ->init_req(1, slow_i_o);
5038 slow_mem2 ->init_req(1, slow_mem);
5039 slow_reg2 ->init_req(2, control());
5040 slow_i_o2 ->init_req(2, checked_i_o);
5041 slow_mem2 ->init_req(2, checked_mem);
5042
5043 slow_control = _gvn.transform(slow_reg2);
5044 slow_i_o = _gvn.transform(slow_i_o2);
5045 slow_mem = _gvn.transform(slow_mem2);
5046
5047 if (alloc != NULL) {
5048 // We'll restart from the very beginning, after zeroing the whole thing.
5049 // This can cause double writes, but that's OK since dest is brand new.
5050 // So we ignore the low 31 bits of the value returned from the stub.
5051 } else {
5052 // We must continue the copy exactly where it failed, or else
5053 // another thread might see the wrong number of writes to dest.
5054 Node* checked_offset = _gvn.transform( new(C, 3) XorINode(checked_value, intcon(-1)) );
5055 Node* slow_offset = new(C, 3) PhiNode(slow_reg2, TypeInt::INT);
5056 slow_offset->init_req(1, intcon(0));
5057 slow_offset->init_req(2, checked_offset);
5058 slow_offset = _gvn.transform(slow_offset);
5059
5060 // Adjust the arguments by the conditionally incoming offset.
5061 Node* src_off_plus = _gvn.transform( new(C, 3) AddINode(src_offset, slow_offset) );
5062 Node* dest_off_plus = _gvn.transform( new(C, 3) AddINode(dest_offset, slow_offset) );
5063 Node* length_minus = _gvn.transform( new(C, 3) SubINode(copy_length, slow_offset) );
5064
5065 // Tweak the node variables to adjust the code produced below:
5066 src_offset = src_off_plus;
5067 dest_offset = dest_off_plus;
5068 copy_length = length_minus;
5069 }
5070 }
5071
5072 set_control(slow_control);
5073 if (!stopped()) {
5074 // Generate the slow path, if needed.
5075 PreserveJVMState pjvms(this); // replace_in_map may trash the map
5076
5077 set_memory(slow_mem, adr_type);
5078 set_i_o(slow_i_o);
5079
5080 if (dest_uninitialized) {
5081 generate_clear_array(adr_type, dest, basic_elem_type,
5082 intcon(0), NULL,
5083 alloc->in(AllocateNode::AllocSize));
5084 }
5085
5086 generate_slow_arraycopy(adr_type,
5087 src, src_offset, dest, dest_offset,
5088 copy_length, /*dest_uninitialized*/false);
5089
5090 result_region->init_req(slow_call_path, control());
5091 result_i_o ->init_req(slow_call_path, i_o());
5092 result_memory->init_req(slow_call_path, memory(adr_type));
5093 }
5094
5095 // Remove unused edges.
5096 for (uint i = 1; i < result_region->req(); i++) {
5097 if (result_region->in(i) == NULL)
5098 result_region->init_req(i, top());
5099 }
5100
5101 // Finished; return the combined state.
5102 set_control( _gvn.transform(result_region) );
5103 set_i_o( _gvn.transform(result_i_o) );
5104 set_memory( _gvn.transform(result_memory), adr_type );
5105
5106 // The memory edges above are precise in order to model effects around
5107 // array copies accurately to allow value numbering of field loads around
5108 // arraycopy. Such field loads, both before and after, are common in Java
5109 // collections and similar classes involving header/array data structures.
5110 //
5111 // But with low number of register or when some registers are used or killed
5112 // by arraycopy calls it causes registers spilling on stack. See 6544710.
5113 // The next memory barrier is added to avoid it. If the arraycopy can be
5114 // optimized away (which it can, sometimes) then we can manually remove
5115 // the membar also.
5116 //
5117 // Do not let reads from the cloned object float above the arraycopy.
5118 if (alloc != NULL) {
5119 // Do not let stores that initialize this object be reordered with
5120 // a subsequent store that would make this object accessible by
5121 // other threads.
5122 // Record what AllocateNode this StoreStore protects so that
5123 // escape analysis can go from the MemBarStoreStoreNode to the
5124 // AllocateNode and eliminate the MemBarStoreStoreNode if possible
5125 // based on the escape status of the AllocateNode.
5126 insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out(AllocateNode::RawAddress));
5127 } else if (InsertMemBarAfterArraycopy)
5128 insert_mem_bar(Op_MemBarCPUOrder);
5129 }
5130
5131
5132 // Helper function which determines if an arraycopy immediately follows
5133 // an allocation, with no intervening tests or other escapes for the object.
5134 AllocateArrayNode*
5135 LibraryCallKit::tightly_coupled_allocation(Node* ptr,
5136 RegionNode* slow_region) {
5137 if (stopped()) return NULL; // no fast path
5138 if (C->AliasLevel() == 0) return NULL; // no MergeMems around
5139
5140 AllocateArrayNode* alloc = AllocateArrayNode::Ideal_array_allocation(ptr, &_gvn);
5141 if (alloc == NULL) return NULL;
5142
5143 Node* rawmem = memory(Compile::AliasIdxRaw);
5144 // Is the allocation's memory state untouched?
5145 if (!(rawmem->is_Proj() && rawmem->in(0)->is_Initialize())) {
5146 // Bail out if there have been raw-memory effects since the allocation.
5147 // (Example: There might have been a call or safepoint.)
5148 return NULL;
5186 Node* obs = not_ctl->fast_out(j);
5187 if (obs->in(0) == not_ctl && obs->is_Call() &&
5188 (obs->as_Call()->entry_point() == SharedRuntime::uncommon_trap_blob()->entry_point())) {
5189 found_trap = true; break;
5190 }
5191 }
5192 if (found_trap) {
5193 ctl = iff->in(0); // This test feeds a harmless uncommon trap.
5194 continue;
5195 }
5196 }
5197 return NULL;
5198 }
5199
5200 // If we get this far, we have an allocation which immediately
5201 // precedes the arraycopy, and we can take over zeroing the new object.
5202 // The arraycopy will finish the initialization, and provide
5203 // a new control state to which we will anchor the destination pointer.
5204
5205 return alloc;
5206 }
5207
5208 // Helper for initialization of arrays, creating a ClearArray.
5209 // It writes zero bits in [start..end), within the body of an array object.
5210 // The memory effects are all chained onto the 'adr_type' alias category.
5211 //
5212 // Since the object is otherwise uninitialized, we are free
5213 // to put a little "slop" around the edges of the cleared area,
5214 // as long as it does not go back into the array's header,
5215 // or beyond the array end within the heap.
5216 //
5217 // The lower edge can be rounded down to the nearest jint and the
5218 // upper edge can be rounded up to the nearest MinObjAlignmentInBytes.
5219 //
5220 // Arguments:
5221 // adr_type memory slice where writes are generated
5222 // dest oop of the destination array
5223 // basic_elem_type element type of the destination
5224 // slice_idx array index of first element to store
5225 // slice_len number of elements to store (or NULL)
5226 // dest_size total size in bytes of the array object
5227 //
5228 // Exactly one of slice_len or dest_size must be non-NULL.
5229 // If dest_size is non-NULL, zeroing extends to the end of the object.
5230 // If slice_len is non-NULL, the slice_idx value must be a constant.
5231 void
5232 LibraryCallKit::generate_clear_array(const TypePtr* adr_type,
5233 Node* dest,
5234 BasicType basic_elem_type,
5235 Node* slice_idx,
5236 Node* slice_len,
5237 Node* dest_size) {
5238 // one or the other but not both of slice_len and dest_size:
5239 assert((slice_len != NULL? 1: 0) + (dest_size != NULL? 1: 0) == 1, "");
5240 if (slice_len == NULL) slice_len = top();
5241 if (dest_size == NULL) dest_size = top();
5242
5243 // operate on this memory slice:
5244 Node* mem = memory(adr_type); // memory slice to operate on
5245
5246 // scaling and rounding of indexes:
5247 int scale = exact_log2(type2aelembytes(basic_elem_type));
5248 int abase = arrayOopDesc::base_offset_in_bytes(basic_elem_type);
5249 int clear_low = (-1 << scale) & (BytesPerInt - 1);
5250 int bump_bit = (-1 << scale) & BytesPerInt;
5251
5252 // determine constant starts and ends
5253 const intptr_t BIG_NEG = -128;
5254 assert(BIG_NEG + 2*abase < 0, "neg enough");
5255 intptr_t slice_idx_con = (intptr_t) find_int_con(slice_idx, BIG_NEG);
5256 intptr_t slice_len_con = (intptr_t) find_int_con(slice_len, BIG_NEG);
5257 if (slice_len_con == 0) {
5258 return; // nothing to do here
5259 }
5260 intptr_t start_con = (abase + (slice_idx_con << scale)) & ~clear_low;
5261 intptr_t end_con = find_intptr_t_con(dest_size, -1);
5262 if (slice_idx_con >= 0 && slice_len_con >= 0) {
5263 assert(end_con < 0, "not two cons");
5264 end_con = round_to(abase + ((slice_idx_con + slice_len_con) << scale),
5265 BytesPerLong);
5266 }
5267
5268 if (start_con >= 0 && end_con >= 0) {
5269 // Constant start and end. Simple.
5270 mem = ClearArrayNode::clear_memory(control(), mem, dest,
5271 start_con, end_con, &_gvn);
5272 } else if (start_con >= 0 && dest_size != top()) {
5273 // Constant start, pre-rounded end after the tail of the array.
5274 Node* end = dest_size;
5275 mem = ClearArrayNode::clear_memory(control(), mem, dest,
5276 start_con, end, &_gvn);
5277 } else if (start_con >= 0 && slice_len != top()) {
5278 // Constant start, non-constant end. End needs rounding up.
5279 // End offset = round_up(abase + ((slice_idx_con + slice_len) << scale), 8)
5280 intptr_t end_base = abase + (slice_idx_con << scale);
5281 int end_round = (-1 << scale) & (BytesPerLong - 1);
5282 Node* end = ConvI2X(slice_len);
5283 if (scale != 0)
5284 end = _gvn.transform( new(C,3) LShiftXNode(end, intcon(scale) ));
5285 end_base += end_round;
5286 end = _gvn.transform( new(C,3) AddXNode(end, MakeConX(end_base)) );
5287 end = _gvn.transform( new(C,3) AndXNode(end, MakeConX(~end_round)) );
5288 mem = ClearArrayNode::clear_memory(control(), mem, dest,
5289 start_con, end, &_gvn);
5290 } else if (start_con < 0 && dest_size != top()) {
5291 // Non-constant start, pre-rounded end after the tail of the array.
5292 // This is almost certainly a "round-to-end" operation.
5293 Node* start = slice_idx;
5294 start = ConvI2X(start);
5295 if (scale != 0)
5296 start = _gvn.transform( new(C,3) LShiftXNode( start, intcon(scale) ));
5297 start = _gvn.transform( new(C,3) AddXNode(start, MakeConX(abase)) );
5298 if ((bump_bit | clear_low) != 0) {
5299 int to_clear = (bump_bit | clear_low);
5300 // Align up mod 8, then store a jint zero unconditionally
5301 // just before the mod-8 boundary.
5302 if (((abase + bump_bit) & ~to_clear) - bump_bit
5303 < arrayOopDesc::length_offset_in_bytes() + BytesPerInt) {
5304 bump_bit = 0;
5305 assert((abase & to_clear) == 0, "array base must be long-aligned");
5306 } else {
5307 // Bump 'start' up to (or past) the next jint boundary:
5308 start = _gvn.transform( new(C,3) AddXNode(start, MakeConX(bump_bit)) );
5309 assert((abase & clear_low) == 0, "array base must be int-aligned");
5310 }
5311 // Round bumped 'start' down to jlong boundary in body of array.
5312 start = _gvn.transform( new(C,3) AndXNode(start, MakeConX(~to_clear)) );
5313 if (bump_bit != 0) {
5314 // Store a zero to the immediately preceding jint:
5315 Node* x1 = _gvn.transform( new(C,3) AddXNode(start, MakeConX(-bump_bit)) );
5316 Node* p1 = basic_plus_adr(dest, x1);
5317 mem = StoreNode::make(_gvn, control(), mem, p1, adr_type, intcon(0), T_INT);
5318 mem = _gvn.transform(mem);
5319 }
5320 }
5321 Node* end = dest_size; // pre-rounded
5322 mem = ClearArrayNode::clear_memory(control(), mem, dest,
5323 start, end, &_gvn);
5324 } else {
5325 // Non-constant start, unrounded non-constant end.
5326 // (Nobody zeroes a random midsection of an array using this routine.)
5327 ShouldNotReachHere(); // fix caller
5328 }
5329
5330 // Done.
5331 set_memory(mem, adr_type);
5332 }
5333
5334
5335 bool
5336 LibraryCallKit::generate_block_arraycopy(const TypePtr* adr_type,
5337 BasicType basic_elem_type,
5338 AllocateNode* alloc,
5339 Node* src, Node* src_offset,
5340 Node* dest, Node* dest_offset,
5341 Node* dest_size, bool dest_uninitialized) {
5342 // See if there is an advantage from block transfer.
5343 int scale = exact_log2(type2aelembytes(basic_elem_type));
5344 if (scale >= LogBytesPerLong)
5345 return false; // it is already a block transfer
5346
5347 // Look at the alignment of the starting offsets.
5348 int abase = arrayOopDesc::base_offset_in_bytes(basic_elem_type);
5349
5350 intptr_t src_off_con = (intptr_t) find_int_con(src_offset, -1);
5351 intptr_t dest_off_con = (intptr_t) find_int_con(dest_offset, -1);
5352 if (src_off_con < 0 || dest_off_con < 0)
5353 // At present, we can only understand constants.
5354 return false;
5355
5356 intptr_t src_off = abase + (src_off_con << scale);
5357 intptr_t dest_off = abase + (dest_off_con << scale);
5358
5359 if (((src_off | dest_off) & (BytesPerLong-1)) != 0) {
5360 // Non-aligned; too bad.
5361 // One more chance: Pick off an initial 32-bit word.
5362 // This is a common case, since abase can be odd mod 8.
5363 if (((src_off | dest_off) & (BytesPerLong-1)) == BytesPerInt &&
5364 ((src_off ^ dest_off) & (BytesPerLong-1)) == 0) {
5365 Node* sptr = basic_plus_adr(src, src_off);
5366 Node* dptr = basic_plus_adr(dest, dest_off);
5367 Node* sval = make_load(control(), sptr, TypeInt::INT, T_INT, adr_type);
5368 store_to_memory(control(), dptr, sval, T_INT, adr_type);
5369 src_off += BytesPerInt;
5370 dest_off += BytesPerInt;
5371 } else {
5372 return false;
5373 }
5374 }
5375 assert(src_off % BytesPerLong == 0, "");
5376 assert(dest_off % BytesPerLong == 0, "");
5377
5378 // Do this copy by giant steps.
5379 Node* sptr = basic_plus_adr(src, src_off);
5380 Node* dptr = basic_plus_adr(dest, dest_off);
5381 Node* countx = dest_size;
5382 countx = _gvn.transform( new (C, 3) SubXNode(countx, MakeConX(dest_off)) );
5383 countx = _gvn.transform( new (C, 3) URShiftXNode(countx, intcon(LogBytesPerLong)) );
5384
5385 bool disjoint_bases = true; // since alloc != NULL
5386 generate_unchecked_arraycopy(adr_type, T_LONG, disjoint_bases,
5387 sptr, NULL, dptr, NULL, countx, dest_uninitialized);
5388
5389 return true;
5390 }
5391
5392
5393 // Helper function; generates code for the slow case.
5394 // We make a call to a runtime method which emulates the native method,
5395 // but without the native wrapper overhead.
5396 void
5397 LibraryCallKit::generate_slow_arraycopy(const TypePtr* adr_type,
5398 Node* src, Node* src_offset,
5399 Node* dest, Node* dest_offset,
5400 Node* copy_length, bool dest_uninitialized) {
5401 assert(!dest_uninitialized, "Invariant");
5402 Node* call = make_runtime_call(RC_NO_LEAF | RC_UNCOMMON,
5403 OptoRuntime::slow_arraycopy_Type(),
5404 OptoRuntime::slow_arraycopy_Java(),
5405 "slow_arraycopy", adr_type,
5406 src, src_offset, dest, dest_offset,
5407 copy_length);
5408
5409 // Handle exceptions thrown by this fellow:
5410 make_slow_call_ex(call, env()->Throwable_klass(), false);
5411 }
5412
5413 // Helper function; generates code for cases requiring runtime checks.
5414 Node*
5415 LibraryCallKit::generate_checkcast_arraycopy(const TypePtr* adr_type,
5416 Node* dest_elem_klass,
5417 Node* src, Node* src_offset,
5418 Node* dest, Node* dest_offset,
5419 Node* copy_length, bool dest_uninitialized) {
5420 if (stopped()) return NULL;
5421
5422 address copyfunc_addr = StubRoutines::checkcast_arraycopy(dest_uninitialized);
5423 if (copyfunc_addr == NULL) { // Stub was not generated, go slow path.
5424 return NULL;
5425 }
5426
5427 // Pick out the parameters required to perform a store-check
5428 // for the target array. This is an optimistic check. It will
5429 // look in each non-null element's class, at the desired klass's
5430 // super_check_offset, for the desired klass.
5431 int sco_offset = in_bytes(Klass::super_check_offset_offset());
5432 Node* p3 = basic_plus_adr(dest_elem_klass, sco_offset);
5433 Node* n3 = new(C, 3) LoadINode(NULL, memory(p3), p3, _gvn.type(p3)->is_ptr());
5434 Node* check_offset = ConvI2X(_gvn.transform(n3));
5435 Node* check_value = dest_elem_klass;
5436
5437 Node* src_start = array_element_address(src, src_offset, T_OBJECT);
5438 Node* dest_start = array_element_address(dest, dest_offset, T_OBJECT);
5439
5440 // (We know the arrays are never conjoint, because their types differ.)
5441 Node* call = make_runtime_call(RC_LEAF|RC_NO_FP,
5442 OptoRuntime::checkcast_arraycopy_Type(),
5443 copyfunc_addr, "checkcast_arraycopy", adr_type,
5444 // five arguments, of which two are
5445 // intptr_t (jlong in LP64)
5446 src_start, dest_start,
5447 copy_length XTOP,
5448 check_offset XTOP,
5449 check_value);
5450
5451 return _gvn.transform(new (C, 1) ProjNode(call, TypeFunc::Parms));
5452 }
5453
5454
5455 // Helper function; generates code for cases requiring runtime checks.
5456 Node*
5457 LibraryCallKit::generate_generic_arraycopy(const TypePtr* adr_type,
5458 Node* src, Node* src_offset,
5459 Node* dest, Node* dest_offset,
5460 Node* copy_length, bool dest_uninitialized) {
5461 assert(!dest_uninitialized, "Invariant");
5462 if (stopped()) return NULL;
5463 address copyfunc_addr = StubRoutines::generic_arraycopy();
5464 if (copyfunc_addr == NULL) { // Stub was not generated, go slow path.
5465 return NULL;
5466 }
5467
5468 Node* call = make_runtime_call(RC_LEAF|RC_NO_FP,
5469 OptoRuntime::generic_arraycopy_Type(),
5470 copyfunc_addr, "generic_arraycopy", adr_type,
5471 src, src_offset, dest, dest_offset, copy_length);
5472
5473 return _gvn.transform(new (C, 1) ProjNode(call, TypeFunc::Parms));
5474 }
5475
5476 // Helper function; generates the fast out-of-line call to an arraycopy stub.
5477 void
5478 LibraryCallKit::generate_unchecked_arraycopy(const TypePtr* adr_type,
5479 BasicType basic_elem_type,
5480 bool disjoint_bases,
5481 Node* src, Node* src_offset,
5482 Node* dest, Node* dest_offset,
5483 Node* copy_length, bool dest_uninitialized) {
5484 if (stopped()) return; // nothing to do
5485
5486 Node* src_start = src;
5487 Node* dest_start = dest;
5488 if (src_offset != NULL || dest_offset != NULL) {
5489 assert(src_offset != NULL && dest_offset != NULL, "");
5490 src_start = array_element_address(src, src_offset, basic_elem_type);
5491 dest_start = array_element_address(dest, dest_offset, basic_elem_type);
5492 }
5493
5494 // Figure out which arraycopy runtime method to call.
5495 const char* copyfunc_name = "arraycopy";
5496 address copyfunc_addr =
5497 basictype2arraycopy(basic_elem_type, src_offset, dest_offset,
5498 disjoint_bases, copyfunc_name, dest_uninitialized);
5499
5500 // Call it. Note that the count_ix value is not scaled to a byte-size.
5501 make_runtime_call(RC_LEAF|RC_NO_FP,
5502 OptoRuntime::fast_arraycopy_Type(),
5503 copyfunc_addr, copyfunc_name, adr_type,
5504 src_start, dest_start, copy_length XTOP);
5505 }
5506
5507 //----------------------------inline_reference_get----------------------------
5508
5509 bool LibraryCallKit::inline_reference_get() {
5510 const int nargs = 1; // self
5511
5512 guarantee(java_lang_ref_Reference::referent_offset > 0,
5513 "should have already been set");
5514
5515 int referent_offset = java_lang_ref_Reference::referent_offset;
5516
5517 // Restore the stack and pop off the argument
5518 _sp += nargs;
5519 Node *reference_obj = pop();
5520
5521 // Null check on self without removing any arguments.
5522 _sp += nargs;
5523 reference_obj = do_null_check(reference_obj, T_OBJECT);
5524 _sp -= nargs;;
|
73 }
74
75 ciMethod* caller() const { return jvms()->method(); }
76 int bci() const { return jvms()->bci(); }
77 LibraryIntrinsic* intrinsic() const { return _intrinsic; }
78 vmIntrinsics::ID intrinsic_id() const { return _intrinsic->intrinsic_id(); }
79 ciMethod* callee() const { return _intrinsic->method(); }
80 ciSignature* signature() const { return callee()->signature(); }
81 int arg_size() const { return callee()->arg_size(); }
82
83 bool try_to_inline();
84
85 // Helper functions to inline natives
86 void push_result(RegionNode* region, PhiNode* value);
87 Node* generate_guard(Node* test, RegionNode* region, float true_prob);
88 Node* generate_slow_guard(Node* test, RegionNode* region);
89 Node* generate_fair_guard(Node* test, RegionNode* region);
90 Node* generate_negative_guard(Node* index, RegionNode* region,
91 // resulting CastII of index:
92 Node* *pos_index = NULL);
93 Node* generate_limit_guard(Node* offset, Node* subseq_length,
94 Node* array_length,
95 RegionNode* region);
96 Node* generate_current_thread(Node* &tls_output);
97 address basictype2arraycopy(BasicType t, Node *src_offset, Node *dest_offset,
98 bool disjoint_bases, const char* &name, bool dest_uninitialized);
99 Node* load_mirror_from_klass(Node* klass);
100 Node* load_klass_from_mirror_common(Node* mirror, bool never_see_null,
101 int nargs,
102 RegionNode* region, int null_path,
103 int offset);
104 Node* load_klass_from_mirror(Node* mirror, bool never_see_null, int nargs,
105 RegionNode* region, int null_path) {
106 int offset = java_lang_Class::klass_offset_in_bytes();
107 return load_klass_from_mirror_common(mirror, never_see_null, nargs,
108 region, null_path,
109 offset);
110 }
111 Node* load_array_klass_from_mirror(Node* mirror, bool never_see_null,
112 int nargs,
181 bool inline_native_isInterrupted();
182 bool inline_native_Class_query(vmIntrinsics::ID id);
183 bool inline_native_subtype_check();
184
185 bool inline_native_newArray();
186 bool inline_native_getLength();
187 bool inline_array_copyOf(bool is_copyOfRange);
188 bool inline_array_equals();
189 void copy_to_clone(Node* obj, Node* alloc_obj, Node* obj_size, bool is_array, bool card_mark);
190 bool inline_native_clone(bool is_virtual);
191 bool inline_native_Reflection_getCallerClass();
192 bool inline_native_AtomicLong_get();
193 bool inline_native_AtomicLong_attemptUpdate();
194 bool is_method_invoke_or_aux_frame(JVMState* jvms);
195 // Helper function for inlining native object hash method
196 bool inline_native_hashcode(bool is_virtual, bool is_static);
197 bool inline_native_getClass();
198
199 // Helper functions for inlining arraycopy
200 bool inline_arraycopy();
201 AllocateArrayNode* tightly_coupled_allocation(Node* ptr,
202 RegionNode* slow_region);
203 bool inline_unsafe_CAS(BasicType type);
204 bool inline_unsafe_ordered_store(BasicType type);
205 bool inline_fp_conversions(vmIntrinsics::ID id);
206 bool inline_numberOfLeadingZeros(vmIntrinsics::ID id);
207 bool inline_numberOfTrailingZeros(vmIntrinsics::ID id);
208 bool inline_bitCount(vmIntrinsics::ID id);
209 bool inline_reverseBytes(vmIntrinsics::ID id);
210
211 bool inline_reference_get();
212 };
213
214
215 //---------------------------make_vm_intrinsic----------------------------
216 CallGenerator* Compile::make_vm_intrinsic(ciMethod* m, bool is_virtual) {
217 vmIntrinsics::ID id = m->intrinsic_id();
218 assert(id != vmIntrinsics::_none, "must be a VM intrinsic");
219
220 if (DisableIntrinsic[0] != '\0'
221 && strstr(DisableIntrinsic, vmIntrinsics::name_at(id)) != NULL) {
222 // disabled by a user request on the command line:
750 }
751
752 inline Node* LibraryCallKit::generate_negative_guard(Node* index, RegionNode* region,
753 Node* *pos_index) {
754 if (stopped())
755 return NULL; // already stopped
756 if (_gvn.type(index)->higher_equal(TypeInt::POS)) // [0,maxint]
757 return NULL; // index is already adequately typed
758 Node* cmp_lt = _gvn.transform( new (C, 3) CmpINode(index, intcon(0)) );
759 Node* bol_lt = _gvn.transform( new (C, 2) BoolNode(cmp_lt, BoolTest::lt) );
760 Node* is_neg = generate_guard(bol_lt, region, PROB_MIN);
761 if (is_neg != NULL && pos_index != NULL) {
762 // Emulate effect of Parse::adjust_map_after_if.
763 Node* ccast = new (C, 2) CastIINode(index, TypeInt::POS);
764 ccast->set_req(0, control());
765 (*pos_index) = _gvn.transform(ccast);
766 }
767 return is_neg;
768 }
769
770 // Make sure that 'position' is a valid limit index, in [0..length].
771 // There are two equivalent plans for checking this:
772 // A. (offset + copyLength) unsigned<= arrayLength
773 // B. offset <= (arrayLength - copyLength)
774 // We require that all of the values above, except for the sum and
775 // difference, are already known to be non-negative.
776 // Plan A is robust in the face of overflow, if offset and copyLength
777 // are both hugely positive.
778 //
779 // Plan B is less direct and intuitive, but it does not overflow at
780 // all, since the difference of two non-negatives is always
781 // representable. Whenever Java methods must perform the equivalent
782 // check they generally use Plan B instead of Plan A.
783 // For the moment we use Plan A.
784 inline Node* LibraryCallKit::generate_limit_guard(Node* offset,
785 Node* subseq_length,
786 Node* array_length,
787 RegionNode* region) {
788 if (stopped())
789 return NULL; // already stopped
3550 PreserveJVMState pjvms(this);
3551 set_control( _gvn.transform(bailout) );
3552 uncommon_trap(Deoptimization::Reason_intrinsic,
3553 Deoptimization::Action_maybe_recompile);
3554 }
3555
3556 if (!stopped()) {
3557
3558 // How many elements will we copy from the original?
3559 // The answer is MinI(orig_length - start, length).
3560 Node* orig_tail = _gvn.transform( new(C, 3) SubINode(orig_length, start) );
3561 Node* moved = generate_min_max(vmIntrinsics::_min, orig_tail, length);
3562
3563 newcopy = new_array(klass_node, length, 0);
3564
3565 // Generate a direct call to the right arraycopy function(s).
3566 // We know the copy is disjoint but we might not know if the
3567 // oop stores need checking.
3568 // Extreme case: Arrays.copyOf((Integer[])x, 10, String[].class).
3569 // This will fail a store-check if x contains any non-nulls.
3570
3571 Node* alloc = tightly_coupled_allocation(newcopy, NULL);
3572
3573 ArrayCopyNode* ac = ArrayCopyNode::make(this, true, original, start, newcopy, intcon(0), moved, alloc != NULL);
3574 ac->set_copyof();
3575 Node* n = _gvn.transform(ac);
3576 assert(n == ac, "cannot disappear");
3577 ac->connect_outputs(this);
3578 }
3579 } //original reexecute and sp are set back here
3580
3581 if(!stopped()) {
3582 push(newcopy);
3583 }
3584
3585 C->set_has_split_ifs(true); // Has chance for split-if optimization
3586
3587 return true;
3588 }
3589
3590
3591 //----------------------generate_virtual_guard---------------------------
3592 // Helper for hashCode and clone. Peeks inside the vtable to avoid a call.
3593 Node* LibraryCallKit::generate_virtual_guard(Node* obj_klass,
3594 RegionNode* slow_region) {
3595 ciMethod* method = callee();
3596 int vtable_index = method->vtable_index();
3597 // Get the methodOop out of the appropriate vtable entry.
4261 if (base_off % BytesPerLong != 0) {
4262 assert(UseCompressedOops, "");
4263 if (is_array) {
4264 // Exclude length to copy by 8 bytes words.
4265 base_off += sizeof(int);
4266 } else {
4267 // Include klass to copy by 8 bytes words.
4268 base_off = instanceOopDesc::klass_offset_in_bytes();
4269 }
4270 assert(base_off % BytesPerLong == 0, "expect 8 bytes alignment");
4271 }
4272 src = basic_plus_adr(src, base_off);
4273 dest = basic_plus_adr(dest, base_off);
4274
4275 // Compute the length also, if needed:
4276 Node* countx = size;
4277 countx = _gvn.transform( new (C, 3) SubXNode(countx, MakeConX(base_off)) );
4278 countx = _gvn.transform( new (C, 3) URShiftXNode(countx, intcon(LogBytesPerLong) ));
4279
4280 const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
4281
4282 ArrayCopyNode* ac = ArrayCopyNode::make(this, false, src, NULL, dest, NULL, countx, false);
4283 ac->set_clonebasic();
4284 Node* n = _gvn.transform(ac);
4285 assert(n == ac, "cannot disappear");
4286 set_predefined_output_for_runtime_call(ac, ac->in(TypeFunc::Memory), raw_adr_type);
4287
4288 // If necessary, emit some card marks afterwards. (Non-arrays only.)
4289 if (card_mark) {
4290 assert(!is_array, "");
4291 // Put in store barrier for any and all oops we are sticking
4292 // into this object. (We could avoid this if we could prove
4293 // that the object type contains no oop fields at all.)
4294 Node* no_particular_value = NULL;
4295 Node* no_particular_field = NULL;
4296 int raw_adr_idx = Compile::AliasIdxRaw;
4297 post_barrier(control(),
4298 memory(raw_adr_type),
4299 alloc_obj,
4300 no_particular_field,
4301 raw_adr_idx,
4302 no_particular_value,
4303 T_OBJECT,
4304 false);
4305 }
4306
4379 const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
4380 int raw_adr_idx = Compile::AliasIdxRaw;
4381
4382 Node* array_ctl = generate_array_guard(obj_klass, (RegionNode*)NULL);
4383 if (array_ctl != NULL) {
4384 // It's an array.
4385 PreserveJVMState pjvms(this);
4386 set_control(array_ctl);
4387 Node* obj_length = load_array_length(obj);
4388 Node* obj_size = NULL;
4389 Node* alloc_obj = new_array(obj_klass, obj_length, 0, &obj_size);
4390
4391 if (!use_ReduceInitialCardMarks()) {
4392 // If it is an oop array, it requires very special treatment,
4393 // because card marking is required on each card of the array.
4394 Node* is_obja = generate_objArray_guard(obj_klass, (RegionNode*)NULL);
4395 if (is_obja != NULL) {
4396 PreserveJVMState pjvms2(this);
4397 set_control(is_obja);
4398 // Generate a direct call to the right arraycopy function(s).
4399 Node* alloc = tightly_coupled_allocation(alloc_obj, NULL);
4400 ArrayCopyNode* ac = ArrayCopyNode::make(this, true, obj, intcon(0), alloc_obj, intcon(0), obj_length, alloc != NULL);
4401 ac->set_cloneoop();
4402 Node* n = _gvn.transform(ac);
4403 assert(n == ac, "cannot disappear");
4404 ac->connect_outputs(this);
4405
4406 result_reg->init_req(_objArray_path, control());
4407 result_val->init_req(_objArray_path, alloc_obj);
4408 result_i_o ->set_req(_objArray_path, i_o());
4409 result_mem ->set_req(_objArray_path, reset_memory());
4410 }
4411 }
4412 // Otherwise, there are no card marks to worry about.
4413 // (We can dispense with card marks if we know the allocation
4414 // comes out of eden (TLAB)... In fact, ReduceInitialCardMarks
4415 // causes the non-eden paths to take compensating steps to
4416 // simulate a fresh allocation, so that no further
4417 // card marks are required in compiled code to initialize
4418 // the object.)
4419
4420 if (!stopped()) {
4421 copy_to_clone(obj, alloc_obj, obj_size, true, false);
4422
4423 // Present the results of the copy.
4424 result_reg->init_req(_array_path, control());
4425 result_val->init_req(_array_path, alloc_obj);
4525 // This can occur if the offsets are identical non-constants.
4526 disjoint = true;
4527 }
4528
4529 return StubRoutines::select_arraycopy_function(t, aligned, disjoint, name, dest_uninitialized);
4530 }
4531
4532
4533 //------------------------------inline_arraycopy-----------------------
4534 bool LibraryCallKit::inline_arraycopy() {
4535 // Restore the stack and pop off the arguments.
4536 int nargs = 5; // 2 oops, 3 ints, no size_t or long
4537 assert(callee()->signature()->size() == nargs, "copy has 5 arguments");
4538
4539 Node *src = argument(0);
4540 Node *src_offset = argument(1);
4541 Node *dest = argument(2);
4542 Node *dest_offset = argument(3);
4543 Node *length = argument(4);
4544
4545 // The following tests must be performed
4546 // (1) src and dest are arrays.
4547 // (2) src and dest arrays must have elements of the same BasicType
4548 // (3) src and dest must not be null.
4549 // (4) src_offset must not be negative.
4550 // (5) dest_offset must not be negative.
4551 // (6) length must not be negative.
4552 // (7) src_offset + length must not exceed length of src.
4553 // (8) dest_offset + length must not exceed length of dest.
4554 // (9) each element of an oop array must be assignable
4555
4556 // (3) src and dest must not be null.
4557 // always do this here because we need the JVM state for uncommon traps
4558 _sp += nargs;
4559 src = do_null_check(src, T_ARRAY);
4560 dest = do_null_check(dest, T_ARRAY);
4561 _sp -= nargs;
4562
4563 bool notest = false;
4564
4565 if (!too_many_traps(Deoptimization::Reason_intrinsic) && !src->is_top() && !dest->is_top()) {
4566 // validate arguments: enables transformation the ArrayCopyNode
4567 notest = true;
4568
4569 RegionNode* slow_region = new (C, 1) RegionNode(1);
4570 record_for_igvn(slow_region);
4571
4572 // (1) src and dest are arrays.
4573 generate_non_array_guard(load_object_klass(src), slow_region);
4574 generate_non_array_guard(load_object_klass(dest), slow_region);
4575
4576 // (2) src and dest arrays must have elements of the same BasicType
4577 // done at macro expansion or at Ideal transformation time
4578
4579 // (4) src_offset must not be negative.
4580 generate_negative_guard(src_offset, slow_region);
4581
4582 // (5) dest_offset must not be negative.
4583 generate_negative_guard(dest_offset, slow_region);
4584
4585 // (7) src_offset + length must not exceed length of src.
4586 generate_limit_guard(src_offset, length,
4587 load_array_length(src),
4588 slow_region);
4589
4590 // (8) dest_offset + length must not exceed length of dest.
4591 generate_limit_guard(dest_offset, length,
4592 load_array_length(dest),
4593 slow_region);
4594
4595 // (9) each element of an oop array must be assignable
4596 Node* src_klass = load_object_klass(src);
4597 Node* dest_klass = load_object_klass(dest);
4598 Node* not_subtype_ctrl = gen_subtype_check(src_klass, dest_klass);
4599
4600 if (not_subtype_ctrl != top()) {
4601 PreserveJVMState pjvms(this);
4602 set_control(not_subtype_ctrl);
4603 _sp += nargs;
4604 uncommon_trap(Deoptimization::Reason_intrinsic,
4605 Deoptimization::Action_make_not_entrant);
4606 _sp -= nargs;
4607 assert(stopped(), "Should be stopped");
4608 }
4609 {
4610 PreserveJVMState pjvms(this);
4611 set_control(_gvn.transform(slow_region));
4612 _sp += nargs;
4613 uncommon_trap(Deoptimization::Reason_intrinsic,
4614 Deoptimization::Action_make_not_entrant);
4615 _sp -= nargs;
4616 assert(stopped(), "Should be stopped");
4617 }
4618 }
4619
4620 if (stopped()) {
4621 return true;
4622 }
4623
4624 AllocateArrayNode* alloc = tightly_coupled_allocation(dest, NULL);
4625 ArrayCopyNode* ac = ArrayCopyNode::make(this, true, src, src_offset, dest, dest_offset, length, alloc != NULL);
4626
4627 if (notest) {
4628 ac->set_arraycopy_notest();
4629 }
4630
4631 Node* n = _gvn.transform(ac);
4632 assert(n == ac, "cannot disappear");
4633 ac->connect_outputs(this);
4634
4635 return true;
4636 }
4637
4638
4639 // Helper function which determines if an arraycopy immediately follows
4640 // an allocation, with no intervening tests or other escapes for the object.
4641 AllocateArrayNode*
4642 LibraryCallKit::tightly_coupled_allocation(Node* ptr,
4643 RegionNode* slow_region) {
4644 if (stopped()) return NULL; // no fast path
4645 if (C->AliasLevel() == 0) return NULL; // no MergeMems around
4646
4647 AllocateArrayNode* alloc = AllocateArrayNode::Ideal_array_allocation(ptr, &_gvn);
4648 if (alloc == NULL) return NULL;
4649
4650 Node* rawmem = memory(Compile::AliasIdxRaw);
4651 // Is the allocation's memory state untouched?
4652 if (!(rawmem->is_Proj() && rawmem->in(0)->is_Initialize())) {
4653 // Bail out if there have been raw-memory effects since the allocation.
4654 // (Example: There might have been a call or safepoint.)
4655 return NULL;
4693 Node* obs = not_ctl->fast_out(j);
4694 if (obs->in(0) == not_ctl && obs->is_Call() &&
4695 (obs->as_Call()->entry_point() == SharedRuntime::uncommon_trap_blob()->entry_point())) {
4696 found_trap = true; break;
4697 }
4698 }
4699 if (found_trap) {
4700 ctl = iff->in(0); // This test feeds a harmless uncommon trap.
4701 continue;
4702 }
4703 }
4704 return NULL;
4705 }
4706
4707 // If we get this far, we have an allocation which immediately
4708 // precedes the arraycopy, and we can take over zeroing the new object.
4709 // The arraycopy will finish the initialization, and provide
4710 // a new control state to which we will anchor the destination pointer.
4711
4712 return alloc;
4713 }
4714
4715 //----------------------------inline_reference_get----------------------------
4716
4717 bool LibraryCallKit::inline_reference_get() {
4718 const int nargs = 1; // self
4719
4720 guarantee(java_lang_ref_Reference::referent_offset > 0,
4721 "should have already been set");
4722
4723 int referent_offset = java_lang_ref_Reference::referent_offset;
4724
4725 // Restore the stack and pop off the argument
4726 _sp += nargs;
4727 Node *reference_obj = pop();
4728
4729 // Null check on self without removing any arguments.
4730 _sp += nargs;
4731 reference_obj = do_null_check(reference_obj, T_OBJECT);
4732 _sp -= nargs;;
|