14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "opto/arraycopynode.hpp"
27 #include "oops/objArrayKlass.hpp"
28 #include "opto/convertnode.hpp"
29 #include "opto/graphKit.hpp"
30 #include "opto/macro.hpp"
31 #include "opto/runtime.hpp"
32
33
34 void PhaseMacroExpand::insert_mem_bar(Node** ctrl, Node** mem, int opcode, Node* precedent) {
35 MemBarNode* mb = MemBarNode::make(C, opcode, Compile::AliasIdxBot, precedent);
36 mb->init_req(TypeFunc::Control, *ctrl);
37 mb->init_req(TypeFunc::Memory, *mem);
38 transform_later(mb);
39 *ctrl = new ProjNode(mb,TypeFunc::Control);
40 transform_later(*ctrl);
41 Node* mem_proj = new ProjNode(mb,TypeFunc::Memory);
42 transform_later(mem_proj);
43 *mem = mem_proj;
44 }
45
46 Node* PhaseMacroExpand::array_element_address(Node* ary, Node* idx, BasicType elembt) {
47 uint shift = exact_log2(type2aelembytes(elembt));
48 uint header = arrayOopDesc::base_offset_in_bytes(elembt);
49 Node* base = basic_plus_adr(ary, header);
50 #ifdef _LP64
51 // see comment in GraphKit::array_element_address
52 int index_max = max_jint - 1; // array size is max_jint, index is one less
53 const TypeLong* lidxtype = TypeLong::make(CONST64(0), index_max, Type::WidenMax);
54 idx = transform_later( new ConvI2LNode(idx, lidxtype) );
373 if (!length_never_negative) {
374 generate_negative_guard(&local_ctrl, copy_length, slow_region);
375 }
376
377 // copy_length is 0.
378 if (dest_uninitialized) {
379 assert(!local_ctrl->is_top(), "no ctrl?");
380 Node* dest_length = alloc->in(AllocateNode::ALength);
381 if (copy_length->eqv_uncast(dest_length)
382 || _igvn.find_int_con(dest_length, 1) <= 0) {
383 // There is no zeroing to do. No need for a secondary raw memory barrier.
384 } else {
385 // Clear the whole thing since there are no source elements to copy.
386 generate_clear_array(local_ctrl, local_mem,
387 adr_type, dest, basic_elem_type,
388 intcon(0), NULL,
389 alloc->in(AllocateNode::AllocSize));
390 // Use a secondary InitializeNode as raw memory barrier.
391 // Currently it is needed only on this path since other
392 // paths have stub or runtime calls as raw memory barriers.
393 MemBarNode* mb = MemBarNode::make(C, Op_Initialize,
394 Compile::AliasIdxRaw,
395 top());
396 transform_later(mb);
397 mb->set_req(TypeFunc::Control,local_ctrl);
398 mb->set_req(TypeFunc::Memory, local_mem->memory_at(Compile::AliasIdxRaw));
399 local_ctrl = transform_later(new ProjNode(mb, TypeFunc::Control));
400 local_mem->set_memory_at(Compile::AliasIdxRaw, transform_later(new ProjNode(mb, TypeFunc::Memory)));
401
402 InitializeNode* init = mb->as_Initialize();
403 init->set_complete(&_igvn); // (there is no corresponding AllocateNode)
404 }
405 }
406
407 // Present the results of the fast call.
408 result_region->init_req(zero_path, local_ctrl);
409 result_i_o ->init_req(zero_path, local_io);
410 result_memory->init_req(zero_path, local_mem->memory_at(alias_idx));
411 }
412
413 if (!(*ctrl)->is_top() && dest_uninitialized) {
692 // mem no longer guaranteed to stay a MergeMemNode
693 Node* out_mem = mem;
694 DEBUG_ONLY(mem = NULL);
695
696 // The memory edges above are precise in order to model effects around
697 // array copies accurately to allow value numbering of field loads around
698 // arraycopy. Such field loads, both before and after, are common in Java
699 // collections and similar classes involving header/array data structures.
700 //
701 // But with low number of register or when some registers are used or killed
702 // by arraycopy calls it causes registers spilling on stack. See 6544710.
703 // The next memory barrier is added to avoid it. If the arraycopy can be
704 // optimized away (which it can, sometimes) then we can manually remove
705 // the membar also.
706 //
707 // Do not let reads from the cloned object float above the arraycopy.
708 if (alloc != NULL && !alloc->initialization()->does_not_escape()) {
709 // Do not let stores that initialize this object be reordered with
710 // a subsequent store that would make this object accessible by
711 // other threads.
712 insert_mem_bar(ctrl, &out_mem, Op_MemBarStoreStore);
713 } else if (InsertMemBarAfterArraycopy) {
714 insert_mem_bar(ctrl, &out_mem, Op_MemBarCPUOrder);
715 }
716
717 _igvn.replace_node(_memproj_fallthrough, out_mem);
718 _igvn.replace_node(_ioproj_fallthrough, *io);
719 _igvn.replace_node(_fallthroughcatchproj, *ctrl);
720
721 return out_mem;
722 }
723
724 // Helper for initialization of arrays, creating a ClearArray.
725 // It writes zero bits in [start..end), within the body of an array object.
726 // The memory effects are all chained onto the 'adr_type' alias category.
727 //
728 // Since the object is otherwise uninitialized, we are free
729 // to put a little "slop" around the edges of the cleared area,
730 // as long as it does not go back into the array's header,
731 // or beyond the array end within the heap.
732 //
733 // The lower edge can be rounded down to the nearest jint and the
734 // upper edge can be rounded up to the nearest MinObjAlignmentInBytes.
1128 }
1129
1130 assert(ac->is_arraycopy() || ac->is_arraycopy_validated(), "should be an arraycopy");
1131
1132 // Compile time checks. If any of these checks cannot be verified at compile time,
1133 // we do not make a fast path for this call. Instead, we let the call remain as it
1134 // is. The checks we choose to mandate at compile time are:
1135 //
1136 // (1) src and dest are arrays.
1137 const Type* src_type = src->Value(&_igvn);
1138 const Type* dest_type = dest->Value(&_igvn);
1139 const TypeAryPtr* top_src = src_type->isa_aryptr();
1140 const TypeAryPtr* top_dest = dest_type->isa_aryptr();
1141
1142 if (top_src == NULL || top_src->klass() == NULL ||
1143 top_dest == NULL || top_dest->klass() == NULL) {
1144 // Conservatively insert a memory barrier on all memory slices.
1145 // Do not let writes into the source float below the arraycopy.
1146 {
1147 Node* mem = ac->in(TypeFunc::Memory);
1148 insert_mem_bar(&ctrl, &mem, Op_MemBarCPUOrder);
1149
1150 merge_mem = MergeMemNode::make(mem);
1151 transform_later(merge_mem);
1152 }
1153
1154 // Call StubRoutines::generic_arraycopy stub.
1155 Node* mem = generate_arraycopy(ac, NULL, &ctrl, merge_mem, &io,
1156 TypeRawPtr::BOTTOM, T_CONFLICT,
1157 src, src_offset, dest, dest_offset, length);
1158
1159 // Do not let reads from the destination float above the arraycopy.
1160 // Since we cannot type the arrays, we don't know which slices
1161 // might be affected. We could restrict this barrier only to those
1162 // memory slices which pertain to array elements--but don't bother.
1163 if (!InsertMemBarAfterArraycopy) {
1164 // (If InsertMemBarAfterArraycopy, there is already one in place.)
1165 insert_mem_bar(&ctrl, &mem, Op_MemBarCPUOrder);
1166 }
1167 return;
1168 }
1169 // (2) src and dest arrays must have elements of the same BasicType
1170 // Figure out the size and type of the elements we will be copying.
1171 BasicType src_elem = top_src->klass()->as_array_klass()->element_type()->basic_type();
1172 BasicType dest_elem = top_dest->klass()->as_array_klass()->element_type()->basic_type();
1173 if (src_elem == T_ARRAY) src_elem = T_OBJECT;
1174 if (dest_elem == T_ARRAY) dest_elem = T_OBJECT;
1175
1176 if (src_elem != dest_elem || dest_elem == T_VOID) {
1177 // The component types are not the same or are not recognized. Punt.
1178 // (But, avoid the native method wrapper to JVM_ArrayCopy.)
1179 {
1180 Node* mem = ac->in(TypeFunc::Memory);
1181 merge_mem = generate_slow_arraycopy(ac, &ctrl, mem, &io, TypePtr::BOTTOM, src, src_offset, dest, dest_offset, length, false);
1182 }
1183
1184 _igvn.replace_node(_memproj_fallthrough, merge_mem);
1185 _igvn.replace_node(_ioproj_fallthrough, io);
|
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "opto/arraycopynode.hpp"
27 #include "oops/objArrayKlass.hpp"
28 #include "opto/convertnode.hpp"
29 #include "opto/graphKit.hpp"
30 #include "opto/macro.hpp"
31 #include "opto/runtime.hpp"
32
33
34 void PhaseMacroExpand::insert_mem_bar(Node** ctrl, Node** mem, Opcodes opcode, Node* precedent) {
35 MemBarNode* mb = MemBarNode::make(C, opcode, Compile::AliasIdxBot, precedent);
36 mb->init_req(TypeFunc::Control, *ctrl);
37 mb->init_req(TypeFunc::Memory, *mem);
38 transform_later(mb);
39 *ctrl = new ProjNode(mb,TypeFunc::Control);
40 transform_later(*ctrl);
41 Node* mem_proj = new ProjNode(mb,TypeFunc::Memory);
42 transform_later(mem_proj);
43 *mem = mem_proj;
44 }
45
46 Node* PhaseMacroExpand::array_element_address(Node* ary, Node* idx, BasicType elembt) {
47 uint shift = exact_log2(type2aelembytes(elembt));
48 uint header = arrayOopDesc::base_offset_in_bytes(elembt);
49 Node* base = basic_plus_adr(ary, header);
50 #ifdef _LP64
51 // see comment in GraphKit::array_element_address
52 int index_max = max_jint - 1; // array size is max_jint, index is one less
53 const TypeLong* lidxtype = TypeLong::make(CONST64(0), index_max, Type::WidenMax);
54 idx = transform_later( new ConvI2LNode(idx, lidxtype) );
373 if (!length_never_negative) {
374 generate_negative_guard(&local_ctrl, copy_length, slow_region);
375 }
376
377 // copy_length is 0.
378 if (dest_uninitialized) {
379 assert(!local_ctrl->is_top(), "no ctrl?");
380 Node* dest_length = alloc->in(AllocateNode::ALength);
381 if (copy_length->eqv_uncast(dest_length)
382 || _igvn.find_int_con(dest_length, 1) <= 0) {
383 // There is no zeroing to do. No need for a secondary raw memory barrier.
384 } else {
385 // Clear the whole thing since there are no source elements to copy.
386 generate_clear_array(local_ctrl, local_mem,
387 adr_type, dest, basic_elem_type,
388 intcon(0), NULL,
389 alloc->in(AllocateNode::AllocSize));
390 // Use a secondary InitializeNode as raw memory barrier.
391 // Currently it is needed only on this path since other
392 // paths have stub or runtime calls as raw memory barriers.
393 MemBarNode* mb = MemBarNode::make(C, Opcodes::Op_Initialize,
394 Compile::AliasIdxRaw,
395 top());
396 transform_later(mb);
397 mb->set_req(TypeFunc::Control,local_ctrl);
398 mb->set_req(TypeFunc::Memory, local_mem->memory_at(Compile::AliasIdxRaw));
399 local_ctrl = transform_later(new ProjNode(mb, TypeFunc::Control));
400 local_mem->set_memory_at(Compile::AliasIdxRaw, transform_later(new ProjNode(mb, TypeFunc::Memory)));
401
402 InitializeNode* init = mb->as_Initialize();
403 init->set_complete(&_igvn); // (there is no corresponding AllocateNode)
404 }
405 }
406
407 // Present the results of the fast call.
408 result_region->init_req(zero_path, local_ctrl);
409 result_i_o ->init_req(zero_path, local_io);
410 result_memory->init_req(zero_path, local_mem->memory_at(alias_idx));
411 }
412
413 if (!(*ctrl)->is_top() && dest_uninitialized) {
692 // mem no longer guaranteed to stay a MergeMemNode
693 Node* out_mem = mem;
694 DEBUG_ONLY(mem = NULL);
695
696 // The memory edges above are precise in order to model effects around
697 // array copies accurately to allow value numbering of field loads around
698 // arraycopy. Such field loads, both before and after, are common in Java
699 // collections and similar classes involving header/array data structures.
700 //
701 // But with low number of register or when some registers are used or killed
702 // by arraycopy calls it causes registers spilling on stack. See 6544710.
703 // The next memory barrier is added to avoid it. If the arraycopy can be
704 // optimized away (which it can, sometimes) then we can manually remove
705 // the membar also.
706 //
707 // Do not let reads from the cloned object float above the arraycopy.
708 if (alloc != NULL && !alloc->initialization()->does_not_escape()) {
709 // Do not let stores that initialize this object be reordered with
710 // a subsequent store that would make this object accessible by
711 // other threads.
712 insert_mem_bar(ctrl, &out_mem, Opcodes::Op_MemBarStoreStore);
713 } else if (InsertMemBarAfterArraycopy) {
714 insert_mem_bar(ctrl, &out_mem, Opcodes::Op_MemBarCPUOrder);
715 }
716
717 _igvn.replace_node(_memproj_fallthrough, out_mem);
718 _igvn.replace_node(_ioproj_fallthrough, *io);
719 _igvn.replace_node(_fallthroughcatchproj, *ctrl);
720
721 return out_mem;
722 }
723
724 // Helper for initialization of arrays, creating a ClearArray.
725 // It writes zero bits in [start..end), within the body of an array object.
726 // The memory effects are all chained onto the 'adr_type' alias category.
727 //
728 // Since the object is otherwise uninitialized, we are free
729 // to put a little "slop" around the edges of the cleared area,
730 // as long as it does not go back into the array's header,
731 // or beyond the array end within the heap.
732 //
733 // The lower edge can be rounded down to the nearest jint and the
734 // upper edge can be rounded up to the nearest MinObjAlignmentInBytes.
1128 }
1129
1130 assert(ac->is_arraycopy() || ac->is_arraycopy_validated(), "should be an arraycopy");
1131
1132 // Compile time checks. If any of these checks cannot be verified at compile time,
1133 // we do not make a fast path for this call. Instead, we let the call remain as it
1134 // is. The checks we choose to mandate at compile time are:
1135 //
1136 // (1) src and dest are arrays.
1137 const Type* src_type = src->Value(&_igvn);
1138 const Type* dest_type = dest->Value(&_igvn);
1139 const TypeAryPtr* top_src = src_type->isa_aryptr();
1140 const TypeAryPtr* top_dest = dest_type->isa_aryptr();
1141
1142 if (top_src == NULL || top_src->klass() == NULL ||
1143 top_dest == NULL || top_dest->klass() == NULL) {
1144 // Conservatively insert a memory barrier on all memory slices.
1145 // Do not let writes into the source float below the arraycopy.
1146 {
1147 Node* mem = ac->in(TypeFunc::Memory);
1148 insert_mem_bar(&ctrl, &mem, Opcodes::Op_MemBarCPUOrder);
1149
1150 merge_mem = MergeMemNode::make(mem);
1151 transform_later(merge_mem);
1152 }
1153
1154 // Call StubRoutines::generic_arraycopy stub.
1155 Node* mem = generate_arraycopy(ac, NULL, &ctrl, merge_mem, &io,
1156 TypeRawPtr::BOTTOM, T_CONFLICT,
1157 src, src_offset, dest, dest_offset, length);
1158
1159 // Do not let reads from the destination float above the arraycopy.
1160 // Since we cannot type the arrays, we don't know which slices
1161 // might be affected. We could restrict this barrier only to those
1162 // memory slices which pertain to array elements--but don't bother.
1163 if (!InsertMemBarAfterArraycopy) {
1164 // (If InsertMemBarAfterArraycopy, there is already one in place.)
1165 insert_mem_bar(&ctrl, &mem, Opcodes::Op_MemBarCPUOrder);
1166 }
1167 return;
1168 }
1169 // (2) src and dest arrays must have elements of the same BasicType
1170 // Figure out the size and type of the elements we will be copying.
1171 BasicType src_elem = top_src->klass()->as_array_klass()->element_type()->basic_type();
1172 BasicType dest_elem = top_dest->klass()->as_array_klass()->element_type()->basic_type();
1173 if (src_elem == T_ARRAY) src_elem = T_OBJECT;
1174 if (dest_elem == T_ARRAY) dest_elem = T_OBJECT;
1175
1176 if (src_elem != dest_elem || dest_elem == T_VOID) {
1177 // The component types are not the same or are not recognized. Punt.
1178 // (But, avoid the native method wrapper to JVM_ArrayCopy.)
1179 {
1180 Node* mem = ac->in(TypeFunc::Memory);
1181 merge_mem = generate_slow_arraycopy(ac, &ctrl, mem, &io, TypePtr::BOTTOM, src, src_offset, dest, dest_offset, length, false);
1182 }
1183
1184 _igvn.replace_node(_memproj_fallthrough, merge_mem);
1185 _igvn.replace_node(_ioproj_fallthrough, io);
|