src/share/vm/opto/library_call.cpp
Index
Unified diffs
Context diffs
Sdiffs
Patch
New
Old
Previous File
Next File
g1-bulk-zeroing-reduction Cdiff src/share/vm/opto/library_call.cpp
src/share/vm/opto/library_call.cpp
Print this page
*** 1,7 ****
/*
! * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
--- 1,7 ----
/*
! * Copyright (c) 1999, 2011, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*** 95,105 ****
Node* generate_limit_guard(Node* offset, Node* subseq_length,
Node* array_length,
RegionNode* region);
Node* generate_current_thread(Node* &tls_output);
address basictype2arraycopy(BasicType t, Node *src_offset, Node *dest_offset,
! bool disjoint_bases, const char* &name);
Node* load_mirror_from_klass(Node* klass);
Node* load_klass_from_mirror_common(Node* mirror, bool never_see_null,
int nargs,
RegionNode* region, int null_path,
int offset);
--- 95,105 ----
Node* generate_limit_guard(Node* offset, Node* subseq_length,
Node* array_length,
RegionNode* region);
Node* generate_current_thread(Node* &tls_output);
address basictype2arraycopy(BasicType t, Node *src_offset, Node *dest_offset,
! bool disjoint_bases, const char* &name, bool dest_uninitialized);
Node* load_mirror_from_klass(Node* klass);
Node* load_klass_from_mirror_common(Node* mirror, bool never_see_null,
int nargs,
RegionNode* region, int null_path,
int offset);
*** 210,239 ****
bool generate_block_arraycopy(const TypePtr* adr_type,
BasicType basic_elem_type,
AllocateNode* alloc,
Node* src, Node* src_offset,
Node* dest, Node* dest_offset,
! Node* dest_size);
void generate_slow_arraycopy(const TypePtr* adr_type,
Node* src, Node* src_offset,
Node* dest, Node* dest_offset,
! Node* copy_length);
Node* generate_checkcast_arraycopy(const TypePtr* adr_type,
Node* dest_elem_klass,
Node* src, Node* src_offset,
Node* dest, Node* dest_offset,
! Node* copy_length);
Node* generate_generic_arraycopy(const TypePtr* adr_type,
Node* src, Node* src_offset,
Node* dest, Node* dest_offset,
! Node* copy_length);
void generate_unchecked_arraycopy(const TypePtr* adr_type,
BasicType basic_elem_type,
bool disjoint_bases,
Node* src, Node* src_offset,
Node* dest, Node* dest_offset,
! Node* copy_length);
bool inline_unsafe_CAS(BasicType type);
bool inline_unsafe_ordered_store(BasicType type);
bool inline_fp_conversions(vmIntrinsics::ID id);
bool inline_numberOfLeadingZeros(vmIntrinsics::ID id);
bool inline_numberOfTrailingZeros(vmIntrinsics::ID id);
--- 210,239 ----
bool generate_block_arraycopy(const TypePtr* adr_type,
BasicType basic_elem_type,
AllocateNode* alloc,
Node* src, Node* src_offset,
Node* dest, Node* dest_offset,
! Node* dest_size, bool dest_uninitialized);
void generate_slow_arraycopy(const TypePtr* adr_type,
Node* src, Node* src_offset,
Node* dest, Node* dest_offset,
! Node* copy_length, bool dest_uninitialized);
Node* generate_checkcast_arraycopy(const TypePtr* adr_type,
Node* dest_elem_klass,
Node* src, Node* src_offset,
Node* dest, Node* dest_offset,
! Node* copy_length, bool dest_uninitialized);
Node* generate_generic_arraycopy(const TypePtr* adr_type,
Node* src, Node* src_offset,
Node* dest, Node* dest_offset,
! Node* copy_length, bool dest_uninitialized);
void generate_unchecked_arraycopy(const TypePtr* adr_type,
BasicType basic_elem_type,
bool disjoint_bases,
Node* src, Node* src_offset,
Node* dest, Node* dest_offset,
! Node* copy_length, bool dest_uninitialized);
bool inline_unsafe_CAS(BasicType type);
bool inline_unsafe_ordered_store(BasicType type);
bool inline_fp_conversions(vmIntrinsics::ID id);
bool inline_numberOfLeadingZeros(vmIntrinsics::ID id);
bool inline_numberOfTrailingZeros(vmIntrinsics::ID id);
*** 4079,4089 ****
countx = _gvn.transform( new (C, 3) URShiftXNode(countx, intcon(LogBytesPerLong) ));
const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
bool disjoint_bases = true;
generate_unchecked_arraycopy(raw_adr_type, T_LONG, disjoint_bases,
! src, NULL, dest, NULL, countx);
// If necessary, emit some card marks afterwards. (Non-arrays only.)
if (card_mark) {
assert(!is_array, "");
// Put in store barrier for any and all oops we are sticking
--- 4079,4090 ----
countx = _gvn.transform( new (C, 3) URShiftXNode(countx, intcon(LogBytesPerLong) ));
const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
bool disjoint_bases = true;
generate_unchecked_arraycopy(raw_adr_type, T_LONG, disjoint_bases,
! src, NULL, dest, NULL, countx,
! /*dest_uninitialized*/false);
// If necessary, emit some card marks afterwards. (Non-arrays only.)
if (card_mark) {
assert(!is_array, "");
// Put in store barrier for any and all oops we are sticking
*** 4293,4311 ****
};
// Note: The condition "disjoint" applies also for overlapping copies
// where an descending copy is permitted (i.e., dest_offset <= src_offset).
static address
! select_arraycopy_function(BasicType t, bool aligned, bool disjoint, const char* &name) {
int selector =
(aligned ? COPYFUNC_ALIGNED : COPYFUNC_UNALIGNED) +
(disjoint ? COPYFUNC_DISJOINT : COPYFUNC_CONJOINT);
#define RETURN_STUB(xxx_arraycopy) { \
name = #xxx_arraycopy; \
return StubRoutines::xxx_arraycopy(); }
switch (t) {
case T_BYTE:
case T_BOOLEAN:
switch (selector) {
case COPYFUNC_CONJOINT | COPYFUNC_UNALIGNED: RETURN_STUB(jbyte_arraycopy);
--- 4294,4316 ----
};
// Note: The condition "disjoint" applies also for overlapping copies
// where an descending copy is permitted (i.e., dest_offset <= src_offset).
static address
! select_arraycopy_function(BasicType t, bool aligned, bool disjoint, const char* &name, bool dest_uninitialized) {
int selector =
(aligned ? COPYFUNC_ALIGNED : COPYFUNC_UNALIGNED) +
(disjoint ? COPYFUNC_DISJOINT : COPYFUNC_CONJOINT);
#define RETURN_STUB(xxx_arraycopy) { \
name = #xxx_arraycopy; \
return StubRoutines::xxx_arraycopy(); }
+ #define RETURN_STUB_PARM(xxx_arraycopy, parm) { \
+ name = #xxx_arraycopy; \
+ return StubRoutines::xxx_arraycopy(parm); }
+
switch (t) {
case T_BYTE:
case T_BOOLEAN:
switch (selector) {
case COPYFUNC_CONJOINT | COPYFUNC_UNALIGNED: RETURN_STUB(jbyte_arraycopy);
*** 4338,4366 ****
case COPYFUNC_DISJOINT | COPYFUNC_ALIGNED: RETURN_STUB(arrayof_jlong_disjoint_arraycopy);
}
case T_ARRAY:
case T_OBJECT:
switch (selector) {
! case COPYFUNC_CONJOINT | COPYFUNC_UNALIGNED: RETURN_STUB(oop_arraycopy);
! case COPYFUNC_CONJOINT | COPYFUNC_ALIGNED: RETURN_STUB(arrayof_oop_arraycopy);
! case COPYFUNC_DISJOINT | COPYFUNC_UNALIGNED: RETURN_STUB(oop_disjoint_arraycopy);
! case COPYFUNC_DISJOINT | COPYFUNC_ALIGNED: RETURN_STUB(arrayof_oop_disjoint_arraycopy);
}
default:
ShouldNotReachHere();
return NULL;
}
#undef RETURN_STUB
}
//------------------------------basictype2arraycopy----------------------------
address LibraryCallKit::basictype2arraycopy(BasicType t,
Node* src_offset,
Node* dest_offset,
bool disjoint_bases,
! const char* &name) {
const TypeInt* src_offset_inttype = gvn().find_int_type(src_offset);;
const TypeInt* dest_offset_inttype = gvn().find_int_type(dest_offset);;
bool aligned = false;
bool disjoint = disjoint_bases;
--- 4343,4373 ----
case COPYFUNC_DISJOINT | COPYFUNC_ALIGNED: RETURN_STUB(arrayof_jlong_disjoint_arraycopy);
}
case T_ARRAY:
case T_OBJECT:
switch (selector) {
! case COPYFUNC_CONJOINT | COPYFUNC_UNALIGNED: RETURN_STUB_PARM(oop_arraycopy, dest_uninitialized);
! case COPYFUNC_CONJOINT | COPYFUNC_ALIGNED: RETURN_STUB_PARM(arrayof_oop_arraycopy, dest_uninitialized);
! case COPYFUNC_DISJOINT | COPYFUNC_UNALIGNED: RETURN_STUB_PARM(oop_disjoint_arraycopy, dest_uninitialized);
! case COPYFUNC_DISJOINT | COPYFUNC_ALIGNED: RETURN_STUB_PARM(arrayof_oop_disjoint_arraycopy, dest_uninitialized);
}
default:
ShouldNotReachHere();
return NULL;
}
#undef RETURN_STUB
+ #undef RETURN_STUB_PARM
}
//------------------------------basictype2arraycopy----------------------------
address LibraryCallKit::basictype2arraycopy(BasicType t,
Node* src_offset,
Node* dest_offset,
bool disjoint_bases,
! const char* &name,
! bool dest_uninitialized) {
const TypeInt* src_offset_inttype = gvn().find_int_type(src_offset);;
const TypeInt* dest_offset_inttype = gvn().find_int_type(dest_offset);;
bool aligned = false;
bool disjoint = disjoint_bases;
*** 4382,4392 ****
} else if (src_offset == dest_offset && src_offset != NULL) {
// This can occur if the offsets are identical non-constants.
disjoint = true;
}
! return select_arraycopy_function(t, aligned, disjoint, name);
}
//------------------------------inline_arraycopy-----------------------
bool LibraryCallKit::inline_arraycopy() {
--- 4389,4399 ----
} else if (src_offset == dest_offset && src_offset != NULL) {
// This can occur if the offsets are identical non-constants.
disjoint = true;
}
! return select_arraycopy_function(t, aligned, disjoint, name, dest_uninitialized);
}
//------------------------------inline_arraycopy-----------------------
bool LibraryCallKit::inline_arraycopy() {
*** 4438,4448 ****
if (src_elem != dest_elem || dest_elem == T_VOID) {
// The component types are not the same or are not recognized. Punt.
// (But, avoid the native method wrapper to JVM_ArrayCopy.)
generate_slow_arraycopy(TypePtr::BOTTOM,
! src, src_offset, dest, dest_offset, length);
return true;
}
//---------------------------------------------------------------------------
// We will make a fast path for this call to arraycopy.
--- 4445,4456 ----
if (src_elem != dest_elem || dest_elem == T_VOID) {
// The component types are not the same or are not recognized. Punt.
// (But, avoid the native method wrapper to JVM_ArrayCopy.)
generate_slow_arraycopy(TypePtr::BOTTOM,
! src, src_offset, dest, dest_offset, length,
! /*uninitialized_target*/false);
return true;
}
//---------------------------------------------------------------------------
// We will make a fast path for this call to arraycopy.
*** 4582,4591 ****
--- 4590,4603 ----
alloc = NULL;
//original_dest = dest;
//must_clear_dest = false;
}
+ // We must choose different arraycopy stubs if the target array are not initialized,
+ // so the the pre-barriers wouldn't peek into the old values. See CR 6627983.
+ const bool& dest_uninitialized = must_clear_dest;
+
// Results are placed here:
enum { fast_path = 1, // normal void-returning assembly stub
checked_path = 2, // special assembly stub with cleanup
slow_call_path = 3, // something went wrong; call the VM
zero_path = 4, // bypass when length of copy is zero
*** 4614,4624 ****
if (basic_elem_type == T_CONFLICT) {
assert(!must_clear_dest, "");
Node* cv = generate_generic_arraycopy(adr_type,
src, src_offset, dest, dest_offset,
! copy_length);
if (cv == NULL) cv = intcon(-1); // failure (no stub available)
checked_control = control();
checked_i_o = i_o();
checked_mem = memory(adr_type);
checked_value = cv;
--- 4626,4636 ----
if (basic_elem_type == T_CONFLICT) {
assert(!must_clear_dest, "");
Node* cv = generate_generic_arraycopy(adr_type,
src, src_offset, dest, dest_offset,
! copy_length, dest_uninitialized);
if (cv == NULL) cv = intcon(-1); // failure (no stub available)
checked_control = control();
checked_i_o = i_o();
checked_mem = memory(adr_type);
checked_value = cv;
*** 4696,4706 ****
// There is no tail. Try an upgrade to a 64-bit copy.
bool didit = false;
{ PreserveJVMState pjvms(this);
didit = generate_block_arraycopy(adr_type, basic_elem_type, alloc,
src, src_offset, dest, dest_offset,
! dest_size);
if (didit) {
// Present the results of the block-copying fast call.
result_region->init_req(bcopy_path, control());
result_i_o ->init_req(bcopy_path, i_o());
result_memory->init_req(bcopy_path, memory(adr_type));
--- 4708,4718 ----
// There is no tail. Try an upgrade to a 64-bit copy.
bool didit = false;
{ PreserveJVMState pjvms(this);
didit = generate_block_arraycopy(adr_type, basic_elem_type, alloc,
src, src_offset, dest, dest_offset,
! dest_size, dest_uninitialized);
if (didit) {
// Present the results of the block-copying fast call.
result_region->init_req(bcopy_path, control());
result_i_o ->init_req(bcopy_path, i_o());
result_memory->init_req(bcopy_path, memory(adr_type));
*** 4772,4782 ****
Node* n1 = LoadKlassNode::make(_gvn, immutable_memory(), p1, TypeRawPtr::BOTTOM);
Node* dest_elem_klass = _gvn.transform(n1);
Node* cv = generate_checkcast_arraycopy(adr_type,
dest_elem_klass,
src, src_offset, dest, dest_offset,
! ConvI2X(copy_length));
if (cv == NULL) cv = intcon(-1); // failure (no stub available)
checked_control = control();
checked_i_o = i_o();
checked_mem = memory(adr_type);
checked_value = cv;
--- 4784,4794 ----
Node* n1 = LoadKlassNode::make(_gvn, immutable_memory(), p1, TypeRawPtr::BOTTOM);
Node* dest_elem_klass = _gvn.transform(n1);
Node* cv = generate_checkcast_arraycopy(adr_type,
dest_elem_klass,
src, src_offset, dest, dest_offset,
! ConvI2X(copy_length), dest_uninitialized);
if (cv == NULL) cv = intcon(-1); // failure (no stub available)
checked_control = control();
checked_i_o = i_o();
checked_mem = memory(adr_type);
checked_value = cv;
*** 4795,4805 ****
if (!stopped()) {
// Generate the fast path, if possible.
PreserveJVMState pjvms(this);
generate_unchecked_arraycopy(adr_type, copy_type, disjoint_bases,
src, src_offset, dest, dest_offset,
! ConvI2X(copy_length));
// Present the results of the fast call.
result_region->init_req(fast_path, control());
result_i_o ->init_req(fast_path, i_o());
result_memory->init_req(fast_path, memory(adr_type));
--- 4807,4817 ----
if (!stopped()) {
// Generate the fast path, if possible.
PreserveJVMState pjvms(this);
generate_unchecked_arraycopy(adr_type, copy_type, disjoint_bases,
src, src_offset, dest, dest_offset,
! ConvI2X(copy_length), dest_uninitialized);
// Present the results of the fast call.
result_region->init_req(fast_path, control());
result_i_o ->init_req(fast_path, i_o());
result_memory->init_req(fast_path, memory(adr_type));
*** 4882,4892 ****
alloc->in(AllocateNode::AllocSize));
}
generate_slow_arraycopy(adr_type,
src, src_offset, dest, dest_offset,
! copy_length);
result_region->init_req(slow_call_path, control());
result_i_o ->init_req(slow_call_path, i_o());
result_memory->init_req(slow_call_path, memory(adr_type));
}
--- 4894,4904 ----
alloc->in(AllocateNode::AllocSize));
}
generate_slow_arraycopy(adr_type,
src, src_offset, dest, dest_offset,
! copy_length, /*dest_uninitialized*/false);
result_region->init_req(slow_call_path, control());
result_i_o ->init_req(slow_call_path, i_o());
result_memory->init_req(slow_call_path, memory(adr_type));
}
*** 5126,5136 ****
LibraryCallKit::generate_block_arraycopy(const TypePtr* adr_type,
BasicType basic_elem_type,
AllocateNode* alloc,
Node* src, Node* src_offset,
Node* dest, Node* dest_offset,
! Node* dest_size) {
// See if there is an advantage from block transfer.
int scale = exact_log2(type2aelembytes(basic_elem_type));
if (scale >= LogBytesPerLong)
return false; // it is already a block transfer
--- 5138,5148 ----
LibraryCallKit::generate_block_arraycopy(const TypePtr* adr_type,
BasicType basic_elem_type,
AllocateNode* alloc,
Node* src, Node* src_offset,
Node* dest, Node* dest_offset,
! Node* dest_size, bool dest_uninitialized) {
// See if there is an advantage from block transfer.
int scale = exact_log2(type2aelembytes(basic_elem_type));
if (scale >= LogBytesPerLong)
return false; // it is already a block transfer
*** 5171,5181 ****
countx = _gvn.transform( new (C, 3) SubXNode(countx, MakeConX(dest_off)) );
countx = _gvn.transform( new (C, 3) URShiftXNode(countx, intcon(LogBytesPerLong)) );
bool disjoint_bases = true; // since alloc != NULL
generate_unchecked_arraycopy(adr_type, T_LONG, disjoint_bases,
! sptr, NULL, dptr, NULL, countx);
return true;
}
--- 5183,5193 ----
countx = _gvn.transform( new (C, 3) SubXNode(countx, MakeConX(dest_off)) );
countx = _gvn.transform( new (C, 3) URShiftXNode(countx, intcon(LogBytesPerLong)) );
bool disjoint_bases = true; // since alloc != NULL
generate_unchecked_arraycopy(adr_type, T_LONG, disjoint_bases,
! sptr, NULL, dptr, NULL, countx, dest_uninitialized);
return true;
}
*** 5184,5194 ****
// but without the native wrapper overhead.
void
LibraryCallKit::generate_slow_arraycopy(const TypePtr* adr_type,
Node* src, Node* src_offset,
Node* dest, Node* dest_offset,
! Node* copy_length) {
Node* call = make_runtime_call(RC_NO_LEAF | RC_UNCOMMON,
OptoRuntime::slow_arraycopy_Type(),
OptoRuntime::slow_arraycopy_Java(),
"slow_arraycopy", adr_type,
src, src_offset, dest, dest_offset,
--- 5196,5207 ----
// but without the native wrapper overhead.
void
LibraryCallKit::generate_slow_arraycopy(const TypePtr* adr_type,
Node* src, Node* src_offset,
Node* dest, Node* dest_offset,
! Node* copy_length, bool dest_uninitialized) {
! assert(!dest_uninitialized, "Invariant");
Node* call = make_runtime_call(RC_NO_LEAF | RC_UNCOMMON,
OptoRuntime::slow_arraycopy_Type(),
OptoRuntime::slow_arraycopy_Java(),
"slow_arraycopy", adr_type,
src, src_offset, dest, dest_offset,
*** 5202,5215 ****
Node*
LibraryCallKit::generate_checkcast_arraycopy(const TypePtr* adr_type,
Node* dest_elem_klass,
Node* src, Node* src_offset,
Node* dest, Node* dest_offset,
! Node* copy_length) {
if (stopped()) return NULL;
! address copyfunc_addr = StubRoutines::checkcast_arraycopy();
if (copyfunc_addr == NULL) { // Stub was not generated, go slow path.
return NULL;
}
// Pick out the parameters required to perform a store-check
--- 5215,5228 ----
Node*
LibraryCallKit::generate_checkcast_arraycopy(const TypePtr* adr_type,
Node* dest_elem_klass,
Node* src, Node* src_offset,
Node* dest, Node* dest_offset,
! Node* copy_length, bool dest_uninitialized) {
if (stopped()) return NULL;
! address copyfunc_addr = StubRoutines::checkcast_arraycopy(dest_uninitialized);
if (copyfunc_addr == NULL) { // Stub was not generated, go slow path.
return NULL;
}
// Pick out the parameters required to perform a store-check
*** 5243,5255 ****
// Helper function; generates code for cases requiring runtime checks.
Node*
LibraryCallKit::generate_generic_arraycopy(const TypePtr* adr_type,
Node* src, Node* src_offset,
Node* dest, Node* dest_offset,
! Node* copy_length) {
if (stopped()) return NULL;
-
address copyfunc_addr = StubRoutines::generic_arraycopy();
if (copyfunc_addr == NULL) { // Stub was not generated, go slow path.
return NULL;
}
--- 5256,5268 ----
// Helper function; generates code for cases requiring runtime checks.
Node*
LibraryCallKit::generate_generic_arraycopy(const TypePtr* adr_type,
Node* src, Node* src_offset,
Node* dest, Node* dest_offset,
! Node* copy_length, bool dest_uninitialized) {
! assert(!dest_uninitialized, "Invariant");
if (stopped()) return NULL;
address copyfunc_addr = StubRoutines::generic_arraycopy();
if (copyfunc_addr == NULL) { // Stub was not generated, go slow path.
return NULL;
}
*** 5266,5276 ****
LibraryCallKit::generate_unchecked_arraycopy(const TypePtr* adr_type,
BasicType basic_elem_type,
bool disjoint_bases,
Node* src, Node* src_offset,
Node* dest, Node* dest_offset,
! Node* copy_length) {
if (stopped()) return; // nothing to do
Node* src_start = src;
Node* dest_start = dest;
if (src_offset != NULL || dest_offset != NULL) {
--- 5279,5289 ----
LibraryCallKit::generate_unchecked_arraycopy(const TypePtr* adr_type,
BasicType basic_elem_type,
bool disjoint_bases,
Node* src, Node* src_offset,
Node* dest, Node* dest_offset,
! Node* copy_length, bool dest_uninitialized) {
if (stopped()) return; // nothing to do
Node* src_start = src;
Node* dest_start = dest;
if (src_offset != NULL || dest_offset != NULL) {
*** 5281,5291 ****
// Figure out which arraycopy runtime method to call.
const char* copyfunc_name = "arraycopy";
address copyfunc_addr =
basictype2arraycopy(basic_elem_type, src_offset, dest_offset,
! disjoint_bases, copyfunc_name);
// Call it. Note that the count_ix value is not scaled to a byte-size.
make_runtime_call(RC_LEAF|RC_NO_FP,
OptoRuntime::fast_arraycopy_Type(),
copyfunc_addr, copyfunc_name, adr_type,
--- 5294,5304 ----
// Figure out which arraycopy runtime method to call.
const char* copyfunc_name = "arraycopy";
address copyfunc_addr =
basictype2arraycopy(basic_elem_type, src_offset, dest_offset,
! disjoint_bases, copyfunc_name, dest_uninitialized);
// Call it. Note that the count_ix value is not scaled to a byte-size.
make_runtime_call(RC_LEAF|RC_NO_FP,
OptoRuntime::fast_arraycopy_Type(),
copyfunc_addr, copyfunc_name, adr_type,
src/share/vm/opto/library_call.cpp
Index
Unified diffs
Context diffs
Sdiffs
Patch
New
Old
Previous File
Next File