< prev index next >
src/hotspot/share/opto/graphKit.cpp
BarrierSetC2
#include "precompiled.hpp"
#include "ci/ciUtilities.hpp"
#include "compiler/compileLog.hpp"
#include "gc/shared/barrierSet.hpp"
! #include "gc/shared/cardTable.hpp"
! #include "gc/shared/cardTableBarrierSet.hpp"
! #include "gc/shared/collectedHeap.hpp"
#include "interpreter/interpreter.hpp"
#include "memory/resourceArea.hpp"
#include "opto/addnode.hpp"
#include "opto/castnode.hpp"
#include "opto/convertnode.hpp"
#include "precompiled.hpp"
#include "ci/ciUtilities.hpp"
#include "compiler/compileLog.hpp"
#include "gc/shared/barrierSet.hpp"
! #include "gc/shared/c2/barrierSetC2.hpp"
#include "interpreter/interpreter.hpp"
#include "memory/resourceArea.hpp"
#include "opto/addnode.hpp"
#include "opto/castnode.hpp"
#include "opto/convertnode.hpp"
***************
#include "opto/parse.hpp"
#include "opto/rootnode.hpp"
#include "opto/runtime.hpp"
#include "runtime/deoptimization.hpp"
#include "runtime/sharedRuntime.hpp"
- #if INCLUDE_G1GC
- #include "gc/g1/g1CardTable.hpp"
- #include "gc/g1/g1ThreadLocalData.hpp"
- #include "gc/g1/heapRegion.hpp"
- #endif // INCLUDE_ALL_GCS
//----------------------------GraphKit-----------------------------------------
// Main utility constructor.
GraphKit::GraphKit(JVMState* jvms)
: Phase(Phase::Parser),
_env(C->env()),
! _gvn(*C->initial_gvn())
{
_exceptions = jvms->map()->next_exception();
if (_exceptions != NULL) jvms->map()->set_next_exception(NULL);
set_jvms(jvms);
}
// Private constructor for parser.
GraphKit::GraphKit()
: Phase(Phase::Parser),
_env(C->env()),
! _gvn(*C->initial_gvn())
{
_exceptions = NULL;
set_map(NULL);
debug_only(_sp = -99);
debug_only(set_bci(-99));
#include "opto/parse.hpp"
#include "opto/rootnode.hpp"
#include "opto/runtime.hpp"
#include "runtime/deoptimization.hpp"
#include "runtime/sharedRuntime.hpp"
//----------------------------GraphKit-----------------------------------------
// Main utility constructor.
GraphKit::GraphKit(JVMState* jvms)
: Phase(Phase::Parser),
_env(C->env()),
! _gvn(*C->initial_gvn()),
! _barrier_set(BarrierSet::barrier_set()->barrier_set_c2())
{
_exceptions = jvms->map()->next_exception();
if (_exceptions != NULL) jvms->map()->set_next_exception(NULL);
set_jvms(jvms);
}
// Private constructor for parser.
GraphKit::GraphKit()
: Phase(Phase::Parser),
_env(C->env()),
! _gvn(*C->initial_gvn()),
! _barrier_set(BarrierSet::barrier_set()->barrier_set_c2())
{
_exceptions = NULL;
set_map(NULL);
debug_only(_sp = -99);
debug_only(set_bci(-99));
***************
int offset = java_lang_Throwable::get_detailMessage_offset();
const TypePtr* adr_typ = ex_con->add_offset(offset);
Node *adr = basic_plus_adr(ex_node, ex_node, offset);
const TypeOopPtr* val_type = TypeOopPtr::make_from_klass(env()->String_klass());
! // Conservatively release stores of object references.
! Node *store = store_oop_to_object(control(), ex_node, adr, adr_typ, null(), val_type, T_OBJECT, MemNode::release);
add_exception_state(make_exception_state(ex_node));
return;
}
}
int offset = java_lang_Throwable::get_detailMessage_offset();
const TypePtr* adr_typ = ex_con->add_offset(offset);
Node *adr = basic_plus_adr(ex_node, ex_node, offset);
const TypeOopPtr* val_type = TypeOopPtr::make_from_klass(env()->String_klass());
! Node *store = access_store_at(control(), ex_node, adr, adr_typ, null(), val_type, T_OBJECT, IN_HEAP);
add_exception_state(make_exception_state(ex_node));
return;
}
}
***************
record_for_igvn(st);
return st;
}
- void GraphKit::pre_barrier(bool do_load,
- Node* ctl,
- Node* obj,
- Node* adr,
- uint adr_idx,
- Node* val,
- const TypeOopPtr* val_type,
- Node* pre_val,
- BasicType bt) {
-
- BarrierSet* bs = BarrierSet::barrier_set();
set_control(ctl);
! switch (bs->kind()) {
!
! #if INCLUDE_G1GC
! case BarrierSet::G1BarrierSet:
! g1_write_barrier_pre(do_load, obj, adr, adr_idx, val, val_type, pre_val, bt);
! break;
! #endif
!
! case BarrierSet::CardTableBarrierSet:
! break;
! default :
! ShouldNotReachHere();
}
}
! bool GraphKit::can_move_pre_barrier() const {
! BarrierSet* bs = BarrierSet::barrier_set();
! switch (bs->kind()) {
!
! #if INCLUDE_G1GC
! case BarrierSet::G1BarrierSet:
! return true; // Can move it if no safepoint
! #endif
!
! case BarrierSet::CardTableBarrierSet:
! return true; // There is no pre-barrier
! default :
! ShouldNotReachHere();
}
- return false;
}
! void GraphKit::post_barrier(Node* ctl,
! Node* store,
! Node* obj,
! Node* adr,
! uint adr_idx,
! Node* val,
! BasicType bt,
! bool use_precise) {
! BarrierSet* bs = BarrierSet::barrier_set();
set_control(ctl);
! switch (bs->kind()) {
! #if INCLUDE_G1GC
! case BarrierSet::G1BarrierSet:
! g1_write_barrier_post(store, obj, adr, adr_idx, val, bt, use_precise);
! break;
! #endif
!
! case BarrierSet::CardTableBarrierSet:
! write_barrier_post(store, obj, adr, adr_idx, val, use_precise);
! break;
!
! default :
! ShouldNotReachHere();
!
}
}
! Node* GraphKit::store_oop(Node* ctl,
! Node* obj,
! Node* adr,
! const TypePtr* adr_type,
! Node* val,
! const TypeOopPtr* val_type,
! BasicType bt,
! bool use_precise,
! MemNode::MemOrd mo,
! bool mismatched) {
! // Transformation of a value which could be NULL pointer (CastPP #NULL)
! // could be delayed during Parse (for example, in adjust_map_after_if()).
! // Execute transformation here to avoid barrier generation in such case.
! if (_gvn.type(val) == TypePtr::NULL_PTR)
! val = _gvn.makecon(TypePtr::NULL_PTR);
!
set_control(ctl);
! if (stopped()) return top(); // Dead path ?
!
! assert(bt == T_OBJECT, "sanity");
! assert(val != NULL, "not dead path");
! uint adr_idx = C->get_alias_index(adr_type);
! assert(adr_idx != Compile::AliasIdxTop, "use other store_to_memory factory" );
! pre_barrier(true /* do_load */,
! control(), obj, adr, adr_idx, val, val_type,
! NULL /* pre_val */,
! bt);
!
! Node* store = store_to_memory(control(), adr, val, bt, adr_idx, mo, mismatched);
! post_barrier(control(), store, obj, adr, adr_idx, val, bt, use_precise);
! return store;
! }
!
! // Could be an array or object we don't know at compile time (unsafe ref.)
! Node* GraphKit::store_oop_to_unknown(Node* ctl,
! Node* obj, // containing obj
! Node* adr, // actual adress to store val at
! const TypePtr* adr_type,
! Node* val,
! BasicType bt,
! MemNode::MemOrd mo,
! bool mismatched) {
! Compile::AliasType* at = C->alias_type(adr_type);
! const TypeOopPtr* val_type = NULL;
! if (adr_type->isa_instptr()) {
! if (at->field() != NULL) {
! // known field. This code is a copy of the do_put_xxx logic.
! ciField* field = at->field();
! if (!field->type()->is_loaded()) {
! val_type = TypeInstPtr::BOTTOM;
! } else {
! val_type = TypeOopPtr::make_from_klass(field->type()->as_klass());
! }
! }
! } else if (adr_type->isa_aryptr()) {
! val_type = adr_type->is_aryptr()->elem()->make_oopptr();
}
! if (val_type == NULL) {
! val_type = TypeInstPtr::BOTTOM;
}
- return store_oop(ctl, obj, adr, adr_type, val, val_type, bt, true, mo, mismatched);
}
//-------------------------array_element_address-------------------------
Node* GraphKit::array_element_address(Node* ary, Node* idx, BasicType elembt,
const TypeInt* sizetype, Node* ctrl) {
uint shift = exact_log2(type2aelembytes(elembt));
record_for_igvn(st);
return st;
}
+ Node* GraphKit::access_store_at(Node* ctl,
+ Node* obj,
+ Node* adr,
+ const TypePtr* adr_type,
+ Node* val,
+ const Type* val_type,
+ BasicType bt,
+ DecoratorSet decorators) {
+ // Transformation of a value which could be NULL pointer (CastPP #NULL)
+ // could be delayed during Parse (for example, in adjust_map_after_if()).
+ // Execute transformation here to avoid barrier generation in such case.
+ if (_gvn.type(val) == TypePtr::NULL_PTR) {
+ val = _gvn.makecon(TypePtr::NULL_PTR);
+ }
set_control(ctl);
! if (stopped()) {
! return top(); // Dead path ?
! }
! assert(val != NULL, "not dead path");
+ C2AccessValuePtr addr(adr, adr_type);
+ C2AccessValue value(val, val_type);
+ C2Access access(this, decorators | C2_WRITE_ACCESS, bt, obj, addr);
+ if (access.is_raw()) {
+ return _barrier_set->BarrierSetC2::store_at(access, value);
+ } else {
+ return _barrier_set->store_at(access, value);
}
}
! Node* GraphKit::access_load_at(Node* obj, // containing obj
! Node* adr, // actual adress to store val at
! const TypePtr* adr_type,
! const Type* val_type,
! BasicType bt,
! DecoratorSet decorators) {
! if (stopped()) {
! return top(); // Dead path ?
! }
! C2AccessValuePtr addr(adr, adr_type);
! C2Access access(this, decorators | C2_READ_ACCESS, bt, obj, addr);
! if (access.is_raw()) {
! return _barrier_set->BarrierSetC2::load_at(access, val_type);
! } else {
! return _barrier_set->load_at(access, val_type);
}
}
! Node* GraphKit::access_atomic_cmpxchg_val_at(Node* ctl,
! Node* obj,
! Node* adr,
! const TypePtr* adr_type,
! int alias_idx,
! Node* expected_val,
! Node* new_val,
! const Type* value_type,
! BasicType bt,
! DecoratorSet decorators) {
set_control(ctl);
! C2AccessValuePtr addr(adr, adr_type);
! C2AtomicAccess access(this, decorators | C2_READ_ACCESS | C2_WRITE_ACCESS,
! bt, obj, addr, alias_idx);
! if (access.is_raw()) {
! return _barrier_set->BarrierSetC2::atomic_cmpxchg_val_at(access, expected_val, new_val, value_type);
! } else {
! return _barrier_set->atomic_cmpxchg_val_at(access, expected_val, new_val, value_type);
}
}
! Node* GraphKit::access_atomic_cmpxchg_bool_at(Node* ctl,
! Node* obj,
! Node* adr,
! const TypePtr* adr_type,
! int alias_idx,
! Node* expected_val,
! Node* new_val,
! const Type* value_type,
! BasicType bt,
! DecoratorSet decorators) {
set_control(ctl);
! C2AccessValuePtr addr(adr, adr_type);
! C2AtomicAccess access(this, decorators | C2_READ_ACCESS | C2_WRITE_ACCESS,
! bt, obj, addr, alias_idx);
! if (access.is_raw()) {
! return _barrier_set->BarrierSetC2::atomic_cmpxchg_bool_at(access, expected_val, new_val, value_type);
! } else {
! return _barrier_set->atomic_cmpxchg_bool_at(access, expected_val, new_val, value_type);
! }
! }
! Node* GraphKit::access_atomic_xchg_at(Node* ctl,
! Node* obj,
! Node* adr,
! const TypePtr* adr_type,
! int alias_idx,
! Node* new_val,
! const Type* value_type,
! BasicType bt,
! DecoratorSet decorators) {
! set_control(ctl);
! C2AccessValuePtr addr(adr, adr_type);
! C2AtomicAccess access(this, decorators | C2_READ_ACCESS | C2_WRITE_ACCESS,
! bt, obj, addr, alias_idx);
! if (access.is_raw()) {
! return _barrier_set->BarrierSetC2::atomic_xchg_at(access, new_val, value_type);
! } else {
! return _barrier_set->atomic_xchg_at(access, new_val, value_type);
}
! }
!
! Node* GraphKit::access_atomic_add_at(Node* ctl,
! Node* obj,
! Node* adr,
! const TypePtr* adr_type,
! int alias_idx,
! Node* new_val,
! const Type* value_type,
! BasicType bt,
! DecoratorSet decorators) {
! set_control(ctl);
! C2AccessValuePtr addr(adr, adr_type);
! C2AtomicAccess access(this, decorators | C2_READ_ACCESS | C2_WRITE_ACCESS, bt, obj, addr, alias_idx);
! if (access.is_raw()) {
! return _barrier_set->BarrierSetC2::atomic_add_at(access, new_val, value_type);
! } else {
! return _barrier_set->atomic_add_at(access, new_val, value_type);
}
}
+ void GraphKit::access_clone(Node* ctl, Node* src, Node* dst, Node* size, bool is_array) {
+ set_control(ctl);
+ return _barrier_set->clone(this, src, dst, size, is_array);
+ }
//-------------------------array_element_address-------------------------
Node* GraphKit::array_element_address(Node* ary, Node* idx, BasicType elembt,
const TypeInt* sizetype, Node* ctrl) {
uint shift = exact_log2(type2aelembytes(elembt));
***************
}
// loop's limit check predicate should be near the loop.
add_predicate_impl(Deoptimization::Reason_loop_limit_check, nargs);
}
- //----------------------------- store barriers ----------------------------
- #define __ ideal.
-
- bool GraphKit::use_ReduceInitialCardMarks() {
- BarrierSet *bs = BarrierSet::barrier_set();
- return bs->is_a(BarrierSet::CardTableBarrierSet)
- && barrier_set_cast<CardTableBarrierSet>(bs)->can_elide_tlab_store_barriers()
- && ReduceInitialCardMarks;
- }
-
void GraphKit::sync_kit(IdealKit& ideal) {
! set_all_memory(__ merged_memory());
! set_i_o(__ i_o());
! set_control(__ ctrl());
}
void GraphKit::final_sync(IdealKit& ideal) {
// Final sync IdealKit and graphKit.
sync_kit(ideal);
}
- Node* GraphKit::byte_map_base_node() {
- // Get base of card map
- jbyte* card_table_base = ci_card_table_address();
- if (card_table_base != NULL) {
- return makecon(TypeRawPtr::make((address)card_table_base));
- } else {
- return null();
- }
- }
-
- // vanilla/CMS post barrier
- // Insert a write-barrier store. This is to let generational GC work; we have
- // to flag all oop-stores before the next GC point.
- void GraphKit::write_barrier_post(Node* oop_store,
- Node* obj,
- Node* adr,
- uint adr_idx,
- Node* val,
- bool use_precise) {
- // No store check needed if we're storing a NULL or an old object
- // (latter case is probably a string constant). The concurrent
- // mark sweep garbage collector, however, needs to have all nonNull
- // oop updates flagged via card-marks.
- if (val != NULL && val->is_Con()) {
- // must be either an oop or NULL
- const Type* t = val->bottom_type();
- if (t == TypePtr::NULL_PTR || t == Type::TOP)
- // stores of null never (?) need barriers
- return;
- }
-
- if (use_ReduceInitialCardMarks()
- && obj == just_allocated_object(control())) {
- // We can skip marks on a freshly-allocated object in Eden.
- // Keep this code in sync with new_deferred_store_barrier() in runtime.cpp.
- // That routine informs GC to take appropriate compensating steps,
- // upon a slow-path allocation, so as to make this card-mark
- // elision safe.
- return;
- }
-
- if (!use_precise) {
- // All card marks for a (non-array) instance are in one place:
- adr = obj;
- }
- // (Else it's an array (or unknown), and we want more precise card marks.)
- assert(adr != NULL, "");
-
- IdealKit ideal(this, true);
-
- // Convert the pointer to an int prior to doing math on it
- Node* cast = __ CastPX(__ ctrl(), adr);
-
- // Divide by card size
- assert(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet),
- "Only one we handle so far.");
- Node* card_offset = __ URShiftX( cast, __ ConI(CardTable::card_shift) );
-
- // Combine card table base and card offset
- Node* card_adr = __ AddP(__ top(), byte_map_base_node(), card_offset );
-
- // Get the alias_index for raw card-mark memory
- int adr_type = Compile::AliasIdxRaw;
- Node* zero = __ ConI(0); // Dirty card value
- BasicType bt = T_BYTE;
-
- if (UseConcMarkSweepGC && UseCondCardMark) {
- insert_mem_bar(Op_MemBarVolatile); // StoreLoad barrier
- __ sync_kit(this);
- }
-
- if (UseCondCardMark) {
- // The classic GC reference write barrier is typically implemented
- // as a store into the global card mark table. Unfortunately
- // unconditional stores can result in false sharing and excessive
- // coherence traffic as well as false transactional aborts.
- // UseCondCardMark enables MP "polite" conditional card mark
- // stores. In theory we could relax the load from ctrl() to
- // no_ctrl, but that doesn't buy much latitude.
- Node* card_val = __ load( __ ctrl(), card_adr, TypeInt::BYTE, bt, adr_type);
- __ if_then(card_val, BoolTest::ne, zero);
- }
-
- // Smash zero into card
- if( !UseConcMarkSweepGC ) {
- __ store(__ ctrl(), card_adr, zero, bt, adr_type, MemNode::unordered);
- } else {
- // Specialized path for CM store barrier
- __ storeCM(__ ctrl(), card_adr, zero, oop_store, adr_idx, bt, adr_type);
- }
-
- if (UseCondCardMark) {
- __ end_if();
- }
-
- // Final sync IdealKit and GraphKit.
- final_sync(ideal);
- }
-
- #if INCLUDE_G1GC
-
- /*
- * Determine if the G1 pre-barrier can be removed. The pre-barrier is
- * required by SATB to make sure all objects live at the start of the
- * marking are kept alive, all reference updates need to any previous
- * reference stored before writing.
- *
- * If the previous value is NULL there is no need to save the old value.
- * References that are NULL are filtered during runtime by the barrier
- * code to avoid unnecessary queuing.
- *
- * However in the case of newly allocated objects it might be possible to
- * prove that the reference about to be overwritten is NULL during compile
- * time and avoid adding the barrier code completely.
- *
- * The compiler needs to determine that the object in which a field is about
- * to be written is newly allocated, and that no prior store to the same field
- * has happened since the allocation.
- *
- * Returns true if the pre-barrier can be removed
- */
- bool GraphKit::g1_can_remove_pre_barrier(PhaseTransform* phase, Node* adr,
- BasicType bt, uint adr_idx) {
- intptr_t offset = 0;
- Node* base = AddPNode::Ideal_base_and_offset(adr, phase, offset);
- AllocateNode* alloc = AllocateNode::Ideal_allocation(base, phase);
-
- if (offset == Type::OffsetBot) {
- return false; // cannot unalias unless there are precise offsets
- }
-
- if (alloc == NULL) {
- return false; // No allocation found
- }
-
- intptr_t size_in_bytes = type2aelembytes(bt);
-
- Node* mem = memory(adr_idx); // start searching here...
-
- for (int cnt = 0; cnt < 50; cnt++) {
-
- if (mem->is_Store()) {
-
- Node* st_adr = mem->in(MemNode::Address);
- intptr_t st_offset = 0;
- Node* st_base = AddPNode::Ideal_base_and_offset(st_adr, phase, st_offset);
-
- if (st_base == NULL) {
- break; // inscrutable pointer
- }
-
- // Break we have found a store with same base and offset as ours so break
- if (st_base == base && st_offset == offset) {
- break;
- }
-
- if (st_offset != offset && st_offset != Type::OffsetBot) {
- const int MAX_STORE = BytesPerLong;
- if (st_offset >= offset + size_in_bytes ||
- st_offset <= offset - MAX_STORE ||
- st_offset <= offset - mem->as_Store()->memory_size()) {
- // Success: The offsets are provably independent.
- // (You may ask, why not just test st_offset != offset and be done?
- // The answer is that stores of different sizes can co-exist
- // in the same sequence of RawMem effects. We sometimes initialize
- // a whole 'tile' of array elements with a single jint or jlong.)
- mem = mem->in(MemNode::Memory);
- continue; // advance through independent store memory
- }
- }
-
- if (st_base != base
- && MemNode::detect_ptr_independence(base, alloc, st_base,
- AllocateNode::Ideal_allocation(st_base, phase),
- phase)) {
- // Success: The bases are provably independent.
- mem = mem->in(MemNode::Memory);
- continue; // advance through independent store memory
- }
- } else if (mem->is_Proj() && mem->in(0)->is_Initialize()) {
-
- InitializeNode* st_init = mem->in(0)->as_Initialize();
- AllocateNode* st_alloc = st_init->allocation();
-
- // Make sure that we are looking at the same allocation site.
- // The alloc variable is guaranteed to not be null here from earlier check.
- if (alloc == st_alloc) {
- // Check that the initialization is storing NULL so that no previous store
- // has been moved up and directly write a reference
- Node* captured_store = st_init->find_captured_store(offset,
- type2aelembytes(T_OBJECT),
- phase);
- if (captured_store == NULL || captured_store == st_init->zero_memory()) {
- return true;
- }
- }
- }
-
- // Unless there is an explicit 'continue', we must bail out here,
- // because 'mem' is an inscrutable memory state (e.g., a call).
- break;
- }
-
- return false;
- }
-
- // G1 pre/post barriers
- void GraphKit::g1_write_barrier_pre(bool do_load,
- Node* obj,
- Node* adr,
- uint alias_idx,
- Node* val,
- const TypeOopPtr* val_type,
- Node* pre_val,
- BasicType bt) {
-
- // Some sanity checks
- // Note: val is unused in this routine.
-
- if (do_load) {
- // We need to generate the load of the previous value
- assert(obj != NULL, "must have a base");
- assert(adr != NULL, "where are loading from?");
- assert(pre_val == NULL, "loaded already?");
- assert(val_type != NULL, "need a type");
-
- if (use_ReduceInitialCardMarks()
- && g1_can_remove_pre_barrier(&_gvn, adr, bt, alias_idx)) {
- return;
- }
-
- } else {
- // In this case both val_type and alias_idx are unused.
- assert(pre_val != NULL, "must be loaded already");
- // Nothing to be done if pre_val is null.
- if (pre_val->bottom_type() == TypePtr::NULL_PTR) return;
- assert(pre_val->bottom_type()->basic_type() == T_OBJECT, "or we shouldn't be here");
- }
- assert(bt == T_OBJECT, "or we shouldn't be here");
-
- IdealKit ideal(this, true);
-
- Node* tls = __ thread(); // ThreadLocalStorage
-
- Node* no_ctrl = NULL;
- Node* no_base = __ top();
- Node* zero = __ ConI(0);
- Node* zeroX = __ ConX(0);
-
- float likely = PROB_LIKELY(0.999);
- float unlikely = PROB_UNLIKELY(0.999);
-
- BasicType active_type = in_bytes(SATBMarkQueue::byte_width_of_active()) == 4 ? T_INT : T_BYTE;
- assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 4 || in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "flag width");
-
- // Offsets into the thread
- const int marking_offset = in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset());
- const int index_offset = in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset());
- const int buffer_offset = in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset());
-
- // Now the actual pointers into the thread
- Node* marking_adr = __ AddP(no_base, tls, __ ConX(marking_offset));
- Node* buffer_adr = __ AddP(no_base, tls, __ ConX(buffer_offset));
- Node* index_adr = __ AddP(no_base, tls, __ ConX(index_offset));
-
- // Now some of the values
- Node* marking = __ load(__ ctrl(), marking_adr, TypeInt::INT, active_type, Compile::AliasIdxRaw);
-
- // if (!marking)
- __ if_then(marking, BoolTest::ne, zero, unlikely); {
- BasicType index_bt = TypeX_X->basic_type();
- assert(sizeof(size_t) == type2aelembytes(index_bt), "Loading G1 SATBMarkQueue::_index with wrong size.");
- Node* index = __ load(__ ctrl(), index_adr, TypeX_X, index_bt, Compile::AliasIdxRaw);
-
- if (do_load) {
- // load original value
- // alias_idx correct??
- pre_val = __ load(__ ctrl(), adr, val_type, bt, alias_idx);
- }
-
- // if (pre_val != NULL)
- __ if_then(pre_val, BoolTest::ne, null()); {
- Node* buffer = __ load(__ ctrl(), buffer_adr, TypeRawPtr::NOTNULL, T_ADDRESS, Compile::AliasIdxRaw);
-
- // is the queue for this thread full?
- __ if_then(index, BoolTest::ne, zeroX, likely); {
-
- // decrement the index
- Node* next_index = _gvn.transform(new SubXNode(index, __ ConX(sizeof(intptr_t))));
-
- // Now get the buffer location we will log the previous value into and store it
- Node *log_addr = __ AddP(no_base, buffer, next_index);
- __ store(__ ctrl(), log_addr, pre_val, T_OBJECT, Compile::AliasIdxRaw, MemNode::unordered);
- // update the index
- __ store(__ ctrl(), index_adr, next_index, index_bt, Compile::AliasIdxRaw, MemNode::unordered);
-
- } __ else_(); {
-
- // logging buffer is full, call the runtime
- const TypeFunc *tf = OptoRuntime::g1_wb_pre_Type();
- __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), "g1_wb_pre", pre_val, tls);
- } __ end_if(); // (!index)
- } __ end_if(); // (pre_val != NULL)
- } __ end_if(); // (!marking)
-
- // Final sync IdealKit and GraphKit.
- final_sync(ideal);
- }
-
- /*
- * G1 similar to any GC with a Young Generation requires a way to keep track of
- * references from Old Generation to Young Generation to make sure all live
- * objects are found. G1 also requires to keep track of object references
- * between different regions to enable evacuation of old regions, which is done
- * as part of mixed collections. References are tracked in remembered sets and
- * is continuously updated as reference are written to with the help of the
- * post-barrier.
- *
- * To reduce the number of updates to the remembered set the post-barrier
- * filters updates to fields in objects located in the Young Generation,
- * the same region as the reference, when the NULL is being written or
- * if the card is already marked as dirty by an earlier write.
- *
- * Under certain circumstances it is possible to avoid generating the
- * post-barrier completely if it is possible during compile time to prove
- * the object is newly allocated and that no safepoint exists between the
- * allocation and the store.
- *
- * In the case of slow allocation the allocation code must handle the barrier
- * as part of the allocation in the case the allocated object is not located
- * in the nursery, this would happen for humongous objects. This is similar to
- * how CMS is required to handle this case, see the comments for the method
- * CardTableBarrierSet::on_allocation_slowpath_exit and OptoRuntime::new_deferred_store_barrier.
- * A deferred card mark is required for these objects and handled in the above
- * mentioned methods.
- *
- * Returns true if the post barrier can be removed
- */
- bool GraphKit::g1_can_remove_post_barrier(PhaseTransform* phase, Node* store,
- Node* adr) {
- intptr_t offset = 0;
- Node* base = AddPNode::Ideal_base_and_offset(adr, phase, offset);
- AllocateNode* alloc = AllocateNode::Ideal_allocation(base, phase);
-
- if (offset == Type::OffsetBot) {
- return false; // cannot unalias unless there are precise offsets
- }
-
- if (alloc == NULL) {
- return false; // No allocation found
- }
-
- // Start search from Store node
- Node* mem = store->in(MemNode::Control);
- if (mem->is_Proj() && mem->in(0)->is_Initialize()) {
-
- InitializeNode* st_init = mem->in(0)->as_Initialize();
- AllocateNode* st_alloc = st_init->allocation();
-
- // Make sure we are looking at the same allocation
- if (alloc == st_alloc) {
- return true;
- }
- }
-
- return false;
- }
-
- //
- // Update the card table and add card address to the queue
- //
- void GraphKit::g1_mark_card(IdealKit& ideal,
- Node* card_adr,
- Node* oop_store,
- uint oop_alias_idx,
- Node* index,
- Node* index_adr,
- Node* buffer,
- const TypeFunc* tf) {
-
- Node* zero = __ ConI(0);
- Node* zeroX = __ ConX(0);
- Node* no_base = __ top();
- BasicType card_bt = T_BYTE;
- // Smash zero into card. MUST BE ORDERED WRT TO STORE
- __ storeCM(__ ctrl(), card_adr, zero, oop_store, oop_alias_idx, card_bt, Compile::AliasIdxRaw);
-
- // Now do the queue work
- __ if_then(index, BoolTest::ne, zeroX); {
-
- Node* next_index = _gvn.transform(new SubXNode(index, __ ConX(sizeof(intptr_t))));
- Node* log_addr = __ AddP(no_base, buffer, next_index);
-
- // Order, see storeCM.
- __ store(__ ctrl(), log_addr, card_adr, T_ADDRESS, Compile::AliasIdxRaw, MemNode::unordered);
- __ store(__ ctrl(), index_adr, next_index, TypeX_X->basic_type(), Compile::AliasIdxRaw, MemNode::unordered);
-
- } __ else_(); {
- __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_post), "g1_wb_post", card_adr, __ thread());
- } __ end_if();
-
- }
-
- void GraphKit::g1_write_barrier_post(Node* oop_store,
- Node* obj,
- Node* adr,
- uint alias_idx,
- Node* val,
- BasicType bt,
- bool use_precise) {
- // If we are writing a NULL then we need no post barrier
-
- if (val != NULL && val->is_Con() && val->bottom_type() == TypePtr::NULL_PTR) {
- // Must be NULL
- const Type* t = val->bottom_type();
- assert(t == Type::TOP || t == TypePtr::NULL_PTR, "must be NULL");
- // No post barrier if writing NULLx
- return;
- }
-
- if (use_ReduceInitialCardMarks() && obj == just_allocated_object(control())) {
- // We can skip marks on a freshly-allocated object in Eden.
- // Keep this code in sync with new_deferred_store_barrier() in runtime.cpp.
- // That routine informs GC to take appropriate compensating steps,
- // upon a slow-path allocation, so as to make this card-mark
- // elision safe.
- return;
- }
-
- if (use_ReduceInitialCardMarks()
- && g1_can_remove_post_barrier(&_gvn, oop_store, adr)) {
- return;
- }
-
- if (!use_precise) {
- // All card marks for a (non-array) instance are in one place:
- adr = obj;
- }
- // (Else it's an array (or unknown), and we want more precise card marks.)
- assert(adr != NULL, "");
-
- IdealKit ideal(this, true);
-
- Node* tls = __ thread(); // ThreadLocalStorage
-
- Node* no_base = __ top();
- float likely = PROB_LIKELY(0.999);
- float unlikely = PROB_UNLIKELY(0.999);
- Node* young_card = __ ConI((jint)G1CardTable::g1_young_card_val());
- Node* dirty_card = __ ConI((jint)CardTable::dirty_card_val());
- Node* zeroX = __ ConX(0);
-
- // Get the alias_index for raw card-mark memory
- const TypePtr* card_type = TypeRawPtr::BOTTOM;
-
- const TypeFunc *tf = OptoRuntime::g1_wb_post_Type();
-
- // Offsets into the thread
- const int index_offset = in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset());
- const int buffer_offset = in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset());
-
- // Pointers into the thread
-
- Node* buffer_adr = __ AddP(no_base, tls, __ ConX(buffer_offset));
- Node* index_adr = __ AddP(no_base, tls, __ ConX(index_offset));
-
- // Now some values
- // Use ctrl to avoid hoisting these values past a safepoint, which could
- // potentially reset these fields in the JavaThread.
- Node* index = __ load(__ ctrl(), index_adr, TypeX_X, TypeX_X->basic_type(), Compile::AliasIdxRaw);
- Node* buffer = __ load(__ ctrl(), buffer_adr, TypeRawPtr::NOTNULL, T_ADDRESS, Compile::AliasIdxRaw);
-
- // Convert the store obj pointer to an int prior to doing math on it
- // Must use ctrl to prevent "integerized oop" existing across safepoint
- Node* cast = __ CastPX(__ ctrl(), adr);
-
- // Divide pointer by card size
- Node* card_offset = __ URShiftX( cast, __ ConI(CardTable::card_shift) );
-
- // Combine card table base and card offset
- Node* card_adr = __ AddP(no_base, byte_map_base_node(), card_offset );
-
- // If we know the value being stored does it cross regions?
-
- if (val != NULL) {
- // Does the store cause us to cross regions?
-
- // Should be able to do an unsigned compare of region_size instead of
- // and extra shift. Do we have an unsigned compare??
- // Node* region_size = __ ConI(1 << HeapRegion::LogOfHRGrainBytes);
- Node* xor_res = __ URShiftX ( __ XorX( cast, __ CastPX(__ ctrl(), val)), __ ConI(HeapRegion::LogOfHRGrainBytes));
-
- // if (xor_res == 0) same region so skip
- __ if_then(xor_res, BoolTest::ne, zeroX); {
-
- // No barrier if we are storing a NULL
- __ if_then(val, BoolTest::ne, null(), unlikely); {
-
- // Ok must mark the card if not already dirty
-
- // load the original value of the card
- Node* card_val = __ load(__ ctrl(), card_adr, TypeInt::INT, T_BYTE, Compile::AliasIdxRaw);
-
- __ if_then(card_val, BoolTest::ne, young_card); {
- sync_kit(ideal);
- // Use Op_MemBarVolatile to achieve the effect of a StoreLoad barrier.
- insert_mem_bar(Op_MemBarVolatile, oop_store);
- __ sync_kit(this);
-
- Node* card_val_reload = __ load(__ ctrl(), card_adr, TypeInt::INT, T_BYTE, Compile::AliasIdxRaw);
- __ if_then(card_val_reload, BoolTest::ne, dirty_card); {
- g1_mark_card(ideal, card_adr, oop_store, alias_idx, index, index_adr, buffer, tf);
- } __ end_if();
- } __ end_if();
- } __ end_if();
- } __ end_if();
- } else {
- // The Object.clone() intrinsic uses this path if !ReduceInitialCardMarks.
- // We don't need a barrier here if the destination is a newly allocated object
- // in Eden. Otherwise, GC verification breaks because we assume that cards in Eden
- // are set to 'g1_young_gen' (see G1CardTable::verify_g1_young_region()).
- assert(!use_ReduceInitialCardMarks(), "can only happen with card marking");
- Node* card_val = __ load(__ ctrl(), card_adr, TypeInt::INT, T_BYTE, Compile::AliasIdxRaw);
- __ if_then(card_val, BoolTest::ne, young_card); {
- g1_mark_card(ideal, card_adr, oop_store, alias_idx, index, index_adr, buffer, tf);
- } __ end_if();
- }
-
- // Final sync IdealKit and GraphKit.
- final_sync(ideal);
- }
- #undef __
-
- #endif // INCLUDE_G1GC
-
Node* GraphKit::load_String_length(Node* ctrl, Node* str) {
Node* len = load_array_length(load_String_value(ctrl, str));
Node* coder = load_String_coder(ctrl, str);
// Divide length by 2 if coder is UTF16
return _gvn.transform(new RShiftINode(len, coder));
}
// loop's limit check predicate should be near the loop.
add_predicate_impl(Deoptimization::Reason_loop_limit_check, nargs);
}
void GraphKit::sync_kit(IdealKit& ideal) {
! set_all_memory(ideal.merged_memory());
! set_i_o(ideal.i_o());
! set_control(ideal.ctrl());
}
void GraphKit::final_sync(IdealKit& ideal) {
// Final sync IdealKit and graphKit.
sync_kit(ideal);
}
Node* GraphKit::load_String_length(Node* ctrl, Node* str) {
Node* len = load_array_length(load_String_value(ctrl, str));
Node* coder = load_String_coder(ctrl, str);
// Divide length by 2 if coder is UTF16
return _gvn.transform(new RShiftINode(len, coder));
***************
false, NULL, 0);
const TypePtr* value_field_type = string_type->add_offset(value_offset);
const TypeAryPtr* value_type = TypeAryPtr::make(TypePtr::NotNull,
TypeAry::make(TypeInt::BYTE, TypeInt::POS),
ciTypeArrayKlass::make(T_BYTE), true, 0);
! int value_field_idx = C->get_alias_index(value_field_type);
! Node* load = make_load(ctrl, basic_plus_adr(str, str, value_offset),
! value_type, T_OBJECT, value_field_idx, MemNode::unordered);
// String.value field is known to be @Stable.
if (UseImplicitStableValues) {
load = cast_array_to_stable(load, value_type);
}
return load;
false, NULL, 0);
const TypePtr* value_field_type = string_type->add_offset(value_offset);
const TypeAryPtr* value_type = TypeAryPtr::make(TypePtr::NotNull,
TypeAry::make(TypeInt::BYTE, TypeInt::POS),
ciTypeArrayKlass::make(T_BYTE), true, 0);
! Node* p = basic_plus_adr(str, str, value_offset);
! Node* load = access_load_at(str, p, value_field_type, value_type, T_OBJECT,
! IN_HEAP | C2_CONTROL_DEPENDENT_LOAD);
// String.value field is known to be @Stable.
if (UseImplicitStableValues) {
load = cast_array_to_stable(load, value_type);
}
return load;
***************
void GraphKit::store_String_value(Node* ctrl, Node* str, Node* value) {
int value_offset = java_lang_String::value_offset_in_bytes();
const TypeInstPtr* string_type = TypeInstPtr::make(TypePtr::NotNull, C->env()->String_klass(),
false, NULL, 0);
const TypePtr* value_field_type = string_type->add_offset(value_offset);
! store_oop_to_object(ctrl, str, basic_plus_adr(str, value_offset), value_field_type,
! value, TypeAryPtr::BYTES, T_OBJECT, MemNode::unordered);
}
void GraphKit::store_String_coder(Node* ctrl, Node* str, Node* value) {
int coder_offset = java_lang_String::coder_offset_in_bytes();
const TypeInstPtr* string_type = TypeInstPtr::make(TypePtr::NotNull, C->env()->String_klass(),
void GraphKit::store_String_value(Node* ctrl, Node* str, Node* value) {
int value_offset = java_lang_String::value_offset_in_bytes();
const TypeInstPtr* string_type = TypeInstPtr::make(TypePtr::NotNull, C->env()->String_klass(),
false, NULL, 0);
const TypePtr* value_field_type = string_type->add_offset(value_offset);
! access_store_at(ctrl, str, basic_plus_adr(str, value_offset), value_field_type,
! value, TypeAryPtr::BYTES, T_OBJECT, IN_HEAP);
}
void GraphKit::store_String_coder(Node* ctrl, Node* str, Node* value) {
int coder_offset = java_lang_String::coder_offset_in_bytes();
const TypeInstPtr* string_type = TypeInstPtr::make(TypePtr::NotNull, C->env()->String_klass(),
< prev index next >