--- old/src/cpu/aarch64/vm/c1_CodeStubs_aarch64.cpp 2017-09-21 10:12:16.317253491 -0400 +++ new/src/cpu/aarch64/vm/c1_CodeStubs_aarch64.cpp 2017-09-21 10:12:16.052243378 -0400 @@ -350,42 +350,4 @@ __ b(_continuation); } - -///////////////////////////////////////////////////////////////////////////// -#if INCLUDE_ALL_GCS - -void G1PreBarrierStub::emit_code(LIR_Assembler* ce) { - // At this point we know that marking is in progress. - // If do_load() is true then we have to emit the - // load of the previous value; otherwise it has already - // been loaded into _pre_val. - - __ bind(_entry); - assert(pre_val()->is_register(), "Precondition."); - - Register pre_val_reg = pre_val()->as_register(); - - if (do_load()) { - ce->mem2reg(addr(), pre_val(), T_OBJECT, patch_code(), info(), false /*wide*/, false /*unaligned*/); - } - __ cbz(pre_val_reg, _continuation); - ce->store_parameter(pre_val()->as_register(), 0); - __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::g1_pre_barrier_slow_id))); - __ b(_continuation); -} - -void G1PostBarrierStub::emit_code(LIR_Assembler* ce) { - __ bind(_entry); - assert(addr()->is_register(), "Precondition."); - assert(new_val()->is_register(), "Precondition."); - Register new_val_reg = new_val()->as_register(); - __ cbz(new_val_reg, _continuation); - ce->store_parameter(addr()->as_pointer_register(), 0); - __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::g1_post_barrier_slow_id))); - __ b(_continuation); -} - -#endif // INCLUDE_ALL_GCS -///////////////////////////////////////////////////////////////////////////// - #undef __ --- old/src/cpu/aarch64/vm/c1_LIRAssembler_aarch64.cpp 2017-09-21 10:12:17.588301994 -0400 +++ new/src/cpu/aarch64/vm/c1_LIRAssembler_aarch64.cpp 2017-09-21 10:12:17.324291920 -0400 @@ -1595,7 +1595,16 @@ void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) { assert(VM_Version::supports_cx8(), "wrong machine"); - Register addr = as_reg(op->addr()); + Register addr; + if (op->addr()->is_register()) { + addr = as_reg(op->addr()); + } else { + assert(op->addr()->is_address(), "what else?"); + LIR_Address* addr_ptr = op->addr()->as_address_ptr(); + assert(addr_ptr->disp() == 0, "need 0 disp"); + assert(addr_ptr->index() == LIR_OprDesc::illegalOpr(), "need 0 index"); + addr = as_reg(addr_ptr->base()); + } Register newval = as_reg(op->new_value()); Register cmpval = as_reg(op->cmp_value()); Label succeed, fail, around; --- old/src/cpu/aarch64/vm/c1_LIRGenerator_aarch64.cpp 2017-09-21 10:12:18.935353397 -0400 +++ new/src/cpu/aarch64/vm/c1_LIRGenerator_aarch64.cpp 2017-09-21 10:12:18.673343399 -0400 @@ -144,8 +144,22 @@ // accumulate fixed displacements if (index->is_constant()) { - large_disp += (intx)(index->as_constant_ptr()->as_jint()) << shift; - index = LIR_OprFact::illegalOpr; + LIR_Const *constant = index->as_constant_ptr(); + if (constant->type() == T_INT) { + large_disp += index->as_jint() << shift; + } else { + assert(constant->type() == T_LONG, "should be"); + jlong c = index->as_jlong() << shift; + if ((jlong)((jint)c) == c) { + large_disp += c; + index = LIR_OprFact::illegalOpr; + } else { + LIR_Opr tmp = new_register(T_LONG); + __ move(index, tmp); + index = tmp; + // apply shift and displacement below + } + } } if (index->is_register()) { @@ -183,9 +197,8 @@ } } - LIR_Address* LIRGenerator::emit_array_address(LIR_Opr array_opr, LIR_Opr index_opr, - BasicType type, bool needs_card_mark) { + BasicType type) { int offset_in_bytes = arrayOopDesc::base_offset_in_bytes(type); int elem_size = type2aelembytes(type); int shift = exact_log2(elem_size); @@ -206,16 +219,7 @@ LIR_Address::scale(type), offset_in_bytes, type); } - if (needs_card_mark) { - // This store will need a precise card mark, so go ahead and - // compute the full adddres instead of computing once for the - // store and again for the card mark. - LIR_Opr tmp = new_pointer_register(); - __ leal(LIR_OprFact::address(addr), tmp); - return new LIR_Address(tmp, type); - } else { - return addr; - } + return addr; } LIR_Opr LIRGenerator::load_immediate(int x, BasicType type) { @@ -305,87 +309,17 @@ __ store(item, new LIR_Address(FrameMap::sp_opr, in_bytes(offset_from_sp), type)); } -//---------------------------------------------------------------------- -// visitor functions -//---------------------------------------------------------------------- - - -void LIRGenerator::do_StoreIndexed(StoreIndexed* x) { - assert(x->is_pinned(),""); - bool needs_range_check = x->compute_needs_range_check(); - bool use_length = x->length() != NULL; - bool obj_store = x->elt_type() == T_ARRAY || x->elt_type() == T_OBJECT; - bool needs_store_check = obj_store && (x->value()->as_Constant() == NULL || - !get_jobject_constant(x->value())->is_null_object() || - x->should_profile()); - - LIRItem array(x->array(), this); - LIRItem index(x->index(), this); - LIRItem value(x->value(), this); - LIRItem length(this); - - array.load_item(); - index.load_nonconstant(); - - if (use_length && needs_range_check) { - length.set_instruction(x->length()); - length.load_item(); - - } - if (needs_store_check || x->check_boolean()) { - value.load_item(); - } else { - value.load_for_store(x->elt_type()); - } - - set_no_result(x); - - // the CodeEmitInfo must be duplicated for each different - // LIR-instruction because spilling can occur anywhere between two - // instructions and so the debug information must be different - CodeEmitInfo* range_check_info = state_for(x); - CodeEmitInfo* null_check_info = NULL; - if (x->needs_null_check()) { - null_check_info = new CodeEmitInfo(range_check_info); - } - - // emit array address setup early so it schedules better - // FIXME? No harm in this on aarch64, and it might help - LIR_Address* array_addr = emit_array_address(array.result(), index.result(), x->elt_type(), obj_store); - - if (GenerateRangeChecks && needs_range_check) { - if (use_length) { - __ cmp(lir_cond_belowEqual, length.result(), index.result()); - __ branch(lir_cond_belowEqual, T_INT, new RangeCheckStub(range_check_info, index.result())); - } else { - array_range_check(array.result(), index.result(), null_check_info, range_check_info); - // range_check also does the null check - null_check_info = NULL; - } - } - - if (GenerateArrayStoreCheck && needs_store_check) { +void LIRGenerator::array_store_check(LIR_Opr value, LIR_Opr array, CodeEmitInfo* store_check_info, ciMethod* profiled_method, int profiled_bci) { LIR_Opr tmp1 = new_register(objectType); LIR_Opr tmp2 = new_register(objectType); LIR_Opr tmp3 = new_register(objectType); - - CodeEmitInfo* store_check_info = new CodeEmitInfo(range_check_info); - __ store_check(value.result(), array.result(), tmp1, tmp2, tmp3, store_check_info, x->profiled_method(), x->profiled_bci()); - } - - if (obj_store) { - // Needs GC write barriers. - pre_barrier(LIR_OprFact::address(array_addr), LIR_OprFact::illegalOpr /* pre_val */, - true /* do_load */, false /* patch */, NULL); - __ move(value.result(), array_addr, null_check_info); - // Seems to be a precise - post_barrier(LIR_OprFact::address(array_addr), value.result()); - } else { - LIR_Opr result = maybe_mask_boolean(x, array.result(), value.result(), null_check_info); - __ move(result, array_addr, null_check_info); - } + __ store_check(value, array, tmp1, tmp2, tmp3, store_check_info, profiled_method, profiled_bci); } +//---------------------------------------------------------------------- +// visitor functions +//---------------------------------------------------------------------- + void LIRGenerator::do_MonitorEnter(MonitorEnter* x) { assert(x->is_pinned(),""); LIRItem obj(x->obj(), this); @@ -771,76 +705,42 @@ } } -void LIRGenerator::do_CompareAndSwap(Intrinsic* x, ValueType* type) { - assert(x->number_of_arguments() == 4, "wrong type"); - LIRItem obj (x->argument_at(0), this); // object - LIRItem offset(x->argument_at(1), this); // offset of field - LIRItem cmp (x->argument_at(2), this); // value to compare with field - LIRItem val (x->argument_at(3), this); // replace field with val if matches cmp - - assert(obj.type()->tag() == objectTag, "invalid type"); - - // In 64bit the type can be long, sparc doesn't have this assert - // assert(offset.type()->tag() == intTag, "invalid type"); - - assert(cmp.type()->tag() == type->tag(), "invalid type"); - assert(val.type()->tag() == type->tag(), "invalid type"); - - // get address of field - obj.load_item(); - offset.load_nonconstant(); - val.load_item(); - cmp.load_item(); - - LIR_Address* a; - if(offset.result()->is_constant()) { - jlong c = offset.result()->as_jlong(); - if ((jlong)((jint)c) == c) { - a = new LIR_Address(obj.result(), - (jint)c, - as_BasicType(type)); - } else { - LIR_Opr tmp = new_register(T_LONG); - __ move(offset.result(), tmp); - a = new LIR_Address(obj.result(), - tmp, - as_BasicType(type)); - } - } else { - a = new LIR_Address(obj.result(), - offset.result(), - 0, - as_BasicType(type)); - } - LIR_Opr addr = new_pointer_register(); - __ leal(LIR_OprFact::address(a), addr); - - if (type == objectType) { // Write-barrier needed for Object fields. - // Do the pre-write barrier, if any. - pre_barrier(addr, LIR_OprFact::illegalOpr /* pre_val */, - true /* do_load */, false /* patch */, NULL); - } - - LIR_Opr result = rlock_result(x); - +LIR_Opr LIRGenerator::cas(BasicType type, LIR_Opr addr, LIRItem& cmp_value, LIRItem& new_value) { LIR_Opr ill = LIR_OprFact::illegalOpr; // for convenience - if (type == objectType) - __ cas_obj(addr, cmp.result(), val.result(), new_register(T_INT), new_register(T_INT), - result); - else if (type == intType) - __ cas_int(addr, cmp.result(), val.result(), ill, ill); - else if (type == longType) - __ cas_long(addr, cmp.result(), val.result(), ill, ill); - else { + new_value.load_item(); + cmp_value.load_item(); + LIR_Opr result = new_register(T_INT); + if (type == T_OBJECT || type == T_ARRAY) { + __ cas_obj(addr, cmp_value.result(), new_value.result(), new_register(T_INT), new_register(T_INT), result); + } else if (type == T_INT) { + __ cas_int(addr->as_address_ptr()->base(), cmp_value.result(), new_value.result(), ill, ill); + } else if (type == T_LONG) { + __ cas_long(addr->as_address_ptr()->base(), cmp_value.result(), new_value.result(), ill, ill); + } else { ShouldNotReachHere(); + Unimplemented(); } - __ logical_xor(FrameMap::r8_opr, LIR_OprFact::intConst(1), result); + return result; +} - if (type == objectType) { // Write-barrier needed for Object fields. - // Seems to be precise - post_barrier(addr, val.result()); - } +LIR_Opr LIRGenerator::swap(BasicType type, LIR_Opr addr, LIRItem& value) { + bool is_oop = type == T_OBJECT || type == T_ARRAY; + LIR_Opr result = new_register(type); + value.load_item(); + assert(type == T_INT || is_oop LP64_ONLY( || type == T_LONG ), "unexpected type"); + LIR_Opr tmp = new_register(T_INT); + __ xchg(addr, value.result(), result, tmp); + return result; +} + +LIR_Opr LIRGenerator::add(BasicType type, LIR_Opr addr, LIRItem& value) { + LIR_Opr result = new_register(type); + value.load_item(); + assert(type == T_INT LP64_ONLY( || type == T_LONG ), "unexpected type"); + LIR_Opr tmp = new_register(T_INT); + __ xadd(addr, value.result(), result, tmp); + return result; } void LIRGenerator::do_MathIntrinsic(Intrinsic* x) { @@ -1359,84 +1259,3 @@ __ volatile_load_mem_reg(address, result, info); } - -void LIRGenerator::get_Object_unsafe(LIR_Opr dst, LIR_Opr src, LIR_Opr offset, - BasicType type, bool is_volatile) { - LIR_Address* addr = new LIR_Address(src, offset, type); - __ load(addr, dst); -} - - -void LIRGenerator::put_Object_unsafe(LIR_Opr src, LIR_Opr offset, LIR_Opr data, - BasicType type, bool is_volatile) { - LIR_Address* addr = new LIR_Address(src, offset, type); - bool is_obj = (type == T_ARRAY || type == T_OBJECT); - if (is_obj) { - // Do the pre-write barrier, if any. - pre_barrier(LIR_OprFact::address(addr), LIR_OprFact::illegalOpr /* pre_val */, - true /* do_load */, false /* patch */, NULL); - __ move(data, addr); - assert(src->is_register(), "must be register"); - // Seems to be a precise address - post_barrier(LIR_OprFact::address(addr), data); - } else { - __ move(data, addr); - } -} - -void LIRGenerator::do_UnsafeGetAndSetObject(UnsafeGetAndSetObject* x) { - BasicType type = x->basic_type(); - LIRItem src(x->object(), this); - LIRItem off(x->offset(), this); - LIRItem value(x->value(), this); - - src.load_item(); - off.load_nonconstant(); - - // We can cope with a constant increment in an xadd - if (! (x->is_add() - && value.is_constant() - && can_inline_as_constant(x->value()))) { - value.load_item(); - } - - LIR_Opr dst = rlock_result(x, type); - LIR_Opr data = value.result(); - bool is_obj = (type == T_ARRAY || type == T_OBJECT); - LIR_Opr offset = off.result(); - - if (data == dst) { - LIR_Opr tmp = new_register(data->type()); - __ move(data, tmp); - data = tmp; - } - - LIR_Address* addr; - if (offset->is_constant()) { - jlong l = offset->as_jlong(); - assert((jlong)((jint)l) == l, "offset too large for constant"); - jint c = (jint)l; - addr = new LIR_Address(src.result(), c, type); - } else { - addr = new LIR_Address(src.result(), offset, type); - } - - LIR_Opr tmp = new_register(T_INT); - LIR_Opr ptr = LIR_OprFact::illegalOpr; - - if (x->is_add()) { - __ xadd(LIR_OprFact::address(addr), data, dst, tmp); - } else { - if (is_obj) { - // Do the pre-write barrier, if any. - ptr = new_pointer_register(); - __ add(src.result(), off.result(), ptr); - pre_barrier(ptr, LIR_OprFact::illegalOpr /* pre_val */, - true /* do_load */, false /* patch */, NULL); - } - __ xchg(LIR_OprFact::address(addr), data, dst, tmp); - if (is_obj) { - post_barrier(ptr, data); - } - } -} --- old/src/cpu/aarch64/vm/c1_MacroAssembler_aarch64.cpp 2017-09-21 10:12:20.229402778 -0400 +++ new/src/cpu/aarch64/vm/c1_MacroAssembler_aarch64.cpp 2017-09-21 10:12:19.965392704 -0400 @@ -358,6 +358,16 @@ void C1_MacroAssembler::verified_entry() { } +void C1_MacroAssembler::load_parameter(int offset_in_words, Register reg) { + // rbp, + 0: link + // + 1: return address + // + 2: argument with offset 0 + // + 3: argument with offset 1 + // + 4: ... + + ldr(reg, Address(rfp, (offset_in_words + 2) * BytesPerWord)); +} + #ifndef PRODUCT void C1_MacroAssembler::verify_stack_oop(int stack_offset) { --- old/src/cpu/aarch64/vm/c1_MacroAssembler_aarch64.hpp 2017-09-21 10:12:21.461449793 -0400 +++ new/src/cpu/aarch64/vm/c1_MacroAssembler_aarch64.hpp 2017-09-21 10:12:21.201439871 -0400 @@ -109,4 +109,6 @@ // This platform only uses signal-based null checks. The Label is not needed. void null_check(Register r, Label *Lnull = NULL) { MacroAssembler::null_check(r); } + void load_parameter(int offset_in_words, Register reg); + #endif // CPU_AARCH64_VM_C1_MACROASSEMBLER_AARCH64_HPP --- old/src/cpu/aarch64/vm/c1_Runtime1_aarch64.cpp 2017-09-21 10:12:22.682496388 -0400 +++ new/src/cpu/aarch64/vm/c1_Runtime1_aarch64.cpp 2017-09-21 10:12:22.417486275 -0400 @@ -172,31 +172,32 @@ ~StubFrame(); };; +void StubAssembler::prologue(const char* name, bool must_gc_arguments) { + set_info(name, must_gc_arguments); + enter(); +} + +void StubAssembler::epilogue() { + leave(); + ret(lr); +} #define __ _sasm-> StubFrame::StubFrame(StubAssembler* sasm, const char* name, bool must_gc_arguments) { _sasm = sasm; - __ set_info(name, must_gc_arguments); - __ enter(); + __ prologue(name, must_gc_arguments); } // load parameters that were stored with LIR_Assembler::store_parameter // Note: offsets for store_parameter and load_argument must match void StubFrame::load_argument(int offset_in_words, Register reg) { - // rbp, + 0: link - // + 1: return address - // + 2: argument with offset 0 - // + 3: argument with offset 1 - // + 4: ... - - __ ldr(reg, Address(rfp, (offset_in_words + 2) * BytesPerWord)); + __ load_parameter(offset_in_words, reg); } StubFrame::~StubFrame() { - __ leave(); - __ ret(lr); + __ epilogue(); } #undef __ @@ -1152,139 +1153,6 @@ } break; -#if INCLUDE_ALL_GCS - - case g1_pre_barrier_slow_id: - { - StubFrame f(sasm, "g1_pre_barrier", dont_gc_arguments); - // arg0 : previous value of memory - - BarrierSet* bs = Universe::heap()->barrier_set(); - if (bs->kind() != BarrierSet::G1BarrierSet) { - __ mov(r0, (int)id); - __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, unimplemented_entry), r0); - __ should_not_reach_here(); - break; - } - - const Register pre_val = r0; - const Register thread = rthread; - const Register tmp = rscratch1; - - Address in_progress(thread, in_bytes(JavaThread::satb_mark_queue_offset() + - SATBMarkQueue::byte_offset_of_active())); - - Address queue_index(thread, in_bytes(JavaThread::satb_mark_queue_offset() + - SATBMarkQueue::byte_offset_of_index())); - Address buffer(thread, in_bytes(JavaThread::satb_mark_queue_offset() + - SATBMarkQueue::byte_offset_of_buf())); - - Label done; - Label runtime; - - // Is marking still active? - if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) { - __ ldrw(tmp, in_progress); - } else { - assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption"); - __ ldrb(tmp, in_progress); - } - __ cbzw(tmp, done); - - // Can we store original value in the thread's buffer? - __ ldr(tmp, queue_index); - __ cbz(tmp, runtime); - - __ sub(tmp, tmp, wordSize); - __ str(tmp, queue_index); - __ ldr(rscratch2, buffer); - __ add(tmp, tmp, rscratch2); - f.load_argument(0, rscratch2); - __ str(rscratch2, Address(tmp, 0)); - __ b(done); - - __ bind(runtime); - __ push_call_clobbered_registers(); - f.load_argument(0, pre_val); - __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), pre_val, thread); - __ pop_call_clobbered_registers(); - __ bind(done); - } - break; - case g1_post_barrier_slow_id: - { - StubFrame f(sasm, "g1_post_barrier", dont_gc_arguments); - - // arg0: store_address - Address store_addr(rfp, 2*BytesPerWord); - - BarrierSet* bs = Universe::heap()->barrier_set(); - CardTableModRefBS* ctbs = barrier_set_cast(bs); - CardTable* ct = ctbs->card_table(); - assert(sizeof(*ct->byte_map_base()) == sizeof(jbyte), "adjust this code"); - - Label done; - Label runtime; - - // At this point we know new_value is non-NULL and the new_value crosses regions. - // Must check to see if card is already dirty - - const Register thread = rthread; - - Address queue_index(thread, in_bytes(JavaThread::dirty_card_queue_offset() + - DirtyCardQueue::byte_offset_of_index())); - Address buffer(thread, in_bytes(JavaThread::dirty_card_queue_offset() + - DirtyCardQueue::byte_offset_of_buf())); - - const Register card_offset = rscratch2; - // LR is free here, so we can use it to hold the byte_map_base. - const Register byte_map_base = lr; - - assert_different_registers(card_offset, byte_map_base, rscratch1); - - f.load_argument(0, card_offset); - __ lsr(card_offset, card_offset, CardTable::card_shift); - __ load_byte_map_base(byte_map_base); - __ ldrb(rscratch1, Address(byte_map_base, card_offset)); - __ cmpw(rscratch1, (int)G1CardTable::g1_young_card_val()); - __ br(Assembler::EQ, done); - - assert((int)CardTable::dirty_card_val() == 0, "must be 0"); - - __ membar(Assembler::StoreLoad); - __ ldrb(rscratch1, Address(byte_map_base, card_offset)); - __ cbzw(rscratch1, done); - - // storing region crossing non-NULL, card is clean. - // dirty card and log. - __ strb(zr, Address(byte_map_base, card_offset)); - - // Convert card offset into an address in card_addr - Register card_addr = card_offset; - __ add(card_addr, byte_map_base, card_addr); - - __ ldr(rscratch1, queue_index); - __ cbz(rscratch1, runtime); - __ sub(rscratch1, rscratch1, wordSize); - __ str(rscratch1, queue_index); - - // Reuse LR to hold buffer_addr - const Register buffer_addr = lr; - - __ ldr(buffer_addr, buffer); - __ str(card_addr, Address(buffer_addr, rscratch1)); - __ b(done); - - __ bind(runtime); - __ push_call_clobbered_registers(); - __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_post), card_addr, thread); - __ pop_call_clobbered_registers(); - __ bind(done); - - } - break; -#endif - case predicate_failed_trap_id: { StubFrame f(sasm, "predicate_failed_trap", dont_gc_arguments); --- old/src/cpu/aarch64/vm/interp_masm_aarch64.cpp 2017-09-21 10:12:23.964545311 -0400 +++ new/src/cpu/aarch64/vm/interp_masm_aarch64.cpp 2017-09-21 10:12:23.702535312 -0400 @@ -24,6 +24,7 @@ */ #include "precompiled.hpp" +#include "gc/shared/barrierSetCodeGen.hpp" #include "interp_masm_aarch64.hpp" #include "interpreter/interpreter.hpp" #include "interpreter/interpreterRuntime.hpp" @@ -275,7 +276,8 @@ resolve_oop_handle(result); // Add in the index add(result, result, tmp); - load_heap_oop(result, Address(result, arrayOopDesc::base_offset_in_bytes(T_OBJECT))); + BarrierSetCodeGen *code_gen = BarrierSet::barrier_set()->code_gen(); + code_gen->load_at(this, ACCESS_IN_HEAP, T_OBJECT, result, Address(result, arrayOopDesc::base_offset_in_bytes(T_OBJECT)), /*tmp1*/ noreg, /*tmp_thread*/ noreg); } void InterpreterMacroAssembler::load_resolved_klass_at_offset( --- old/src/cpu/aarch64/vm/macroAssembler_aarch64.cpp 2017-09-21 10:12:25.260594768 -0400 +++ new/src/cpu/aarch64/vm/macroAssembler_aarch64.cpp 2017-09-21 10:12:24.995584655 -0400 @@ -29,6 +29,7 @@ #include "asm/assembler.hpp" #include "asm/assembler.inline.hpp" #include "gc/shared/cardTable.hpp" +#include "gc/shared/barrierSetCodeGen.hpp" #include "gc/shared/cardTableModRefBS.hpp" #include "interpreter/interpreter.hpp" #include "compiler/disassembler.hpp" @@ -2002,6 +2003,28 @@ } #endif +void MacroAssembler::resolve_jobject(Register value, Register thread, Register tmp) { + BarrierSetCodeGen *code_gen = Universe::heap()->barrier_set()->code_gen(); + Label done, not_weak; + cbz(value, done); // Use NULL as-is. + + STATIC_ASSERT(JNIHandles::weak_tag_mask == 1u); + tbz(r0, 0, not_weak); // Test for jweak tag. + + // Resolve jweak. + code_gen->load_at(this, ACCESS_IN_ROOT | ACCESS_ON_PHANTOM_OOP_REF, T_OBJECT, + value, Address(value, -JNIHandles::weak_tag_value), tmp, thread); + verify_oop(value); + b(done); + + bind(not_weak); + // Resolve (untagged) jobject. + code_gen->load_at(this, ACCESS_IN_ROOT | ACCESS_ON_STRONG_OOP_REF, T_OBJECT, + value, Address(value, 0), tmp, thread); + verify_oop(value); + bind(done); +} + void MacroAssembler::stop(const char* msg) { address ip = pc(); pusha(); @@ -3235,43 +3258,6 @@ cmp(src1, rscratch1); } -void MacroAssembler::store_check(Register obj, Address dst) { - store_check(obj); -} - -void MacroAssembler::store_check(Register obj) { - // Does a store check for the oop in register obj. The content of - // register obj is destroyed afterwards. - - BarrierSet* bs = Universe::heap()->barrier_set(); - assert(bs->kind() == BarrierSet::CardTableModRef, - "Wrong barrier set kind"); - - CardTableModRefBS* ctbs = barrier_set_cast(bs); - CardTable* ct = ctbs->card_table(); - assert(sizeof(*ct->byte_map_base()) == sizeof(jbyte), "adjust this code"); - - lsr(obj, obj, CardTable::card_shift); - - assert(CardTable::dirty_card_val() == 0, "must be"); - - load_byte_map_base(rscratch1); - - if (UseCondCardMark) { - Label L_already_dirty; - membar(StoreLoad); - ldrb(rscratch2, Address(obj, rscratch1)); - cbz(rscratch2, L_already_dirty); - strb(zr, Address(obj, rscratch1)); - bind(L_already_dirty); - } else { - if (UseConcMarkSweepGC && CMSPrecleaningEnabled) { - membar(StoreStore); - } - strb(zr, Address(obj, rscratch1)); - } -} - void MacroAssembler::load_klass(Register dst, Register src) { if (UseCompressedClassPointers) { ldrw(dst, Address(src, oopDesc::klass_offset_in_bytes())); @@ -3631,190 +3617,6 @@ str(zr, dst); } -#if INCLUDE_ALL_GCS -/* - * g1_write_barrier_pre -- G1GC pre-write barrier for store of new_val at - * store_addr. - * - * Allocates rscratch1 - */ -void MacroAssembler::g1_write_barrier_pre(Register obj, - Register pre_val, - Register thread, - Register tmp, - bool tosca_live, - bool expand_call) { - // If expand_call is true then we expand the call_VM_leaf macro - // directly to skip generating the check by - // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp. - - assert(thread == rthread, "must be"); - - Label done; - Label runtime; - - assert_different_registers(obj, pre_val, tmp, rscratch1); - assert(pre_val != noreg && tmp != noreg, "expecting a register"); - - Address in_progress(thread, in_bytes(JavaThread::satb_mark_queue_offset() + - SATBMarkQueue::byte_offset_of_active())); - Address index(thread, in_bytes(JavaThread::satb_mark_queue_offset() + - SATBMarkQueue::byte_offset_of_index())); - Address buffer(thread, in_bytes(JavaThread::satb_mark_queue_offset() + - SATBMarkQueue::byte_offset_of_buf())); - - - // Is marking active? - if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) { - ldrw(tmp, in_progress); - } else { - assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption"); - ldrb(tmp, in_progress); - } - cbzw(tmp, done); - - // Do we need to load the previous value? - if (obj != noreg) { - load_heap_oop(pre_val, Address(obj, 0)); - } - - // Is the previous value null? - cbz(pre_val, done); - - // Can we store original value in the thread's buffer? - // Is index == 0? - // (The index field is typed as size_t.) - - ldr(tmp, index); // tmp := *index_adr - cbz(tmp, runtime); // tmp == 0? - // If yes, goto runtime - - sub(tmp, tmp, wordSize); // tmp := tmp - wordSize - str(tmp, index); // *index_adr := tmp - ldr(rscratch1, buffer); - add(tmp, tmp, rscratch1); // tmp := tmp + *buffer_adr - - // Record the previous value - str(pre_val, Address(tmp, 0)); - b(done); - - bind(runtime); - // save the live input values - push(r0->bit(tosca_live) | obj->bit(obj != noreg) | pre_val->bit(true), sp); - - // Calling the runtime using the regular call_VM_leaf mechanism generates - // code (generated by InterpreterMacroAssember::call_VM_leaf_base) - // that checks that the *(rfp+frame::interpreter_frame_last_sp) == NULL. - // - // If we care generating the pre-barrier without a frame (e.g. in the - // intrinsified Reference.get() routine) then ebp might be pointing to - // the caller frame and so this check will most likely fail at runtime. - // - // Expanding the call directly bypasses the generation of the check. - // So when we do not have have a full interpreter frame on the stack - // expand_call should be passed true. - - if (expand_call) { - assert(pre_val != c_rarg1, "smashed arg"); - pass_arg1(this, thread); - pass_arg0(this, pre_val); - MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), 2); - } else { - call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), pre_val, thread); - } - - pop(r0->bit(tosca_live) | obj->bit(obj != noreg) | pre_val->bit(true), sp); - - bind(done); -} - -/* - * g1_write_barrier_post -- G1GC post-write barrier for store of new_val at - * store_addr - * - * Allocates rscratch1 - */ -void MacroAssembler::g1_write_barrier_post(Register store_addr, - Register new_val, - Register thread, - Register tmp, - Register tmp2) { - assert(thread == rthread, "must be"); - assert_different_registers(store_addr, new_val, thread, tmp, tmp2, - rscratch1); - assert(store_addr != noreg && new_val != noreg && tmp != noreg - && tmp2 != noreg, "expecting a register"); - - Address queue_index(thread, in_bytes(JavaThread::dirty_card_queue_offset() + - DirtyCardQueue::byte_offset_of_index())); - Address buffer(thread, in_bytes(JavaThread::dirty_card_queue_offset() + - DirtyCardQueue::byte_offset_of_buf())); - - BarrierSet* bs = Universe::heap()->barrier_set(); - CardTableModRefBS* ctbs = barrier_set_cast(bs); - CardTable* ct = ctbs->card_table(); - assert(sizeof(*ct->byte_map_base()) == sizeof(jbyte), "adjust this code"); - - Label done; - Label runtime; - - // Does store cross heap regions? - - eor(tmp, store_addr, new_val); - lsr(tmp, tmp, HeapRegion::LogOfHRGrainBytes); - cbz(tmp, done); - - // crosses regions, storing NULL? - - cbz(new_val, done); - - // storing region crossing non-NULL, is card already dirty? - - ExternalAddress cardtable((address) ct->byte_map_base()); - assert(sizeof(*ct->byte_map_base()) == sizeof(jbyte), "adjust this code"); - const Register card_addr = tmp; - - lsr(card_addr, store_addr, CardTable::card_shift); - - // get the address of the card - load_byte_map_base(tmp2); - add(card_addr, card_addr, tmp2); - ldrb(tmp2, Address(card_addr)); - cmpw(tmp2, (int)G1CardTable::g1_young_card_val()); - br(Assembler::EQ, done); - - assert((int)CardTable::dirty_card_val() == 0, "must be 0"); - - membar(Assembler::StoreLoad); - - ldrb(tmp2, Address(card_addr)); - cbzw(tmp2, done); - - // storing a region crossing, non-NULL oop, card is clean. - // dirty card and log. - - strb(zr, Address(card_addr)); - - ldr(rscratch1, queue_index); - cbz(rscratch1, runtime); - sub(rscratch1, rscratch1, wordSize); - str(rscratch1, queue_index); - - ldr(tmp2, buffer); - str(card_addr, Address(tmp2, rscratch1)); - b(done); - - bind(runtime); - // save the live input values - push(store_addr->bit(true) | new_val->bit(true), sp); - call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_post), card_addr, thread); - pop(store_addr->bit(true) | new_val->bit(true), sp); - - bind(done); -} - -#endif // INCLUDE_ALL_GCS - Address MacroAssembler::allocate_metadata_address(Metadata* obj) { assert(oop_recorder() != NULL, "this assembler needs a Recorder"); int index = oop_recorder()->allocate_metadata_index(obj); @@ -4241,8 +4043,7 @@ void MacroAssembler::load_byte_map_base(Register reg) { jbyte *byte_map_base = - ((CardTableModRefBS*)(Universe::heap()->barrier_set()))->byte_map_base; - + ((CardTableModRefBS*)(Universe::heap()->barrier_set()))->card_table()->byte_map_base(); if (is_valid_AArch64_address((address)byte_map_base)) { // Strictly speaking the byte_map_base isn't an address at all, // and it might even be negative. --- old/src/cpu/aarch64/vm/macroAssembler_aarch64.hpp 2017-09-21 10:12:26.688649262 -0400 +++ new/src/cpu/aarch64/vm/macroAssembler_aarch64.hpp 2017-09-21 10:12:26.425639225 -0400 @@ -768,6 +768,8 @@ void store_check(Register obj); // store check for obj - register is destroyed afterwards void store_check(Register obj, Address dst); // same as above, dst is exact store location (reg. is destroyed) + void resolve_jobject(Register value, Register thread, Register tmp); + #if INCLUDE_ALL_GCS void g1_write_barrier_pre(Register obj, --- old/src/cpu/aarch64/vm/sharedRuntime_aarch64.cpp 2017-09-21 10:12:27.962697879 -0400 +++ new/src/cpu/aarch64/vm/sharedRuntime_aarch64.cpp 2017-09-21 10:12:27.699687843 -0400 @@ -2055,29 +2055,7 @@ // Unbox oop result, e.g. JNIHandles::resolve result. if (ret_type == T_OBJECT || ret_type == T_ARRAY) { - Label done, not_weak; - __ cbz(r0, done); // Use NULL as-is. - STATIC_ASSERT(JNIHandles::weak_tag_mask == 1u); - __ tbz(r0, 0, not_weak); // Test for jweak tag. - // Resolve jweak. - __ ldr(r0, Address(r0, -JNIHandles::weak_tag_value)); - __ verify_oop(r0); -#if INCLUDE_ALL_GCS - if (UseG1GC) { - __ g1_write_barrier_pre(noreg /* obj */, - r0 /* pre_val */, - rthread /* thread */, - rscratch2 /* tmp */, - true /* tosca_live */, - true /* expand_call */); - } -#endif // INCLUDE_ALL_GCS - __ b(done); - __ bind(not_weak); - // Resolve (untagged) jobject. - __ ldr(r0, Address(r0, 0)); - __ verify_oop(r0); - __ bind(done); + __ resolve_jobject(r0, rthread, rscratch2); } if (CheckJNICalls) { --- old/src/cpu/aarch64/vm/stubGenerator_aarch64.cpp 2017-09-21 10:12:29.301748977 -0400 +++ new/src/cpu/aarch64/vm/stubGenerator_aarch64.cpp 2017-09-21 10:12:29.037738903 -0400 @@ -26,6 +26,7 @@ #include "precompiled.hpp" #include "asm/macroAssembler.hpp" #include "asm/macroAssembler.inline.hpp" +#include "gc/shared/barrierSetCodeGen.hpp" #include "gc/shared/cardTable.hpp" #include "gc/shared/cardTableModRefBS.hpp" #include "interpreter/interpreter.hpp" @@ -620,111 +621,6 @@ void array_overlap_test(Label& L_no_overlap, Address::sxtw sf) { __ b(L_no_overlap); } - // Generate code for an array write pre barrier - // - // addr - starting address - // count - element count - // tmp - scratch register - // saved_regs - registers to be saved before calling static_write_ref_array_pre - // - // Callers must specify which registers to preserve in saved_regs. - // Clobbers: r0-r18, v0-v7, v16-v31, except saved_regs. - // - void gen_write_ref_array_pre_barrier(Register addr, Register count, bool dest_uninitialized, RegSet saved_regs) { - BarrierSet* bs = Universe::heap()->barrier_set(); - switch (bs->kind()) { - case BarrierSet::G1BarrierSet: - // With G1, don't generate the call if we statically know that the target in uninitialized - if (!dest_uninitialized) { - __ push(saved_regs, sp); - if (count == c_rarg0) { - if (addr == c_rarg1) { - // exactly backwards!! - __ mov(rscratch1, c_rarg0); - __ mov(c_rarg0, c_rarg1); - __ mov(c_rarg1, rscratch1); - } else { - __ mov(c_rarg1, count); - __ mov(c_rarg0, addr); - } - } else { - __ mov(c_rarg0, addr); - __ mov(c_rarg1, count); - } - __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_pre), 2); - __ pop(saved_regs, sp); - break; - case BarrierSet::CardTableModRef: - break; - default: - ShouldNotReachHere(); - - } - } - } - - // - // Generate code for an array write post barrier - // - // Input: - // start - register containing starting address of destination array - // end - register containing ending address of destination array - // scratch - scratch register - // saved_regs - registers to be saved before calling static_write_ref_array_post - // - // The input registers are overwritten. - // The ending address is inclusive. - // Callers must specify which registers to preserve in saved_regs. - // Clobbers: r0-r18, v0-v7, v16-v31, except saved_regs. - void gen_write_ref_array_post_barrier(Register start, Register end, Register scratch, RegSet saved_regs) { - assert_different_registers(start, end, scratch); - BarrierSet* bs = Universe::heap()->barrier_set(); - switch (bs->kind()) { - case BarrierSet::G1BarrierSet: - - { - __ push(saved_regs, sp); - // must compute element count unless barrier set interface is changed (other platforms supply count) - assert_different_registers(start, end, scratch); - __ lea(scratch, Address(end, BytesPerHeapOop)); - __ sub(scratch, scratch, start); // subtract start to get #bytes - __ lsr(scratch, scratch, LogBytesPerHeapOop); // convert to element count - __ mov(c_rarg0, start); - __ mov(c_rarg1, scratch); - __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_post), 2); - __ pop(saved_regs, sp); - } - break; - case BarrierSet::CardTableModRef: - { - CardTableModRefBS* ctbs = barrier_set_cast(bs); - CardTable* ct = ctbs->card_table(); - assert(sizeof(*ct->byte_map_base()) == sizeof(jbyte), "adjust this code"); - - Label L_loop; - - __ lsr(start, start, CardTable::card_shift); - __ lsr(end, end, CardTable::card_shift); - __ sub(end, end, start); // number of bytes to copy - - const Register count = end; // 'end' register contains bytes count now - __ load_byte_map_base(scratch); - __ add(start, start, scratch); - if (UseConcMarkSweepGC) { - __ membar(__ StoreStore); - } - __ BIND(L_loop); - __ strb(zr, Address(start, count)); - __ subs(count, count, 1); - __ br(Assembler::GE, L_loop); - } - break; - default: - ShouldNotReachHere(); - - } - } - // The inner part of zero_words(). This is the bulk operation, // zeroing words in blocks, possibly using DC ZVA to do it. The // caller is responsible for zeroing the last few words. @@ -1456,20 +1352,33 @@ BLOCK_COMMENT("Entry:"); } + BarrierSetCodeGen *bs = Universe::heap()->barrier_set()->code_gen(); + DecoratorSet decorators = ARRAYCOPY_DISJOINT; + if (dest_uninitialized) { + decorators |= DEST_NOT_INITIALIZED; + } + if (aligned) { + decorators |= ARRAYCOPY_ALIGNED; + } + + bs->arraycopy_prologue(_masm, decorators, is_oop, d, count, saved_reg); + if (is_oop) { - gen_write_ref_array_pre_barrier(d, count, dest_uninitialized, saved_reg); // save regs before copy_memory __ push(RegSet::of(d, count), sp); } copy_memory(aligned, s, d, count, rscratch1, size); + if (is_oop) { __ pop(RegSet::of(d, count), sp); if (VerifyOops) verify_oop_array(size, d, count, r16); __ sub(count, count, 1); // make an inclusive end pointer __ lea(count, Address(d, count, Address::lsl(exact_log2(size)))); - gen_write_ref_array_post_barrier(d, count, rscratch1, RegSet()); } + + bs->arraycopy_epilogue(_masm, decorators, is_oop, d, count, rscratch1, RegSet()); + __ leave(); __ mov(r0, zr); // return 0 __ ret(lr); @@ -1517,8 +1426,17 @@ __ cmp(rscratch1, count, Assembler::LSL, exact_log2(size)); __ br(Assembler::HS, nooverlap_target); + BarrierSetCodeGen *bs = Universe::heap()->barrier_set()->code_gen(); + DecoratorSet decorators = DECORATOR_DEFAULT; + if (dest_uninitialized) { + decorators |= DEST_NOT_INITIALIZED; + } + if (aligned) { + decorators |= ARRAYCOPY_ALIGNED; + } + bs->arraycopy_prologue(_masm, decorators, is_oop, d, count, saved_regs); + if (is_oop) { - gen_write_ref_array_pre_barrier(d, count, dest_uninitialized, saved_regs); // save regs before copy_memory __ push(RegSet::of(d, count), sp); } @@ -1529,8 +1447,8 @@ verify_oop_array(size, d, count, r16); __ sub(count, count, 1); // make an inclusive end pointer __ lea(count, Address(d, count, Address::lsl(exact_log2(size)))); - gen_write_ref_array_post_barrier(d, count, rscratch1, RegSet()); } + bs->arraycopy_epilogue(_masm, decorators, is_oop, d, count, rscratch1, RegSet()); __ leave(); __ mov(r0, zr); // return 0 __ ret(lr); @@ -1871,7 +1789,14 @@ } #endif //ASSERT - gen_write_ref_array_pre_barrier(to, count, dest_uninitialized, wb_pre_saved_regs); + BarrierSetCodeGen *bs = Universe::heap()->barrier_set()->code_gen(); + DecoratorSet decorators = ARRAYCOPY_CONTRAVARIANT | ARRAYCOPY_DISJOINT; + bool is_oop = true; + if (dest_uninitialized) { + decorators |= DEST_NOT_INITIALIZED; + } + + bs->arraycopy_prologue(_masm, decorators, is_oop, to, count, wb_pre_saved_regs); // save the original count __ mov(count_save, count); @@ -1915,7 +1840,7 @@ __ BIND(L_do_card_marks); __ add(to, to, -heapOopSize); // make an inclusive end pointer - gen_write_ref_array_post_barrier(start_to, to, rscratch1, wb_post_saved_regs); + bs->arraycopy_epilogue(_masm, decorators, is_oop, start_to, to, rscratch1, wb_post_saved_regs); __ bind(L_done_pop); __ pop(RegSet::of(r18, r19, r20, r21), sp); --- old/src/cpu/aarch64/vm/templateInterpreterGenerator_aarch64.cpp 2017-09-21 10:12:30.735803701 -0400 +++ new/src/cpu/aarch64/vm/templateInterpreterGenerator_aarch64.cpp 2017-09-21 10:12:30.471793626 -0400 @@ -25,6 +25,7 @@ #include "precompiled.hpp" #include "asm/macroAssembler.hpp" +#include "gc/shared/barrierSetCodeGen.hpp" #include "interpreter/bytecodeHistogram.hpp" #include "interpreter/interpreter.hpp" #include "interpreter/interpreterRuntime.hpp" @@ -875,7 +876,6 @@ // Method entry for java.lang.ref.Reference.get. address TemplateInterpreterGenerator::generate_Reference_get_entry(void) { -#if INCLUDE_ALL_GCS // Code: _aload_0, _getfield, _areturn // parameter size = 1 // @@ -909,43 +909,29 @@ const int referent_offset = java_lang_ref_Reference::referent_offset; guarantee(referent_offset > 0, "referent offset not initialized"); - if (UseG1GC) { - Label slow_path; - const Register local_0 = c_rarg0; - // Check if local 0 != NULL - // If the receiver is null then it is OK to jump to the slow path. - __ ldr(local_0, Address(esp, 0)); - __ cbz(local_0, slow_path); - - // Load the value of the referent field. - const Address field_address(local_0, referent_offset); - __ load_heap_oop(local_0, field_address); - - __ mov(r19, r13); // Move senderSP to a callee-saved register - // Generate the G1 pre-barrier code to log the value of - // the referent field in an SATB buffer. - __ enter(); // g1_write may call runtime - __ g1_write_barrier_pre(noreg /* obj */, - local_0 /* pre_val */, - rthread /* thread */, - rscratch2 /* tmp */, - true /* tosca_live */, - true /* expand_call */); - __ leave(); - // areturn - __ andr(sp, r19, -16); // done with stack - __ ret(lr); + Label slow_path; + const Register local_0 = c_rarg0; + // Check if local 0 != NULL + // If the receiver is null then it is OK to jump to the slow path. + __ ldr(local_0, Address(esp, 0)); + __ cbz(local_0, slow_path); + + __ mov(r19, r13); // Move senderSP to a callee-saved register + + // Load the value of the referent field. + const Address field_address(local_0, referent_offset); + BarrierSetCodeGen *code_gen = Universe::heap()->barrier_set()->code_gen(); + code_gen->load_at(_masm, ACCESS_IN_HEAP | ACCESS_ON_WEAK_OOP_REF, T_OBJECT, local_0, field_address, /*tmp1*/ rscratch2, /*tmp2*/ rscratch1); - // generate a vanilla interpreter entry as the slow path - __ bind(slow_path); - __ jump_to_entry(Interpreter::entry_for_kind(Interpreter::zerolocals)); - return entry; - } -#endif // INCLUDE_ALL_GCS + // areturn + __ andr(sp, r19, -16); // done with stack + __ ret(lr); + + // generate a vanilla interpreter entry as the slow path + __ bind(slow_path); + __ jump_to_entry(Interpreter::entry_for_kind(Interpreter::zerolocals)); + return entry; - // If G1 is not enabled then attempt to go through the accessor entry point - // Reference.get is an accessor - return NULL; } /** @@ -1401,28 +1387,7 @@ __ br(Assembler::NE, no_oop); // Unbox oop result, e.g. JNIHandles::resolve result. __ pop(ltos); - __ cbz(r0, store_result); // Use NULL as-is. - STATIC_ASSERT(JNIHandles::weak_tag_mask == 1u); - __ tbz(r0, 0, not_weak); // Test for jweak tag. - // Resolve jweak. - __ ldr(r0, Address(r0, -JNIHandles::weak_tag_value)); -#if INCLUDE_ALL_GCS - if (UseG1GC) { - __ enter(); // Barrier may call runtime. - __ g1_write_barrier_pre(noreg /* obj */, - r0 /* pre_val */, - rthread /* thread */, - t /* tmp */, - true /* tosca_live */, - true /* expand_call */); - __ leave(); - } -#endif // INCLUDE_ALL_GCS - __ b(store_result); - __ bind(not_weak); - // Resolve (untagged) jobject. - __ ldr(r0, Address(r0, 0)); - __ bind(store_result); + __ resolve_jobject(r0, rthread, t); __ str(r0, Address(rfp, frame::interpreter_frame_oop_temp_offset*wordSize)); // keep stack depth as expected by pushing oop which will eventually be discarded __ push(ltos); --- old/src/cpu/aarch64/vm/templateTable_aarch64.cpp 2017-09-21 10:12:32.043853616 -0400 +++ new/src/cpu/aarch64/vm/templateTable_aarch64.cpp 2017-09-21 10:12:31.781843617 -0400 @@ -25,6 +25,7 @@ #include "precompiled.hpp" #include "asm/macroAssembler.hpp" +#include "gc/shared/barrierSetCodeGen.hpp" #include "interpreter/interpreter.hpp" #include "interpreter/interpreterRuntime.hpp" #include "interpreter/interp_masm.hpp" @@ -141,76 +142,20 @@ // Store an oop (or NULL) at the Address described by obj. // If val == noreg this means store a NULL static void do_oop_store(InterpreterMacroAssembler* _masm, - Address obj, + Address dst, Register val, - BarrierSet::Name barrier, - bool precise) { + DecoratorSet decorators) { assert(val == noreg || val == r0, "parameter is just for looks"); - switch (barrier) { -#if INCLUDE_ALL_GCS - case BarrierSet::G1BarrierSet: - { - // flatten object address if needed - if (obj.index() == noreg && obj.offset() == 0) { - if (obj.base() != r3) { - __ mov(r3, obj.base()); - } - } else { - __ lea(r3, obj); - } - __ g1_write_barrier_pre(r3 /* obj */, - r1 /* pre_val */, - rthread /* thread */, - r10 /* tmp */, - val != noreg /* tosca_live */, - false /* expand_call */); - if (val == noreg) { - __ store_heap_oop_null(Address(r3, 0)); - } else { - // G1 barrier needs uncompressed oop for region cross check. - Register new_val = val; - if (UseCompressedOops) { - new_val = rscratch2; - __ mov(new_val, val); - } - __ store_heap_oop(Address(r3, 0), val); - __ g1_write_barrier_post(r3 /* store_adr */, - new_val /* new_val */, - rthread /* thread */, - r10 /* tmp */, - r1 /* tmp2 */); - } - - } - break; -#endif // INCLUDE_ALL_GCS - case BarrierSet::CardTableModRef: - { - if (val == noreg) { - __ store_heap_oop_null(obj); - } else { - __ store_heap_oop(obj, val); - // flatten object address if needed - if (!precise || (obj.index() == noreg && obj.offset() == 0)) { - __ store_check(obj.base()); - } else { - __ lea(r3, obj); - __ store_check(r3); - } - } - } - break; - case BarrierSet::ModRef: - if (val == noreg) { - __ store_heap_oop_null(obj); - } else { - __ store_heap_oop(obj, val); - } - break; - default : - ShouldNotReachHere(); + BarrierSetCodeGen *code_gen = Universe::heap()->barrier_set()->code_gen(); + code_gen->store_at(_masm, decorators, T_OBJECT, dst, val, /*tmp1*/ r10, /*tmp2*/ r1); +} - } +static void do_oop_load(InterpreterMacroAssembler* _masm, + Address src, + Register dst, + DecoratorSet decorators) { + BarrierSetCodeGen *code_gen = Universe::heap()->barrier_set()->code_gen(); + code_gen->load_at(_masm, decorators, T_OBJECT, dst, src, /*tmp1*/ r10, /*tmp_thread*/ r1); } Address TemplateTable::at_bcp(int offset) { @@ -727,7 +672,10 @@ index_check(r0, r1); // leaves index in r1, kills rscratch1 int s = (UseCompressedOops ? 2 : 3); __ lea(r1, Address(r0, r1, Address::uxtw(s))); - __ load_heap_oop(r0, Address(r1, arrayOopDesc::base_offset_in_bytes(T_OBJECT))); + do_oop_load(_masm, + Address(r1, arrayOopDesc::base_offset_in_bytes(T_OBJECT)), + r0, + ACCESS_IN_HEAP | ACCESS_IN_HEAP_ARRAY); } void TemplateTable::baload() @@ -1055,7 +1003,7 @@ // Get the value we will store __ ldr(r0, at_tos()); // Now store using the appropriate barrier - do_oop_store(_masm, element_address, r0, _bs->kind(), true); + do_oop_store(_masm, element_address, r0, ACCESS_IN_HEAP | ACCESS_IN_HEAP_ARRAY); __ b(done); // Have a NULL in r0, r3=array, r2=index. Store NULL at ary[idx] @@ -1063,7 +1011,7 @@ __ profile_null_seen(r2); // Store a NULL - do_oop_store(_masm, element_address, noreg, _bs->kind(), true); + do_oop_store(_masm, element_address, noreg, ACCESS_IN_HEAP | ACCESS_IN_HEAP_ARRAY); // Pop stack arguments __ bind(done); @@ -2455,7 +2403,7 @@ __ cmp(flags, atos); __ br(Assembler::NE, notObj); // atos - __ load_heap_oop(r0, field); + do_oop_load(_masm, field, r0, ACCESS_IN_HEAP); __ push(atos); if (rc == may_rewrite) { patch_bytecode(Bytecodes::_fast_agetfield, bc, r1); @@ -2698,7 +2646,7 @@ __ pop(atos); if (!is_static) pop_and_check_object(obj); // Store into the field - do_oop_store(_masm, field, r0, _bs->kind(), false); + do_oop_store(_masm, field, r0, ACCESS_IN_HEAP); if (rc == may_rewrite) { patch_bytecode(Bytecodes::_fast_aputfield, bc, r1, true, byte_no); } @@ -2918,7 +2866,7 @@ // access field switch (bytecode()) { case Bytecodes::_fast_aputfield: - do_oop_store(_masm, field, r0, _bs->kind(), false); + do_oop_store(_masm, field, r0, ACCESS_IN_HEAP); break; case Bytecodes::_fast_lputfield: __ str(r0, field); @@ -3010,7 +2958,7 @@ // access field switch (bytecode()) { case Bytecodes::_fast_agetfield: - __ load_heap_oop(r0, field); + do_oop_load(_masm, field, r0, ACCESS_IN_HEAP); __ verify_oop(r0); break; case Bytecodes::_fast_lgetfield: @@ -3080,7 +3028,7 @@ __ ldrw(r0, Address(r0, r1, Address::lsl(0))); break; case atos: - __ load_heap_oop(r0, Address(r0, r1, Address::lsl(0))); + do_oop_load(_masm, Address(r0, r1, Address::lsl(0)), r0, ACCESS_IN_HEAP); __ verify_oop(r0); break; case ftos: --- old/src/cpu/x86/vm/templateTable_x86.cpp 2017-09-21 10:12:33.426906393 -0400 +++ new/src/cpu/x86/vm/templateTable_x86.cpp 2017-09-21 10:12:33.162896318 -0400 @@ -999,7 +999,7 @@ // Get the value we will store __ movptr(rax, at_tos()); // Now store using the appropriate barrier - do_oop_store(_masm, Address(rdx, 0), rax, ACCESS_IN_HEAP); + do_oop_store(_masm, Address(rdx, 0), rax, ACCESS_IN_HEAP | ACCESS_IN_HEAP_ARRAY); __ jmp(done); // Have a NULL in rax, rdx=array, ecx=index. Store NULL at ary[idx] @@ -1007,7 +1007,7 @@ __ profile_null_seen(rbx); // Store a NULL - do_oop_store(_masm, element_address, noreg, ACCESS_IN_HEAP); + do_oop_store(_masm, element_address, noreg, ACCESS_IN_HEAP | ACCESS_IN_HEAP_ARRAY); // Pop stack arguments __ bind(done); @@ -3017,7 +3017,7 @@ __ pop(atos); if (!is_static) pop_and_check_object(obj); // Store into the field - do_oop_store(_masm, field, rax, ACCESS_IN_HEAP | ACCESS_IN_HEAP_ARRAY); + do_oop_store(_masm, field, rax, ACCESS_IN_HEAP); if (!is_static && rc == may_rewrite) { patch_bytecode(Bytecodes::_fast_aputfield, bc, rbx, true, byte_no); } @@ -3270,7 +3270,7 @@ // access field switch (bytecode()) { case Bytecodes::_fast_aputfield: - do_oop_store(_masm, field, rax, ACCESS_IN_HEAP | ACCESS_IN_HEAP_ARRAY); + do_oop_store(_masm, field, rax, ACCESS_IN_HEAP); break; case Bytecodes::_fast_lputfield: #ifdef _LP64 --- old/src/share/vm/classfile/classLoaderData.cpp 2017-09-21 10:12:34.823959704 -0400 +++ new/src/share/vm/classfile/classLoaderData.cpp 2017-09-21 10:12:34.544949057 -0400 @@ -76,7 +76,7 @@ #include "utilities/macros.hpp" #include "utilities/ostream.hpp" #if INCLUDE_ALL_GCS -#include "gc/g1/g1SATBCardTableModRefBS.hpp" +#include "gc/g1/g1BarrierSet.hpp" #endif // INCLUDE_ALL_GCS #if INCLUDE_TRACE #include "trace/tracing.hpp" @@ -778,7 +778,7 @@ if (UseG1GC) { oop obj = *ptr; if (obj != NULL) { - G1SATBCardTableModRefBS::enqueue(obj); + G1BarrierSet::satb_enqueue(obj); } } #endif --- old/src/share/vm/gc/g1/g1CollectedHeap.cpp 2017-09-21 10:12:36.115008970 -0400 +++ new/src/share/vm/gc/g1/g1CollectedHeap.cpp 2017-09-21 10:12:35.826997980 -0400 @@ -1722,8 +1722,8 @@ G1RegionToSpaceMapper* cardtable_storage = create_aux_memory_mapper("Card Table", - G1SATBCardTableLoggingModRefBS::compute_size(g1_rs.size() / HeapWordSize), - G1SATBCardTableLoggingModRefBS::heap_map_factor()); + G1CardTable::compute_size(g1_rs.size() / HeapWordSize), + G1CardTable::heap_map_factor()); G1RegionToSpaceMapper* card_counts_storage = create_aux_memory_mapper("Card Counts Table", --- old/src/share/vm/gc/shared/c2_BarrierSetCodeGen.cpp 2017-09-21 10:12:37.532063045 -0400 +++ new/src/share/vm/gc/shared/c2_BarrierSetCodeGen.cpp 2017-09-21 10:12:37.255052474 -0400 @@ -273,16 +273,7 @@ } } - pin_atomic_op(kit, load_store, alias_idx); - - Node* result = load_store; -#ifdef _LP64 - if (is_obj && adr->bottom_type()->is_ptr_to_narrowoop()) { - result = kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type())); - } -#endif - - return result; + return load_store; } Node* C2BarrierSetCodeGen::cas_bool_at_resolved(GraphKit* kit, Node* obj, Node* adr, const TypePtr* adr_type, int alias_idx, @@ -352,8 +343,6 @@ } } - pin_atomic_op(kit, load_store, alias_idx); - return load_store; } @@ -435,6 +424,16 @@ Node* mem = atomic_op_membar_prologue(kit, decorators, alias_idx); Node* load_store = NULL; Node* result = cas_val_at_resolved(kit, obj, adr, adr_type, alias_idx, expected_val, new_val, value_type, mem, load_store, bt, decorators); + + pin_atomic_op(kit, result, alias_idx); + +#ifdef _LP64 + bool is_obj = bt == T_OBJECT || bt == T_ARRAY; + if (is_obj && adr->bottom_type()->is_ptr_to_narrowoop()) { + result = kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type())); + } +#endif + atomic_op_membar_epilogue(kit, decorators); return result; } @@ -445,6 +444,9 @@ decorators = fixup_decorators(decorators); Node* mem = atomic_op_membar_prologue(kit, decorators, alias_idx); Node* result = cas_bool_at_resolved(kit, obj, adr, adr_type, alias_idx, expected_val, new_val, value_type, mem, bt, decorators); + + pin_atomic_op(kit, result, alias_idx); + atomic_op_membar_epilogue(kit, decorators); return result; } --- /dev/null 2017-09-13 16:57:49.748706170 -0400 +++ new/src/cpu/aarch64/vm/gc/g1/g1BSCodeGen_aarch64.cpp 2017-09-21 10:12:38.497099870 -0400 @@ -0,0 +1,472 @@ +/* + * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "c1/c1_LIRAssembler.hpp" +#include "c1/c1_MacroAssembler.hpp" +#include "gc/g1/c1_G1BSCodeGen.hpp" +#include "gc/g1/g1BSCodeGen.hpp" +#include "gc/g1/g1BarrierSet.hpp" +#include "gc/g1/g1CardTable.hpp" + +#define __ masm-> + +void G1BSCodeGen::gen_write_ref_array_pre_barrier(MacroAssembler* masm, DecoratorSet decorators, + Register addr, Register count, RegSet saved_regs) { + bool dest_uninitialized = (decorators & DEST_NOT_INITIALIZED) != 0; + if (!dest_uninitialized) { + __ push(saved_regs, sp); + if (count == c_rarg0) { + if (addr == c_rarg1) { + // exactly backwards!! + __ mov(rscratch1, c_rarg0); + __ mov(c_rarg0, c_rarg1); + __ mov(c_rarg1, rscratch1); + } else { + __ mov(c_rarg1, count); + __ mov(c_rarg0, addr); + } + } else { + __ mov(c_rarg0, addr); + __ mov(c_rarg1, count); + } + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ModRefBarrierSet::static_write_ref_array_pre), 2); + __ pop(saved_regs, sp); + } +} + +void G1BSCodeGen::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators, + Register start, Register end, Register scratch, RegSet saved_regs) { + __ push(saved_regs, sp); + // must compute element count unless barrier set interface is changed (other platforms supply count) + assert_different_registers(start, end, scratch); + __ lea(scratch, Address(end, BytesPerHeapOop)); + __ sub(scratch, scratch, start); // subtract start to get #bytes + __ lsr(scratch, scratch, LogBytesPerHeapOop); // convert to element count + __ mov(c_rarg0, start); + __ mov(c_rarg1, scratch); + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ModRefBarrierSet::static_write_ref_array_post), 2); + __ pop(saved_regs, sp); +} + +void G1BSCodeGen::g1_write_barrier_pre(MacroAssembler* masm, + Register obj, + Register pre_val, + Register thread, + Register tmp, + bool tosca_live, + bool expand_call) { + // If expand_call is true then we expand the call_VM_leaf macro + // directly to skip generating the check by + // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp. + + assert(thread == rthread, "must be"); + + Label done; + Label runtime; + + assert_different_registers(obj, pre_val, tmp, rscratch1); + assert(pre_val != noreg && tmp != noreg, "expecting a register"); + + Address in_progress(thread, in_bytes(JavaThread::satb_mark_queue_offset() + + SATBMarkQueue::byte_offset_of_active())); + Address index(thread, in_bytes(JavaThread::satb_mark_queue_offset() + + SATBMarkQueue::byte_offset_of_index())); + Address buffer(thread, in_bytes(JavaThread::satb_mark_queue_offset() + + SATBMarkQueue::byte_offset_of_buf())); + + + // Is marking active? + if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) { + __ ldrw(tmp, in_progress); + } else { + assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption"); + __ ldrb(tmp, in_progress); + } + __ cbzw(tmp, done); + + // Do we need to load the previous value? + if (obj != noreg) { + __ load_heap_oop(pre_val, Address(obj, 0)); + } + + // Is the previous value null? + __ cbz(pre_val, done); + + // Can we store original value in the thread's buffer? + // Is index == 0? + // (The index field is typed as size_t.) + + __ ldr(tmp, index); // tmp := *index_adr + __ cbz(tmp, runtime); // tmp == 0? + // If yes, goto runtime + + __ sub(tmp, tmp, wordSize); // tmp := tmp - wordSize + __ str(tmp, index); // *index_adr := tmp + __ ldr(rscratch1, buffer); + __ add(tmp, tmp, rscratch1); // tmp := tmp + *buffer_adr + + // Record the previous value + __ str(pre_val, Address(tmp, 0)); + __ b(done); + + __ bind(runtime); + // save the live input values + RegSet saved = RegSet::of(pre_val); + if (tosca_live) saved += RegSet::of(r0); + if (obj != noreg) saved += RegSet::of(obj); + + __ push(saved, sp); + + // Calling the runtime using the regular call_VM_leaf mechanism generates + // code (generated by InterpreterMacroAssember::call_VM_leaf_base) + // that checks that the *(rfp+frame::interpreter_frame_last_sp) == NULL. + // + // If we care generating the pre-barrier without a frame (e.g. in the + // intrinsified Reference.get() routine) then ebp might be pointing to + // the caller frame and so this check will most likely fail at runtime. + // + // Expanding the call directly bypasses the generation of the check. + // So when we do not have have a full interpreter frame on the stack + // expand_call should be passed true. + + if (expand_call) { + assert(pre_val != c_rarg1, "smashed arg"); + __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSet::g1_wb_pre), pre_val, thread); + } else { + __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSet::g1_wb_pre), pre_val, thread); + } + + __ pop(saved, sp); + + __ bind(done); + +} + +void G1BSCodeGen::g1_write_barrier_post(MacroAssembler* masm, + Register store_addr, + Register new_val, + Register thread, + Register tmp, + Register tmp2) { + assert(thread == rthread, "must be"); + assert_different_registers(store_addr, new_val, thread, tmp, tmp2, + rscratch1); + assert(store_addr != noreg && new_val != noreg && tmp != noreg + && tmp2 != noreg, "expecting a register"); + + Address queue_index(thread, in_bytes(JavaThread::dirty_card_queue_offset() + + DirtyCardQueue::byte_offset_of_index())); + Address buffer(thread, in_bytes(JavaThread::dirty_card_queue_offset() + + DirtyCardQueue::byte_offset_of_buf())); + + BarrierSet* bs = Universe::heap()->barrier_set(); + CardTableModRefBS* ctbs = barrier_set_cast(bs); + CardTable* ct = ctbs->card_table(); + assert(sizeof(*ct->byte_map_base()) == sizeof(jbyte), "adjust this code"); + + Label done; + Label runtime; + + // Does store cross heap regions? + + __ eor(tmp, store_addr, new_val); + __ lsr(tmp, tmp, HeapRegion::LogOfHRGrainBytes); + __ cbz(tmp, done); + + // crosses regions, storing NULL? + + __ cbz(new_val, done); + + // storing region crossing non-NULL, is card already dirty? + + ExternalAddress cardtable((address) ct->byte_map_base()); + assert(sizeof(*ct->byte_map_base()) == sizeof(jbyte), "adjust this code"); + const Register card_addr = tmp; + + __ lsr(card_addr, store_addr, CardTable::card_shift); + + // get the address of the card + __ load_byte_map_base(tmp2); + __ add(card_addr, card_addr, tmp2); + __ ldrb(tmp2, Address(card_addr)); + __ cmpw(tmp2, (int)G1CardTable::g1_young_card_val()); + __ br(Assembler::EQ, done); + + assert((int)CardTable::dirty_card_val() == 0, "must be 0"); + + __ membar(Assembler::StoreLoad); + + __ ldrb(tmp2, Address(card_addr)); + __ cbzw(tmp2, done); + + // storing a region crossing, non-NULL oop, card is clean. + // dirty card and log. + + __ strb(zr, Address(card_addr)); + + __ ldr(rscratch1, queue_index); + __ cbz(rscratch1, runtime); + __ sub(rscratch1, rscratch1, wordSize); + __ str(rscratch1, queue_index); + + __ ldr(tmp2, buffer); + __ str(card_addr, Address(tmp2, rscratch1)); + __ b(done); + + __ bind(runtime); + // save the live input values + RegSet saved = RegSet::of(store_addr, new_val); + __ push(saved, sp); + __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSet::g1_wb_post), card_addr, thread); + __ pop(saved, sp); + + __ bind(done); +} + +void G1BSCodeGen::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, + Register dst, Address src, Register tmp1, Register tmp_thread) { + bool on_oop = type == T_OBJECT || type == T_ARRAY; + bool on_weak = (decorators & ACCESS_ON_WEAK_OOP_REF) != 0; + bool on_phantom = (decorators & ACCESS_ON_PHANTOM_OOP_REF) != 0; + bool on_reference = on_weak || on_phantom; + ModRefBSCodeGen::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread); + if (on_oop && on_reference) { + // Generate the G1 pre-barrier code to log the value of + // the referent field in an SATB buffer. + g1_write_barrier_pre(masm /* masm */, + noreg /* obj */, + dst /* pre_val */, + rthread /* thread */, + tmp1 /* tmp */, + true /* tosca_live */, + true /* expand_call */); + } +} + +void G1BSCodeGen::oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, + Address dst, Register val, Register tmp1, Register tmp2) { + // flatten object address if needed + if (dst.index() == noreg && dst.offset() == 0) { + if (dst.base() != r3) { + __ mov(r3, dst.base()); + } + } else { + __ lea(r3, dst); + } + + g1_write_barrier_pre(masm, + r3 /* obj */, + tmp2 /* pre_val */, + rthread /* thread */, + tmp1 /* tmp */, + val != noreg /* tosca_live */, + false /* expand_call */); + + if (val == noreg) { + __ store_heap_oop_null(Address(r3, 0)); + } else { + // G1 barrier needs uncompressed oop for region cross check. + Register new_val = val; + if (UseCompressedOops) { + new_val = rscratch2; + __ mov(new_val, val); + } + __ store_heap_oop(Address(r3, 0), val); + g1_write_barrier_post(masm, + r3 /* store_adr */, + new_val /* new_val */, + rthread /* thread */, + tmp1 /* tmp */, + tmp2 /* tmp2 */); + } + +} + +#undef __ + +#define __ ce->masm()-> + +void G1BSCodeGen::gen_g1_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrierStub* stub) { + C1G1BSCodeGen* code_gen = (C1G1BSCodeGen*)Universe::heap()->barrier_set()->c1_code_gen(); + // At this point we know that marking is in progress. + // If do_load() is true then we have to emit the + // load of the previous value; otherwise it has already + // been loaded into _pre_val. + + __ bind(stub->_entry); + assert(stub->pre_val()->is_register(), "Precondition."); + + Register pre_val_reg = stub->pre_val()->as_register(); + + if (stub->do_load()) { + ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/, false /*unaligned*/); + } + __ cbz(pre_val_reg, stub->_continuation); + ce->store_parameter(stub->pre_val()->as_register(), 0); + __ far_call(RuntimeAddress(code_gen->pre_barrier_c1_runtime_code_blob()->code_begin())); + __ b(stub->_continuation); +} + +void G1BSCodeGen::gen_g1_post_barrier_stub(LIR_Assembler* ce, G1PostBarrierStub* stub) { + C1G1BSCodeGen* code_gen = (C1G1BSCodeGen*)Universe::heap()->barrier_set()->c1_code_gen(); + __ bind(stub->_entry); + assert(stub->addr()->is_register(), "Precondition."); + assert(stub->new_val()->is_register(), "Precondition."); + Register new_val_reg = stub->new_val()->as_register(); + __ cbz(new_val_reg, stub->_continuation); + ce->store_parameter(stub->addr()->as_pointer_register(), 0); + __ far_call(RuntimeAddress(code_gen->post_barrier_c1_runtime_code_blob()->code_begin())); + __ b(stub->_continuation); +} + +#undef __ + +#define __ sasm-> + +void G1BSCodeGen::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) { + __ prologue("g1_pre_barrier", false); + + // arg0 : previous value of memory + + BarrierSet* bs = Universe::heap()->barrier_set(); + + const Register pre_val = r0; + const Register thread = rthread; + const Register tmp = rscratch1; + + Address in_progress(thread, in_bytes(JavaThread::satb_mark_queue_offset() + + SATBMarkQueue::byte_offset_of_active())); + + Address queue_index(thread, in_bytes(JavaThread::satb_mark_queue_offset() + + SATBMarkQueue::byte_offset_of_index())); + Address buffer(thread, in_bytes(JavaThread::satb_mark_queue_offset() + + SATBMarkQueue::byte_offset_of_buf())); + + Label done; + Label runtime; + + // Is marking still active? + if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) { + __ ldrw(tmp, in_progress); + } else { + assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption"); + __ ldrb(tmp, in_progress); + } + __ cbzw(tmp, done); + + // Can we store original value in the thread's buffer? + __ ldr(tmp, queue_index); + __ cbz(tmp, runtime); + + __ sub(tmp, tmp, wordSize); + __ str(tmp, queue_index); + __ ldr(rscratch2, buffer); + __ add(tmp, tmp, rscratch2); + __ load_parameter(0, rscratch2); + __ str(rscratch2, Address(tmp, 0)); + __ b(done); + + __ bind(runtime); + __ push_call_clobbered_registers(); + __ load_parameter(0, pre_val); + __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSet::g1_wb_pre), pre_val, thread); + __ pop_call_clobbered_registers(); + __ bind(done); + + __ epilogue(); +} + +void G1BSCodeGen::generate_c1_post_barrier_runtime_stub(StubAssembler* sasm) { + __ prologue("g1_post_barrier", false); + + // arg0: store_address + Address store_addr(rfp, 2*BytesPerWord); + + BarrierSet* bs = Universe::heap()->barrier_set(); + CardTableModRefBS* ctbs = barrier_set_cast(bs); + CardTable* ct = ctbs->card_table(); + assert(sizeof(*ct->byte_map_base()) == sizeof(jbyte), "adjust this code"); + + Label done; + Label runtime; + + // At this point we know new_value is non-NULL and the new_value crosses regions. + // Must check to see if card is already dirty + + const Register thread = rthread; + + Address queue_index(thread, in_bytes(JavaThread::dirty_card_queue_offset() + + DirtyCardQueue::byte_offset_of_index())); + Address buffer(thread, in_bytes(JavaThread::dirty_card_queue_offset() + + DirtyCardQueue::byte_offset_of_buf())); + + const Register card_offset = rscratch2; + // LR is free here, so we can use it to hold the byte_map_base. + const Register byte_map_base = lr; + + assert_different_registers(card_offset, byte_map_base, rscratch1); + + __ load_parameter(0, card_offset); + __ lsr(card_offset, card_offset, CardTable::card_shift); + __ load_byte_map_base(byte_map_base); + __ ldrb(rscratch1, Address(byte_map_base, card_offset)); + __ cmpw(rscratch1, (int)G1CardTable::g1_young_card_val()); + __ br(Assembler::EQ, done); + + assert((int)CardTable::dirty_card_val() == 0, "must be 0"); + + __ membar(Assembler::StoreLoad); + __ ldrb(rscratch1, Address(byte_map_base, card_offset)); + __ cbzw(rscratch1, done); + + // storing region crossing non-NULL, card is clean. + // dirty card and log. + __ strb(zr, Address(byte_map_base, card_offset)); + + // Convert card offset into an address in card_addr + Register card_addr = card_offset; + __ add(card_addr, byte_map_base, card_addr); + + __ ldr(rscratch1, queue_index); + __ cbz(rscratch1, runtime); + __ sub(rscratch1, rscratch1, wordSize); + __ str(rscratch1, queue_index); + + // Reuse LR to hold buffer_addr + const Register buffer_addr = lr; + + __ ldr(buffer_addr, buffer); + __ str(card_addr, Address(buffer_addr, rscratch1)); + __ b(done); + + __ bind(runtime); + __ push_call_clobbered_registers(); + __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSet::g1_wb_post), card_addr, thread); + __ pop_call_clobbered_registers(); + __ bind(done); + __ epilogue(); +} + +#undef __ --- /dev/null 2017-09-13 16:57:49.748706170 -0400 +++ new/src/cpu/aarch64/vm/gc/g1/g1BSCodeGen_aarch64.hpp 2017-09-21 10:12:39.682145091 -0400 @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_AARCH64_VM_GC_G1_G1BSCODEGEN_AARCH64_HPP +#define CPU_AARCH64_VM_GC_G1_G1BSCODEGEN_AARCH64_HPP + +#include "asm/macroAssembler.hpp" +#include "gc/shared/modRefBSCodeGen.hpp" + +class LIR_Assembler; +class StubAssembler; +class G1PreBarrierStub; +class G1PostBarrierStub; + +class G1BSCodeGen: public ModRefBSCodeGen { +protected: + void gen_write_ref_array_pre_barrier(MacroAssembler* masm, DecoratorSet decorators, + Register addr, Register count, RegSet saved_regs); + void gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators, + Register start, Register end, Register tmp, RegSet saved_regs); + + void g1_write_barrier_pre(MacroAssembler* masm, + Register obj, + Register pre_val, + Register thread, + Register tmp, + bool tosca_live, + bool expand_call); + + void g1_write_barrier_post(MacroAssembler* masm, + Register store_addr, + Register new_val, + Register thread, + Register tmp, + Register tmp2); + + virtual void oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, + Address dst, Register val, Register tmp1, Register tmp2); + +public: + void gen_g1_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrierStub* stub); + void gen_g1_post_barrier_stub(LIR_Assembler* ce, G1PostBarrierStub* stub); + + void generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm); + void generate_c1_post_barrier_runtime_stub(StubAssembler* sasm); + + void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, + Register dst, Address src, Register tmp1, Register tmp_thread); +}; + +#endif // CPU_AARCH64_VM_GC_G1_G1BSCODEGEN_AARCH64_HPP --- /dev/null 2017-09-13 16:57:49.748706170 -0400 +++ new/src/cpu/aarch64/vm/gc/shared/barrierSetCodeGen_aarch64.cpp 2017-09-21 10:12:40.866190274 -0400 @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/shared/barrierSetCodeGen.hpp" + +#define __ masm-> + +void BarrierSetCodeGen::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, + Register dst, Address src, Register tmp1, Register tmp_thread) { + bool on_heap = (decorators & ACCESS_IN_HEAP) != 0; + bool on_root = (decorators & ACCESS_IN_ROOT) != 0; + switch (type) { + case T_OBJECT: + case T_ARRAY: { + if (on_heap) { + __ load_heap_oop(dst, src); + } else { + assert(on_root, "why else?"); + __ ldr(dst, src); + } + break; + } + default: Unimplemented(); + } +} + +void BarrierSetCodeGen::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, + Address dst, Register val, Register tmp1, Register tmp2) { + bool on_heap = (decorators & ACCESS_IN_HEAP) != 0; + bool on_root = (decorators & ACCESS_IN_ROOT) != 0; + switch (type) { + case T_OBJECT: + case T_ARRAY: { + if (on_heap) { + __ store_heap_oop(dst, val); + } else { + assert(on_root, "why else?"); + __ str(val, dst); + } + break; + } + default: Unimplemented(); + } +} --- /dev/null 2017-09-13 16:57:49.748706170 -0400 +++ new/src/cpu/aarch64/vm/gc/shared/barrierSetCodeGen_aarch64.hpp 2017-09-21 10:12:42.048235381 -0400 @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_AARCH64_VM_GC_SHARED_BARRIERSETCODEGEN_AARCH64_HPP +#define CPU_AARCH64_VM_GC_SHARED_BARRIERSETCODEGEN_AARCH64_HPP + +#include "memory/allocation.hpp" + +class BarrierSetCodeGen: public CHeapObj { +public: + virtual void arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop, + Register addr, Register count, RegSet saved_regs) {} + virtual void arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop, + Register start, Register end, Register tmp, RegSet saved_regs) {} + virtual void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, + Register dst, Address src, Register tmp1, Register tmp_thread); + virtual void store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, + Address dst, Register val, Register tmp1, Register tmp2); +}; + +#endif // CPU_AARCH64_VM_GC_SHARED_BARRIERSETCODEGEN_AARCH64_HPP + --- /dev/null 2017-09-13 16:57:49.748706170 -0400 +++ new/src/cpu/aarch64/vm/gc/shared/cardTableModRefBSCodeGen_aarch64.cpp 2017-09-21 10:12:43.232280564 -0400 @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/shared/cardTableModRefBSCodeGen.hpp" + +#define __ masm-> + +void CardTableModRefBSCodeGen::store_check(MacroAssembler* masm, Register obj, Address dst) { + // Does a store check for the oop in register obj. The content of + // register obj is destroyed afterwards. + + BarrierSet* bs = Universe::heap()->barrier_set(); + assert(bs->kind() == BarrierSet::CardTableModRef, "Wrong barrier set kind"); + + CardTableModRefBS* ctbs = barrier_set_cast(bs); + CardTable* ct = ctbs->card_table(); + assert(sizeof(*ct->byte_map_base()) == sizeof(jbyte), "adjust this code"); + + __ lsr(obj, obj, CardTable::card_shift); + + assert(CardTable::dirty_card_val() == 0, "must be"); + + __ load_byte_map_base(rscratch1); + + if (UseCondCardMark) { + Label L_already_dirty; + __ membar(Assembler::StoreLoad); + __ ldrb(rscratch2, Address(obj, rscratch1)); + __ cbz(rscratch2, L_already_dirty); + __ strb(zr, Address(obj, rscratch1)); + __ bind(L_already_dirty); + } else { + if (UseConcMarkSweepGC && CMSPrecleaningEnabled) { + __ membar(Assembler::StoreStore); + } + __ strb(zr, Address(obj, rscratch1)); + } +} + +void CardTableModRefBSCodeGen::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators, + Register start, Register end, Register scratch, RegSet saved_regs) { + + BarrierSet* bs = Universe::heap()->barrier_set(); + CardTableModRefBS* ctbs = barrier_set_cast(bs); + CardTable* ct = ctbs->card_table(); + assert(sizeof(*ct->byte_map_base()) == sizeof(jbyte), "adjust this code"); + + Label L_loop; + + __ lsr(start, start, CardTable::card_shift); + __ lsr(end, end, CardTable::card_shift); + __ sub(end, end, start); // number of bytes to copy + + const Register count = end; // 'end' register contains bytes count now + __ load_byte_map_base(scratch); + __ add(start, start, scratch); + if (UseConcMarkSweepGC) { + __ membar(__ StoreStore); + } + __ bind(L_loop); + __ strb(zr, Address(start, count)); + __ subs(count, count, 1); + __ br(Assembler::GE, L_loop); +} + +void CardTableModRefBSCodeGen::oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, + Address dst, Register val, Register tmp1, Register tmp2) { + + bool on_array = (decorators & ACCESS_IN_HEAP_ARRAY) != 0; + bool on_anonymous = (decorators & ACCESS_ON_UNKNOWN_OOP_REF) != 0; + bool precise = on_array || on_anonymous; + if (val == noreg) { + __ store_heap_oop_null(dst); + } else { + __ store_heap_oop(dst, val); + // flatten object address if needed + if (!precise || (dst.index() == noreg && dst.offset() == 0)) { + store_check(masm, dst.base(), dst); + } else { + __ lea(r3, dst); + store_check(masm, r3, dst); + } + } +} + --- /dev/null 2017-09-13 16:57:49.748706170 -0400 +++ new/src/cpu/aarch64/vm/gc/shared/cardTableModRefBSCodeGen_aarch64.hpp 2017-09-21 10:12:44.418325823 -0400 @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_AARCH64_VM_GC_SHARED_CARDTABLEMODREFBSCODEGEN_AARCH64_HPP +#define CPU_AARCH64_VM_GC_SHARED_CARDTABLEMODREFBSCODEGEN_AARCH64_HPP + +#include "gc/shared/modRefBSCodeGen.hpp" + +class CardTableModRefBSCodeGen: public ModRefBSCodeGen { +protected: + void store_check(MacroAssembler* masm, Register obj, Address dst); + + virtual void gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators, + Register start, Register end, Register tmp, RegSet saved_regs); + virtual void oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, + Address dst, Register val, Register tmp1, Register tmp2); + +}; + +#endif // #ifndef CPU_AARCH64_VM_GC_SHARED_CARDTABLEMODREFBSCODEGEN_AARCH64_HPP --- /dev/null 2017-09-13 16:57:49.748706170 -0400 +++ new/src/cpu/aarch64/vm/gc/shared/modRefBSCodeGen_aarch64.cpp 2017-09-21 10:12:45.604371083 -0400 @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "asm/macroAssembler.inline.hpp" +#include "gc/shared/modRefBSCodeGen.hpp" + +#define __ masm-> + +void ModRefBSCodeGen::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop, + Register addr, Register count, RegSet saved_regs) { + + if (is_oop) { + gen_write_ref_array_pre_barrier(masm, decorators, addr, count, saved_regs); + } +} + +void ModRefBSCodeGen::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop, + Register start, Register end, Register tmp, + RegSet saved_regs) { + if (is_oop) { + gen_write_ref_array_post_barrier(masm, decorators, start, end, tmp, saved_regs); + } +} + + +void ModRefBSCodeGen::oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, + Address dst, Register val, Register tmp1, Register tmp2) { + BarrierSetCodeGen::store_at(masm, decorators, type, dst, val, tmp1, tmp2); +} + +void ModRefBSCodeGen::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, + Address dst, Register val, Register tmp1, Register tmp2) { + if (type == T_OBJECT || type == T_ARRAY) { + oop_store_at(masm, decorators, type, dst, val, tmp1, tmp2); + } else { + BarrierSetCodeGen::store_at(masm, decorators, type, dst, val, tmp1, tmp2); + } +} --- /dev/null 2017-09-13 16:57:49.748706170 -0400 +++ new/src/cpu/aarch64/vm/gc/shared/modRefBSCodeGen_aarch64.hpp 2017-09-21 10:12:46.788416266 -0400 @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_AARCH64_VM_GC_SHARED_MODREFBSCODEGEN_AARCH64_HPP +#define CPU_AARCH64_VM_GC_SHARED_MODREFBSCODEGEN_AARCH64_HPP + +#include "asm/macroAssembler.hpp" +#include "gc/shared/barrierSetCodeGen.hpp" + +class ModRefBSCodeGen: public BarrierSetCodeGen { +protected: + virtual void gen_write_ref_array_pre_barrier(MacroAssembler* masm, DecoratorSet decorators, + Register addr, Register count, RegSet saved_regs) {} + virtual void gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators, + Register start, Register end, Register tmp, RegSet saved_regs) {} + + virtual void oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, + Address dst, Register val, Register tmp1, Register tmp2); + +public: + virtual void arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop, + Register addr, Register count, RegSet saved_regs); + virtual void arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop, + Register start, Register end, Register tmp, RegSet saved_regs); + virtual void store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, + Address dst, Register val, Register tmp1, Register tmp2); +}; + +#endif // CPU_AARCH64_VM_GC_SHARED_MODREFBSCODEGEN_AARCH64_HPP