--- old/src/hotspot/cpu/x86/gc/shared/barrierSetAssembler_x86.cpp 2019-11-22 13:54:03.839972016 +0100 +++ new/src/hotspot/cpu/x86/gc/shared/barrierSetAssembler_x86.cpp 2019-11-22 13:54:03.587967756 +0100 @@ -23,9 +23,11 @@ */ #include "precompiled.hpp" +#include "asm/macroAssembler.inline.hpp" #include "gc/shared/barrierSet.hpp" #include "gc/shared/barrierSetAssembler.hpp" #include "gc/shared/barrierSetNMethod.hpp" +#include "gc/shared/barrierSetRuntime.hpp" #include "gc/shared/collectedHeap.hpp" #include "interpreter/interp_masm.hpp" #include "memory/universe.hpp" @@ -195,6 +197,20 @@ } } +void BarrierSetAssembler::value_copy(MacroAssembler* masm, DecoratorSet decorators, + Register src, Register dst, Register value_klass) { + // value_copy implementation is fairly complex, and there are not any + // "short-cuts" to be made from asm. What there is, appears to have the same + // cost in C++, so just "call_VM_leaf" for now rather than maintain hundreds + // of hand-rolled instructions... + if (decorators & IS_DEST_UNINITIALIZED) { + __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSetRuntime::value_copy_is_dest_uninitialized), src, dst, value_klass); + } else { + __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSetRuntime::value_copy), src, dst, value_klass); + } +} + + #ifndef _LP64 void BarrierSetAssembler::obj_equals(MacroAssembler* masm, Address obj1, jobject obj2) { --- old/src/hotspot/cpu/x86/gc/shared/barrierSetAssembler_x86.hpp 2019-11-22 13:54:04.475982767 +0100 +++ new/src/hotspot/cpu/x86/gc/shared/barrierSetAssembler_x86.hpp 2019-11-22 13:54:04.227978575 +0100 @@ -49,6 +49,9 @@ virtual void store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Address dst, Register val, Register tmp1, Register tmp2, Register tmp3 = noreg); + virtual void value_copy(MacroAssembler* masm, DecoratorSet decorators, + Register src, Register dst, Register value_klass); + #ifndef _LP64 virtual void obj_equals(MacroAssembler* masm, Address obj1, jobject obj2); --- old/src/hotspot/cpu/x86/interp_masm_x86.cpp 2019-11-22 13:54:05.239995682 +0100 +++ new/src/hotspot/cpu/x86/interp_masm_x86.cpp 2019-11-22 13:54:04.987991423 +0100 @@ -1198,6 +1198,65 @@ bind(has_counters); } +void InterpreterMacroAssembler::allocate_instance(Register klass, Register new_obj, + Register t1, Register t2, + bool clear_fields, Label& alloc_failed) { + MacroAssembler::allocate_instance(klass, new_obj, t1, t2, clear_fields, alloc_failed); + { + SkipIfEqual skip_if(this, &DTraceAllocProbes, 0); + // Trigger dtrace event for fastpath + push(atos); + call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_object_alloc), new_obj); + pop(atos); + } +} + + +void InterpreterMacroAssembler::read_flattened_field(Register holder_klass, + Register field_index, Register field_offset, + Register obj) { + Label alloc_failed, empty_value, done; + const Register src = field_offset; + const Register alloc_temp = LP64_ONLY(rscratch1) NOT_LP64(rsi); + const Register dst_temp = LP64_ONLY(rscratch2) NOT_LP64(rdi); + assert_different_registers(obj, holder_klass, field_index, field_offset, dst_temp); + + // Grap the inline field klass + push(holder_klass); + const Register field_klass = holder_klass; + get_value_field_klass(holder_klass, field_index, field_klass); + + //check for empty value klass + test_klass_is_empty_value(field_klass, dst_temp, empty_value); + + // allocate buffer + push(obj); // save holder + allocate_instance(field_klass, obj, alloc_temp, dst_temp, false, alloc_failed); + + // Have a oop instance buffer, copy into it + data_for_oop(obj, dst_temp, field_klass); + pop(alloc_temp); // restore holder + lea(src, Address(alloc_temp, field_offset)); + // call_VM_leaf, clobbers a few regs, save restore new obj + push(obj); + access_value_copy(IS_DEST_UNINITIALIZED, src, dst_temp, field_klass); + pop(obj); + pop(holder_klass); + jmp(done); + + bind(empty_value); + get_empty_value_oop(field_klass, dst_temp, obj); + pop(holder_klass); + jmp(done); + + bind(alloc_failed); + pop(obj); + pop(holder_klass); + call_VM(obj, CAST_FROM_FN_PTR(address, InterpreterRuntime::read_flattened_field), + obj, field_index, holder_klass); + + bind(done); +} // Lock object // --- old/src/hotspot/cpu/x86/interp_masm_x86.hpp 2019-11-22 13:54:06.284013330 +0100 +++ new/src/hotspot/cpu/x86/interp_masm_x86.hpp 2019-11-22 13:54:05.996008461 +0100 @@ -223,6 +223,20 @@ bool notify_jvmdi = true); void get_method_counters(Register method, Register mcs, Label& skip); + // Kills t1 and t2, perserves klass, return allocation in new_obj + void allocate_instance(Register klass, Register new_obj, + Register t1, Register t2, + bool clear_fields, Label& alloc_failed); + // Allocate value buffer in new_obj and read in flattened field + // NOTES: + // - input holder object via "obj", which must be rax, + // will return new value buffer obj via the same reg + // - assumes holder_klass and valueKlass field klass have both been resolved + // - 32 bits: kills rdi and rsi + void read_flattened_field(Register holder_klass, + Register field_index, Register field_offset, + Register obj = rax); + // Object locking void lock_object (Register lock_reg); void unlock_object(Register lock_reg); --- old/src/hotspot/cpu/x86/macroAssembler_x86.cpp 2019-11-22 13:54:07.280030166 +0100 +++ new/src/hotspot/cpu/x86/macroAssembler_x86.cpp 2019-11-22 13:54:06.996025366 +0100 @@ -3428,6 +3428,20 @@ jcc(Assembler::notZero, is_value); } +void MacroAssembler::test_klass_is_empty_value(Register klass, Register temp_reg, Label& is_empty_value) { +#ifdef ASSERT + { + Label done_check; + test_klass_is_value(klass, temp_reg, done_check); + stop("test_klass_is_empty_value with none value klass"); + bind(done_check); + } +#endif + movb(temp_reg, Address(klass, InstanceKlass::extra_flags_offset())); + testb(temp_reg, InstanceKlass::_extra_is_empty_value); + jcc(Assembler::notZero, is_empty_value); +} + void MacroAssembler::test_field_is_flattenable(Register flags, Register temp_reg, Label& is_flattenable) { movl(temp_reg, flags); shrl(temp_reg, ConstantPoolCacheEntry::is_flattenable_field_shift); @@ -4304,6 +4318,134 @@ LP64_ONLY(testq(dst, src)) NOT_LP64(testl(dst, src)); } +// Object / value buffer allocation... +// +// Kills klass and rsi on LP64 +void MacroAssembler::allocate_instance(Register klass, Register new_obj, + Register t1, Register t2, + bool clear_fields, Label& alloc_failed) +{ + Label done, initialize_header, initialize_object, slow_case, slow_case_no_pop; + Register layout_size = t1; + assert(new_obj == rax, "needs to be rax, according to barrier asm eden_allocate"); + assert_different_registers(klass, new_obj, t1, t2); + +#ifdef ASSERT + { + Label L; + cmpb(Address(klass, InstanceKlass::init_state_offset()), InstanceKlass::fully_initialized); + jcc(Assembler::equal, L); + stop("klass not initialized"); + bind(L); + } +#endif + + // get instance_size in InstanceKlass (scaled to a count of bytes) + movl(layout_size, Address(klass, Klass::layout_helper_offset())); + // test to see if it has a finalizer or is malformed in some way + testl(layout_size, Klass::_lh_instance_slow_path_bit); + jcc(Assembler::notZero, slow_case_no_pop); + + // Allocate the instance: + // If TLAB is enabled: + // Try to allocate in the TLAB. + // If fails, go to the slow path. + // Else If inline contiguous allocations are enabled: + // Try to allocate in eden. + // If fails due to heap end, go to slow path. + // + // If TLAB is enabled OR inline contiguous is enabled: + // Initialize the allocation. + // Exit. + // + // Go to slow path. + const bool allow_shared_alloc = + Universe::heap()->supports_inline_contig_alloc(); + + push(klass); + const Register thread = LP64_ONLY(r15_thread) NOT_LP64(klass); +#ifndef _LP64 + if (UseTLAB || allow_shared_alloc) { + get_thread(thread); + } +#endif // _LP64 + + if (UseTLAB) { + tlab_allocate(thread, new_obj, layout_size, 0, klass, t2, slow_case); + if (ZeroTLAB || (!clear_fields)) { + // the fields have been already cleared + jmp(initialize_header); + } else { + // initialize both the header and fields + jmp(initialize_object); + } + } else { + // Allocation in the shared Eden, if allowed. + // + eden_allocate(thread, new_obj, layout_size, 0, t2, slow_case); + } + + // If UseTLAB or allow_shared_alloc are true, the object is created above and + // there is an initialize need. Otherwise, skip and go to the slow path. + if (UseTLAB || allow_shared_alloc) { + if (clear_fields) { + // The object is initialized before the header. If the object size is + // zero, go directly to the header initialization. + bind(initialize_object); + decrement(layout_size, sizeof(oopDesc)); + jcc(Assembler::zero, initialize_header); + + // Initialize topmost object field, divide size by 8, check if odd and + // test if zero. + Register zero = klass; + xorl(zero, zero); // use zero reg to clear memory (shorter code) + shrl(layout_size, LogBytesPerLong); // divide by 2*oopSize and set carry flag if odd + + #ifdef ASSERT + // make sure instance_size was multiple of 8 + Label L; + // Ignore partial flag stall after shrl() since it is debug VM + jcc(Assembler::carryClear, L); + stop("object size is not multiple of 2 - adjust this code"); + bind(L); + // must be > 0, no extra check needed here + #endif + + // initialize remaining object fields: instance_size was a multiple of 8 + { + Label loop; + bind(loop); + movptr(Address(new_obj, layout_size, Address::times_8, sizeof(oopDesc) - 1*oopSize), zero); + NOT_LP64(movptr(Address(new_obj, layout_size, Address::times_8, sizeof(oopDesc) - 2*oopSize), zero)); + decrement(layout_size); + jcc(Assembler::notZero, loop); + } + } // clear_fields + + // initialize object header only. + bind(initialize_header); + pop(klass); + Register mark_word = t2; + movptr(mark_word, Address(klass, Klass::prototype_header_offset())); + movptr(Address(new_obj, oopDesc::mark_offset_in_bytes ()), mark_word); +#ifdef _LP64 + xorl(rsi, rsi); // use zero reg to clear memory (shorter code) + store_klass_gap(new_obj, rsi); // zero klass gap for compressed oops +#endif + movptr(t2, klass); // preserve klass + store_klass(new_obj, t2); // src klass reg is potentially compressed + + jmp(done); + } + + bind(slow_case); + pop(klass); + bind(slow_case_no_pop); + jmp(alloc_failed); + + bind(done); +} + // Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes. void MacroAssembler::tlab_allocate(Register thread, Register obj, Register var_size_in_bytes, @@ -4381,6 +4523,56 @@ bind(done); } +void MacroAssembler::get_value_field_klass(Register klass, Register index, Register value_klass) { + movptr(value_klass, Address(klass, InstanceKlass::value_field_klasses_offset())); +#ifdef ASSERT + { + Label done; + cmpptr(value_klass, 0); + jcc(Assembler::notEqual, done); + stop("get_value_field_klass contains no inline klasses"); + bind(done); + } +#endif + movptr(value_klass, Address(value_klass, index, Address::times_ptr)); +} + +void MacroAssembler::get_default_value_oop(Register value_klass, Register temp_reg, Register obj) { +#ifdef ASSERT + { + Label done_check; + test_klass_is_value(value_klass, temp_reg, done_check); + stop("get_default_value_oop from non-value klass"); + bind(done_check); + } +#endif + Register offset = temp_reg; + // Getting the offset of the pre-allocated default value + movptr(offset, Address(value_klass, in_bytes(InstanceKlass::adr_valueklass_fixed_block_offset()))); + movl(offset, Address(offset, in_bytes(ValueKlass::default_value_offset_offset()))); + + // Getting the mirror + movptr(obj, Address(value_klass, in_bytes(Klass::java_mirror_offset()))); + resolve_oop_handle(obj, value_klass); + + // Getting the pre-allocated default value from the mirror + Address field(obj, offset, Address::times_1); + load_heap_oop(obj, field); +} + +void MacroAssembler::get_empty_value_oop(Register value_klass, Register temp_reg, Register obj) { +#ifdef ASSERT + { + Label done_check; + test_klass_is_empty_value(value_klass, temp_reg, done_check); + stop("get_empty_value from non-empty value klass"); + bind(done_check); + } +#endif + get_default_value_oop(value_klass, temp_reg, obj); +} + + // Look up the method for a megamorphic invokeinterface call. // The target method is determined by . // The receiver klass is in recv_klass. @@ -5396,6 +5588,28 @@ } } +void MacroAssembler::access_value_copy(DecoratorSet decorators, Register src, Register dst, + Register value_klass) { + BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler(); + bs->value_copy(this, decorators, src, dst, value_klass); +} + +void MacroAssembler::first_field_offset(Register value_klass, Register offset) { + movptr(offset, Address(value_klass, InstanceKlass::adr_valueklass_fixed_block_offset())); + movl(offset, Address(offset, ValueKlass::first_field_offset_offset())); +} + +void MacroAssembler::data_for_oop(Register oop, Register data, Register value_klass) { + // ((address) (void*) o) + vk->first_field_offset(); + Register offset = (data == oop) ? rscratch1 : data; + first_field_offset(value_klass, offset); + if (data == oop) { + addptr(data, offset); + } else { + lea(data, Address(oop, offset)); + } +} + void MacroAssembler::resolve(DecoratorSet decorators, Register obj) { // Use stronger ACCESS_WRITE|ACCESS_READ by default. if ((decorators & (ACCESS_READ | ACCESS_WRITE)) == 0) { --- old/src/hotspot/cpu/x86/macroAssembler_x86.hpp 2019-11-22 13:54:08.352048288 +0100 +++ new/src/hotspot/cpu/x86/macroAssembler_x86.hpp 2019-11-22 13:54:08.060043352 +0100 @@ -101,7 +101,15 @@ static bool needs_explicit_null_check(intptr_t offset); static bool uses_implicit_null_check(void* address); + // valueKlass queries, kills temp_reg void test_klass_is_value(Register klass, Register temp_reg, Label& is_value); + void test_klass_is_empty_value(Register klass, Register temp_reg, Label& is_empty_value); + + // Get the default value oop for the given ValueKlass + void get_default_value_oop(Register value_klass, Register temp_reg, Register obj); + // The empty value oop, for the given ValueKlass ("empty" as in no instance fields) + // get_default_value_oop with extra assertion for empty value klass + void get_empty_value_oop(Register value_klass, Register temp_reg, Register obj); void test_field_is_flattenable(Register flags, Register temp_reg, Label& is_flattenable); void test_field_is_not_flattenable(Register flags, Register temp_reg, Label& notFlattenable); @@ -346,6 +354,13 @@ void access_store_at(BasicType type, DecoratorSet decorators, Address dst, Register src, Register tmp1, Register tmp2, Register tmp3 = noreg); + void access_value_copy(DecoratorSet decorators, Register src, Register dst, Register value_klass); + + // value type data payload offsets... + void first_field_offset(Register value_klass, Register offset); + void data_for_oop(Register oop, Register data, Register value_klass); + + // Resolves obj access. Result is placed in the same register. // All other registers are preserved. void resolve(DecoratorSet decorators, Register obj); @@ -534,6 +549,15 @@ void pop_callee_saved_registers(); // allocation + + // Object / value buffer allocation... + // Allocate instance of klass, assumes klass initialized by caller + // new_obj prefers to be rax + // Kills t1 and t2, perserves klass, return allocation in new_obj (rsi on LP64) + void allocate_instance(Register klass, Register new_obj, + Register t1, Register t2, + bool clear_fields, Label& alloc_failed); + void eden_allocate( Register thread, // Current thread Register obj, // result: pointer to object after successful allocation @@ -553,6 +577,9 @@ ); void zero_memory(Register address, Register length_in_bytes, int offset_in_bytes, Register temp); + // For field "index" within "klass", return value_klass ... + void get_value_field_klass(Register klass, Register index, Register value_klass); + // interface method calling void lookup_interface_method(Register recv_klass, Register intf_klass, --- old/src/hotspot/cpu/x86/templateTable_x86.cpp 2019-11-22 13:54:09.316064584 +0100 +++ new/src/hotspot/cpu/x86/templateTable_x86.cpp 2019-11-22 13:54:09.032059783 +0100 @@ -3818,6 +3818,7 @@ __ verify_oop(rax); __ jmp(Done); __ bind(isFlattened); + __ push(rdx); // save offset __ movl(rdx, Address(rcx, rbx, Address::times_ptr, in_bytes(ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::flags_offset()))); @@ -3825,10 +3826,10 @@ __ movptr(rcx, Address(rcx, rbx, Address::times_ptr, in_bytes(ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::f1_offset()))); - call_VM(rax, CAST_FROM_FN_PTR(address, InterpreterRuntime::read_flattened_field), - rax, rdx, rcx); - __ verify_oop(rax); + __ pop(rbx); // restore offset + __ read_flattened_field(rcx, rdx, rbx, rax); __ bind(Done); + __ verify_oop(rax); } break; case Bytecodes::_fast_agetfield: @@ -4304,10 +4305,7 @@ transition(vtos, atos); __ get_unsigned_2_byte_index_at_bcp(rdx, 1); Label slow_case; - Label slow_case_no_pop; Label done; - Label initialize_header; - Label initialize_object; // including clearing the fields __ get_cpool_and_tags(rcx, rax); @@ -4316,130 +4314,20 @@ // how Constant Pool is updated (see ConstantPool::klass_at_put) const int tags_offset = Array::base_offset_in_bytes(); __ cmpb(Address(rax, rdx, Address::times_1, tags_offset), JVM_CONSTANT_Class); - __ jcc(Assembler::notEqual, slow_case_no_pop); + __ jcc(Assembler::notEqual, slow_case); // get InstanceKlass __ load_resolved_klass_at_index(rcx, rcx, rdx); - __ push(rcx); // save the contexts of klass for initializing the header // make sure klass is initialized & doesn't have finalizer - // make sure klass is fully initialized __ cmpb(Address(rcx, InstanceKlass::init_state_offset()), InstanceKlass::fully_initialized); __ jcc(Assembler::notEqual, slow_case); - // get instance_size in InstanceKlass (scaled to a count of bytes) - __ movl(rdx, Address(rcx, Klass::layout_helper_offset())); - // test to see if it has a finalizer or is malformed in some way - __ testl(rdx, Klass::_lh_instance_slow_path_bit); - __ jcc(Assembler::notZero, slow_case); - - // Allocate the instance: - // If TLAB is enabled: - // Try to allocate in the TLAB. - // If fails, go to the slow path. - // Else If inline contiguous allocations are enabled: - // Try to allocate in eden. - // If fails due to heap end, go to slow path. - // - // If TLAB is enabled OR inline contiguous is enabled: - // Initialize the allocation. - // Exit. - // - // Go to slow path. - - const bool allow_shared_alloc = - Universe::heap()->supports_inline_contig_alloc(); - - const Register thread = LP64_ONLY(r15_thread) NOT_LP64(rcx); -#ifndef _LP64 - if (UseTLAB || allow_shared_alloc) { - __ get_thread(thread); - } -#endif // _LP64 - - if (UseTLAB) { - __ tlab_allocate(thread, rax, rdx, 0, rcx, rbx, slow_case); - if (ZeroTLAB) { - // the fields have been already cleared - __ jmp(initialize_header); - } else { - // initialize both the header and fields - __ jmp(initialize_object); - } - } else { - // Allocation in the shared Eden, if allowed. - // - // rdx: instance size in bytes - __ eden_allocate(thread, rax, rdx, 0, rbx, slow_case); - } - - // If UseTLAB or allow_shared_alloc are true, the object is created above and - // there is an initialize need. Otherwise, skip and go to the slow path. - if (UseTLAB || allow_shared_alloc) { - // The object is initialized before the header. If the object size is - // zero, go directly to the header initialization. - __ bind(initialize_object); - __ decrement(rdx, sizeof(oopDesc)); - __ jcc(Assembler::zero, initialize_header); - - // Initialize topmost object field, divide rdx by 8, check if odd and - // test if zero. - __ xorl(rcx, rcx); // use zero reg to clear memory (shorter code) - __ shrl(rdx, LogBytesPerLong); // divide by 2*oopSize and set carry flag if odd - - // rdx must have been multiple of 8 -#ifdef ASSERT - // make sure rdx was multiple of 8 - Label L; - // Ignore partial flag stall after shrl() since it is debug VM - __ jcc(Assembler::carryClear, L); - __ stop("object size is not multiple of 2 - adjust this code"); - __ bind(L); - // rdx must be > 0, no extra check needed here -#endif - - // initialize remaining object fields: rdx was a multiple of 8 - { Label loop; - __ bind(loop); - __ movptr(Address(rax, rdx, Address::times_8, sizeof(oopDesc) - 1*oopSize), rcx); - NOT_LP64(__ movptr(Address(rax, rdx, Address::times_8, sizeof(oopDesc) - 2*oopSize), rcx)); - __ decrement(rdx); - __ jcc(Assembler::notZero, loop); - } - - // initialize object header only. - __ bind(initialize_header); - if (UseBiasedLocking) { - __ pop(rcx); // get saved klass back in the register. - __ movptr(rbx, Address(rcx, Klass::prototype_header_offset())); - __ movptr(Address(rax, oopDesc::mark_offset_in_bytes ()), rbx); - } else { - __ movptr(Address(rax, oopDesc::mark_offset_in_bytes ()), - (intptr_t)markWord::prototype().value()); // header - __ pop(rcx); // get saved klass back in the register. - } -#ifdef _LP64 - __ xorl(rsi, rsi); // use zero reg to clear memory (shorter code) - __ store_klass_gap(rax, rsi); // zero klass gap for compressed oops -#endif - __ store_klass(rax, rcx); // klass - - { - SkipIfEqual skip_if(_masm, &DTraceAllocProbes, 0); - // Trigger dtrace event for fastpath - __ push(atos); - __ call_VM_leaf( - CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_object_alloc), rax); - __ pop(atos); - } - - __ jmp(done); - } + __ allocate_instance(rcx, rax, rdx, rbx, true, slow_case); + __ jmp(done); // slow case __ bind(slow_case); - __ pop(rcx); // restore stack pointer to what it was when we came in. - __ bind(slow_case_no_pop); Register rarg1 = LP64_ONLY(c_rarg1) NOT_LP64(rax); Register rarg2 = LP64_ONLY(c_rarg2) NOT_LP64(rdx); @@ -4476,19 +4364,8 @@ __ cmpb(Address(rcx, InstanceKlass::init_state_offset()), InstanceKlass::fully_initialized); __ jcc(Assembler::notEqual, slow_case); - // Getting the offset of the pre-allocated default value - __ movptr(rdx, Address(rcx, in_bytes(InstanceKlass::adr_valueklass_fixed_block_offset()))); - __ movl(rdx, Address(rdx, in_bytes(ValueKlass::default_value_offset_offset()))); - - // Getting the mirror - __ movptr(rbx, Address(rcx, in_bytes(Klass::java_mirror_offset()))); - __ resolve_oop_handle(rbx, rcx); - __ verify_oop(rbx); - - // Getting the pre-allocated default value from the mirror - Address field(rbx, rdx, Address::times_1); - do_oop_load(_masm, field, rax); - + // have a resolved ValueKlass in rcx, return the default value oop from it + __ get_default_value_oop(rcx, rdx, rax); __ jmp(done); __ bind(slow_case); --- old/src/hotspot/share/gc/shared/collectedHeap.hpp 2019-11-22 13:54:10.268080677 +0100 +++ new/src/hotspot/share/gc/shared/collectedHeap.hpp 2019-11-22 13:54:10.024076552 +0100 @@ -244,6 +244,7 @@ GCCause::Cause gc_cause() { return _gc_cause; } oop obj_allocate(Klass* klass, int size, TRAPS); + oop obj_buffer_allocate(Klass* klass, int size, TRAPS); // doesn't clear memory virtual oop array_allocate(Klass* klass, int size, int length, bool do_zero, TRAPS); oop class_allocate(Klass* klass, int size, TRAPS); --- old/src/hotspot/share/gc/shared/collectedHeap.inline.hpp 2019-11-22 13:54:11.116095012 +0100 +++ new/src/hotspot/share/gc/shared/collectedHeap.inline.hpp 2019-11-22 13:54:10.820090008 +0100 @@ -72,6 +72,11 @@ return allocator.allocate(); } +inline oop CollectedHeap::obj_buffer_allocate(Klass* klass, int size, TRAPS) { + ObjBufferAllocator allocator(klass, size, THREAD); + return allocator.allocate(); +} + inline oop CollectedHeap::array_allocate(Klass* klass, int size, int length, bool do_zero, TRAPS) { ObjArrayAllocator allocator(klass, size, length, do_zero, THREAD); return allocator.allocate(); --- old/src/hotspot/share/gc/shared/memAllocator.cpp 2019-11-22 13:54:11.716105155 +0100 +++ new/src/hotspot/share/gc/shared/memAllocator.cpp 2019-11-22 13:54:11.472101030 +0100 @@ -401,6 +401,12 @@ return finish(mem); } +oop ObjBufferAllocator::initialize(HeapWord* mem) const { + oopDesc::set_klass_gap(mem, 0); + return finish(mem); +} + + MemRegion ObjArrayAllocator::obj_memory_range(oop obj) const { if (_do_zero) { return MemAllocator::obj_memory_range(obj); --- old/src/hotspot/share/gc/shared/memAllocator.hpp 2019-11-22 13:54:12.416116988 +0100 +++ new/src/hotspot/share/gc/shared/memAllocator.hpp 2019-11-22 13:54:12.132112187 +0100 @@ -83,6 +83,14 @@ virtual oop initialize(HeapWord* mem) const; }; +class ObjBufferAllocator: public MemAllocator { +public: + ObjBufferAllocator(Klass* klass, size_t word_size, Thread* thread = Thread::current()) + : MemAllocator(klass, word_size, thread) {} + virtual oop initialize(HeapWord* mem) const; +}; + + class ObjArrayAllocator: public MemAllocator { const int _length; const bool _do_zero; --- old/src/hotspot/share/oops/instanceKlass.cpp 2019-11-22 13:54:13.200130241 +0100 +++ new/src/hotspot/share/oops/instanceKlass.cpp 2019-11-22 13:54:12.952126048 +0100 @@ -478,6 +478,7 @@ _init_thread(NULL), _init_state(allocated), _reference_type(parser.reference_type()), + _value_field_klasses(NULL), _adr_valueklass_fixed_block(NULL) { set_vtable_length(parser.vtable_size()); @@ -504,6 +505,9 @@ if (UseBiasedLocking && BiasedLocking::enabled()) { set_prototype_header(markWord::biased_locking_prototype()); } + if (has_value_fields()) { + _value_field_klasses = (const Klass**) adr_value_fields_klasses(); + } } void InstanceKlass::deallocate_methods(ClassLoaderData* loader_data, --- old/src/hotspot/share/oops/instanceKlass.hpp 2019-11-22 13:54:14.204147213 +0100 +++ new/src/hotspot/share/oops/instanceKlass.hpp 2019-11-22 13:54:13.956143020 +0100 @@ -361,6 +361,7 @@ // [generic signature index] // ... Array* _fields; + const Klass** _value_field_klasses; // For "inline class" fields, NULL if none present const ValueKlassFixedBlock* _adr_valueklass_fixed_block; @@ -1066,6 +1067,7 @@ JFR_ONLY(DEFINE_KLASS_TRACE_ID_OFFSET;) static ByteSize init_thread_offset() { return in_ByteSize(offset_of(InstanceKlass, _init_thread)); } + static ByteSize value_field_klasses_offset() { return in_ByteSize(offset_of(InstanceKlass, _value_field_klasses)); } static ByteSize adr_valueklass_fixed_block_offset() { return in_ByteSize(offset_of(InstanceKlass, _adr_valueklass_fixed_block)); } // subclass/subinterface checks --- old/src/hotspot/share/oops/valueKlass.cpp 2019-11-22 13:54:15.216164320 +0100 +++ new/src/hotspot/share/oops/valueKlass.cpp 2019-11-22 13:54:14.924159384 +0100 @@ -130,6 +130,14 @@ return oop; } +instanceOop ValueKlass::allocate_instance_buffer(TRAPS) { + int size = size_helper(); // Query before forming handle. + + instanceOop oop = (instanceOop)Universe::heap()->obj_buffer_allocate(this, size, CHECK_NULL); + assert(oop->mark().is_always_locked(), "Unlocked value type"); + return oop; +} + bool ValueKlass::is_atomic() { return (nonstatic_field_size() * heapOopSize) <= longSize; } @@ -153,7 +161,7 @@ res = (instanceOop)default_value(); } else { Handle obj_h(THREAD, obj); - res = allocate_instance(CHECK_NULL); + res = allocate_instance_buffer(CHECK_NULL); value_copy_payload_to_new_oop(((char*)(oopDesc*)obj_h()) + offset, res); } assert(res != NULL, "Must be set in one of two paths above"); --- old/src/hotspot/share/oops/valueKlass.hpp 2019-11-22 13:54:16.004177641 +0100 +++ new/src/hotspot/share/oops/valueKlass.hpp 2019-11-22 13:54:15.756173449 +0100 @@ -194,7 +194,12 @@ void array_klasses_do(void f(Klass* k)); // allocate_instance() allocates a stand alone value in the Java heap + // initialized to default value (cleared memory) instanceOop allocate_instance(TRAPS); + // allocates a stand alone value buffer in the Java heap + // DOES NOT have memory cleared, user MUST initialize payload before + // returning to Java (i.e.: value_copy) + instanceOop allocate_instance_buffer(TRAPS); // minimum number of bytes occupied by nonstatic fields, HeapWord aligned or pow2 int raw_value_byte_size(); @@ -273,6 +278,10 @@ return byte_offset_of(ValueKlassFixedBlock, _default_value_offset); } + static ByteSize first_field_offset_offset() { + return byte_offset_of(ValueKlassFixedBlock, _first_field_offset); + } + void set_default_value_offset(int offset) { *((int*)adr_default_value_offset()) = offset; } --- /dev/null 2019-11-14 09:41:41.408352399 +0100 +++ new/src/hotspot/share/gc/shared/barrierSetRuntime.cpp 2019-11-22 13:54:16.356183591 +0100 @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/shared/barrierSetRuntime.hpp" +#include "oops/access.inline.hpp" +#include "runtime/interfaceSupport.inline.hpp" +#include "utilities/macros.hpp" + +JRT_LEAF(void, BarrierSetRuntime::value_copy(void* src, void* dst, ValueKlass* md)) + assert(md->is_value_type_klass(), "invariant"); + HeapAccess<>::value_copy(src, dst, md); +JRT_END + +JRT_LEAF(void, BarrierSetRuntime::value_copy_is_dest_uninitialized(void* src, void* dst, ValueKlass* md)) + assert(md->is_value_type_klass(), "invariant"); + HeapAccess::value_copy(src, dst, md); +JRT_END --- /dev/null 2019-11-14 09:41:41.408352399 +0100 +++ new/src/hotspot/share/gc/shared/barrierSetRuntime.hpp 2019-11-22 13:54:16.992194343 +0100 @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_GC_SHARED_BARRIERSETRUNTIME_HPP +#define SHARE_GC_SHARED_BARRIERSETRUNTIME_HPP + +#include "memory/allocation.hpp" +#include "oops/oopsHierarchy.hpp" +#include "oops/valueKlass.hpp" +#include "utilities/globalDefinitions.hpp" +#include "utilities/macros.hpp" + +class oopDesc; +class JavaThread; + +class BarrierSetRuntime: public AllStatic { +public: + // Template interpreter... + static void value_copy(void* src, void* dst, ValueKlass* md); + static void value_copy_is_dest_uninitialized(void* src, void* dst, ValueKlass* md); +}; + +#endif // SHARE_GC_SHARED_BARRIERSETRUNTIME_HPP