--- old/src/hotspot/cpu/x86/templateTable_x86.cpp 2019-03-11 14:24:58.802356087 +0100 +++ new/src/hotspot/cpu/x86/templateTable_x86.cpp 2019-03-11 14:24:58.598356089 +0100 @@ -177,6 +177,7 @@ Label L_patch_done; switch (bc) { + case Bytecodes::_fast_qputfield: case Bytecodes::_fast_aputfield: case Bytecodes::_fast_bputfield: case Bytecodes::_fast_zputfield: @@ -369,6 +370,7 @@ // get type __ movzbl(rdx, Address(rax, rbx, Address::times_1, tags_offset)); + __ andl(rdx, ~JVM_CONSTANT_QDESC_BIT); // unresolved class - get the resolved class __ cmpl(rdx, JVM_CONSTANT_UnresolvedClass); @@ -819,15 +821,32 @@ void TemplateTable::aaload() { transition(itos, atos); - // rax: index - // rdx: array - index_check(rdx, rax); // kills rbx - do_oop_load(_masm, - Address(rdx, rax, - UseCompressedOops ? Address::times_4 : Address::times_ptr, - arrayOopDesc::base_offset_in_bytes(T_OBJECT)), - rax, - IS_ARRAY); + + Register array = rcx; + Register index = rax; + + index_check(array, index); // kills rbx + if (ValueArrayFlatten) { + Label is_flat_array, done; + __ test_flat_array_oop(array, rbx, is_flat_array); + do_oop_load(_masm, + Address(array, index, + UseCompressedOops ? Address::times_4 : Address::times_ptr, + arrayOopDesc::base_offset_in_bytes(T_OBJECT)), + rax, + IS_ARRAY); + __ jmp(done); + __ bind(is_flat_array); + __ call_VM(rax, CAST_FROM_FN_PTR(address, InterpreterRuntime::value_array_load), array, index); + __ bind(done); + } else { + do_oop_load(_masm, + Address(array, index, + UseCompressedOops ? Address::times_4 : Address::times_ptr, + arrayOopDesc::base_offset_in_bytes(T_OBJECT)), + rax, + IS_ARRAY); + } } void TemplateTable::baload() { @@ -1113,7 +1132,7 @@ } void TemplateTable::aastore() { - Label is_null, ok_is_subtype, done; + Label is_null, is_flat_array, ok_is_subtype, done; transition(vtos, vtos); // stack: ..., array, index, value __ movptr(rax, at_tos()); // value @@ -1125,18 +1144,25 @@ arrayOopDesc::base_offset_in_bytes(T_OBJECT)); index_check_without_pop(rdx, rcx); // kills rbx + __ testptr(rax, rax); __ jcc(Assembler::zero, is_null); + // Move array class to rdi + __ load_klass(rdi, rdx); + if (ValueArrayFlatten) { + __ test_flat_array_klass(rdi, rbx, is_flat_array); + } + // Move subklass into rbx __ load_klass(rbx, rax); - // Move superklass into rax - __ load_klass(rax, rdx); - __ movptr(rax, Address(rax, + // Move array element superklass into rax + __ movptr(rax, Address(rdi, ObjArrayKlass::element_klass_offset())); // Generate subtype check. Blows rcx, rdi // Superklass in rax. Subklass in rbx. + // is "rbx <: rax" ? (value subclass <: array element superclass) __ gen_subtype_check(rbx, ok_is_subtype); // Come here on failure @@ -1156,10 +1182,53 @@ // Have a NULL in rax, rdx=array, ecx=index. Store NULL at ary[idx] __ bind(is_null); __ profile_null_seen(rbx); + if (EnableValhalla) { + Label is_null_into_value_array_npe, store_null; + + __ load_klass(rdi, rdx); + // No way to store null in flat array + __ test_flat_array_klass(rdi, rbx, is_null_into_value_array_npe); + + // Use case for storing values in objArray where element_klass is specifically + // a value type because they could not be flattened "for reasons", + // these need to have the same semantics as flat arrays, i.e. NPE + __ movptr(rdi, Address(rdi, ObjArrayKlass::element_klass_offset())); + __ test_klass_is_value(rdi, rdi, is_null_into_value_array_npe); + __ jmp(store_null); + + __ bind(is_null_into_value_array_npe); + __ jump(ExternalAddress(Interpreter::_throw_NullPointerException_entry)); + __ bind(store_null); + } // Store a NULL do_oop_store(_masm, element_address, noreg, IS_ARRAY); + __ jmp(done); + if (EnableValhalla) { + Label is_type_ok; + __ bind(is_flat_array); // Store non-null value to flat + + // Simplistic type check... + + // Profile the not-null value's klass. + __ load_klass(rbx, rax); + __ profile_typecheck(rcx, rbx, rax); // blows rcx, and rax + // Move element klass into rax + __ movptr(rax, Address(rdi, ArrayKlass::element_klass_offset())); + // flat value array needs exact type match + // is "rax == rbx" (value subclass == array element superclass) + __ cmpptr(rax, rbx); + __ jccb(Assembler::equal, is_type_ok); + + __ profile_typecheck_failed(rcx); + __ jump(ExternalAddress(Interpreter::_throw_ArrayStoreException_entry)); + + __ bind(is_type_ok); + __ movptr(rax, at_tos()); // value + __ movl(rcx, at_tos_p1()); // index + __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::value_array_store), rax, rdx, rcx); + } // Pop stack arguments __ bind(done); __ addptr(rsp, 3 * Interpreter::stackElementSize); @@ -2405,10 +2474,37 @@ void TemplateTable::if_acmp(Condition cc) { transition(atos, vtos); // assume branch is more often taken than not (loops use backward branches) - Label not_taken; + Label taken, not_taken; __ pop_ptr(rdx); + + const int is_value_mask = markOopDesc::always_locked_pattern; + if (EnableValhalla && ACmpOnValues == 1) { + Label is_null; + __ testptr(rdx, rdx); + __ jcc(Assembler::zero, is_null); + __ movptr(rbx, Address(rdx, oopDesc::mark_offset_in_bytes())); + __ andptr(rbx, is_value_mask); + __ cmpl(rbx, is_value_mask); + __ setb(Assembler::equal, rbx); + __ movzbl(rbx, rbx); + __ orptr(rdx, rbx); + __ bind(is_null); + } + __ cmpoop(rdx, rax); + + if (EnableValhalla && ACmpOnValues != 1) { + __ jcc(Assembler::notEqual, (cc == not_equal) ? taken : not_taken); + __ testptr(rdx, rdx); + __ jcc(Assembler::zero, (cc == equal) ? taken : not_taken); + __ movptr(rbx, Address(rdx, oopDesc::mark_offset_in_bytes())); + __ andptr(rbx, is_value_mask); + __ cmpl(rbx, is_value_mask); + cc = (cc == equal) ? not_equal : equal; + } + __ jcc(j_not(cc), not_taken); + __ bind(taken); branch(false, false); __ bind(not_taken); __ profile_not_taken_branch(rax); @@ -2679,7 +2775,8 @@ if (state == itos) { __ narrow(rax); } - __ remove_activation(state, rbcp); + + __ remove_activation(state, rbcp, true, true, true); __ jmp(rbcp); } @@ -2866,16 +2963,23 @@ const Register off = rbx; const Register flags = rax; const Register bc = LP64_ONLY(c_rarg3) NOT_LP64(rcx); // uses same reg as obj, so don't mix them + const Register flags2 = rdx; resolve_cache_and_index(byte_no, cache, index, sizeof(u2)); jvmti_post_field_access(cache, index, is_static, false); load_field_cp_cache_entry(obj, cache, index, off, flags, is_static); - if (!is_static) pop_and_check_object(obj); - const Address field(obj, off, Address::times_1, 0*wordSize); - Label Done, notByte, notBool, notInt, notShort, notChar, notLong, notFloat, notObj; + Label Done, notByte, notBool, notInt, notShort, notChar, notLong, notFloat, notObj, notValueType; + + if (!is_static) { + __ movptr(rcx, Address(cache, index, Address::times_ptr, + in_bytes(ConstantPoolCache::base_offset() + + ConstantPoolCacheEntry::f1_offset()))); + } + + __ movl(flags2, flags); __ shrl(flags, ConstantPoolCacheEntry::tos_state_shift); // Make sure we don't need to mask edx after the above shift @@ -2885,6 +2989,7 @@ __ jcc(Assembler::notZero, notByte); // btos + if (!is_static) pop_and_check_object(obj); __ access_load_at(T_BYTE, IN_HEAP, rax, field, noreg, noreg); __ push(btos); // Rewrite bytecode to be faster @@ -2894,9 +2999,10 @@ __ jmp(Done); __ bind(notByte); + __ cmpl(flags, ztos); __ jcc(Assembler::notEqual, notBool); - + if (!is_static) pop_and_check_object(obj); // ztos (same code as btos) __ access_load_at(T_BOOLEAN, IN_HEAP, rax, field, noreg, noreg); __ push(ztos); @@ -2911,14 +3017,80 @@ __ cmpl(flags, atos); __ jcc(Assembler::notEqual, notObj); // atos - do_oop_load(_masm, field, rax); - __ push(atos); - if (!is_static && rc == may_rewrite) { - patch_bytecode(Bytecodes::_fast_agetfield, bc, rbx); + if (!EnableValhalla) { + if (!is_static) pop_and_check_object(obj); + do_oop_load(_masm, field, rax); + __ push(atos); + if (!is_static && rc == may_rewrite) { + patch_bytecode(Bytecodes::_fast_agetfield, bc, rbx); + } + __ jmp(Done); + } else { + if (is_static) { + __ load_heap_oop(rax, field); + Label isFlattenable, uninitialized; + // Issue below if the static field has not been initialized yet + __ test_field_is_flattenable(flags2, rscratch1, isFlattenable); + // Not flattenable case + __ push(atos); + __ jmp(Done); + // Flattenable case, must not return null even if uninitialized + __ bind(isFlattenable); + __ testptr(rax, rax); + __ jcc(Assembler::zero, uninitialized); + __ push(atos); + __ jmp(Done); + __ bind(uninitialized); + __ andl(flags2, ConstantPoolCacheEntry::field_index_mask); + __ call_VM(rax, CAST_FROM_FN_PTR(address, InterpreterRuntime::uninitialized_static_value_field), + obj, flags2); + __ verify_oop(rax); + __ push(atos); + __ jmp(Done); + } else { + Label isFlattened, nonnull, isFlattenable, rewriteFlattenable; + __ test_field_is_flattenable(flags2, rscratch1, isFlattenable); + // Non-flattenable field case, also covers the object case + pop_and_check_object(obj); + __ load_heap_oop(rax, field); + __ push(atos); + if (rc == may_rewrite) { + patch_bytecode(Bytecodes::_fast_agetfield, bc, rbx); + } + __ jmp(Done); + __ bind(isFlattenable); + __ test_field_is_flattened(flags2, rscratch1, isFlattened); + // Non-flattened field case + pop_and_check_object(obj); + __ load_heap_oop(rax, field); + __ testptr(rax, rax); + __ jcc(Assembler::notZero, nonnull); + __ andl(flags2, ConstantPoolCacheEntry::field_index_mask); + __ call_VM(rax, CAST_FROM_FN_PTR(address, InterpreterRuntime::uninitialized_instance_value_field), + obj, flags2); + __ bind(nonnull); + __ verify_oop(rax); + __ push(atos); + __ jmp(rewriteFlattenable); + __ bind(isFlattened); + __ andl(flags2, ConstantPoolCacheEntry::field_index_mask); + pop_and_check_object(rbx); + call_VM(rax, CAST_FROM_FN_PTR(address, InterpreterRuntime::read_flattened_field), + rbx, flags2, rcx); + __ verify_oop(rax); + __ push(atos); + __ bind(rewriteFlattenable); + if (rc == may_rewrite) { + patch_bytecode(Bytecodes::_fast_qgetfield, bc, rbx); + } + __ jmp(Done); + } } - __ jmp(Done); __ bind(notObj); + + if (!is_static) pop_and_check_object(obj); + __ cmpl(flags, itos); __ jcc(Assembler::notEqual, notInt); // itos @@ -3017,6 +3189,21 @@ getfield_or_static(byte_no, true); } +void TemplateTable::withfield() { + transition(vtos, atos); + + Register cache = LP64_ONLY(c_rarg1) NOT_LP64(rcx); + Register index = LP64_ONLY(c_rarg2) NOT_LP64(rdx); + + resolve_cache_and_index(f2_byte, cache, index, sizeof(u2)); + + call_VM(rbx, CAST_FROM_FN_PTR(address, InterpreterRuntime::withfield), cache); + // new value type is returned in rbx + // stack adjustement is returned in rax + __ verify_oop(rbx); + __ addptr(rsp, rax); + __ movptr(rax, rbx); +} // The registers cache and index expected to be set before call. // The function may destroy various registers, just not the cache and index registers. @@ -3112,6 +3299,7 @@ const Register obj = rcx; const Register off = rbx; const Register flags = rax; + const Register flags2 = rdx; resolve_cache_and_index(byte_no, cache, index, sizeof(u2)); jvmti_post_field_mod(cache, index, is_static); @@ -3128,28 +3316,29 @@ // Check for volatile store __ testl(rdx, rdx); + __ movl(flags2, flags); __ jcc(Assembler::zero, notVolatile); - putfield_or_static_helper(byte_no, is_static, rc, obj, off, flags); + putfield_or_static_helper(byte_no, is_static, rc, obj, off, flags, flags2); volatile_barrier(Assembler::Membar_mask_bits(Assembler::StoreLoad | Assembler::StoreStore)); __ jmp(Done); __ bind(notVolatile); - putfield_or_static_helper(byte_no, is_static, rc, obj, off, flags); + putfield_or_static_helper(byte_no, is_static, rc, obj, off, flags, flags2); __ bind(Done); } void TemplateTable::putfield_or_static_helper(int byte_no, bool is_static, RewriteControl rc, - Register obj, Register off, Register flags) { + Register obj, Register off, Register flags, Register flags2) { // field addresses const Address field(obj, off, Address::times_1, 0*wordSize); NOT_LP64( const Address hi(obj, off, Address::times_1, 1*wordSize);) Label notByte, notBool, notInt, notShort, notChar, - notLong, notFloat, notObj; + notLong, notFloat, notObj, notValueType; Label Done; const Register bc = LP64_ONLY(c_rarg3) NOT_LP64(rcx); @@ -3192,14 +3381,56 @@ // atos { - __ pop(atos); - if (!is_static) pop_and_check_object(obj); - // Store into the field - do_oop_store(_masm, field, rax); - if (!is_static && rc == may_rewrite) { - patch_bytecode(Bytecodes::_fast_aputfield, bc, rbx, true, byte_no); + if (!EnableValhalla) { + __ pop(atos); + if (!is_static) pop_and_check_object(obj); + // Store into the field + do_oop_store(_masm, field, rax); + if (!is_static && rc == may_rewrite) { + patch_bytecode(Bytecodes::_fast_aputfield, bc, rbx, true, byte_no); + } + __ jmp(Done); + } else { + __ pop(atos); + if (is_static) { + Label notFlattenable, notBuffered; + __ test_field_is_not_flattenable(flags2, rscratch1, notFlattenable); + __ null_check(rax); + __ bind(notFlattenable); + do_oop_store(_masm, field, rax); + __ jmp(Done); + } else { + Label isFlattenable, isFlattened, notBuffered, notBuffered2, rewriteNotFlattenable, rewriteFlattenable; + __ test_field_is_flattenable(flags2, rscratch1, isFlattenable); + // Not flattenable case, covers not flattenable values and objects + pop_and_check_object(obj); + // Store into the field + do_oop_store(_masm, field, rax); + __ bind(rewriteNotFlattenable); + if (rc == may_rewrite) { + patch_bytecode(Bytecodes::_fast_aputfield, bc, rbx, true, byte_no); + } + __ jmp(Done); + // Implementation of the flattenable semantic + __ bind(isFlattenable); + __ null_check(rax); + __ test_field_is_flattened(flags2, rscratch1, isFlattened); + // Not flattened case + pop_and_check_object(obj); + // Store into the field + do_oop_store(_masm, field, rax); + __ jmp(rewriteFlattenable); + __ bind(isFlattened); + pop_and_check_object(obj); + call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::write_flattened_value), + rax, off, obj); + __ bind(rewriteFlattenable); + if (rc == may_rewrite) { + patch_bytecode(Bytecodes::_fast_qputfield, bc, rbx, true, byte_no); + } + __ jmp(Done); + } } - __ jmp(Done); } __ bind(notObj); @@ -3336,6 +3567,7 @@ // to do it for every data type, we use the saved values as the // jvalue object. switch (bytecode()) { // load values into the jvalue object + case Bytecodes::_fast_qputfield: //fall through case Bytecodes::_fast_aputfield: __ push_ptr(rax); break; case Bytecodes::_fast_bputfield: // fall through case Bytecodes::_fast_zputfield: // fall through @@ -3361,6 +3593,7 @@ NOT_LP64(__ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::post_field_modification), rbx, rax, rcx)); switch (bytecode()) { // restore tos values + case Bytecodes::_fast_qputfield: // fall through case Bytecodes::_fast_aputfield: __ pop_ptr(rax); break; case Bytecodes::_fast_bputfield: // fall through case Bytecodes::_fast_zputfield: // fall through @@ -3400,6 +3633,10 @@ // Assembler::StoreStore)); Label notVolatile, Done; + if (bytecode() == Bytecodes::_fast_qputfield) { + __ movl(rscratch2, rdx); + } + __ shrl(rdx, ConstantPoolCacheEntry::is_volatile_shift); __ andl(rdx, 0x1); @@ -3428,8 +3665,24 @@ // access field switch (bytecode()) { + case Bytecodes::_fast_qputfield: + { + Label isFlattened, done; + __ null_check(rax); + __ test_field_is_flattened(rscratch2, rscratch1, isFlattened); + // No Flattened case + do_oop_store(_masm, field, rax); + __ jmp(done); + __ bind(isFlattened); + call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::write_flattened_value), + rax, rbx, rcx); + __ bind(done); + } + break; case Bytecodes::_fast_aputfield: - do_oop_store(_masm, field, rax); + { + do_oop_store(_masm, field, rax); + } break; case Bytecodes::_fast_lputfield: #ifdef _LP64 @@ -3499,17 +3752,53 @@ // __ shrl(rdx, ConstantPoolCacheEntry::is_volatile_shift); // __ andl(rdx, 0x1); // - __ movptr(rbx, Address(rcx, rbx, Address::times_ptr, + __ movptr(rdx, Address(rcx, rbx, Address::times_ptr, in_bytes(ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::f2_offset()))); // rax: object __ verify_oop(rax); __ null_check(rax); - Address field(rax, rbx, Address::times_1); + Address field(rax, rdx, Address::times_1); // access field switch (bytecode()) { + case Bytecodes::_fast_qgetfield: + { + Label isFlattened, nonnull, Done; + __ movptr(rscratch1, Address(rcx, rbx, Address::times_ptr, + in_bytes(ConstantPoolCache::base_offset() + + ConstantPoolCacheEntry::flags_offset()))); + __ test_field_is_flattened(rscratch1, rscratch2, isFlattened); + // Non-flattened field case + __ movptr(rscratch1, rax); + __ load_heap_oop(rax, field); + __ testptr(rax, rax); + __ jcc(Assembler::notZero, nonnull); + __ movptr(rax, rscratch1); + __ movl(rcx, Address(rcx, rbx, Address::times_ptr, + in_bytes(ConstantPoolCache::base_offset() + + ConstantPoolCacheEntry::flags_offset()))); + __ andl(rcx, ConstantPoolCacheEntry::field_index_mask); + __ call_VM(rax, CAST_FROM_FN_PTR(address, InterpreterRuntime::uninitialized_instance_value_field), + rax, rcx); + __ bind(nonnull); + __ verify_oop(rax); + __ jmp(Done); + __ bind(isFlattened); + __ movl(rdx, Address(rcx, rbx, Address::times_ptr, + in_bytes(ConstantPoolCache::base_offset() + + ConstantPoolCacheEntry::flags_offset()))); + __ andl(rdx, ConstantPoolCacheEntry::field_index_mask); + __ movptr(rcx, Address(rcx, rbx, Address::times_ptr, + in_bytes(ConstantPoolCache::base_offset() + + ConstantPoolCacheEntry::f1_offset()))); + call_VM(rax, CAST_FROM_FN_PTR(address, InterpreterRuntime::read_flattened_field), + rax, rdx, rcx); + __ verify_oop(rax); + __ bind(Done); + } + break; case Bytecodes::_fast_agetfield: do_oop_load(_masm, field, rax); __ verify_oop(rax); @@ -4134,6 +4423,20 @@ __ bind(done); } +void TemplateTable::defaultvalue() { + transition(vtos, atos); + + Register rarg1 = LP64_ONLY(c_rarg1) NOT_LP64(rcx); + Register rarg2 = LP64_ONLY(c_rarg2) NOT_LP64(rdx); + + __ get_unsigned_2_byte_index_at_bcp(rarg2, 1); + __ get_constant_pool(rarg1); + + call_VM(rax, CAST_FROM_FN_PTR(address, InterpreterRuntime::defaultvalue), + rarg1, rarg2); + __ verify_oop(rax); +} + void TemplateTable::newarray() { transition(itos, atos); Register rarg1 = LP64_ONLY(c_rarg1) NOT_LP64(rdx); @@ -4170,10 +4473,11 @@ __ get_cpool_and_tags(rcx, rdx); // rcx=cpool, rdx=tags array __ get_unsigned_2_byte_index_at_bcp(rbx, 1); // rbx=index // See if bytecode has already been quicked - __ cmpb(Address(rdx, rbx, - Address::times_1, - Array::base_offset_in_bytes()), - JVM_CONSTANT_Class); + __ movzbl(rdx, Address(rdx, rbx, + Address::times_1, + Array::base_offset_in_bytes())); + __ andl (rdx, ~JVM_CONSTANT_QDESC_BIT); + __ cmpl(rdx, JVM_CONSTANT_Class); __ jcc(Assembler::equal, quicked); __ push(atos); // save receiver for result, and for GC call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::quicken_io_cc)); @@ -4211,15 +4515,29 @@ // Come here on success __ bind(ok_is_subtype); __ mov(rax, rdx); // Restore object in rdx + __ jmp(done); + + __ bind(is_null); // Collect counts on whether this check-cast sees NULLs a lot or not. if (ProfileInterpreter) { - __ jmp(done); - __ bind(is_null); __ profile_null_seen(rcx); - } else { - __ bind(is_null); // same as 'done' } + + if (EnableValhalla) { + // Get cpool & tags index + __ get_cpool_and_tags(rcx, rdx); // rcx=cpool, rdx=tags array + __ get_unsigned_2_byte_index_at_bcp(rbx, 1); // rbx=index + // See if CP entry is a Q-descriptor + __ movzbl(rcx, Address(rdx, rbx, + Address::times_1, + Array::base_offset_in_bytes())); + __ andl (rcx, JVM_CONSTANT_QDESC_BIT); + __ cmpl(rcx, JVM_CONSTANT_QDESC_BIT); + __ jcc(Assembler::notEqual, done); + __ jump(ExternalAddress(Interpreter::_throw_NullPointerException_entry)); + } + __ bind(done); } @@ -4233,10 +4551,11 @@ __ get_cpool_and_tags(rcx, rdx); // rcx=cpool, rdx=tags array __ get_unsigned_2_byte_index_at_bcp(rbx, 1); // rbx=index // See if bytecode has already been quicked - __ cmpb(Address(rdx, rbx, - Address::times_1, - Array::base_offset_in_bytes()), - JVM_CONSTANT_Class); + __ movzbl(rdx, Address(rdx, rbx, + Address::times_1, + Array::base_offset_in_bytes())); + __ andl (rdx, ~JVM_CONSTANT_QDESC_BIT); + __ cmpl(rdx, JVM_CONSTANT_Class); __ jcc(Assembler::equal, quicked); __ push(atos); // save receiver for result, and for GC @@ -4288,7 +4607,6 @@ // rax = 1: obj != NULL and obj is an instanceof the specified klass } - //---------------------------------------------------------------------------------------------------- // Breakpoints void TemplateTable::_breakpoint() {