--- old/src/hotspot/share/ci/ciMetadata.hpp 2019-09-17 12:56:16.000000000 +0200 +++ new/src/hotspot/share/ci/ciMetadata.hpp 2019-09-17 12:56:11.000000000 +0200 @@ -63,6 +63,7 @@ virtual bool is_obj_array_klass() const { return false; } virtual bool is_type_array_klass() const { return false; } virtual bool is_wrapper() const { return false; } + virtual bool flatten_array() const { return false; } virtual void dump_replay_data(outputStream* st) { /* do nothing */ } ciMethod* as_method() { --- old/src/hotspot/share/opto/callnode.cpp 2019-09-17 12:56:21.000000000 +0200 +++ new/src/hotspot/share/opto/callnode.cpp 2019-09-17 12:56:16.000000000 +0200 @@ -1015,6 +1015,46 @@ return CallNode::cmp(call) && _method == call._method && _override_symbolic_info == call._override_symbolic_info; } + +void CallJavaNode::copy_call_debug_info(PhaseIterGVN* phase, CallNode *oldcall) { + // Copy debug information and adjust JVMState information + uint old_dbg_start = oldcall->tf()->domain_sig()->cnt(); + uint new_dbg_start = tf()->domain_sig()->cnt(); + int jvms_adj = new_dbg_start - old_dbg_start; + assert (new_dbg_start == req(), "argument count mismatch"); + Compile* C = phase->C; + + // SafePointScalarObject node could be referenced several times in debug info. + // Use Dict to record cloned nodes. + Dict* sosn_map = new Dict(cmpkey,hashkey); + for (uint i = old_dbg_start; i < oldcall->req(); i++) { + Node* old_in = oldcall->in(i); + // Clone old SafePointScalarObjectNodes, adjusting their field contents. + if (old_in != NULL && old_in->is_SafePointScalarObject()) { + SafePointScalarObjectNode* old_sosn = old_in->as_SafePointScalarObject(); + uint old_unique = C->unique(); + Node* new_in = old_sosn->clone(sosn_map); + if (old_unique != C->unique()) { // New node? + new_in->set_req(0, C->root()); // reset control edge + new_in = phase->transform(new_in); // Register new node. + } + old_in = new_in; + } + add_req(old_in); + } + + // JVMS may be shared so clone it before we modify it + set_jvms(oldcall->jvms() != NULL ? oldcall->jvms()->clone_deep(C) : NULL); + for (JVMState *jvms = this->jvms(); jvms != NULL; jvms = jvms->caller()) { + jvms->set_map(this); + jvms->set_locoff(jvms->locoff()+jvms_adj); + jvms->set_stkoff(jvms->stkoff()+jvms_adj); + jvms->set_monoff(jvms->monoff()+jvms_adj); + jvms->set_scloff(jvms->scloff()+jvms_adj); + jvms->set_endoff(jvms->endoff()+jvms_adj); + } +} + #ifdef ASSERT bool CallJavaNode::validate_symbolic_info() const { if (method() == NULL) { @@ -1078,6 +1118,149 @@ return call->in(TypeFunc::Parms)->bottom_type()->is_int()->get_con(); } +bool CallStaticJavaNode::remove_useless_allocation(PhaseGVN *phase, Node* ctl, Node* mem, Node* unc_arg) { + // Split if can cause the flattened array branch of an array load to + // end in an uncommon trap. In that case, the allocation of the + // loaded value and its initialization is useless. Eliminate it. use + // the jvm state of the allocation to create a new uncommon trap + // call at the load. + if (ctl == NULL || ctl->is_top() || mem == NULL || mem->is_top() || !mem->is_MergeMem()) { + return false; + } + PhaseIterGVN* igvn = phase->is_IterGVN(); + if (ctl->is_Region()) { + bool res = false; + for (uint i = 1; i < ctl->req(); i++) { + MergeMemNode* mm = mem->clone()->as_MergeMem(); + for (MergeMemStream mms(mm); mms.next_non_empty(); ) { + Node* m = mms.memory(); + if (m->is_Phi() && m->in(0) == ctl) { + mms.set_memory(m->in(i)); + } + } + if (remove_useless_allocation(phase, ctl->in(i), mm, unc_arg)) { + res = true; + if (!ctl->in(i)->is_Region()) { + igvn->replace_input_of(ctl, i, phase->C->top()); + } + } + igvn->remove_dead_node(mm); + } + return res; + } + // verify the control flow is ok + Node* c = ctl; + Node* copy = NULL; + Node* alloc = NULL; + for (;;) { + if (c == NULL || c->is_top()) { + return false; + } + if (c->is_Proj() || c->is_Catch() || c->is_MemBar()) { + c = c->in(0); + } else if (c->Opcode() == Op_CallLeaf && + c->as_Call()->entry_point() == CAST_FROM_FN_PTR(address, OptoRuntime::load_unknown_value)) { + copy = c; + c = c->in(0); + } else if (c->is_Allocate()) { + Node* new_obj = c->as_Allocate()->result_cast(); + if (copy == NULL || new_obj == NULL) { + return false; + } + Node* copy_dest = copy->in(TypeFunc::Parms + 2); + if (copy_dest != new_obj) { + return false; + } + alloc = c; + break; + } else { + return false; + } + } + + JVMState* jvms = alloc->jvms(); + if (phase->C->too_many_traps(jvms->method(), jvms->bci(), Deoptimization::trap_request_reason(uncommon_trap_request()))) { + return false; + } + + Node* alloc_mem = alloc->in(TypeFunc::Memory); + if (alloc_mem == NULL || alloc_mem->is_top()) { + return false; + } + if (!alloc_mem->is_MergeMem()) { + alloc_mem = MergeMemNode::make(alloc_mem); + } + + // and that there's no unexpected side effect + for (MergeMemStream mms2(mem->as_MergeMem(), alloc_mem->as_MergeMem()); mms2.next_non_empty2(); ) { + Node* m1 = mms2.is_empty() ? mms2.base_memory() : mms2.memory(); + Node* m2 = mms2.memory2(); + + for (uint i = 0; i < 100; i++) { + if (m1 == m2) { + break; + } else if (m1->is_Proj()) { + m1 = m1->in(0); + } else if (m1->is_MemBar()) { + m1 = m1->in(TypeFunc::Memory); + } else if (m1->Opcode() == Op_CallLeaf && + m1->as_Call()->entry_point() == CAST_FROM_FN_PTR(address, OptoRuntime::load_unknown_value)) { + if (m1 != copy) { + return false; + } + m1 = m1->in(TypeFunc::Memory); + } else if (m1->is_Allocate()) { + if (m1 != alloc) { + return false; + } + break; + } else { + return false; + } + } + } + if (alloc_mem->outcnt() == 0) { + igvn->remove_dead_node(alloc_mem); + } + + address call_addr = SharedRuntime::uncommon_trap_blob()->entry_point(); + CallNode* unc = new CallStaticJavaNode(OptoRuntime::uncommon_trap_Type(), call_addr, "uncommon_trap", + jvms->bci(), NULL); + unc->init_req(TypeFunc::Control, alloc->in(0)); + unc->init_req(TypeFunc::I_O, alloc->in(TypeFunc::I_O)); + unc->init_req(TypeFunc::Memory, alloc->in(TypeFunc::Memory)); + unc->init_req(TypeFunc::FramePtr, alloc->in(TypeFunc::FramePtr)); + unc->init_req(TypeFunc::ReturnAdr, alloc->in(TypeFunc::ReturnAdr)); + unc->init_req(TypeFunc::Parms+0, unc_arg); + unc->set_cnt(PROB_UNLIKELY_MAG(4)); + unc->copy_call_debug_info(igvn, alloc->as_Allocate()); + + igvn->replace_input_of(alloc, 0, phase->C->top()); + + igvn->register_new_node_with_optimizer(unc); + + Node* ctrl = phase->transform(new ProjNode(unc, TypeFunc::Control)); + Node* halt = phase->transform(new HaltNode(ctrl, alloc->in(TypeFunc::FramePtr))); + phase->C->root()->add_req(halt); + + return true; +} + + +Node* CallStaticJavaNode::Ideal(PhaseGVN *phase, bool can_reshape) { + if (can_reshape && uncommon_trap_request() != 0) { + if (remove_useless_allocation(phase, in(0), in(TypeFunc::Memory), in(TypeFunc::Parms))) { + if (!in(0)->is_Region()) { + PhaseIterGVN* igvn = phase->is_IterGVN(); + igvn->replace_input_of(this, 0, phase->C->top()); + } + return this; + } + } + return CallNode::Ideal(phase, can_reshape); +} + + #ifndef PRODUCT void CallStaticJavaNode::dump_spec(outputStream *st) const { st->print("# Static "); --- old/src/hotspot/share/opto/callnode.hpp 2019-09-17 12:56:27.000000000 +0200 +++ new/src/hotspot/share/opto/callnode.hpp 2019-09-17 12:56:22.000000000 +0200 @@ -660,6 +660,8 @@ bool is_call_to_arraycopystub() const; + virtual void copy_call_debug_info(PhaseIterGVN* phase, CallNode *oldcall) {} + #ifndef PRODUCT virtual void dump_req(outputStream *st = tty) const; virtual void dump_spec(outputStream *st) const; @@ -703,6 +705,8 @@ void set_override_symbolic_info(bool f) { _override_symbolic_info = f; } bool override_symbolic_info() const { return _override_symbolic_info; } + void copy_call_debug_info(PhaseIterGVN* phase, CallNode *oldcall); + DEBUG_ONLY( bool validate_symbolic_info() const; ) #ifndef PRODUCT @@ -718,6 +722,9 @@ class CallStaticJavaNode : public CallJavaNode { virtual bool cmp( const Node &n ) const; virtual uint size_of() const; // Size is bigger + + bool remove_useless_allocation(PhaseGVN *phase, Node* ctl, Node* mem, Node* unc_arg); + public: CallStaticJavaNode(Compile* C, const TypeFunc* tf, address addr, ciMethod* method, int bci) : CallJavaNode(tf, addr, method, bci) { @@ -772,6 +779,8 @@ } } + virtual Node *Ideal(PhaseGVN *phase, bool can_reshape); + virtual int Opcode() const; #ifndef PRODUCT virtual void dump_spec(outputStream *st) const; --- old/src/hotspot/share/opto/classes.hpp 2019-09-17 12:56:32.000000000 +0200 +++ new/src/hotspot/share/opto/classes.hpp 2019-09-17 12:56:27.000000000 +0200 @@ -424,3 +424,4 @@ macro(UpperCase) macro(Whitespace) macro(GetNullFreeProperty) +macro(GetFlattenedProperty) --- old/src/hotspot/share/opto/compile.cpp 2019-09-17 12:56:38.000000000 +0200 +++ new/src/hotspot/share/opto/compile.cpp 2019-09-17 12:56:32.000000000 +0200 @@ -1580,7 +1580,7 @@ // No constant oop pointers (such as Strings); they alias with // unknown strings. assert(!is_known_inst, "not scalarizable allocation"); - tj = to = TypeInstPtr::make(TypePtr::BotPTR,to->klass(),false,0,Type::Offset(offset)); + tj = to = TypeInstPtr::make(TypePtr::BotPTR,to->klass(),false,0,Type::Offset(offset), to->klass()->flatten_array()); } } else if( is_known_inst ) { tj = to; // Keep NotNull and klass_is_exact for instance type @@ -1588,17 +1588,17 @@ // During the 2nd round of IterGVN, NotNull castings are removed. // Make sure the Bottom and NotNull variants alias the same. // Also, make sure exact and non-exact variants alias the same. - tj = to = TypeInstPtr::make(TypePtr::BotPTR,to->klass(),false,0,Type::Offset(offset)); + tj = to = TypeInstPtr::make(TypePtr::BotPTR,to->klass(),false,0,Type::Offset(offset), to->klass()->flatten_array()); } if (to->speculative() != NULL) { - tj = to = TypeInstPtr::make(to->ptr(),to->klass(),to->klass_is_exact(),to->const_oop(),Type::Offset(to->offset()), to->instance_id()); + tj = to = TypeInstPtr::make(to->ptr(),to->klass(),to->klass_is_exact(),to->const_oop(),Type::Offset(to->offset()), to->klass()->flatten_array(), to->instance_id()); } // Canonicalize the holder of this field if (offset >= 0 && offset < instanceOopDesc::base_offset_in_bytes()) { // First handle header references such as a LoadKlassNode, even if the // object's klass is unloaded at compile time (4965979). if (!is_known_inst) { // Do it only for non-instance types - tj = to = TypeInstPtr::make(TypePtr::BotPTR, env()->Object_klass(), false, NULL, Type::Offset(offset)); + tj = to = TypeInstPtr::make(TypePtr::BotPTR, env()->Object_klass(), false, NULL, Type::Offset(offset), false); } } else if (BarrierSet::barrier_set()->barrier_set_c2()->flatten_gc_alias_type(tj)) { to = tj->is_instptr(); @@ -1614,9 +1614,9 @@ ciInstanceKlass *canonical_holder = k->get_canonical_holder(offset); if (!k->equals(canonical_holder) || tj->offset() != offset) { if( is_known_inst ) { - tj = to = TypeInstPtr::make(to->ptr(), canonical_holder, true, NULL, Type::Offset(offset), to->instance_id()); + tj = to = TypeInstPtr::make(to->ptr(), canonical_holder, true, NULL, Type::Offset(offset), canonical_holder->flatten_array(), to->instance_id()); } else { - tj = to = TypeInstPtr::make(to->ptr(), canonical_holder, false, NULL, Type::Offset(offset)); + tj = to = TypeInstPtr::make(to->ptr(), canonical_holder, false, NULL, Type::Offset(offset), canonical_holder->flatten_array()); } } } @@ -1633,7 +1633,8 @@ tj = tk = TypeKlassPtr::make(TypePtr::NotNull, TypeKlassPtr::OBJECT->klass(), - Type::Offset(offset)); + Type::Offset(offset), + false); } ciKlass* klass = tk->klass(); @@ -1641,7 +1642,7 @@ ciKlass* k = TypeAryPtr::OOPS->klass(); if( !k || !k->is_loaded() ) // Only fails for some -Xcomp runs k = TypeInstPtr::BOTTOM->klass(); - tj = tk = TypeKlassPtr::make(TypePtr::NotNull, k, Type::Offset(offset)); + tj = tk = TypeKlassPtr::make(TypePtr::NotNull, k, Type::Offset(offset), false); } // Check for precise loads from the primary supertype array and force them @@ -1657,7 +1658,7 @@ offset < (int)(primary_supers_offset + Klass::primary_super_limit() * wordSize)) || offset == (int)in_bytes(Klass::secondary_super_cache_offset())) { offset = in_bytes(Klass::secondary_super_cache_offset()); - tj = tk = TypeKlassPtr::make(TypePtr::NotNull, tk->klass(), Type::Offset(offset)); + tj = tk = TypeKlassPtr::make(TypePtr::NotNull, tk->klass(), Type::Offset(offset), tk->flatten_array()); } } @@ -3437,9 +3438,10 @@ if (EnableValhalla && (nop == Op_LoadKlass || nop == Op_LoadNKlass)) { const TypeKlassPtr* tk = n->bottom_type()->make_ptr()->is_klassptr(); assert(!tk->klass_is_exact(), "should have been folded"); - if (tk->klass()->can_be_value_array_klass()) { + if (tk->klass()->can_be_value_array_klass() && n->as_Mem()->adr_type()->offset() == oopDesc::klass_offset_in_bytes()) { // Array load klass needs to filter out property bits (but not - // GetNullFreePropertyNode which needs to extract the null free bits) + // GetNullFreePropertyNode or GetFlattenedPropertyNode which + // needs to extract the storage property bits) uint last = unique(); Node* pointer = NULL; if (nop == Op_LoadKlass) { @@ -3455,7 +3457,7 @@ } for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { Node* u = n->fast_out(i); - if (u->_idx < last && u->Opcode() != Op_GetNullFreeProperty) { + if (u->_idx < last && u->Opcode() != Op_GetNullFreeProperty && u->Opcode() != Op_GetFlattenedProperty) { // If user is a comparison with a klass that can't be a value type // array klass, we don't need to clear the storage property bits. Node* cmp = (u->is_DecodeNKlass() && u->outcnt() == 1) ? u->unique_out() : u; @@ -3995,19 +3997,21 @@ break; } #endif - case Op_GetNullFreeProperty: { + case Op_GetNullFreeProperty: + case Op_GetFlattenedProperty: { // Extract the null free bits uint last = unique(); Node* null_free = NULL; + int bit = nop == Op_GetNullFreeProperty ? ArrayStorageProperties::null_free_bit : ArrayStorageProperties::flattened_bit; if (n->in(1)->Opcode() == Op_LoadKlass) { Node* cast = new CastP2XNode(NULL, n->in(1)); - null_free = new AndLNode(cast, new ConLNode(TypeLong::make(((jlong)1)<<(oopDesc::wide_storage_props_shift + ArrayStorageProperties::null_free_bit)))); + null_free = new AndLNode(cast, new ConLNode(TypeLong::make(((jlong)1)<<(oopDesc::wide_storage_props_shift + bit)))); } else { assert(n->in(1)->Opcode() == Op_LoadNKlass, "not a compressed klass?"); Node* cast = new CastN2INode(n->in(1)); - null_free = new AndINode(cast, new ConINode(TypeInt::make(1<<(oopDesc::narrow_storage_props_shift + ArrayStorageProperties::null_free_bit)))); + null_free = new AndINode(cast, new ConINode(TypeInt::make(1<<(oopDesc::narrow_storage_props_shift + bit)))); } - n->replace_by(null_free); + n->subsume_by(null_free, this); break; } default: --- old/src/hotspot/share/opto/graphKit.cpp 2019-09-17 12:56:43.000000000 +0200 +++ new/src/hotspot/share/opto/graphKit.cpp 2019-09-17 12:56:38.000000000 +0200 @@ -3426,8 +3426,8 @@ } } else if (obj->is_Phi()) { Node* region = obj->in(0); - if (region->req() == 3 && region->in(2) != NULL) { - IfNode* iff = region->in(2)->in(0)->isa_If(); + if (region->req() == 3 && region->in(1) != NULL) { + IfNode* iff = region->in(1)->in(0)->isa_If(); if (iff != NULL) { iff->is_flattened_array_check(&_gvn, array); } @@ -3513,6 +3513,20 @@ return _gvn.transform(new BoolNode(cmp, BoolTest::eq)); } +Node* GraphKit::gen_flattened_array_test(Node* ary) { + assert(EnableValhalla, "should only be used if value types are enabled"); + // Extract flattened property from klass pointer + Node* k_adr = basic_plus_adr(ary, oopDesc::klass_offset_in_bytes()); + const TypePtr* k_adr_type = k_adr->bottom_type()->isa_ptr(); + Node* klass = NULL; + if (k_adr_type->is_ptr_to_narrowklass()) { + klass = _gvn.transform(new LoadNKlassNode(NULL, immutable_memory(), k_adr, TypeInstPtr::KLASS, TypeKlassPtr::OBJECT->make_narrowklass(), MemNode::unordered)); + } else { + klass = _gvn.transform(new LoadKlassNode(NULL, immutable_memory(), k_adr, TypeInstPtr::KLASS, TypeKlassPtr::OBJECT, MemNode::unordered)); + } + return _gvn.transform(new GetFlattenedPropertyNode(klass)); +} + // Deoptimize if 'ary' is a null-free value type array and 'val' is null Node* GraphKit::gen_value_array_null_guard(Node* ary, Node* val, int nargs, bool safe_for_replace) { const Type* val_t = _gvn.type(val); @@ -4396,7 +4410,7 @@ Node* GraphKit::load_String_value(Node* str, bool set_ctrl) { int value_offset = java_lang_String::value_offset_in_bytes(); const TypeInstPtr* string_type = TypeInstPtr::make(TypePtr::NotNull, C->env()->String_klass(), - false, NULL, Type::Offset(0)); + false, NULL, Type::Offset(0), false); const TypePtr* value_field_type = string_type->add_offset(value_offset); const TypeAryPtr* value_type = TypeAryPtr::make(TypePtr::NotNull, TypeAry::make(TypeInt::BYTE, TypeInt::POS, false, true, true), @@ -4413,7 +4427,7 @@ } int coder_offset = java_lang_String::coder_offset_in_bytes(); const TypeInstPtr* string_type = TypeInstPtr::make(TypePtr::NotNull, C->env()->String_klass(), - false, NULL, Type::Offset(0)); + false, NULL, Type::Offset(0), false); const TypePtr* coder_field_type = string_type->add_offset(coder_offset); Node* p = basic_plus_adr(str, str, coder_offset); @@ -4425,7 +4439,7 @@ void GraphKit::store_String_value(Node* str, Node* value) { int value_offset = java_lang_String::value_offset_in_bytes(); const TypeInstPtr* string_type = TypeInstPtr::make(TypePtr::NotNull, C->env()->String_klass(), - false, NULL, Type::Offset(0)); + false, NULL, Type::Offset(0), false); const TypePtr* value_field_type = string_type->add_offset(value_offset); access_store_at(str, basic_plus_adr(str, value_offset), value_field_type, @@ -4435,7 +4449,7 @@ void GraphKit::store_String_coder(Node* str, Node* value) { int coder_offset = java_lang_String::coder_offset_in_bytes(); const TypeInstPtr* string_type = TypeInstPtr::make(TypePtr::NotNull, C->env()->String_klass(), - false, NULL, Type::Offset(0)); + false, NULL, Type::Offset(0), false); const TypePtr* coder_field_type = string_type->add_offset(coder_offset); access_store_at(str, basic_plus_adr(str, coder_offset), coder_field_type, --- old/src/hotspot/share/opto/graphKit.hpp 2019-09-17 12:56:50.000000000 +0200 +++ new/src/hotspot/share/opto/graphKit.hpp 2019-09-17 12:56:44.000000000 +0200 @@ -860,6 +860,7 @@ Node* is_value_mirror(Node* mirror); void gen_value_type_guard(Node* obj, int nargs = 0); Node* gen_null_free_array_check(Node* ary); + Node* gen_flattened_array_test(Node* ary); Node* gen_value_array_null_guard(Node* ary, Node* val, int nargs, bool safe_for_replace = false); Node* load_lh_array_tag(Node* kls); Node* gen_lh_array_test(Node* kls, unsigned int lh_value); --- old/src/hotspot/share/opto/ifnode.cpp 2019-09-17 12:56:55.000000000 +0200 +++ new/src/hotspot/share/opto/ifnode.cpp 2019-09-17 12:56:50.000000000 +0200 @@ -1220,38 +1220,29 @@ return false; } Node* cmp = bol->in(1); - if (cmp->Opcode() != Op_CmpI) { + if (cmp->Opcode() != Op_CmpI && cmp->Opcode() != Op_CmpL) { return false; } Node* cmp_in1 = cmp->in(1); Node* cmp_in2 = cmp->in(2); - if ((unsigned int)cmp_in2->find_int_con(0) != Klass::_lh_array_tag_vt_value) { + + if (cmp_in1->Opcode() != Op_GetFlattenedProperty) { return false; } - if (cmp_in1->Opcode() != Op_RShiftI) { - return false; - } - Node* shift_in1 = cmp_in1->in(1); - Node* shift_in2 = cmp_in1->in(2); - if ((unsigned int)shift_in2->find_int_con(0) != Klass::_lh_array_tag_shift) { - return false; - } - if (shift_in1->Opcode() != Op_LoadI) { - return false; - } - intptr_t offset; - Node* ptr = shift_in1->in(MemNode::Address); - Node* addr = AddPNode::Ideal_base_and_offset(ptr, phase, offset); - if (addr == NULL || offset != in_bytes(Klass::layout_helper_offset())) { - return false; + + jlong in2 = -1; + if (cmp->Opcode() == Op_CmpI) { + in2 = cmp_in2->find_int_con(-1); + } else { + in2 = cmp_in2->find_long_con(-1); } - if (!phase->type(addr)->isa_klassptr()) { + + if (in2 != 0) { return false; } - Node* klass_load = ptr->as_AddP()->in(AddPNode::Base)->uncast(); - if (klass_load->is_DecodeNKlass()) { - klass_load = klass_load->in(1); - } + + Node* klass_load = cmp_in1->in(1); + if (klass_load->is_Load()) { Node* address = klass_load->in(MemNode::Address); array = address->as_AddP()->in(AddPNode::Base); --- old/src/hotspot/share/opto/library_call.cpp 2019-09-17 12:57:01.000000000 +0200 +++ new/src/hotspot/share/opto/library_call.cpp 2019-09-17 12:56:55.000000000 +0200 @@ -4071,7 +4071,7 @@ if (not_objArray != NULL) { // Improve the klass node's type from the new optimistic assumption: ciKlass* ak = ciArrayKlass::make(env()->Object_klass()); - const Type* akls = TypeKlassPtr::make(TypePtr::NotNull, ak, Type::Offset(0)); + const Type* akls = TypeKlassPtr::make(TypePtr::NotNull, ak, Type::Offset(0), false); Node* cast = new CastPPNode(klass_node, akls); cast->init_req(0, control()); klass_node = _gvn.transform(cast); --- old/src/hotspot/share/opto/loopUnswitch.cpp 2019-09-17 12:57:06.000000000 +0200 +++ new/src/hotspot/share/opto/loopUnswitch.cpp 2019-09-17 12:57:01.000000000 +0200 @@ -130,7 +130,11 @@ flattened_checks.push(n); } } - unswitch_iff = NULL; + if (flattened_checks.size() > 1) { + unswitch_iff = NULL; + } else { + flattened_checks.clear(); + } } return unswitch_iff; @@ -196,7 +200,9 @@ int nct = head->unswitch_count() + 1; head->set_unswitch_count(nct); head_clone->set_unswitch_count(nct); - head_clone->mark_flattened_arrays(); + if (flattened_checks.size() > 0) { + head->mark_flattened_arrays(); + } // Add test to new "if" outside of loop IfNode* invar_iff = proj_true->in(0)->as_If(); @@ -220,12 +226,14 @@ Node* in1 = NULL; for (uint i = 0; i < flattened_checks.size(); i++) { Node* v = flattened_checks.at(i)->in(1)->in(1)->in(1); - v = new AndINode(v, _igvn.intcon(Klass::_lh_array_tag_vt_value)); - register_new_node(v, invar_iff->in(0)); if (in1 == NULL) { in1 = v; } else { - in1 = new OrINode(in1, v); + if (cmp->Opcode() == Op_CmpL) { + in1 = new OrLNode(in1, v); + } else { + in1 = new OrINode(in1, v); + } register_new_node(in1, invar_iff->in(0)); } } @@ -243,25 +251,50 @@ Node_List worklist; - for (DUIterator_Fast imax, i = unswitch_iff->fast_outs(imax); i < imax; i++) { - ProjNode* proj= unswitch_iff->fast_out(i)->as_Proj(); - // Copy to a worklist for easier manipulation - for (DUIterator_Fast jmax, j = proj->fast_outs(jmax); j < jmax; j++) { - Node* use = proj->fast_out(j); - if (use->Opcode() == Op_CheckCastPP && loop->is_invariant(use->in(1))) { - worklist.push(use); + if (flattened_checks.size() > 0) { + for (uint i = 0; i < flattened_checks.size(); i++) { + IfNode* iff = flattened_checks.at(i)->as_If(); + ProjNode* proj= iff->proj_out(0)->as_Proj(); + // Copy to a worklist for easier manipulation + for (DUIterator_Fast jmax, j = proj->fast_outs(jmax); j < jmax; j++) { + Node* use = proj->fast_out(j); + if (use->Opcode() == Op_CheckCastPP && loop->is_invariant(use->in(1))) { + worklist.push(use); + } + } + ProjNode* invar_proj = invar_iff->proj_out(proj->_con)->as_Proj(); + while (worklist.size() > 0) { + Node* use = worklist.pop(); + Node* nuse = use->clone(); + nuse->set_req(0, invar_proj); + _igvn.replace_input_of(use, 1, nuse); + register_new_node(nuse, invar_proj); + // Same for the clone + Node* use_clone = old_new[use->_idx]; + _igvn.replace_input_of(use_clone, 1, nuse); } } - ProjNode* invar_proj = invar_iff->proj_out(proj->_con)->as_Proj(); - while (worklist.size() > 0) { - Node* use = worklist.pop(); - Node* nuse = use->clone(); - nuse->set_req(0, invar_proj); - _igvn.replace_input_of(use, 1, nuse); - register_new_node(nuse, invar_proj); - // Same for the clone - Node* use_clone = old_new[use->_idx]; - _igvn.replace_input_of(use_clone, 1, nuse); + } else { + for (DUIterator_Fast imax, i = unswitch_iff->fast_outs(imax); i < imax; i++) { + ProjNode* proj= unswitch_iff->fast_out(i)->as_Proj(); + // Copy to a worklist for easier manipulation + for (DUIterator_Fast jmax, j = proj->fast_outs(jmax); j < jmax; j++) { + Node* use = proj->fast_out(j); + if (use->Opcode() == Op_CheckCastPP && loop->is_invariant(use->in(1))) { + worklist.push(use); + } + } + ProjNode* invar_proj = invar_iff->proj_out(proj->_con)->as_Proj(); + while (worklist.size() > 0) { + Node* use = worklist.pop(); + Node* nuse = use->clone(); + nuse->set_req(0, invar_proj); + _igvn.replace_input_of(use, 1, nuse); + register_new_node(nuse, invar_proj); + // Same for the clone + Node* use_clone = old_new[use->_idx]; + _igvn.replace_input_of(use_clone, 1, nuse); + } } } @@ -270,7 +303,7 @@ for (uint i = 0; i < flattened_checks.size(); i++) { IfNode* iff = flattened_checks.at(i)->as_If(); _igvn.rehash_node_delayed(iff); - short_circuit_if(iff, proj_true); + short_circuit_if(old_new[iff->_idx]->as_If(), proj_false); } } else { // Hardwire the control paths in the loops into if(true) and if(false) --- old/src/hotspot/share/opto/loopnode.hpp 2019-09-17 12:57:12.000000000 +0200 +++ new/src/hotspot/share/opto/loopnode.hpp 2019-09-17 12:57:07.000000000 +0200 @@ -1349,6 +1349,7 @@ Node* try_move_store_before_loop(Node* n, Node *n_ctrl); void try_move_store_after_loop(Node* n); bool identical_backtoback_ifs(Node *n); + bool flatten_array_element_type_check(Node *n); bool can_split_if(Node *n_ctrl); // Determine if a method is too big for a/another round of split-if, based on --- old/src/hotspot/share/opto/loopopts.cpp 2019-09-17 12:57:17.000000000 +0200 +++ new/src/hotspot/share/opto/loopopts.cpp 2019-09-17 12:57:12.000000000 +0200 @@ -1214,12 +1214,112 @@ return out_le; } +bool PhaseIdealLoop::flatten_array_element_type_check(Node *n) { + // If the CmpP is a subtype check for a value that has just been + // loaded from an array, the subtype checks guarantees the value + // can't be stored in a flattened array and the load of the value + // happens with a flattened array check then: push the type check + // through the phi of the flattened array check. This needs special + // logic because the subtype check's input is not a phi but a + // LoadKlass that must first be cloned through the phi. + if (n->Opcode() != Op_CmpP) { + return false; + } + + Node* klassptr = n->in(1); + Node* klasscon = n->in(2); + + if (klassptr->is_DecodeNarrowPtr()) { + klassptr = klassptr->in(1); + } + + if (klassptr->Opcode() != Op_LoadKlass && klassptr->Opcode() != Op_LoadNKlass) { + return false; + } + + if (!klasscon->is_Con()) { + return false; + } + + Node* addr = klassptr->in(MemNode::Address); + + if (!addr->is_AddP()) { + return false; + } + + intptr_t offset; + Node* obj = AddPNode::Ideal_base_and_offset(addr, &_igvn, offset); + + if (obj == NULL) { + return false; + } + + assert(obj != NULL && addr->in(AddPNode::Base) == addr->in(AddPNode::Address), "malformed AddP?"); + if (obj->Opcode() == Op_CastPP) { + obj = obj->in(1); + } + + if (!obj->is_Phi()) { + return false; + } + + Node* region = obj->in(0); + + Node* phi = PhiNode::make_blank(region, n->in(1)); + for (uint i = 1; i < region->req(); i++) { + Node* in = obj->in(i); + Node* ctrl = get_ctrl(in); + if (addr->in(AddPNode::Base) != obj) { + Node* cast = addr->in(AddPNode::Base); + assert(cast->Opcode() == Op_CastPP && cast->in(0) != NULL, "inconsistent subgraph"); + Node* cast_clone = cast->clone(); + cast_clone->set_req(0, region->in(i)); + cast_clone->set_req(1, in); + register_new_node(cast_clone, region->in(i)); + _igvn.set_type(cast_clone, cast_clone->Value(&_igvn)); + in = cast_clone; + } + Node* addr_clone = addr->clone(); + addr_clone->set_req(AddPNode::Base, in); + addr_clone->set_req(AddPNode::Address, in); + register_new_node(addr_clone, ctrl); + _igvn.set_type(addr_clone, addr_clone->Value(&_igvn)); + Node* klassptr_clone = klassptr->clone(); + klassptr_clone->set_req(2, addr_clone); + register_new_node(klassptr_clone, ctrl); + _igvn.set_type(klassptr_clone, klassptr_clone->Value(&_igvn)); + if (klassptr != n->in(1)) { + Node* decode = n->in(1); + assert(decode->is_DecodeNarrowPtr(), "inconcistent subgraph"); + Node* decode_clone = decode->clone(); + decode_clone->set_req(1, klassptr_clone); + register_new_node(decode_clone, ctrl); + _igvn.set_type(decode_clone, decode_clone->Value(&_igvn)); + klassptr_clone = decode_clone; + } + phi->set_req(i, klassptr_clone); + } + register_new_node(phi, region); + Node* orig = n->in(1); + _igvn.replace_input_of(n, 1, phi); + split_if_with_blocks_post(n); + if (n->outcnt() != 0) { + _igvn.replace_input_of(n, 1, orig); + _igvn.remove_dead_node(phi); + } + return true; +} + //------------------------------split_if_with_blocks_post---------------------- // Do the real work in a non-recursive function. CFG hackery wants to be // in the post-order, so it can dirty the I-DOM info and not use the dirtied // info. void PhaseIdealLoop::split_if_with_blocks_post(Node *n) { + if (flatten_array_element_type_check(n)) { + return; + } + // Cloning Cmp through Phi's involves the split-if transform. // FastLock is not used by an If if (n->is_Cmp() && !n->is_FastLock()) { --- old/src/hotspot/share/opto/macro.cpp 2019-09-17 12:57:23.000000000 +0200 +++ new/src/hotspot/share/opto/macro.cpp 2019-09-17 12:57:17.000000000 +0200 @@ -80,44 +80,6 @@ return nreplacements; } -void PhaseMacroExpand::copy_call_debug_info(CallNode *oldcall, CallNode * newcall) { - // Copy debug information and adjust JVMState information - uint old_dbg_start = oldcall->tf()->domain_sig()->cnt(); - uint new_dbg_start = newcall->tf()->domain_sig()->cnt(); - int jvms_adj = new_dbg_start - old_dbg_start; - assert (new_dbg_start == newcall->req(), "argument count mismatch"); - - // SafePointScalarObject node could be referenced several times in debug info. - // Use Dict to record cloned nodes. - Dict* sosn_map = new Dict(cmpkey,hashkey); - for (uint i = old_dbg_start; i < oldcall->req(); i++) { - Node* old_in = oldcall->in(i); - // Clone old SafePointScalarObjectNodes, adjusting their field contents. - if (old_in != NULL && old_in->is_SafePointScalarObject()) { - SafePointScalarObjectNode* old_sosn = old_in->as_SafePointScalarObject(); - uint old_unique = C->unique(); - Node* new_in = old_sosn->clone(sosn_map); - if (old_unique != C->unique()) { // New node? - new_in->set_req(0, C->root()); // reset control edge - new_in = transform_later(new_in); // Register new node. - } - old_in = new_in; - } - newcall->add_req(old_in); - } - - // JVMS may be shared so clone it before we modify it - newcall->set_jvms(oldcall->jvms() != NULL ? oldcall->jvms()->clone_deep(C) : NULL); - for (JVMState *jvms = newcall->jvms(); jvms != NULL; jvms = jvms->caller()) { - jvms->set_map(newcall); - jvms->set_locoff(jvms->locoff()+jvms_adj); - jvms->set_stkoff(jvms->stkoff()+jvms_adj); - jvms->set_monoff(jvms->monoff()+jvms_adj); - jvms->set_scloff(jvms->scloff()+jvms_adj); - jvms->set_endoff(jvms->endoff()+jvms_adj); - } -} - Node* PhaseMacroExpand::opt_bits_test(Node* ctrl, Node* region, int edge, Node* word, int mask, int bits, bool return_fast_path) { Node* cmp; if (mask != 0) { @@ -170,7 +132,7 @@ if (parm0 != NULL) call->init_req(TypeFunc::Parms+0, parm0); if (parm1 != NULL) call->init_req(TypeFunc::Parms+1, parm1); if (parm2 != NULL) call->init_req(TypeFunc::Parms+2, parm2); - copy_call_debug_info(oldcall, call); + call->copy_call_debug_info(&_igvn, oldcall); call->set_cnt(PROB_UNLIKELY_MAG(4)); // Same effect as RC_UNCOMMON. _igvn.replace_node(oldcall, call); transform_later(call); @@ -1592,7 +1554,7 @@ // Copy debug information and adjust JVMState information, then replace // allocate node with the call - copy_call_debug_info((CallNode *) alloc, call); + call->copy_call_debug_info(&_igvn, alloc); if (!always_slow) { call->set_cnt(PROB_UNLIKELY_MAG(4)); // Same effect as RC_UNCOMMON. } else { @@ -2472,7 +2434,7 @@ unc->init_req(TypeFunc::ReturnAdr, lock->in(TypeFunc::ReturnAdr)); unc->init_req(TypeFunc::Parms+0, _igvn.intcon(trap_request)); unc->set_cnt(PROB_UNLIKELY_MAG(4)); - copy_call_debug_info(lock, unc); + unc->copy_call_debug_info(&_igvn, lock); assert(unc->peek_monitor_box() == box, "wrong monitor"); assert(unc->peek_monitor_obj() == obj, "wrong monitor"); @@ -2742,7 +2704,7 @@ } // We can safepoint at that new call - copy_call_debug_info(call, slow_call); + slow_call->copy_call_debug_info(&_igvn, call); transform_later(slow_call); transform_later(handler_call); --- old/src/hotspot/share/opto/macro.hpp 2019-09-17 12:57:28.000000000 +0200 +++ new/src/hotspot/share/opto/macro.hpp 2019-09-17 12:57:23.000000000 +0200 @@ -192,7 +192,6 @@ void expand_arraycopy_node(ArrayCopyNode *ac); int replace_input(Node *use, Node *oldref, Node *newref); - void copy_call_debug_info(CallNode *oldcall, CallNode * newcall); Node* opt_bits_test(Node* ctrl, Node* region, int edge, Node* word, int mask, int bits, bool return_fast_path = false); void copy_predefined_input_for_runtime_call(Node * ctrl, CallNode* oldcall, CallNode* call); CallNode* make_slow_call(CallNode *oldcall, const TypeFunc* slow_call_type, address slow_call, --- old/src/hotspot/share/opto/macroArrayCopy.cpp 2019-09-17 12:57:34.000000000 +0200 +++ new/src/hotspot/share/opto/macroArrayCopy.cpp 2019-09-17 12:57:28.000000000 +0200 @@ -1030,7 +1030,7 @@ call->init_req(TypeFunc::Parms+2, dest); call->init_req(TypeFunc::Parms+3, dest_offset); call->init_req(TypeFunc::Parms+4, copy_length); - copy_call_debug_info(ac, call); + call->copy_call_debug_info(&_igvn, ac); call->set_cnt(PROB_UNLIKELY_MAG(4)); // Same effect as RC_UNCOMMON. _igvn.replace_node(ac, call); --- old/src/hotspot/share/opto/memnode.cpp 2019-09-17 12:57:39.000000000 +0200 +++ new/src/hotspot/share/opto/memnode.cpp 2019-09-17 12:57:34.000000000 +0200 @@ -2258,7 +2258,7 @@ } // Return root of possible klass - return TypeKlassPtr::make(TypePtr::NotNull, ik, Type::Offset(0)); + return TypeKlassPtr::make(TypePtr::NotNull, ik, Type::Offset(0), tinst->flatten_array()); } } @@ -2293,7 +2293,7 @@ return TypeKlassPtr::make(ak); } } - return TypeKlassPtr::make(TypePtr::NotNull, ak, Type::Offset(0)); + return TypeKlassPtr::make(TypePtr::NotNull, ak, Type::Offset(0), false); } else if (ak->is_type_array_klass()) { //assert(!UseExactTypes, "this code should be useless with exact types"); return TypeKlassPtr::make(ak); // These are always precise @@ -2317,11 +2317,11 @@ // The array's TypeKlassPtr was declared 'precise' or 'not precise' // according to the element type's subclassing. - return TypeKlassPtr::make(tkls->ptr(), elem, Type::Offset(0)); + return TypeKlassPtr::make(tkls->ptr(), elem, Type::Offset(0), elem->flatten_array()); } else if (klass->is_value_array_klass() && tkls->offset() == in_bytes(ObjArrayKlass::element_klass_offset())) { ciKlass* elem = klass->as_value_array_klass()->element_klass(); - return TypeKlassPtr::make(tkls->ptr(), elem, Type::Offset(0)); + return TypeKlassPtr::make(tkls->ptr(), elem, Type::Offset(0), true); } if( klass->is_instance_klass() && tkls->klass_is_exact() && tkls->offset() == in_bytes(Klass::super_offset())) { @@ -2343,7 +2343,7 @@ return klass_identity_common(phase); } -const Type* GetNullFreePropertyNode::Value(PhaseGVN* phase) const { +const Type* GetStoragePropertyNode::Value(PhaseGVN* phase) const { if (in(1) != NULL) { const Type* in1_t = phase->type(in(1)); if (in1_t == Type::TOP) { @@ -2355,7 +2355,11 @@ if (tk->klass_is_exact() || (!elem->is_java_lang_Object() && !elem->is_interface() && !elem->is_valuetype())) { int props_shift = in1_t->isa_narrowklass() ? oopDesc::narrow_storage_props_shift : oopDesc::wide_storage_props_shift; ArrayStorageProperties props = ak->storage_properties(); - intptr_t storage_properties = props.encode(props_shift); + intptr_t storage_properties = 0; + if ((Opcode() == Op_GetFlattenedProperty && props.is_flattened()) || + (Opcode() == Op_GetNullFreeProperty && props.is_null_free())){ + storage_properties = 1; + } if (in1_t->isa_narrowklass()) { return TypeInt::make((int)storage_properties); } @@ -2365,7 +2369,7 @@ return bottom_type(); } -Node* GetNullFreePropertyNode::Ideal(PhaseGVN *phase, bool can_reshape) { +Node* GetStoragePropertyNode::Ideal(PhaseGVN *phase, bool can_reshape) { if (!can_reshape) { return NULL; } @@ -2376,7 +2380,9 @@ for (uint i = 1; i < r->req(); i++) { Node* in = phi->in(i); if (in == NULL) continue; - new_phi->init_req(i, phase->transform(new GetNullFreePropertyNode(in))); + Node* n = clone(); + n->set_req(1, in); + new_phi->init_req(i, phase->transform(n)); } return new_phi; } --- old/src/hotspot/share/opto/memnode.hpp 2019-09-17 12:57:45.000000000 +0200 +++ new/src/hotspot/share/opto/memnode.hpp 2019-09-17 12:57:40.000000000 +0200 @@ -534,24 +534,24 @@ virtual bool depends_only_on_test() const { return true; } }; -// Retrieve the null free property from an array klass. This is -// treated a bit like a field that would be read from the klass +// Retrieve the null free/flattened property from an array klass. This +// is treated a bit like a field that would be read from the klass // structure at runtime except, the implementation encodes the // property as a bit in the klass header field of the array. This // implementation detail is hidden under this node so it doesn't make // a difference for high level optimizations. At final graph reshaping // time, this node is turned into the actual logical operations that // extract the property from the klass pointer. For this to work -// correctly, GetNullFreePropertyNode must take a LoadKlass/LoadNKlass -// input. The Ideal transformation splits the GetNullFreePropertyNode +// correctly, GeStoragePropertyNodes must take a LoadKlass/LoadNKlass +// input. The Ideal transformation splits the GetStoragePropertyNode // through phis, Value returns a constant if the node's input is a -// constant. These 2 should guarantee GetNullFreePropertyNode does +// constant. These 2 should guarantee GetStoragePropertyNode does // indeed have a LoadKlass/LoadNKlass input at final graph reshaping // time. -class GetNullFreePropertyNode : public Node { +class GetStoragePropertyNode : public Node { +protected: + GetStoragePropertyNode(Node* klass) : Node(NULL, klass) {} public: - GetNullFreePropertyNode(Node* klass) : Node(NULL, klass) {} - virtual int Opcode() const; virtual const Type* Value(PhaseGVN* phase) const; virtual Node* Ideal(PhaseGVN *phase, bool can_reshape); virtual const Type* bottom_type() const { @@ -562,6 +562,19 @@ } }; + +class GetNullFreePropertyNode : public GetStoragePropertyNode { +public: + GetNullFreePropertyNode(Node* klass) : GetStoragePropertyNode(klass) {} + virtual int Opcode() const; +}; + +class GetFlattenedPropertyNode : public GetStoragePropertyNode { +public: + GetFlattenedPropertyNode(Node* klass) : GetStoragePropertyNode(klass) {} + virtual int Opcode() const; +}; + //------------------------------StoreNode-------------------------------------- // Store value; requires Store, Address and Value class StoreNode : public MemNode { --- old/src/hotspot/share/opto/parse2.cpp 2019-09-17 12:57:50.000000000 +0200 +++ new/src/hotspot/share/opto/parse2.cpp 2019-09-17 12:57:45.000000000 +0200 @@ -81,17 +81,8 @@ IdealKit ideal(this); IdealVariable res(ideal); ideal.declarations_done(); - Node* kls = load_object_klass(ary); - Node* tag = load_lh_array_tag(kls); - ideal.if_then(tag, BoolTest::ne, intcon(Klass::_lh_array_tag_vt_value)); { - // non-flattened - sync_kit(ideal); - const TypeAryPtr* adr_type = TypeAryPtr::get_array_body_type(bt); - Node* ld = access_load_at(ary, adr, adr_type, elemptr, bt, - IN_HEAP | IS_ARRAY | C2_CONTROL_DEPENDENT_LOAD, ctl); - ideal.sync_kit(this); - ideal.set(res, ld); - } ideal.else_(); { + Node* flattened = gen_flattened_array_test(ary); + ideal.if_then(flattened, BoolTest::ne, zerocon(flattened->bottom_type()->basic_type())); { // flattened sync_kit(ideal); if (elemptr->is_valuetypeptr()) { @@ -101,11 +92,12 @@ ciArrayKlass* array_klass = ciArrayKlass::make(vk, /* never_null */ true); const TypeAryPtr* arytype = TypeOopPtr::make_from_klass(array_klass)->isa_aryptr(); Node* cast = _gvn.transform(new CheckCastPPNode(control(), ary, arytype)); - adr = array_element_address(cast, idx, T_VALUETYPE, ary_t->size(), control()); - Node* vt = ValueTypeNode::make_from_flattened(this, vk, cast, adr)->allocate(this, false, false)->get_oop(); + Node* casted_adr = array_element_address(cast, idx, T_VALUETYPE, ary_t->size(), control()); + Node* vt = ValueTypeNode::make_from_flattened(this, vk, cast, casted_adr)->allocate(this, false, false)->get_oop(); ideal.set(res, vt); ideal.sync_kit(this); } else { + Node* kls = load_object_klass(ary); // Element type is unknown, emit runtime call Node* k_adr = basic_plus_adr(kls, in_bytes(ArrayKlass::element_klass_offset())); Node* elem_klass = _gvn.transform(LoadKlassNode::make(_gvn, NULL, immutable_memory(), k_adr, TypeInstPtr::KLASS)); @@ -166,10 +158,22 @@ alloc_obj->set_req(0, control()); alloc_obj = _gvn.transform(alloc_obj); + const Type* unknown_value = TypeInstPtr::BOTTOM->cast_to_flatten_array(); + + alloc_obj = _gvn.transform(new CheckCastPPNode(control(), alloc_obj, unknown_value)); + ideal.sync_kit(this); ideal.set(res, alloc_obj); } + } ideal.else_(); { + // non-flattened + sync_kit(ideal); + const TypeAryPtr* adr_type = TypeAryPtr::get_array_body_type(bt); + Node* ld = access_load_at(ary, adr, adr_type, elemptr, bt, + IN_HEAP | IS_ARRAY | C2_CONTROL_DEPENDENT_LOAD, ctl); + ideal.sync_kit(this); + ideal.set(res, ld); } ideal.end_if(); sync_kit(ideal); push_node(bt, _gvn.transform(ideal.value(res))); @@ -269,24 +273,15 @@ assert(ValueArrayFlatten && !not_flattenable && elemtype->is_oopptr()->can_be_value_type() && !ary_t->klass_is_exact() && !ary_t->is_not_null_free(), "array can't be flattened"); IdealKit ideal(this); - Node* kls = load_object_klass(ary); - Node* layout_val = load_lh_array_tag(kls); - ideal.if_then(layout_val, BoolTest::ne, intcon(Klass::_lh_array_tag_vt_value)); - { - // non-flattened - sync_kit(ideal); - gen_value_array_null_guard(ary, cast_val, 3); - access_store_at(ary, adr, adr_type, cast_val, elemtype, bt, MO_UNORDERED | IN_HEAP | IS_ARRAY, false, false); - ideal.sync_kit(this); - } - ideal.else_(); - { + Node* flattened = gen_flattened_array_test(ary); + ideal.if_then(flattened, BoolTest::ne, zerocon(flattened->bottom_type()->basic_type())); { + Node* val = cast_val; // flattened - if (!cast_val->is_ValueType() && tval->maybe_null()) { + if (!val->is_ValueType() && tval->maybe_null()) { // Add null check sync_kit(ideal); Node* null_ctl = top(); - cast_val = null_check_oop(cast_val, &null_ctl); + val = null_check_oop(val, &null_ctl); if (null_ctl != top()) { PreserveJVMState pjvms(this); inc_sp(3); @@ -305,19 +300,20 @@ } else if (elemtype->is_valuetypeptr()) { vk = elemtype->value_klass(); } + Node* casted_ary = ary; if (vk != NULL && !stopped()) { // Element type is known, cast and store to flattened representation sync_kit(ideal); assert(vk->flatten_array() && elemtype->maybe_null(), "must be a flattenable and nullable array"); ciArrayKlass* array_klass = ciArrayKlass::make(vk, /* never_null */ true); const TypeAryPtr* arytype = TypeOopPtr::make_from_klass(array_klass)->isa_aryptr(); - ary = _gvn.transform(new CheckCastPPNode(control(), ary, arytype)); - adr = array_element_address(ary, idx, T_OBJECT, arytype->size(), control()); - if (!cast_val->is_ValueType()) { - assert(!gvn().type(cast_val)->maybe_null(), "value type array elements should never be null"); - cast_val = ValueTypeNode::make_from_oop(this, cast_val, vk); + casted_ary = _gvn.transform(new CheckCastPPNode(control(), casted_ary, arytype)); + Node* casted_adr = array_element_address(casted_ary, idx, T_OBJECT, arytype->size(), control()); + if (!val->is_ValueType()) { + assert(!gvn().type(val)->maybe_null(), "value type array elements should never be null"); + val = ValueTypeNode::make_from_oop(this, val, vk); } - cast_val->as_ValueType()->store_flattened(this, ary, adr); + val->as_ValueType()->store_flattened(this, casted_ary, casted_adr); ideal.sync_kit(this); } else if (!ideal.ctrl()->is_top()) { // Element type is unknown, emit runtime call @@ -332,16 +328,24 @@ ideal.make_leaf_call(OptoRuntime::store_unknown_value_Type(), CAST_FROM_FN_PTR(address, OptoRuntime::store_unknown_value), "store_unknown_value", - cast_val, ary, idx); + val, casted_ary, idx); sync_kit(ideal); // Same as MemBarCPUOrder above: keep this unknown // flattened array access correctly ordered with other - // flattened array access + // flattened array accesses. insert_mem_bar_volatile(Op_MemBarCPUOrder, C->get_alias_index(TypeAryPtr::VALUES)); ideal.sync_kit(this); } } + ideal.else_(); + { + // non-flattened + sync_kit(ideal); + gen_value_array_null_guard(ary, cast_val, 3); + access_store_at(ary, adr, adr_type, cast_val, elemtype, bt, MO_UNORDERED | IN_HEAP | IS_ARRAY, false, false); + ideal.sync_kit(this); + } ideal.end_if(); sync_kit(ideal); return; --- old/src/hotspot/share/opto/phaseX.cpp 2019-09-17 12:57:56.000000000 +0200 +++ new/src/hotspot/share/opto/phaseX.cpp 2019-09-17 12:57:51.000000000 +0200 @@ -1692,6 +1692,15 @@ } BarrierSet::barrier_set()->barrier_set_c2()->igvn_add_users_to_worklist(this, use); + if (use->is_Region()) { + Node* c = use; + do { + c = c->unique_ctrl_out(); + } while (c != NULL && c->is_Region()); + if (c != NULL && c->is_CallStaticJava() && c->as_CallStaticJava()->uncommon_trap_request() != 0) { + _worklist.push(c); + } + } } } --- old/src/hotspot/share/opto/subnode.cpp 2019-09-17 12:58:01.000000000 +0200 +++ new/src/hotspot/share/opto/subnode.cpp 2019-09-17 12:57:56.000000000 +0200 @@ -903,7 +903,15 @@ } } } - + const TypeKlassPtr* k0 = r0->isa_klassptr(); + const TypeKlassPtr* k1 = r1->isa_klassptr(); + if (k0 && k1) { + if ((k0->flatten_array() && (!k1->can_be_value_type() || (k1->klass()->is_valuetype() && !k1->klass()->flatten_array()))) || + (k1->flatten_array() && (!k0->can_be_value_type() || (k0->klass()->is_valuetype() && !k0->klass()->flatten_array())))) { + return TypeInt::CC_GT; + } + } + // Known constants can be compared exactly // Null can be distinguished from any NotNull pointers // Unknown inputs makes an unknown result --- old/src/hotspot/share/opto/type.cpp 2019-09-17 12:58:07.000000000 +0200 +++ new/src/hotspot/share/opto/type.cpp 2019-09-17 12:58:02.000000000 +0200 @@ -594,9 +594,9 @@ TypeInstPtr::BOTTOM = TypeInstPtr::make(TypePtr::BotPTR, current->env()->Object_klass()); TypeInstPtr::MIRROR = TypeInstPtr::make(TypePtr::NotNull, current->env()->Class_klass()); TypeInstPtr::MARK = TypeInstPtr::make(TypePtr::BotPTR, current->env()->Object_klass(), - false, 0, Offset(oopDesc::mark_offset_in_bytes())); + false, 0, Offset(oopDesc::mark_offset_in_bytes()), false); TypeInstPtr::KLASS = TypeInstPtr::make(TypePtr::BotPTR, current->env()->Object_klass(), - false, 0, Offset(oopDesc::klass_offset_in_bytes())); + false, 0, Offset(oopDesc::klass_offset_in_bytes()), false); TypeOopPtr::BOTTOM = TypeOopPtr::make(TypePtr::BotPTR, Offset::bottom, TypeOopPtr::InstanceBot); TypeMetadataPtr::BOTTOM = TypeMetadataPtr::make(TypePtr::BotPTR, NULL, Offset::bottom); @@ -655,8 +655,8 @@ TypeAryPtr::_array_body_type[T_FLOAT] = TypeAryPtr::FLOATS; TypeAryPtr::_array_body_type[T_DOUBLE] = TypeAryPtr::DOUBLES; - TypeKlassPtr::OBJECT = TypeKlassPtr::make(TypePtr::NotNull, current->env()->Object_klass(), Offset(0) ); - TypeKlassPtr::OBJECT_OR_NULL = TypeKlassPtr::make(TypePtr::BotPTR, current->env()->Object_klass(), Offset(0) ); + TypeKlassPtr::OBJECT = TypeKlassPtr::make(TypePtr::NotNull, current->env()->Object_klass(), Offset(0), false); + TypeKlassPtr::OBJECT_OR_NULL = TypeKlassPtr::make(TypePtr::BotPTR, current->env()->Object_klass(), Offset(0), false ); const Type **fi2c = TypeTuple::fields(2); fi2c[TypeFunc::Parms+0] = TypeInstPtr::BOTTOM; // Method* @@ -3348,7 +3348,7 @@ if (k == NULL) return TypeKlassPtr::OBJECT; else - return TypeKlassPtr::make(xk? Constant: NotNull, k, Offset(0)); + return TypeKlassPtr::make(xk? Constant: NotNull, k, Offset(0), isa_instptr() && is_instptr()->flatten_array()); } //------------------------------meet------------------------------------------- @@ -3462,7 +3462,7 @@ } } } - return TypeInstPtr::make(TypePtr::BotPTR, klass, klass_is_exact, NULL, Offset(0)); + return TypeInstPtr::make(TypePtr::BotPTR, klass, klass_is_exact, NULL, Offset(0), klass->flatten_array()); } else if (klass->is_obj_array_klass()) { // Element is an object or value array. Recursively call ourself. const TypeOopPtr* etype = TypeOopPtr::make_from_klass_common(klass->as_array_klass()->element_klass(), false, try_for_exact); @@ -3514,7 +3514,7 @@ if (make_constant) { return TypeInstPtr::make(o); } else { - return TypeInstPtr::make(TypePtr::NotNull, klass, true, NULL, Offset(0)); + return TypeInstPtr::make(TypePtr::NotNull, klass, true, NULL, Offset(0), klass->flatten_array()); } } else if (klass->is_obj_array_klass()) { // Element is an object array. Recursively call ourself. @@ -3771,12 +3771,15 @@ //------------------------------TypeInstPtr------------------------------------- TypeInstPtr::TypeInstPtr(PTR ptr, ciKlass* k, bool xk, ciObject* o, Offset off, - int instance_id, const TypePtr* speculative, int inline_depth) + bool flatten_array, int instance_id, const TypePtr* speculative, + int inline_depth) : TypeOopPtr(InstPtr, ptr, k, xk, o, off, Offset::bottom, instance_id, speculative, inline_depth), - _name(k->name()) { + _name(k->name()), _flatten_array(flatten_array) { assert(k != NULL && (k->is_loaded() || o == NULL), "cannot have constants with non-loaded klass"); + assert(!klass()->is_valuetype() || !klass()->flatten_array() || flatten_array, "incorrect flatten array bit"); + assert(!flatten_array || can_be_value_type(), "incorrect flatten array bit"); }; //------------------------------make------------------------------------------- @@ -3785,6 +3788,7 @@ bool xk, ciObject* o, Offset offset, + bool flatten_array, int instance_id, const TypePtr* speculative, int inline_depth) { @@ -3808,7 +3812,7 @@ // Now hash this baby TypeInstPtr *result = - (TypeInstPtr*)(new TypeInstPtr(ptr, k, xk, o ,offset, instance_id, speculative, inline_depth))->hashcons(); + (TypeInstPtr*)(new TypeInstPtr(ptr, k, xk, o ,offset, flatten_array, instance_id, speculative, inline_depth))->hashcons(); return result; } @@ -3841,7 +3845,7 @@ if( ptr == _ptr ) return this; // Reconstruct _sig info here since not a problem with later lazy // construction, _sig will show up on demand. - return make(ptr, klass(), klass_is_exact(), const_oop(), _offset, _instance_id, _speculative, _inline_depth); + return make(ptr, klass(), klass_is_exact(), const_oop(), _offset, _flatten_array, _instance_id, _speculative, _inline_depth); } @@ -3853,18 +3857,18 @@ ciInstanceKlass* ik = _klass->as_instance_klass(); if( (ik->is_final() || _const_oop) ) return this; // cannot clear xk if( ik->is_interface() ) return this; // cannot set xk - return make(ptr(), klass(), klass_is_exact, const_oop(), _offset, _instance_id, _speculative, _inline_depth); + return make(ptr(), klass(), klass_is_exact, const_oop(), _offset, _flatten_array, _instance_id, _speculative, _inline_depth); } //-----------------------------cast_to_instance_id---------------------------- const TypeOopPtr *TypeInstPtr::cast_to_instance_id(int instance_id) const { if( instance_id == _instance_id ) return this; - return make(_ptr, klass(), _klass_is_exact, const_oop(), _offset, instance_id, _speculative, _inline_depth); + return make(_ptr, klass(), _klass_is_exact, const_oop(), _offset, _flatten_array, instance_id, _speculative, _inline_depth); } const TypeOopPtr *TypeInstPtr::cast_to_nonconst() const { if (const_oop() == NULL) return this; - return make(NotNull, klass(), _klass_is_exact, NULL, _offset, _instance_id, _speculative, _inline_depth); + return make(NotNull, klass(), _klass_is_exact, NULL, _offset, _flatten_array, _instance_id, _speculative, _inline_depth); } //------------------------------xmeet_unloaded--------------------------------- @@ -3896,7 +3900,7 @@ assert(loaded->ptr() != TypePtr::Null, "insanity check"); // if( loaded->ptr() == TypePtr::TopPTR ) { return unloaded; } - else if (loaded->ptr() == TypePtr::AnyNull) { return TypeInstPtr::make(ptr, unloaded->klass(), false, NULL, off, instance_id, speculative, depth); } + else if (loaded->ptr() == TypePtr::AnyNull) { return TypeInstPtr::make(ptr, unloaded->klass(), false, NULL, off, false, instance_id, speculative, depth); } else if (loaded->ptr() == TypePtr::BotPTR ) { return TypeInstPtr::BOTTOM; } else if (loaded->ptr() == TypePtr::Constant || loaded->ptr() == TypePtr::NotNull) { if (unloaded->ptr() == TypePtr::BotPTR ) { return TypeInstPtr::BOTTOM; } @@ -3966,7 +3970,7 @@ // cannot subclass, so the meet has to fall badly below the centerline ptr = NotNull; instance_id = InstanceBot; - return TypeInstPtr::make( ptr, ciEnv::current()->Object_klass(), false, NULL, offset, instance_id, speculative, depth); + return TypeInstPtr::make( ptr, ciEnv::current()->Object_klass(), false, NULL, offset, false, instance_id, speculative, depth); } case Constant: case NotNull: @@ -3989,7 +3993,7 @@ if( ptr == Constant ) ptr = NotNull; instance_id = InstanceBot; - return make(ptr, ciEnv::current()->Object_klass(), false, NULL, offset, instance_id, speculative, depth); + return make(ptr, ciEnv::current()->Object_klass(), false, NULL, offset, false, instance_id, speculative, depth); default: typerr(t); } } @@ -4006,7 +4010,7 @@ const TypePtr* speculative = xmeet_speculative(tp); int depth = meet_inline_depth(tp->inline_depth()); return make(ptr, klass(), klass_is_exact(), - (ptr == Constant ? const_oop() : NULL), offset, instance_id, speculative, depth); + (ptr == Constant ? const_oop() : NULL), offset, flatten_array(), instance_id, speculative, depth); } case NotNull: case BotPTR: { @@ -4034,7 +4038,7 @@ case TopPTR: case AnyNull: { return make(ptr, klass(), klass_is_exact(), - (ptr == Constant ? const_oop() : NULL), offset, instance_id, speculative, depth); + (ptr == Constant ? const_oop() : NULL), offset, flatten_array(), instance_id, speculative, depth); } case NotNull: case BotPTR: @@ -4072,8 +4076,9 @@ // If we have constants, then we created oops so classes are loaded // and we can handle the constants further down. This case handles // both-not-loaded or both-loaded classes - if (ptr != Constant && klass()->equals(tinst->klass()) && klass_is_exact() == tinst->klass_is_exact()) { - return make(ptr, klass(), klass_is_exact(), NULL, off, instance_id, speculative, depth); + if (ptr != Constant && klass()->equals(tinst->klass()) && klass_is_exact() == tinst->klass_is_exact() && + flatten_array() == tinst->flatten_array()) { + return make(ptr, klass(), klass_is_exact(), NULL, off, flatten_array(), instance_id, speculative, depth); } // Classes require inspection in the Java klass hierarchy. Must be loaded. @@ -4081,6 +4086,8 @@ ciKlass* this_klass = this->klass(); bool tinst_xk = tinst->klass_is_exact(); bool this_xk = this->klass_is_exact(); + bool tinst_flatten_array = tinst->flatten_array(); + bool this_flatten_array = this->flatten_array(); if (!tinst_klass->is_loaded() || !this_klass->is_loaded() ) { // One of these classes has not been loaded const TypeInstPtr *unloaded_meet = xmeet_unloaded(tinst); @@ -4103,6 +4110,9 @@ bool tmp2 = tinst_xk; tinst_xk = this_xk; this_xk = tmp2; + tmp2 = tinst_flatten_array; + tinst_flatten_array = this_flatten_array; + this_flatten_array = tmp2; } if (tinst_klass->is_interface() && !(this_klass->is_interface() || @@ -4114,6 +4124,7 @@ // See if the oop subtypes (implements) interface. ciKlass *k; bool xk; + bool value; if( this_klass->is_subtype_of( tinst_klass ) ) { // Oop indeed subtypes. Now keep oop or interface depending // on whether we are both above the centerline or either is @@ -4122,12 +4133,14 @@ k = below_centerline(ptr) ? tinst_klass : this_klass; // If we are keeping this_klass, keep its exactness too. xk = below_centerline(ptr) ? tinst_xk : this_xk; + value = below_centerline(ptr) ? tinst_flatten_array : this_flatten_array; } else { // Does not implement, fall to Object // Oop does not implement interface, so mixing falls to Object // just like the verifier does (if both are above the // centerline fall to interface) k = above_centerline(ptr) ? tinst_klass : ciEnv::current()->Object_klass(); xk = above_centerline(ptr) ? tinst_xk : false; + value = above_centerline(ptr) ? tinst_flatten_array : false; // Watch out for Constant vs. AnyNull interface. if (ptr == Constant) ptr = NotNull; // forget it was a constant instance_id = InstanceBot; @@ -4137,7 +4150,7 @@ // Find out which constant. o = (this_klass == klass()) ? const_oop() : tinst->const_oop(); } - return make(ptr, k, xk, o, off, instance_id, speculative, depth); + return make(ptr, k, xk, o, off, value, instance_id, speculative, depth); } // Either oop vs oop or interface vs interface or interface vs Object @@ -4169,29 +4182,37 @@ // Check for subtyping: ciKlass *subtype = NULL; bool subtype_exact = false; + bool flatten_array = false; if( tinst_klass->equals(this_klass) ) { subtype = this_klass; subtype_exact = below_centerline(ptr) ? (this_xk && tinst_xk) : (this_xk || tinst_xk); + flatten_array = below_centerline(ptr) ? (this_flatten_array && tinst_flatten_array) : (this_flatten_array || tinst_flatten_array); } else if( !tinst_xk && this_klass->is_subtype_of( tinst_klass ) ) { subtype = this_klass; // Pick subtyping class subtype_exact = this_xk; + flatten_array = this_flatten_array; } else if( !this_xk && tinst_klass->is_subtype_of( this_klass ) ) { subtype = tinst_klass; // Pick subtyping class subtype_exact = tinst_xk; + flatten_array = tinst_flatten_array; } if( subtype ) { if( above_centerline(ptr) ) { // both are up? this_klass = tinst_klass = subtype; this_xk = tinst_xk = subtype_exact; + this_flatten_array = tinst_flatten_array = flatten_array; } else if( above_centerline(this ->_ptr) && !above_centerline(tinst->_ptr) ) { this_klass = tinst_klass; // tinst is down; keep down man this_xk = tinst_xk; + this_flatten_array = tinst_flatten_array; } else if( above_centerline(tinst->_ptr) && !above_centerline(this ->_ptr) ) { tinst_klass = this_klass; // this is down; keep down man tinst_xk = this_xk; + tinst_flatten_array = this_flatten_array; } else { this_xk = subtype_exact; // either they are equal, or we'll do an LCA + this_flatten_array = flatten_array; } } @@ -4214,7 +4235,7 @@ else ptr = NotNull; } - return make(ptr, this_klass, this_xk, o, off, instance_id, speculative, depth); + return make(ptr, this_klass, this_xk, o, off, this_flatten_array, instance_id, speculative, depth); } // Else classes are not equal // Since klasses are different, we require a LCA in the Java @@ -4226,7 +4247,7 @@ // Now we find the LCA of Java classes ciKlass* k = this_klass->least_common_ancestor(tinst_klass); - return make(ptr, k, false, NULL, off, instance_id, speculative, depth); + return make(ptr, k, false, NULL, off, false, instance_id, speculative, depth); } // End of case InstPtr case ValueType: { @@ -4271,7 +4292,7 @@ // Dual: do NOT dual on klasses. This means I do NOT understand the Java // inheritance mechanism. const Type *TypeInstPtr::xdual() const { - return new TypeInstPtr(dual_ptr(), klass(), klass_is_exact(), const_oop(), dual_offset(), dual_instance_id(), dual_speculative(), dual_inline_depth()); + return new TypeInstPtr(dual_ptr(), klass(), klass_is_exact(), const_oop(), dual_offset(), flatten_array(), dual_instance_id(), dual_speculative(), dual_inline_depth()); } //------------------------------eq--------------------------------------------- @@ -4280,13 +4301,14 @@ const TypeInstPtr *p = t->is_instptr(); return klass()->equals(p->klass()) && + flatten_array() == p->flatten_array() && TypeOopPtr::eq(p); // Check sub-type stuff } //------------------------------hash------------------------------------------- // Type-specific hashing function. int TypeInstPtr::hash(void) const { - int hash = java_add((jint)klass()->hash(), (jint)TypeOopPtr::hash()); + int hash = java_add(java_add((jint)klass()->hash(), (jint)TypeOopPtr::hash()), (jint)flatten_array()); return hash; } @@ -4321,6 +4343,11 @@ _offset.dump2(st); st->print(" *"); + + if (flatten_array() && !klass()->is_valuetype()) { + st->print(" (flatten array)"); + } + if (_instance_id == InstanceTop) st->print(",iid=top"); else if (_instance_id != InstanceBot) @@ -4333,7 +4360,7 @@ //------------------------------add_offset------------------------------------- const TypePtr *TypeInstPtr::add_offset(intptr_t offset) const { - return make(_ptr, klass(), klass_is_exact(), const_oop(), xadd_offset(offset), + return make(_ptr, klass(), klass_is_exact(), const_oop(), xadd_offset(offset), flatten_array(), _instance_id, add_offset_speculative(offset), _inline_depth); } @@ -4342,7 +4369,7 @@ return this; } assert(_inline_depth == InlineDepthTop || _inline_depth == InlineDepthBottom, "non speculative type shouldn't have inline depth"); - return make(_ptr, klass(), klass_is_exact(), const_oop(), _offset, + return make(_ptr, klass(), klass_is_exact(), const_oop(), _offset, flatten_array(), _instance_id, NULL, _inline_depth); } @@ -4350,14 +4377,19 @@ if (!UseInlineDepthForSpeculativeTypes) { return this; } - return make(_ptr, klass(), klass_is_exact(), const_oop(), _offset, _instance_id, _speculative, depth); + return make(_ptr, klass(), klass_is_exact(), const_oop(), _offset, flatten_array(), _instance_id, _speculative, depth); } const TypePtr *TypeInstPtr::with_instance_id(int instance_id) const { assert(is_known_instance(), "should be known"); - return make(_ptr, klass(), klass_is_exact(), const_oop(), _offset, instance_id, _speculative, _inline_depth); + return make(_ptr, klass(), klass_is_exact(), const_oop(), _offset, flatten_array(), instance_id, _speculative, _inline_depth); +} + +const TypeInstPtr *TypeInstPtr::cast_to_flatten_array() const { + return make(_ptr, klass(), klass_is_exact(), const_oop(), _offset, true, _instance_id, _speculative, _inline_depth); } + //============================================================================= // Convenience common pre-built types. const TypeAryPtr *TypeAryPtr::RANGE; @@ -4765,7 +4797,7 @@ // cannot subclass, so the meet has to fall badly below the centerline ptr = NotNull; instance_id = InstanceBot; - return TypeInstPtr::make(ptr, ciEnv::current()->Object_klass(), false, NULL, offset, instance_id, speculative, depth); + return TypeInstPtr::make(ptr, ciEnv::current()->Object_klass(), false, NULL, offset, false, instance_id, speculative, depth); } case Constant: case NotNull: @@ -4788,7 +4820,7 @@ if( ptr == Constant ) ptr = NotNull; instance_id = InstanceBot; - return TypeInstPtr::make(ptr, ciEnv::current()->Object_klass(), false, NULL, offset, instance_id, speculative, depth); + return TypeInstPtr::make(ptr, ciEnv::current()->Object_klass(), false, NULL, offset, false, instance_id, speculative, depth); default: typerr(t); } } @@ -5326,28 +5358,30 @@ const TypeKlassPtr *TypeKlassPtr::OBJECT_OR_NULL; //------------------------------TypeKlassPtr----------------------------------- -TypeKlassPtr::TypeKlassPtr( PTR ptr, ciKlass* klass, Offset offset ) - : TypePtr(KlassPtr, ptr, offset), _klass(klass), _klass_is_exact(ptr == Constant) { +TypeKlassPtr::TypeKlassPtr(PTR ptr, ciKlass* klass, Offset offset, bool flatten_array) + : TypePtr(KlassPtr, ptr, offset), _klass(klass), _klass_is_exact(ptr == Constant), _flatten_array(flatten_array) { + assert(!klass->is_valuetype() || !klass->flatten_array() || flatten_array, "incorrect flatten array bit"); + assert(!flatten_array || can_be_value_type(), "incorrect flatten array bit"); } //------------------------------make------------------------------------------- // ptr to klass 'k', if Constant, or possibly to a sub-klass if not a Constant -const TypeKlassPtr* TypeKlassPtr::make(PTR ptr, ciKlass* k, Offset offset) { +const TypeKlassPtr* TypeKlassPtr::make(PTR ptr, ciKlass* k, Offset offset, bool value) { assert(k == NULL || k->is_instance_klass() || k->is_array_klass(), "Incorrect type of klass oop"); - return (TypeKlassPtr*)(new TypeKlassPtr(ptr, k, offset))->hashcons(); + return (TypeKlassPtr*)(new TypeKlassPtr(ptr, k, offset, value))->hashcons(); } //------------------------------eq--------------------------------------------- // Structural equality check for Type representations bool TypeKlassPtr::eq( const Type *t ) const { const TypeKlassPtr *p = t->is_klassptr(); - return klass() == p->klass() && TypePtr::eq(p); + return klass() == p->klass() && TypePtr::eq(p) && flatten_array() == p->flatten_array(); } //------------------------------hash------------------------------------------- // Type-specific hashing function. int TypeKlassPtr::hash(void) const { - return java_add(klass() != NULL ? klass()->hash() : (jint)0, (jint)TypePtr::hash()); + return java_add(java_add(klass() != NULL ? klass()->hash() : (jint)0, (jint)TypePtr::hash()), (jint)flatten_array()); } //------------------------------singleton-------------------------------------- @@ -5481,14 +5515,14 @@ //------------------------------add_offset------------------------------------- // Access internals of klass object const TypePtr *TypeKlassPtr::add_offset( intptr_t offset ) const { - return make( _ptr, klass(), xadd_offset(offset) ); + return make(_ptr, klass(), xadd_offset(offset), flatten_array()); } //------------------------------cast_to_ptr_type------------------------------- const Type *TypeKlassPtr::cast_to_ptr_type(PTR ptr) const { assert(_base == KlassPtr, "subclass must override cast_to_ptr_type"); if( ptr == _ptr ) return this; - return make(ptr, _klass, _offset); + return make(ptr, _klass, _offset, _flatten_array); } @@ -5496,7 +5530,7 @@ const Type *TypeKlassPtr::cast_to_exactness(bool klass_is_exact) const { if( klass_is_exact == _klass_is_exact ) return this; if (!UseExactTypes) return this; - return make(klass_is_exact ? Constant : NotNull, _klass, _offset); + return make(klass_is_exact ? Constant : NotNull, _klass, _offset, _flatten_array); } @@ -5511,6 +5545,9 @@ const TypeOopPtr* toop = TypeOopPtr::make_from_klass_raw(k); guarantee(toop != NULL, "need type for given klass"); toop = toop->cast_to_ptr_type(TypePtr::NotNull)->is_oopptr(); + if (flatten_array() && !klass()->is_valuetype()) { + toop = toop->is_instptr()->cast_to_flatten_array(); + } return toop->cast_to_exactness(xk)->is_oopptr(); } @@ -5553,7 +5590,7 @@ case Null: if( ptr == Null ) return TypePtr::make(AnyPtr, ptr, offset, tp->speculative(), tp->inline_depth()); case AnyNull: - return make( ptr, klass(), offset ); + return make(ptr, klass(), offset, flatten_array()); case BotPTR: case NotNull: return TypePtr::make(AnyPtr, ptr, offset, tp->speculative(), tp->inline_depth()); @@ -5594,15 +5631,15 @@ if (ptr == Constant) { k = (klass() == NULL) ? tkls->klass() : klass(); } - return make(ptr, k, off); + return make(ptr, k, off, false); } // Check for easy case; klasses are equal (and perhaps not loaded!) // If we have constants, then we created oops so classes are loaded // and we can handle the constants further down. This case handles // not-loaded classes - if( ptr != Constant && tkls->klass()->equals(klass()) ) { - return make( ptr, klass(), off ); + if (ptr != Constant && tkls->klass()->equals(klass()) && flatten_array() == tkls->flatten_array()) { + return make(ptr, klass(), off, flatten_array()); } // Classes require inspection in the Java klass hierarchy. Must be loaded. @@ -5610,6 +5647,9 @@ ciKlass* this_klass = this->klass(); assert( tkls_klass->is_loaded(), "This class should have been loaded."); assert( this_klass->is_loaded(), "This class should have been loaded."); + bool tkls_flatten_array = tkls->flatten_array(); + bool this_flatten_array = this->flatten_array(); + bool flatten_array = below_centerline(ptr) ? (this_flatten_array && tkls_flatten_array) : (this_flatten_array || tkls_flatten_array); // If 'this' type is above the centerline and is a superclass of the // other, we can treat 'this' as having the same type as the other. @@ -5637,7 +5677,7 @@ else ptr = NotNull; } - return make( ptr, this_klass, off ); + return make(ptr, this_klass, off, flatten_array); } // Else classes are not equal // Since klasses are different, we require the LCA in the Java @@ -5646,7 +5686,7 @@ ptr = NotNull; // Now we find the LCA of Java classes ciKlass* k = this_klass->least_common_ancestor(tkls_klass); - return make( ptr, k, off ); + return make(ptr, k, off, k->is_valuetype() && k->flatten_array()); } // End of case KlassPtr } // End of switch @@ -5656,7 +5696,7 @@ //------------------------------xdual------------------------------------------ // Dual: compute field-by-field dual const Type *TypeKlassPtr::xdual() const { - return new TypeKlassPtr( dual_ptr(), klass(), dual_offset() ); + return new TypeKlassPtr(dual_ptr(), klass(), dual_offset(), flatten_array()); } //------------------------------get_con---------------------------------------- --- old/src/hotspot/share/opto/type.hpp 2019-09-17 12:58:12.000000000 +0200 +++ new/src/hotspot/share/opto/type.hpp 2019-09-17 12:58:07.000000000 +0200 @@ -1139,44 +1139,56 @@ // Class of Java object pointers, pointing either to non-array Java instances // or to a Klass* (including array klasses). class TypeInstPtr : public TypeOopPtr { - TypeInstPtr(PTR ptr, ciKlass* k, bool xk, ciObject* o, Offset offset, int instance_id, - const TypePtr* speculative, int inline_depth); + TypeInstPtr(PTR ptr, ciKlass* k, bool xk, ciObject* o, Offset offset, + bool is_value, int instance_id, const TypePtr* speculative, + int inline_depth); virtual bool eq( const Type *t ) const; virtual int hash() const; // Type specific hashing ciSymbol* _name; // class name - + bool _flatten_array; + + bool meet_flatten_array(bool other_flatten_array) const { + return (_flatten_array && other_flatten_array) ? true : false; + } + public: ciSymbol* name() const { return _name; } + bool flatten_array() const { + assert(!klass()->is_valuetype() || !klass()->as_value_klass()->flatten_array() || _flatten_array, "incorrect value bit"); + assert(!_flatten_array || can_be_value_type(), "incorrect value bit"); + return _flatten_array; + } bool is_loaded() const { return _klass->is_loaded(); } // Make a pointer to a constant oop. static const TypeInstPtr *make(ciObject* o) { - return make(TypePtr::Constant, o->klass(), true, o, Offset(0), InstanceBot); + return make(TypePtr::Constant, o->klass(), true, o, Offset(0), o->klass()->is_valuetype() && o->klass()->as_value_klass()->flatten_array(), InstanceBot); } // Make a pointer to a constant oop with offset. static const TypeInstPtr* make(ciObject* o, Offset offset) { - return make(TypePtr::Constant, o->klass(), true, o, offset, InstanceBot); + return make(TypePtr::Constant, o->klass(), true, o, offset, o->klass()->is_valuetype() && o->klass()->as_value_klass()->flatten_array(), InstanceBot); } // Make a pointer to some value of type klass. static const TypeInstPtr *make(PTR ptr, ciKlass* klass) { - return make(ptr, klass, false, NULL, Offset(0), InstanceBot); + return make(ptr, klass, false, NULL, Offset(0), klass->is_valuetype() && klass->as_value_klass()->flatten_array(), InstanceBot); } // Make a pointer to some non-polymorphic value of exactly type klass. static const TypeInstPtr *make_exact(PTR ptr, ciKlass* klass) { - return make(ptr, klass, true, NULL, Offset(0), InstanceBot); + return make(ptr, klass, true, NULL, Offset(0), klass->is_valuetype() && klass->as_value_klass()->flatten_array(), InstanceBot); } // Make a pointer to some value of type klass with offset. static const TypeInstPtr *make(PTR ptr, ciKlass* klass, Offset offset) { - return make(ptr, klass, false, NULL, offset, InstanceBot); + return make(ptr, klass, false, NULL, offset, klass->is_valuetype() && klass->as_value_klass()->flatten_array(), InstanceBot); } // Make a pointer to an oop. static const TypeInstPtr* make(PTR ptr, ciKlass* k, bool xk, ciObject* o, Offset offset, + bool flatten_array, int instance_id = InstanceBot, const TypePtr* speculative = NULL, int inline_depth = InlineDepthBottom); @@ -1204,6 +1216,8 @@ virtual const TypePtr* with_inline_depth(int depth) const; virtual const TypePtr* with_instance_id(int instance_id) const; + virtual const TypeInstPtr* cast_to_flatten_array() const; + // the core of the computation of the meet of 2 types virtual const Type *xmeet_helper(const Type *t) const; virtual const TypeInstPtr *xmeet_unloaded( const TypeInstPtr *t ) const; @@ -1399,7 +1413,7 @@ //------------------------------TypeKlassPtr----------------------------------- // Class of Java Klass pointers class TypeKlassPtr : public TypePtr { - TypeKlassPtr(PTR ptr, ciKlass* klass, Offset offset); + TypeKlassPtr(PTR ptr, ciKlass* klass, Offset offset, bool flatten_array); protected: virtual const Type *filter_helper(const Type *kills, bool include_speculative) const; @@ -1413,19 +1427,28 @@ // Does the type exclude subclasses of the klass? (Inexact == polymorphic.) bool _klass_is_exact; + bool _flatten_array; public: ciKlass* klass() const { return _klass; } bool klass_is_exact() const { return _klass_is_exact; } + virtual bool can_be_value_type() const { return EnableValhalla && can_be_value_type_raw(); } + virtual bool can_be_value_type_raw() const { return _klass == NULL || !_klass->is_loaded() || _klass->is_valuetype() || ((_klass->is_java_lang_Object() || _klass->is_interface()) && !klass_is_exact()); } + bool flatten_array() const { + assert(!klass()->is_valuetype() || !klass()->as_value_klass()->flatten_array() || _flatten_array, "incorrect value bit"); + assert(!_flatten_array || can_be_value_type(), "incorrect value bit"); + return _flatten_array; + } + bool is_loaded() const { return klass() != NULL && klass()->is_loaded(); } // ptr to klass 'k' - static const TypeKlassPtr* make(ciKlass* k) { return make( TypePtr::Constant, k, Offset(0)); } + static const TypeKlassPtr* make(ciKlass* k) { return make( TypePtr::Constant, k, Offset(0), k->is_valuetype() && k->as_value_klass()->flatten_array()); } // ptr to klass 'k' with offset - static const TypeKlassPtr* make(ciKlass* k, Offset offset) { return make( TypePtr::Constant, k, offset); } + static const TypeKlassPtr* make(ciKlass* k, Offset offset) { return make( TypePtr::Constant, k, offset, k->is_valuetype() && k->as_value_klass()->flatten_array()); } // ptr to klass 'k' or sub-klass - static const TypeKlassPtr* make(PTR ptr, ciKlass* k, Offset offset); + static const TypeKlassPtr* make(PTR ptr, ciKlass* k, Offset offset, bool flatten_array); virtual const Type *cast_to_ptr_type(PTR ptr) const; --- old/test/hotspot/jtreg/compiler/valhalla/valuetypes/TestLWorld.java 2019-09-17 12:58:24.000000000 +0200 +++ new/test/hotspot/jtreg/compiler/valhalla/valuetypes/TestLWorld.java 2019-09-17 12:58:19.000000000 +0200 @@ -2089,6 +2089,7 @@ public void test83_verifier(boolean warmup) { MyValue2[] dst = new MyValue2[1]; test83(dst, testValue2, false); + test83(dst, testValue2, true); if (!warmup) { try { test83(dst, null, true); @@ -2117,8 +2118,8 @@ Asserts.assertTrue(Arrays.equals(src, dst)); } - @Test(valid = G1GCOn, match = { COUNTEDLOOP }, matchCount = { 2 } ) - @Test(valid = G1GCOff, match = { COUNTEDLOOP }, matchCount = { 3 } ) + @Test(valid = G1GCOn, match = { COUNTEDLOOP, LOAD_UNKNOWN_VALUE }, matchCount = { 2, 1 } ) + @Test(valid = G1GCOff, match = { COUNTEDLOOP, LOAD_UNKNOWN_VALUE }, matchCount = { 3, 4 } ) public void test85(Object[] src, Object[] dst) { for (int i = 0; i < src.length; i++) { dst[i] = src[i]; @@ -2232,4 +2233,47 @@ Asserts.assertTrue(test91(new MyValue2[1])); Asserts.assertFalse(test91(new Object())); } + + static inline class Test92Value { + final int field; + public Test92Value() { + field = 0x42; + } + } + + // If the class check succeeds, the flattened array check that + // precedes will never succeed and the flat array branch should + // trigger an uncommon trap. + @Test + @Warmup(10000) + public Object test92(Object[] array) { + Object v = array[0]; + if (v instanceof Integer) { + } + return v; + } + + @DontCompile + public void test92_verifier(boolean warmup) { + if (warmup) { + Object[] array = new Object[1]; + array[0] = 0x42; + Object result = test92(array); + Asserts.assertEquals(result, 0x42); + } else { + Object[] array = new Test92Value[1]; + Method m = tests.get("TestLWorld::test92"); + int extra = 2; + for (int j = 0; j < extra; j++) { + for (int i = 0; i < 10; i++) { + test92(array); + } + boolean compiled = isCompiledByC2(m); + Asserts.assertTrue(!USE_COMPILER || XCOMP || TEST_C1 || compiled == (j == extra-1)); + if (!compiled) { + enqueueMethodForCompilation(m, COMP_LEVEL_FULL_OPTIMIZATION); + } + } + } + } } --- old/test/hotspot/jtreg/compiler/valhalla/valuetypes/TestNullableArrays.java 2019-09-17 12:58:29.000000000 +0200 +++ new/test/hotspot/jtreg/compiler/valhalla/valuetypes/TestNullableArrays.java 2019-09-17 12:58:24.000000000 +0200 @@ -43,14 +43,7 @@ // Unlike C2, C1 intrinsics never deoptimize System.arraycopy. Instead, we fall back to // a normal method invocation when encountering flattened arrays. private static void assertDeoptimizedByC2(Method m) { - int CompLevel_none = 0, // Interpreter - CompLevel_simple = 1, // C1 - CompLevel_limited_profile = 2, // C1, invocation & backedge counters - CompLevel_full_profile = 3, // C1, invocation & backedge counters + mdo - CompLevel_full_optimization = 4; // C2 or JVMCI - - if (USE_COMPILER && !XCOMP && WHITE_BOX.isMethodCompiled(m, false) && - WHITE_BOX.getMethodCompilationLevel(m, false) >= CompLevel_full_optimization) { + if (isCompiledByC2(m)) { throw new RuntimeException("Type check should have caused it to deoptimize"); } } --- old/test/hotspot/jtreg/compiler/valhalla/valuetypes/ValueTypeTest.java 2019-09-17 12:58:35.000000000 +0200 +++ new/test/hotspot/jtreg/compiler/valhalla/valuetypes/ValueTypeTest.java 2019-09-17 12:58:30.000000000 +0200 @@ -753,4 +753,17 @@ } WHITE_BOX.enqueueMethodForCompilation(m, level); } + + // Unlike C2, C1 intrinsics never deoptimize System.arraycopy. Instead, we fall back to + // a normal method invocation when encountering flattened arrays. + static boolean isCompiledByC2(Method m) { + int CompLevel_none = 0, // Interpreter + CompLevel_simple = 1, // C1 + CompLevel_limited_profile = 2, // C1, invocation & backedge counters + CompLevel_full_profile = 3, // C1, invocation & backedge counters + mdo + CompLevel_full_optimization = 4; // C2 or JVMCI + + return USE_COMPILER && !XCOMP && WHITE_BOX.isMethodCompiled(m, false) && + WHITE_BOX.getMethodCompilationLevel(m, false) >= CompLevel_full_optimization; + } }