--- old/src/share/vm/opto/callGenerator.cpp 2017-06-26 12:50:28.699903021 +0200 +++ new/src/share/vm/opto/callGenerator.cpp 2017-06-26 12:50:28.611903025 +0200 @@ -493,11 +493,11 @@ C->set_has_loops(C->has_loops() || _inline_cg->method()->has_loops()); C->env()->notice_inlined_method(_inline_cg->method()); C->set_inlining_progress(true); - + if (return_type->is_valuetype() && return_type != C->env()->___Value_klass()) { if (result->is_ValueType()) { if (!call->tf()->returns_value_type_as_fields()) { - result = result->as_ValueType()->store_to_memory(&kit); + result = result->as_ValueType()->allocate(&kit); } else { // Return of multiple values (the fields of a value type) ValueTypeNode* vt = result->as_ValueType(); @@ -555,7 +555,7 @@ }; bool LateInlineMHCallGenerator::do_late_inline_check(JVMState* jvms) { - + CallGenerator* cg = for_method_handle_inline(jvms, _caller, method(), _input_not_const, AlwaysIncrementalInline); Compile::current()->print_inlining_update_delayed(this); @@ -563,7 +563,7 @@ if (!_input_not_const) { _attempt++; } - + if (cg != NULL && (cg->is_inline() || cg->is_inlined_method_handle_intrinsic(jvms, cg->method()))) { assert(!cg->is_late_inline(), "we're doing late inlining"); _inline_cg = cg; --- old/src/share/vm/opto/callnode.cpp 2017-06-26 12:50:29.007903007 +0200 +++ new/src/share/vm/opto/callnode.cpp 2017-06-26 12:50:28.919903011 +0200 @@ -1381,7 +1381,8 @@ AllocateNode::AllocateNode(Compile* C, const TypeFunc *atype, Node *ctrl, Node *mem, Node *abio, - Node *size, Node *klass_node, Node *initial_test) + Node *size, Node *klass_node, + Node* initial_test, ValueTypeNode* value_node) : CallNode(atype, NULL, TypeRawPtr::BOTTOM) { init_class_id(Class_Allocate); @@ -1400,6 +1401,7 @@ init_req( KlassNode , klass_node); init_req( InitialTest , initial_test); init_req( ALength , topnode); + init_req( ValueNode , value_node); C->add_macro_node(this); } --- old/src/share/vm/opto/callnode.hpp 2017-06-26 12:50:29.327902992 +0200 +++ new/src/share/vm/opto/callnode.hpp 2017-06-26 12:50:29.243902996 +0200 @@ -836,6 +836,7 @@ KlassNode, // type (maybe dynamic) of the obj. InitialTest, // slow-path test (may be constant) ALength, // array length (or TOP if none) + ValueNode, ParmLimit }; @@ -845,6 +846,7 @@ fields[KlassNode] = TypeInstPtr::NOTNULL; fields[InitialTest] = TypeInt::BOOL; fields[ALength] = t; // length (can be a bad length) + fields[ValueNode] = Type::BOTTOM; const TypeTuple *domain = TypeTuple::make(ParmLimit, fields); @@ -865,7 +867,7 @@ virtual uint size_of() const; // Size is bigger AllocateNode(Compile* C, const TypeFunc *atype, Node *ctrl, Node *mem, Node *abio, - Node *size, Node *klass_node, Node *initial_test); + Node *size, Node *klass_node, Node *initial_test, ValueTypeNode* value_node = NULL); // Expansion modifies the JVMState, so we need to clone it virtual void clone_jvms(Compile* C) { if (jvms() != NULL) { --- old/src/share/vm/opto/doCall.cpp 2017-06-26 12:50:29.639902977 +0200 +++ new/src/share/vm/opto/doCall.cpp 2017-06-26 12:50:29.551902982 +0200 @@ -670,7 +670,7 @@ assert(ctype == C->env()->___Value_klass(), "unexpected value type klass"); Node* retnode = pop(); assert(retnode->is_ValueType(), "inconsistent"); - retnode = retnode->as_ValueType()->store_to_memory(this); + retnode = retnode->as_ValueType()->allocate(this); push(retnode); } } else { --- old/src/share/vm/opto/graphKit.cpp 2017-06-26 12:50:29.919902964 +0200 +++ new/src/share/vm/opto/graphKit.cpp 2017-06-26 12:50:29.827902969 +0200 @@ -1512,7 +1512,7 @@ if (bt == T_VALUETYPE) { // Allocate value type and store oop - val = val->as_ValueType()->store_to_memory(this); + val = val->as_ValueType()->allocate(this); } pre_barrier(true /* do_load */, @@ -1610,7 +1610,7 @@ // pass each field of the value type idx += vt->pass_fields(call, idx, *this); } else { - arg = arg->as_ValueType()->store_to_memory(this); + arg = arg->as_ValueType()->allocate(this); call->init_req(idx, arg); idx++; } @@ -1625,7 +1625,7 @@ } else { if (arg->is_ValueType()) { // Pass value type argument via oop to callee - arg = arg->as_ValueType()->store_to_memory(this); + arg = arg->as_ValueType()->allocate(this); } call->init_req(i, arg); } @@ -3343,7 +3343,8 @@ Node* GraphKit::new_instance(Node* klass_node, Node* extra_slow_test, Node* *return_size_val, - bool deoptimize_on_exception) { + bool deoptimize_on_exception, + ValueTypeNode* value_node) { // Compute size in doublewords // The size is always an integral number of doublewords, represented // as a positive bytewise size stored in the klass's layout_helper. @@ -3408,7 +3409,7 @@ AllocateNode* alloc = new AllocateNode(C, AllocateNode::alloc_type(Type::TOP), control(), mem, i_o(), size, klass_node, - initial_slow_test); + initial_slow_test, value_node); return set_output_for_allocation(alloc, oop_type, deoptimize_on_exception); } @@ -3639,7 +3640,7 @@ PreserveJVMState pjvms(this); // Create default value type and store it to memory Node* oop = ValueTypeNode::make_default(gvn(), vk); - oop = oop->as_ValueType()->store_to_memory(this); + oop = oop->as_ValueType()->allocate(this); length = SubI(length, intcon(1)); add_predicate(nargs); --- old/src/share/vm/opto/graphKit.hpp 2017-06-26 12:50:30.231902950 +0200 +++ new/src/share/vm/opto/graphKit.hpp 2017-06-26 12:50:30.143902954 +0200 @@ -876,7 +876,8 @@ Node* new_instance(Node* klass_node, Node* slow_test = NULL, Node* *return_size_val = NULL, - bool deoptimize_on_exception = false); + bool deoptimize_on_exception = false, + ValueTypeNode* value_node = NULL); Node* new_array(Node* klass_node, Node* count_val, int nargs, Node* *return_size_val = NULL, bool deoptimize_on_exception = false); --- old/src/share/vm/opto/loopopts.cpp 2017-06-26 12:50:30.507902937 +0200 +++ new/src/share/vm/opto/loopopts.cpp 2017-06-26 12:50:30.419902941 +0200 @@ -37,6 +37,7 @@ #include "opto/opaquenode.hpp" #include "opto/rootnode.hpp" #include "opto/subnode.hpp" +#include "opto/valuetypenode.hpp" //============================================================================= //------------------------------split_thru_phi--------------------------------- @@ -1359,6 +1360,11 @@ try_move_store_after_loop(n); + // Remove multiple allocations of the same value type + if (n->is_ValueType() && EliminateAllocations) { + n->as_ValueType()->remove_redundant_allocations(&_igvn, this); + } + // Check for Opaque2's who's loop has disappeared - who's input is in the // same loop nest as their output. Remove 'em, they are no longer useful. if( n_op == Op_Opaque2 && --- old/src/share/vm/opto/parse1.cpp 2017-06-26 12:50:30.819902922 +0200 +++ new/src/share/vm/opto/parse1.cpp 2017-06-26 12:50:30.735902926 +0200 @@ -2317,7 +2317,7 @@ !tf()->returns_value_type_as_fields()) { // Returning from root JVMState without multiple returned values, // make sure value type is allocated - value = value->as_ValueType()->store_to_memory(this); + value = value->as_ValueType()->allocate(this); } if (RegisterFinalizersAtInit && --- old/src/share/vm/opto/parse2.cpp 2017-06-26 12:50:31.099902909 +0200 +++ new/src/share/vm/opto/parse2.cpp 2017-06-26 12:50:31.011902914 +0200 @@ -1756,12 +1756,12 @@ const Type* elemtype = arytype->elem(); if (elemtype->isa_valuetype()) { - c->as_ValueType()->store(this, a, d); + c->as_ValueType()->store_flattened(this, a, d); break; } const TypeAryPtr* adr_type = TypeAryPtr::OOPS; - Node* oop = c->as_ValueType()->store_to_memory(this); + Node* oop = c->as_ValueType()->allocate(this); Node* store = store_oop_to_array(control(), a, d, adr_type, oop, elemtype->make_oopptr(), T_OBJECT, StoreNode::release_if_reference(T_OBJECT)); break; --- old/src/share/vm/opto/parse3.cpp 2017-06-26 12:50:31.383902896 +0200 +++ new/src/share/vm/opto/parse3.cpp 2017-06-26 12:50:31.303902900 +0200 @@ -317,7 +317,7 @@ } if (bt == T_VALUETYPE && !field->is_static()) { // Store flattened value type to non-static field - val->as_ValueType()->store(this, obj, obj, field->holder(), offset); + val->as_ValueType()->store_flattened(this, obj, obj, field->holder(), offset); } else { store_oop_to_object(control(), obj, adr, adr_type, val, field_type, bt, mo); } @@ -572,7 +572,7 @@ kill_dead_locals(); ciInstanceKlass* target_vcc_klass = target_klass->as_instance_klass(); - ciInstanceKlass* src_vcc_klass = src_vk->vcc_klass();; + ciInstanceKlass* src_vcc_klass = src_vk->vcc_klass(); // TODO: Extend type check below if (and once) value type class hierarchies become available. // (incl. extension to support dynamic type checks). @@ -592,7 +592,7 @@ // The code below relies on the assumption that the VCC has the // same memory layout as the derived value type. // TODO: Once the layout of the two is not the same, update code below. - vt->as_ValueType()->store_values(this, obj, obj, target_vcc_klass); + vt->as_ValueType()->store(this, obj, obj, target_vcc_klass); // Push the new object onto the stack push(obj); --- old/src/share/vm/opto/valuetypenode.cpp 2017-06-26 12:50:31.647902884 +0200 +++ new/src/share/vm/opto/valuetypenode.cpp 2017-06-26 12:50:31.563902888 +0200 @@ -58,7 +58,9 @@ // values from a heap-allocated version and also save the oop. const TypeValueType* type = gvn.type(oop)->is_valuetypeptr()->value_type(); ValueTypeNode* vt = new ValueTypeNode(type, oop); - vt->load_values(gvn, mem, oop, oop, type->value_klass()); + vt->load(gvn, mem, oop, oop, type->value_klass()); + assert(vt->is_allocated(&gvn), "value type should be allocated"); + assert(oop->is_Con() || oop->is_CheckCastPP() || vt->is_loaded(&gvn, type) != NULL, "value type should be loaded"); return gvn.transform(vt); } @@ -69,11 +71,13 @@ // The value type is flattened into the object without an oop header. Subtract the // offset of the first field to account for the missing header when loading the values. holder_offset -= vk->first_field_offset(); - vt->load_values(gvn, mem, obj, ptr, holder, holder_offset); - return gvn.transform(vt); + vt->load(gvn, mem, obj, ptr, holder, holder_offset); + vt = gvn.transform(vt)->as_ValueType(); + assert(!vt->is_allocated(&gvn), "value type should not be allocated"); + return vt; } -void ValueTypeNode::load_values(PhaseGVN& gvn, Node* mem, Node* base, Node* ptr, ciInstanceKlass* holder, int holder_offset) { +void ValueTypeNode::load(PhaseGVN& gvn, Node* mem, Node* base, Node* ptr, ciInstanceKlass* holder, int holder_offset) { // Initialize the value type by loading its field values from // memory and adding the values as input edges to the node. for (uint i = 0; i < field_count(); ++i) { @@ -118,21 +122,59 @@ } } -void ValueTypeNode::store(GraphKit* kit, Node* obj, Node* ptr, ciInstanceKlass* holder, int holder_offset) const { +Node* ValueTypeNode::is_loaded(PhaseGVN* phase, const TypeValueType* t, Node* base, int holder_offset) { + for (uint i = 0; i < field_count(); ++i) { + int offset = holder_offset + field_offset(i); + Node* value = field_value(i); + if (value->isa_DecodeN()) { + // Skip DecodeN + value = value->in(1); + } + if (value->isa_Load()) { + AddPNode* load_addr = value->in(MemNode::Address)->as_AddP(); + if (base == NULL) { + // Set base and check if pointer type matches + base = load_addr->base_node(); + const TypeValueTypePtr* vtptr = phase->type(base)->isa_valuetypeptr(); + if (vtptr == NULL || !vtptr->value_type()->eq(t)) { + return NULL; + } + } + // Check if base and offset of field load matches + Node* off = load_addr->in(AddPNode::Offset); + int load_offset = LP64_ONLY(off->get_long()) NOT_LP64(off->get_int()); + if (base != load_addr->base_node() || offset != load_offset) { + return NULL; + } + } else if (value->isa_ValueType()) { + // Check value type field load recursively + ValueTypeNode* vt = value->as_ValueType(); + base = vt->is_loaded(phase, t, base, offset - vt->value_klass()->first_field_offset()); + if (base == NULL) { + return NULL; + } + } else { + return NULL; + } + } + return base; +} + +void ValueTypeNode::store_flattened(GraphKit* kit, Node* base, Node* ptr, ciInstanceKlass* holder, int holder_offset) const { // The value type is embedded into the object without an oop header. Subtract the // offset of the first field to account for the missing header when storing the values. holder_offset -= value_klass()->first_field_offset(); - store_values(kit, obj, ptr, holder, holder_offset); + store(kit, base, ptr, holder, holder_offset); } -void ValueTypeNode::store_values(GraphKit* kit, Node* base, Node* ptr, ciInstanceKlass* holder, int holder_offset) const { +void ValueTypeNode::store(GraphKit* kit, Node* base, Node* ptr, ciInstanceKlass* holder, int holder_offset) const { // Write field values to memory for (uint i = 0; i < field_count(); ++i) { int offset = holder_offset + field_offset(i); Node* value = field_value(i); if (value->is_ValueType()) { // Recursively store the flattened value type field - value->isa_ValueType()->store(kit, base, ptr, holder, offset); + value->isa_ValueType()->store_flattened(kit, base, ptr, holder, offset); } else { const Type* base_type = kit->gvn().type(base); const TypePtr* adr_type = NULL; @@ -153,12 +195,11 @@ bool is_array = base_type->isa_aryptr() != NULL; kit->store_oop(kit->control(), base, adr, adr_type, value, ft, bt, is_array, MemNode::unordered); } - } } } -Node* ValueTypeNode::store_to_memory(GraphKit* kit) { +Node* ValueTypeNode::allocate(GraphKit* kit) { Node* in_oop = get_oop(); Node* null_ctl = kit->top(); // Check if value type is already allocated @@ -169,9 +210,7 @@ } // Not able to prove that value type is allocated. // Emit runtime check that may be folded later. - const Type* oop_type = kit->gvn().type(in_oop); - assert(TypePtr::NULL_PTR->higher_equal(oop_type), "should not be allocated"); - + assert(!is_allocated(&kit->gvn()), "should not be allocated"); const TypeValueTypePtr* vtptr_type = TypeValueTypePtr::make(bottom_type()->isa_valuetype(), TypePtr::NotNull); RegionNode* region = new RegionNode(3); PhiNode* oop = new PhiNode(region, vtptr_type); @@ -189,9 +228,9 @@ kit->kill_dead_locals(); ciValueKlass* vk = value_klass(); Node* klass_node = kit->makecon(TypeKlassPtr::make(vk)); - Node* alloc_oop = kit->new_instance(klass_node); + Node* alloc_oop = kit->new_instance(klass_node, NULL, NULL, false, this); // Write field values to memory - store_values(kit, alloc_oop, alloc_oop, vk); + store(kit, alloc_oop, alloc_oop, vk); region->init_req(2, kit->control()); oop ->init_req(2, alloc_oop); io ->init_req(2, kit->i_o()); @@ -214,6 +253,11 @@ return res_oop; } +bool ValueTypeNode::is_allocated(PhaseGVN* phase) const { + const Type* oop_type = phase->type(get_oop()); + return oop_type->meet(TypePtr::NULL_PTR) != oop_type; +} + // Clones the values type to handle control flow merges involving multiple value types. // The inputs are replaced by PhiNodes to represent the merged values for the given region. ValueTypeNode* ValueTypeNode::clone_with_phis(PhaseGVN* gvn, Node* region) { @@ -415,10 +459,19 @@ } Node* ValueTypeNode::Ideal(PhaseGVN* phase, bool can_reshape) { + if (!is_allocated(phase)) { + // Check if this value type is loaded from memory + Node* base = is_loaded(phase, type()->is_valuetype()); + if (base != NULL) { + // Save the oop + set_oop(base); + assert(is_allocated(phase), "should now be allocated"); + } + } + if (can_reshape) { PhaseIterGVN* igvn = phase->is_IterGVN(); - const Type* oop_type = igvn->type(get_oop()); - if (oop_type->meet(TypePtr::NULL_PTR) != oop_type) { + if (is_allocated(igvn)) { // Value type is heap allocated, search for safepoint uses for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) { Node* out = fast_out(i); @@ -430,10 +483,76 @@ } } } - return NULL; } +// Search for multiple allocations of this value type +// and try to replace them by dominating allocations. +void ValueTypeNode::remove_redundant_allocations(PhaseIterGVN* igvn, PhaseIdealLoop* phase) { + assert(EliminateAllocations, "allocation elimination should be enabled"); + Node_List dead_allocations; + // Search for allocations of this value type + for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) { + Node* out1 = fast_out(i); + if (out1->is_Allocate() && out1->in(AllocateNode::ValueNode) == this) { + AllocateNode* alloc = out1->as_Allocate(); + Node* res_dom = NULL; + if (is_allocated(igvn)) { + // The value type is already allocated but still connected to an AllocateNode. + // This can happen with late inlining when we first allocate a value type argument + // but later decide to inline the call with the callee code also allocating. + res_dom = get_oop(); + } else { + // Search for a dominating allocation of the same value type + for (DUIterator_Fast jmax, j = fast_outs(jmax); j < jmax; j++) { + Node* out2 = fast_out(j); + if (alloc != out2 && out2->is_Allocate() && out2->in(AllocateNode::ValueNode) == this && + phase->is_dominator(out2, alloc)) { + AllocateNode* alloc_dom = out2->as_Allocate(); + assert(alloc->in(AllocateNode::KlassNode) == alloc_dom->in(AllocateNode::KlassNode), "klasses should match"); + res_dom = alloc_dom->result_cast(); + break; + } + } + } + if (res_dom != NULL) { + // Found a dominating allocation + Node* res = alloc->result_cast(); + assert(res != NULL, "value type allocation should not be dead"); + // Move users to dominating allocation + igvn->replace_node(res, res_dom); + // The dominated allocation is now dead, remove the + // value type node connection and adjust the iterator. + dead_allocations.push(alloc); + igvn->replace_input_of(alloc, AllocateNode::ValueNode, NULL); + --i; --imax; +#ifdef ASSERT + if (PrintEliminateAllocations) { + tty->print("++++ Eliminated: %d Allocate ", alloc->_idx); + dump_spec(tty); + tty->cr(); + } +#endif + } + } + } + + // Remove dead value type allocations by replacing the projection nodes + for (uint i = 0; i < dead_allocations.size(); ++i) { + CallProjections projs; + AllocateNode* alloc = dead_allocations.at(i)->as_Allocate(); + alloc->extract_projections(&projs, true); + // Use lazy_replace to avoid corrupting the dominator tree of PhaseIdealLoop + phase->lazy_replace(projs.fallthrough_catchproj, alloc->in(TypeFunc::Control)); + phase->lazy_replace(projs.fallthrough_memproj, alloc->in(TypeFunc::Memory)); + phase->lazy_replace(projs.catchall_memproj, phase->C->top()); + phase->lazy_replace(projs.fallthrough_ioproj, alloc->in(TypeFunc::I_O)); + phase->lazy_replace(projs.catchall_ioproj, phase->C->top()); + phase->lazy_replace(projs.catchall_catchproj, phase->C->top()); + phase->lazy_replace(projs.resproj, phase->C->top()); + } +} + // When a call returns multiple values, it has several result // projections, one per field. Replacing the result of the call by a // value type node (after late inlining) requires that for each result --- old/src/share/vm/opto/valuetypenode.hpp 2017-06-26 12:50:31.915902871 +0200 +++ new/src/share/vm/opto/valuetypenode.hpp 2017-06-26 12:50:31.831902875 +0200 @@ -49,7 +49,9 @@ // Get the klass defining the field layout of the value type ciValueKlass* value_klass() const { return type()->is_valuetype()->value_klass(); } // Initialize the value type by loading its field values from memory - void load_values(PhaseGVN& gvn, Node* mem, Node* base, Node* ptr, ciInstanceKlass* holder, int holder_offset = 0); + void load(PhaseGVN& gvn, Node* mem, Node* base, Node* ptr, ciInstanceKlass* holder, int holder_offset = 0); + // Checks if the value type is loaded from memory and if so returns the oop + Node* is_loaded(PhaseGVN* phase, const TypeValueType* t, Node* base = NULL, int holder_offset = 0); enum { Control, // Control input Oop, // Oop of TypeValueTypePtr @@ -73,16 +75,17 @@ bool has_phi_inputs(Node* region); ValueTypeNode* merge_with(PhaseGVN* gvn, const ValueTypeNode* other, int pnum, bool transform); - // Store the value type to memory if not yet allocated and returns the oop - Node* store_to_memory(GraphKit* kit); - // Store the value type to a flattened value type field or array - void store(GraphKit* kit, Node* obj, Node* ptr, ciInstanceKlass* holder = NULL, int holder_offset = 0) const; + // Store the value type as a flattened (headerless) representation + void store_flattened(GraphKit* kit, Node* base, Node* ptr, ciInstanceKlass* holder = NULL, int holder_offset = 0) const; // Store the field values to memory - void store_values(GraphKit* kit, Node* base, Node* ptr, ciInstanceKlass* holder, int holder_offset = 0) const; + void store(GraphKit* kit, Node* base, Node* ptr, ciInstanceKlass* holder, int holder_offset = 0) const; // Get oop for heap allocated value type (may be TypePtr::NULL_PTR) Node* get_oop() const { return in(Oop); } void set_oop(Node* oop) { set_req(Oop, oop); } + // Allocates the value type (if not yet allocated) and returns the oop + Node* allocate(GraphKit* kit); + bool is_allocated(PhaseGVN* phase) const; // Value type fields uint field_count() const { return req() - Values; } @@ -98,6 +101,8 @@ uint pass_fields(Node* call, int base_input, const GraphKit& kit, ciValueKlass* base_vk = NULL, int base_offset = 0); void replace_call_results(Node* call, Compile* C); + // Allocation optimizations + void remove_redundant_allocations(PhaseIterGVN* igvn, PhaseIdealLoop* phase); virtual Node* Ideal(PhaseGVN* phase, bool can_reshape); virtual int Opcode() const; --- old/test/compiler/valhalla/valuetypes/ValueTypeTestBench.java 2017-06-26 12:50:32.167902860 +0200 +++ new/test/compiler/valhalla/valuetypes/ValueTypeTestBench.java 2017-06-26 12:50:32.083902864 +0200 @@ -88,18 +88,6 @@ static final MyValue2 v3 = MyValue2.createWithFieldsInline(ValueTypeTestBench.rI, true); final int c; - private MyValue1(int x, long y, short z, Integer o, int[] oa, MyValue2 v1, MyValue2 v2, int c) { - s = x; - this.x = x; - this.y = y; - this.z = z; - this.o = o; - this.oa = oa; - this.v1 = v1; - this.v2 = v2; - this.c = c; - } - private MyValue1() { s = 0; this.x = 0; @@ -114,7 +102,7 @@ @DontInline __ValueFactory static MyValue1 createDefaultDontInline() { - return __MakeDefault MyValue1(); + return createDefaultInline(); } @ForceInline @@ -124,17 +112,7 @@ @DontInline static MyValue1 createWithFieldsDontInline(int x, long y) { - MyValue1 v = createDefaultInline(); - v = setX(v, x); - v = setY(v, y); - v = setZ(v, (short)x); - v = setO(v, new Integer(x)); - int[] oa = {x}; - v = setOA(v, oa); - v = setV1(v, MyValue2.createWithFieldsInline(x, x < y)); - v = setV2(v, MyValue2.createWithFieldsInline(x, x > y)); - v = setC(v, ValueTypeTestBench.rI); - return v; + return createWithFieldsInline(x, y); } @ForceInline @@ -234,13 +212,6 @@ final boolean b; final long c; - private MyValue2(int x, byte y, boolean b, long c) { - this.x = x; - this.y = y; - this.b = b; - this.c = c; - } - private MyValue2() { this.x = 0; this.y = 0; @@ -322,19 +293,19 @@ final double f8; private MyValue3(char c, - byte bb, - short s, - int i, - long l, - Object o, - float f1, - double f2, - float f3, - double f4, - float f5, - double f6, - float f7, - double f8) { + byte bb, + short s, + int i, + long l, + Object o, + float f1, + double f2, + float f3, + double f4, + float f5, + double f6, + float f7, + double f8) { this.c = c; this.bb = bb; this.s = s; @@ -478,6 +449,32 @@ return v; } + @DontInline + public static MyValue3 createDontInline() { + return create(); + } + + @ForceInline + public static MyValue3 copy(MyValue3 other) { + MyValue3 v = createDefault(); + v = setC(v, other.c); + v = setBB(v, other.bb); + v = setS(v, other.s); + v = setI(v, other.i); + v = setL(v, other.l); + v = setO(v, other.o); + v = setF1(v, other.f1); + v = setF2(v, other.f2); + v = setF3(v, other.f3); + v = setF4(v, other.f4); + v = setF5(v, other.f5); + v = setF6(v, other.f6); + v = setF7(v, other.f7); + v = setF8(v, other.f8); + return v; + } + + @DontInline public void verify(MyValue3 other) { Asserts.assertEQ(c, other.c); Asserts.assertEQ(bb, other.bb); @@ -2166,168 +2163,168 @@ /* Array load out of bounds (upper bound) at compile time.*/ @Test public int test77() { - int arraySize = Math.abs(rI) % 10;; - MyValue1[] va = new MyValue1[arraySize]; + int arraySize = Math.abs(rI) % 10;; + MyValue1[] va = new MyValue1[arraySize]; - for (int i = 0; i < arraySize; i++) { - va[i] = MyValue1.createWithFieldsDontInline(rI + 1, rL); - } - - try { - return va[arraySize + 1].x; - } catch (ArrayIndexOutOfBoundsException e) { - return rI; - } + for (int i = 0; i < arraySize; i++) { + va[i] = MyValue1.createWithFieldsDontInline(rI + 1, rL); + } + + try { + return va[arraySize + 1].x; + } catch (ArrayIndexOutOfBoundsException e) { + return rI; + } } public void test77_verifier(boolean warmup) { - Asserts.assertEQ(test77(), rI); + Asserts.assertEQ(test77(), rI); } /* Array load out of bounds (lower bound) at compile time.*/ @Test public int test78() { - int arraySize = Math.abs(rI) % 10;; - MyValue1[] va = new MyValue1[arraySize]; + int arraySize = Math.abs(rI) % 10;; + MyValue1[] va = new MyValue1[arraySize]; - for (int i = 0; i < arraySize; i++) { - va[i] = MyValue1.createWithFieldsDontInline(rI + i, rL); - } - - try { - return va[-arraySize].x; - } catch (ArrayIndexOutOfBoundsException e) { - return rI; - } + for (int i = 0; i < arraySize; i++) { + va[i] = MyValue1.createWithFieldsDontInline(rI + i, rL); + } + + try { + return va[-arraySize].x; + } catch (ArrayIndexOutOfBoundsException e) { + return rI; + } } public void test78_verifier(boolean warmup) { - Asserts.assertEQ(test78(), rI); + Asserts.assertEQ(test78(), rI); } /* Array load out of bound not known to compiler (both lower and upper bound). */ @Test public int test79(MyValue1[] va, int index) { - return va[index].x; + return va[index].x; } public void test79_verifier(boolean warmup) { - int arraySize = Math.abs(rI) % 10; - MyValue1[] va = new MyValue1[arraySize]; + int arraySize = Math.abs(rI) % 10; + MyValue1[] va = new MyValue1[arraySize]; + + for (int i = 0; i < arraySize; i++) { + va[i] = MyValue1.createWithFieldsDontInline(rI, rL); + } - for (int i = 0; i < arraySize; i++) { - va[i] = MyValue1.createWithFieldsDontInline(rI, rL); - } - - int result; - for (int i = -20; i < 20; i++) { - try { - result = test79(va, i); - } catch (ArrayIndexOutOfBoundsException e) { - result = rI; - } - Asserts.assertEQ(result, rI); - } + int result; + for (int i = -20; i < 20; i++) { + try { + result = test79(va, i); + } catch (ArrayIndexOutOfBoundsException e) { + result = rI; + } + Asserts.assertEQ(result, rI); + } } - /* Array store out of bounds (upper bound) at compile time.*/ + /* Array store out of bounds (upper bound) at compile time.*/ @Test public int test80() { - int arraySize = Math.abs(rI) % 10;; - MyValue1[] va = new MyValue1[arraySize]; + int arraySize = Math.abs(rI) % 10;; + MyValue1[] va = new MyValue1[arraySize]; - try { - for (int i = 0; i <= arraySize; i++) { - va[i] = MyValue1.createWithFieldsDontInline(rI + 1, rL); - } - return rI - 1; - } catch (ArrayIndexOutOfBoundsException e) { - return rI; - } - } - - public void test80_verifier(boolean warmup) { - Asserts.assertEQ(test80(), rI); - } + try { + for (int i = 0; i <= arraySize; i++) { + va[i] = MyValue1.createWithFieldsDontInline(rI + 1, rL); + } + return rI - 1; + } catch (ArrayIndexOutOfBoundsException e) { + return rI; + } + } + + public void test80_verifier(boolean warmup) { + Asserts.assertEQ(test80(), rI); + } /* Array store out of bounds (lower bound) at compile time.*/ @Test public int test81() { - int arraySize = Math.abs(rI) % 10;; - MyValue1[] va = new MyValue1[arraySize]; + int arraySize = Math.abs(rI) % 10;; + MyValue1[] va = new MyValue1[arraySize]; - try { - for (int i = -1; i <= arraySize; i++) { - va[i] = MyValue1.createWithFieldsDontInline(rI + 1, rL); - } - return rI - 1; - } catch (ArrayIndexOutOfBoundsException e) { - return rI; - } + try { + for (int i = -1; i <= arraySize; i++) { + va[i] = MyValue1.createWithFieldsDontInline(rI + 1, rL); + } + return rI - 1; + } catch (ArrayIndexOutOfBoundsException e) { + return rI; + } } public void test81_verifier(boolean warmup) { - Asserts.assertEQ(test81(), rI); + Asserts.assertEQ(test81(), rI); } /* Array store out of bound not known to compiler (both lower and upper bound). */ @Test public int test82(MyValue1[] va, int index, MyValue1 vt) { - va[index] = vt; - return va[index].x; + va[index] = vt; + return va[index].x; } @DontCompile public void test82_verifier(boolean warmup) { - int arraySize = Math.abs(rI) % 10; - MyValue1[] va = new MyValue1[arraySize]; + int arraySize = Math.abs(rI) % 10; + MyValue1[] va = new MyValue1[arraySize]; + + for (int i = 0; i < arraySize; i++) { + va[i] = MyValue1.createWithFieldsDontInline(rI, rL); + } + + MyValue1 vt = MyValue1.createWithFieldsDontInline(rI + 1, rL); + int result; + for (int i = -20; i < 20; i++) { + try { + result = test82(va, i, vt); + } catch (ArrayIndexOutOfBoundsException e) { + result = rI + 1; + } + Asserts.assertEQ(result, rI + 1); + } - for (int i = 0; i < arraySize; i++) { - va[i] = MyValue1.createWithFieldsDontInline(rI, rL); - } - - MyValue1 vt = MyValue1.createWithFieldsDontInline(rI + 1, rL); - int result; - for (int i = -20; i < 20; i++) { - try { - result = test82(va, i, vt); - } catch (ArrayIndexOutOfBoundsException e) { - result = rI + 1; - } - Asserts.assertEQ(result, rI + 1); - } - - for (int i = 0; i < arraySize; i++) { - Asserts.assertEQ(va[i].x, rI + 1); - } + for (int i = 0; i < arraySize; i++) { + Asserts.assertEQ(va[i].x, rI + 1); + } } /* Create a new value type array and store a value type into * it. The test should pass without throwing an exception. */ @Test public void test83() throws Throwable { - vastoreMH.invokeExact(vcc); + vastoreMH.invokeExact(vcc); } - + public void test83_verifier(boolean warmup) throws Throwable { - test83(); + test83(); } private static MethodHandle generateVastore() { return MethodHandleBuilder.loadCode(MethodHandles.lookup(), "Vastore", - MethodType.methodType(void.class, ValueCapableClass1.class), + MethodType.methodType(void.class, ValueCapableClass1.class), CODE -> { CODE. - iconst_1(). - anewarray(ValueType.forClass(ValueCapableClass1.class).valueClass()). - iconst_0(). - aload_0(). - vunbox(ValueType.forClass(ValueCapableClass1.class).valueClass()). - vastore(). - return_(); - } - ); + iconst_1(). + anewarray(ValueType.forClass(ValueCapableClass1.class).valueClass()). + iconst_0(). + aload_0(). + vunbox(ValueType.forClass(ValueCapableClass1.class).valueClass()). + vastore(). + return_(); + } + ); } /* Create a new value type array with element type @@ -2335,33 +2332,33 @@ * ValueCapableClass2 into it. */ @Test public void test84() throws Throwable { - invalidVastoreMH.invokeExact(vcc2); + invalidVastoreMH.invokeExact(vcc2); } - + public void test84_verifier(boolean warmup) throws Throwable { - boolean exceptionThrown = false; - try { - test84(); - } catch (ArrayStoreException e) { - exceptionThrown = true; - } - Asserts.assertTrue(exceptionThrown, "ArrayStoreException must be thrown"); + boolean exceptionThrown = false; + try { + test84(); + } catch (ArrayStoreException e) { + exceptionThrown = true; + } + Asserts.assertTrue(exceptionThrown, "ArrayStoreException must be thrown"); } private static MethodHandle generateInvalidVastore() { return MethodHandleBuilder.loadCode(MethodHandles.lookup(), "Vastore", - MethodType.methodType(void.class, ValueCapableClass2.class), + MethodType.methodType(void.class, ValueCapableClass2.class), CODE -> { CODE. - iconst_1(). - anewarray(ValueType.forClass(ValueCapableClass1.class).valueClass()). - iconst_0(). - aload_0(). - vunbox(ValueType.forClass(ValueCapableClass2.class).valueClass()). - vastore(). - return_(); - } + iconst_1(). + anewarray(ValueType.forClass(ValueCapableClass1.class).valueClass()). + iconst_0(). + aload_0(). + vunbox(ValueType.forClass(ValueCapableClass2.class).valueClass()). + vastore(). + return_(); + } ); } @@ -2426,6 +2423,146 @@ throw new RuntimeException("method handle lookup fails"); } } + + static MyValue3 staticVal3; + static MyValue3 staticVal3_copy; + + // Check elimination of redundant value type allocations + @Test(match = {ALLOC}, matchCount = {1}) + public MyValue3 test87(MyValue3[] va) { + // Create value type and force allocation + MyValue3 vt = MyValue3.create(); + va[0] = vt; + staticVal3 = vt; + vt.verify(staticVal3); + + // Value type is now allocated, make a copy and force allocation. + // Because copy is equal to vt, C2 should remove this redundant allocation. + MyValue3 copy = MyValue3.setC(vt, vt.c); + va[0] = copy; + staticVal3_copy = copy; + copy.verify(staticVal3_copy); + return copy; + } + + @DontCompile + public void test87_verifier(boolean warmup) { + MyValue3[] va = new MyValue3[1]; + MyValue3 vt = test87(va); + staticVal3.verify(vt); + staticVal3.verify(va[0]); + staticVal3_copy.verify(vt); + staticVal3_copy.verify(va[0]); + } + + // Verify that only dominating allocations are re-used + @Test() + public MyValue3 test88(boolean warmup) { + MyValue3 vt = MyValue3.create(); + if (warmup) { + staticVal3 = vt; // Force allocation + } + // Force allocation to verify that above + // non-dominating allocation is not re-used + MyValue3 copy = MyValue3.setC(vt, vt.c); + staticVal3_copy = copy; + copy.verify(vt); + return copy; + } + + @DontCompile + public void test88_verifier(boolean warmup) { + MyValue3 vt = test88(warmup); + if (warmup) { + staticVal3.verify(vt); + } + } + + // Verify that C2 recognizes value type loads and re-uses the oop to avoid allocations + @Test(failOn = ALLOC + ALLOCA + STORE) + public MyValue3 test89(MyValue3[] va) { + // C2 can re-use the oop of staticVal3 because staticVal3 is equal to copy + MyValue3 copy = MyValue3.copy(staticVal3); + va[0] = copy; + staticVal3 = copy; + copy.verify(staticVal3); + return copy; + } + + @DontCompile + public void test89_verifier(boolean warmup) { + staticVal3 = MyValue3.create(); + MyValue3[] va = new MyValue3[1]; + MyValue3 vt = test89(va); + staticVal3.verify(vt); + staticVal3.verify(va[0]); + } + + // Verify that C2 recognizes value type loads and re-uses the oop to avoid allocations + @Test(valid = ValueTypeReturnedAsFieldsOff, failOn = ALLOC + ALLOCA + STORE) + @Test(valid = ValueTypeReturnedAsFieldsOn) + public MyValue3 test90(MyValue3[] va) { + // C2 can re-use the oop returned by createDontInline() + // because the corresponding value type is equal to 'copy'. + MyValue3 copy = MyValue3.copy(MyValue3.createDontInline()); + va[0] = copy; + staticVal3 = copy; + copy.verify(staticVal3); + return copy; + } + + @DontCompile + public void test90_verifier(boolean warmup) { + MyValue3[] va = new MyValue3[1]; + MyValue3 vt = test90(va); + staticVal3.verify(vt); + staticVal3.verify(va[0]); + } + + // Verify that C2 recognizes value type loads and re-uses the oop to avoid allocations + @Test(valid = ValueTypePassFieldsAsArgsOff, failOn = ALLOC + ALLOCA + STORE) + @Test(valid = ValueTypePassFieldsAsArgsOn) + public MyValue3 test91(MyValue3 vt, MyValue3[] va) { + // C2 can re-use the oop of vt because vt is equal to 'copy'. + MyValue3 copy = MyValue3.copy(vt); + va[0] = copy; + staticVal3 = copy; + copy.verify(staticVal3); + return copy; + } + + @DontCompile + public void test91_verifier(boolean warmup) { + MyValue3 vt = MyValue3.create(); + MyValue3[] va = new MyValue3[1]; + MyValue3 result = test91(vt, va); + staticVal3.verify(vt); + va[0].verify(vt); + result.verify(vt); + } + + // Test correct identification of value type copies + @Test() + public MyValue3 test92(MyValue3[] va) { + MyValue3 vt = MyValue3.copy(staticVal3); + vt = MyValue3.setI(vt, (int)vt.c); + // vt is not equal to staticVal3, so C2 should not re-use the oop + va[0] = vt; + staticVal3 = vt; + vt.verify(staticVal3); + return vt; + } + + @DontCompile + public void test92_verifier(boolean warmup) { + staticVal3 = MyValue3.create(); + MyValue3[] va = new MyValue3[1]; + MyValue3 vt = test92(va); + Asserts.assertEQ(staticVal3.i, (int)staticVal3.c); + Asserts.assertEQ(va[0].i, (int)staticVal3.c); + Asserts.assertEQ(vt.i, (int)staticVal3.c); + } + // ========== Test infrastructure ========== private static final WhiteBox WHITE_BOX = WhiteBox.getWhiteBox(); @@ -2500,7 +2637,7 @@ } public static void main(String[] args) throws Throwable { - //tests.values().removeIf(p -> !p.getName().equals("test85")); // Run single test + //tests.values().removeIf(p -> !p.getName().equals("test85")); // Run single test if (args.length == 0) { execute_vm("-XX:+IgnoreUnrecognizedVMOptions", "-XX:-BackgroundCompilation", "-XX:+PrintCompilation", "-XX:+PrintInlining", "-XX:+PrintIdeal", "-XX:+PrintOptoAssembly", @@ -2620,8 +2757,8 @@ public void setup(Method[] methods) { if (XCOMP) { - // Don't control compilation if -Xcomp is enabled - return; + // Don't control compilation if -Xcomp is enabled + return; } for (Method m : methods) { if (m.isAnnotationPresent(Test.class)) { @@ -2649,11 +2786,15 @@ setup(this.getClass().getDeclaredMethods()); setup(MyValue1.class.getDeclaredMethods()); setup(MyValue2.class.getDeclaredMethods()); + setup(MyValue3.class.getDeclaredMethods()); + setup(MyValue4.class.getDeclaredMethods()); // Compile class initializers WHITE_BOX.enqueueInitializerForCompilation(this.getClass(), COMP_LEVEL_FULL_OPTIMIZATION); WHITE_BOX.enqueueInitializerForCompilation(MyValue1.class, COMP_LEVEL_FULL_OPTIMIZATION); WHITE_BOX.enqueueInitializerForCompilation(MyValue2.class, COMP_LEVEL_FULL_OPTIMIZATION); + WHITE_BOX.enqueueInitializerForCompilation(MyValue3.class, COMP_LEVEL_FULL_OPTIMIZATION); + WHITE_BOX.enqueueInitializerForCompilation(MyValue4.class, COMP_LEVEL_FULL_OPTIMIZATION); // Execute tests for (Method test : tests.values()) {