173 assert(index < field_count(), "index out of bounds");
174 return value_klass()->declared_nonstatic_field_at(index)->type();
175 }
176
177 bool ValueTypeBaseNode::field_is_flattened(uint index) const {
178 assert(index < field_count(), "index out of bounds");
179 ciField* field = value_klass()->declared_nonstatic_field_at(index);
180 assert(!field->is_flattened() || field->type()->is_valuetype(), "must be a value type");
181 return field->is_flattened();
182 }
183
184 int ValueTypeBaseNode::make_scalar_in_safepoint(Unique_Node_List& worklist, SafePointNode* sfpt, Node* root, PhaseGVN* gvn) {
185 ciValueKlass* vk = value_klass();
186 uint nfields = vk->nof_nonstatic_fields();
187 JVMState* jvms = sfpt->jvms();
188 int start = jvms->debug_start();
189 int end = jvms->debug_end();
190 // Replace safepoint edge by SafePointScalarObjectNode and add field values
191 assert(jvms != NULL, "missing JVMS");
192 uint first_ind = (sfpt->req() - jvms->scloff());
193 const TypeValueTypePtr* res_type = value_type_ptr();
194 SafePointScalarObjectNode* sobj = new SafePointScalarObjectNode(res_type,
195 #ifdef ASSERT
196 NULL,
197 #endif
198 first_ind, nfields);
199 sobj->init_req(0, root);
200 // Iterate over the value type fields in order of increasing
201 // offset and add the field values to the safepoint.
202 for (uint j = 0; j < nfields; ++j) {
203 int offset = vk->nonstatic_field_at(j)->offset();
204 Node* value = field_value_by_offset(offset, true /* include flattened value type fields */);
205 if (value->is_ValueType()) {
206 if (value->as_ValueType()->is_allocated(gvn)) {
207 value = value->as_ValueType()->get_oop();
208 } else {
209 // Add non-flattened value type field to the worklist to process later
210 worklist.push(value);
211 }
212 }
213 sfpt->add_req(value);
214 }
310 Node* value = NULL;
311 ciType* ft = field_type(i);
312 if (field_is_flattened(i)) {
313 // Recursively load the flattened value type field
314 value = ValueTypeNode::make_from_flattened(gvn, ft->as_value_klass(), ctl, mem, base, ptr, holder, offset);
315 } else {
316 const Type* con_type = NULL;
317 if (base->is_Con()) {
318 // If the oop to the value type is constant (static final field), we can
319 // also treat the fields as constants because the value type is immutable.
320 const TypeOopPtr* oop_ptr = base->bottom_type()->isa_oopptr();
321 ciObject* constant_oop = oop_ptr->const_oop();
322 ciField* field = holder->get_field_by_offset(offset, false);
323 assert(field != NULL, "field not found");
324 ciConstant constant = constant_oop->as_instance()->field_value(field);
325 con_type = Type::make_from_constant(constant, /*require_const=*/ true);
326 }
327 if (con_type != NULL) {
328 // Found a constant field value
329 value = gvn.transform(gvn.makecon(con_type));
330 if (con_type->isa_valuetypeptr()) {
331 // Constant, non-flattened value type field
332 value = ValueTypeNode::make_from_oop(gvn, ctl, mem, value, ft->as_value_klass());
333 }
334 } else {
335 // Load field value from memory
336 const TypePtr* adr_type = field_adr_type(base, offset, holder, gvn);
337 Node* adr = gvn.transform(new AddPNode(base, ptr, gvn.MakeConX(offset)));
338 BasicType bt = type2field[ft->basic_type()];
339 assert(is_java_primitive(bt) || adr->bottom_type()->is_ptr_to_narrowoop() == UseCompressedOops, "inconsistent");
340 const Type* rt = Type::get_const_type(ft);
341 const TypeAryPtr* ary_type = gvn.type(base)->isa_aryptr();
342 bool is_array = ary_type != NULL;
343 Node* load_mem = mem;
344 if (mem->is_MergeMem()) {
345 load_mem = mem->as_MergeMem()->memory_at(gvn.C->get_alias_index(adr_type));
346 }
347 value = gvn.transform(LoadNode::make(gvn, is_array ? ctl : NULL, load_mem, adr, adr_type, rt, bt, MemNode::unordered));
348 if (bt == T_VALUETYPE) {
349 // Non-flattened value type field, check for null
350 value = ValueTypeNode::make_from_oop(gvn, ctl, mem, value, ft->as_value_klass(), /* null_check */ true);
387 bool is_array = ary_type != NULL;
388 kit->store_oop(kit->control(), base, adr, adr_type, value, val_type, bt, is_array, MemNode::unordered, false, deoptimize_on_exception);
389 }
390 }
391 }
392 }
393
394 ValueTypeBaseNode* ValueTypeBaseNode::allocate(GraphKit* kit, bool deoptimize_on_exception) {
395 Node* in_oop = get_oop();
396 Node* null_ctl = kit->top();
397 // Check if value type is already allocated
398 Node* not_null_oop = kit->null_check_oop(in_oop, &null_ctl);
399 if (null_ctl->is_top()) {
400 // Value type is allocated
401 return this;
402 }
403 // Not able to prove that value type is allocated.
404 // Emit runtime check that may be folded later.
405 assert(!is_allocated(&kit->gvn()), "should not be allocated");
406 RegionNode* region = new RegionNode(3);
407 PhiNode* oop = new PhiNode(region, value_type_ptr());
408 PhiNode* io = new PhiNode(region, Type::ABIO);
409 PhiNode* mem = new PhiNode(region, Type::MEMORY, TypePtr::BOTTOM);
410
411 // Oop is non-NULL, use it
412 region->init_req(1, kit->control());
413 oop ->init_req(1, not_null_oop);
414 io ->init_req(1, kit->i_o());
415 mem ->init_req(1, kit->merged_memory());
416
417 // Oop is NULL, allocate value type
418 kit->set_control(null_ctl);
419 kit->kill_dead_locals();
420 ciValueKlass* vk = value_klass();
421 Node* klass_node = kit->makecon(TypeKlassPtr::make(vk));
422 Node* alloc_oop = kit->new_instance(klass_node, NULL, NULL, deoptimize_on_exception, this);
423 // Write field values to memory
424 store(kit, alloc_oop, alloc_oop, vk, 0, deoptimize_on_exception);
425 region->init_req(2, kit->control());
426 oop ->init_req(2, alloc_oop);
427 io ->init_req(2, kit->i_o());
578 assert(vt->is_allocated(&gvn), "value type should be allocated");
579 assert(init_ctl != ctl || oop->is_Con() || oop->is_CheckCastPP() || oop->Opcode() == Op_ValueTypePtr ||
580 vt->is_loaded(&gvn) == oop, "value type should be loaded");
581 }
582
583 if (buffer_check && vk->is_bufferable()) {
584 // Check if oop is in heap bounds or if it points into the vtBuffer:
585 // base <= oop < (base + size) <=> (oop - base) <U size
586 // Discard buffer oops to avoid storing them into fields or arrays.
587 assert(!gvn.type(oop)->isa_narrowoop(), "should not be a narrow oop");
588 Node* heap_base = gvn.MakeConX((intptr_t)Universe::heap()->base());
589 Node* heap_size = gvn.MakeConX(Universe::heap()->max_capacity());
590 Node* sub = gvn.transform(new SubXNode(gvn.transform(new CastP2XNode(NULL, oop)), heap_base));
591 Node* chk = gvn.transform(new CmpUXNode(sub, heap_size));
592 Node* tst = gvn.transform(new BoolNode(chk, BoolTest::lt));
593 IfNode* iff = gvn.transform(new IfNode(ctl, tst, PROB_MAX, COUNT_UNKNOWN))->as_If();
594
595 Node* region = new RegionNode(3);
596 region->init_req(1, gvn.transform(new IfTrueNode(iff)));
597 region->init_req(2, gvn.transform(new IfFalseNode(iff)));
598 Node* new_oop = new PhiNode(region, vt->value_type_ptr());
599 new_oop->init_req(1, oop);
600 new_oop->init_req(2, gvn.zerocon(T_VALUETYPE));
601
602 gvn.hash_delete(vt);
603 vt->set_oop(gvn.transform(new_oop));
604 vt = gvn.transform(vt)->as_ValueType();
605 ctl = gvn.transform(region);
606 }
607
608 return vt;
609 }
610
611 // GraphKit wrapper for the 'make_from_oop' method
612 ValueTypeNode* ValueTypeNode::make_from_oop(GraphKit* kit, Node* oop, ciValueKlass* vk, bool null_check, bool buffer_check) {
613 Node* ctl = kit->control();
614 ValueTypeNode* vt = make_from_oop(kit->gvn(), ctl, kit->merged_memory(), oop, vk, null_check, buffer_check);
615 kit->set_control(ctl);
616 return vt;
617 }
618
649 if (field_count() == 0) {
650 assert(is_allocated(phase), "must be allocated");
651 return get_oop();
652 }
653 for (uint i = 0; i < field_count(); ++i) {
654 int offset = holder_offset + field_offset(i);
655 Node* value = field_value(i);
656 if (value->isa_DecodeN()) {
657 // Skip DecodeN
658 value = value->in(1);
659 }
660 if (value->isa_Load()) {
661 // Check if base and offset of field load matches value type layout
662 intptr_t loffset = 0;
663 Node* lbase = AddPNode::Ideal_base_and_offset(value->in(MemNode::Address), phase, loffset);
664 if (lbase == NULL || (lbase != base && base != NULL) || loffset != offset) {
665 return NULL;
666 } else if (base == NULL) {
667 // Set base and check if pointer type matches
668 base = lbase;
669 const TypeValueTypePtr* vtptr = phase->type(base)->isa_valuetypeptr();
670 if (vtptr == NULL || !vtptr->value_klass()->equals(vk)) {
671 return NULL;
672 }
673 }
674 } else if (value->isa_ValueType()) {
675 // Check value type field load recursively
676 ValueTypeNode* vt = value->as_ValueType();
677 base = vt->is_loaded(phase, vk, base, offset - vt->value_klass()->first_field_offset());
678 if (base == NULL) {
679 return NULL;
680 }
681 } else {
682 return NULL;
683 }
684 }
685 return base;
686 }
687
688 Node* ValueTypeNode::allocate_fields(GraphKit* kit) {
689 ValueTypeNode* vt = clone()->as_ValueType();
690 for (uint i = 0; i < field_count(); i++) {
900
901 ValueTypePtrNode* ValueTypePtrNode::make_from_value_type(PhaseGVN& gvn, ValueTypeNode* vt) {
902 ValueTypePtrNode* vtptr = new ValueTypePtrNode(vt->value_klass(), vt->get_oop());
903 for (uint i = Oop+1; i < vt->req(); i++) {
904 vtptr->init_req(i, vt->in(i));
905 }
906 return gvn.transform(vtptr)->as_ValueTypePtr();
907 }
908
909 ValueTypePtrNode* ValueTypePtrNode::make_from_call(GraphKit* kit, ciValueKlass* vk, CallNode* call) {
910 ValueTypePtrNode* vtptr = new ValueTypePtrNode(vk, kit->zerocon(T_VALUETYPE));
911 Node* ctl = kit->control();
912 vtptr->initialize(&kit->gvn(), ctl, kit->merged_memory(), call, vk);
913 kit->set_control(ctl);
914 return vtptr;
915 }
916
917 ValueTypePtrNode* ValueTypePtrNode::make_from_oop(PhaseGVN& gvn, Node*& ctl, Node* mem, Node* oop) {
918 // Create and initialize a ValueTypePtrNode by loading all field
919 // values from a heap-allocated version and also save the oop.
920 ciValueKlass* vk = gvn.type(oop)->is_valuetypeptr()->value_klass();
921 ValueTypePtrNode* vtptr = new ValueTypePtrNode(vk, oop);
922 vtptr->load(gvn, ctl, mem, oop, oop, vk);
923 return vtptr;
924 }
|
173 assert(index < field_count(), "index out of bounds");
174 return value_klass()->declared_nonstatic_field_at(index)->type();
175 }
176
177 bool ValueTypeBaseNode::field_is_flattened(uint index) const {
178 assert(index < field_count(), "index out of bounds");
179 ciField* field = value_klass()->declared_nonstatic_field_at(index);
180 assert(!field->is_flattened() || field->type()->is_valuetype(), "must be a value type");
181 return field->is_flattened();
182 }
183
184 int ValueTypeBaseNode::make_scalar_in_safepoint(Unique_Node_List& worklist, SafePointNode* sfpt, Node* root, PhaseGVN* gvn) {
185 ciValueKlass* vk = value_klass();
186 uint nfields = vk->nof_nonstatic_fields();
187 JVMState* jvms = sfpt->jvms();
188 int start = jvms->debug_start();
189 int end = jvms->debug_end();
190 // Replace safepoint edge by SafePointScalarObjectNode and add field values
191 assert(jvms != NULL, "missing JVMS");
192 uint first_ind = (sfpt->req() - jvms->scloff());
193 SafePointScalarObjectNode* sobj = new SafePointScalarObjectNode(value_ptr(),
194 #ifdef ASSERT
195 NULL,
196 #endif
197 first_ind, nfields);
198 sobj->init_req(0, root);
199 // Iterate over the value type fields in order of increasing
200 // offset and add the field values to the safepoint.
201 for (uint j = 0; j < nfields; ++j) {
202 int offset = vk->nonstatic_field_at(j)->offset();
203 Node* value = field_value_by_offset(offset, true /* include flattened value type fields */);
204 if (value->is_ValueType()) {
205 if (value->as_ValueType()->is_allocated(gvn)) {
206 value = value->as_ValueType()->get_oop();
207 } else {
208 // Add non-flattened value type field to the worklist to process later
209 worklist.push(value);
210 }
211 }
212 sfpt->add_req(value);
213 }
309 Node* value = NULL;
310 ciType* ft = field_type(i);
311 if (field_is_flattened(i)) {
312 // Recursively load the flattened value type field
313 value = ValueTypeNode::make_from_flattened(gvn, ft->as_value_klass(), ctl, mem, base, ptr, holder, offset);
314 } else {
315 const Type* con_type = NULL;
316 if (base->is_Con()) {
317 // If the oop to the value type is constant (static final field), we can
318 // also treat the fields as constants because the value type is immutable.
319 const TypeOopPtr* oop_ptr = base->bottom_type()->isa_oopptr();
320 ciObject* constant_oop = oop_ptr->const_oop();
321 ciField* field = holder->get_field_by_offset(offset, false);
322 assert(field != NULL, "field not found");
323 ciConstant constant = constant_oop->as_instance()->field_value(field);
324 con_type = Type::make_from_constant(constant, /*require_const=*/ true);
325 }
326 if (con_type != NULL) {
327 // Found a constant field value
328 value = gvn.transform(gvn.makecon(con_type));
329 if (con_type->is_valuetypeptr()) {
330 // Constant, non-flattened value type field
331 value = ValueTypeNode::make_from_oop(gvn, ctl, mem, value, ft->as_value_klass());
332 }
333 } else {
334 // Load field value from memory
335 const TypePtr* adr_type = field_adr_type(base, offset, holder, gvn);
336 Node* adr = gvn.transform(new AddPNode(base, ptr, gvn.MakeConX(offset)));
337 BasicType bt = type2field[ft->basic_type()];
338 assert(is_java_primitive(bt) || adr->bottom_type()->is_ptr_to_narrowoop() == UseCompressedOops, "inconsistent");
339 const Type* rt = Type::get_const_type(ft);
340 const TypeAryPtr* ary_type = gvn.type(base)->isa_aryptr();
341 bool is_array = ary_type != NULL;
342 Node* load_mem = mem;
343 if (mem->is_MergeMem()) {
344 load_mem = mem->as_MergeMem()->memory_at(gvn.C->get_alias_index(adr_type));
345 }
346 value = gvn.transform(LoadNode::make(gvn, is_array ? ctl : NULL, load_mem, adr, adr_type, rt, bt, MemNode::unordered));
347 if (bt == T_VALUETYPE) {
348 // Non-flattened value type field, check for null
349 value = ValueTypeNode::make_from_oop(gvn, ctl, mem, value, ft->as_value_klass(), /* null_check */ true);
386 bool is_array = ary_type != NULL;
387 kit->store_oop(kit->control(), base, adr, adr_type, value, val_type, bt, is_array, MemNode::unordered, false, deoptimize_on_exception);
388 }
389 }
390 }
391 }
392
393 ValueTypeBaseNode* ValueTypeBaseNode::allocate(GraphKit* kit, bool deoptimize_on_exception) {
394 Node* in_oop = get_oop();
395 Node* null_ctl = kit->top();
396 // Check if value type is already allocated
397 Node* not_null_oop = kit->null_check_oop(in_oop, &null_ctl);
398 if (null_ctl->is_top()) {
399 // Value type is allocated
400 return this;
401 }
402 // Not able to prove that value type is allocated.
403 // Emit runtime check that may be folded later.
404 assert(!is_allocated(&kit->gvn()), "should not be allocated");
405 RegionNode* region = new RegionNode(3);
406 PhiNode* oop = new PhiNode(region, value_ptr());
407 PhiNode* io = new PhiNode(region, Type::ABIO);
408 PhiNode* mem = new PhiNode(region, Type::MEMORY, TypePtr::BOTTOM);
409
410 // Oop is non-NULL, use it
411 region->init_req(1, kit->control());
412 oop ->init_req(1, not_null_oop);
413 io ->init_req(1, kit->i_o());
414 mem ->init_req(1, kit->merged_memory());
415
416 // Oop is NULL, allocate value type
417 kit->set_control(null_ctl);
418 kit->kill_dead_locals();
419 ciValueKlass* vk = value_klass();
420 Node* klass_node = kit->makecon(TypeKlassPtr::make(vk));
421 Node* alloc_oop = kit->new_instance(klass_node, NULL, NULL, deoptimize_on_exception, this);
422 // Write field values to memory
423 store(kit, alloc_oop, alloc_oop, vk, 0, deoptimize_on_exception);
424 region->init_req(2, kit->control());
425 oop ->init_req(2, alloc_oop);
426 io ->init_req(2, kit->i_o());
577 assert(vt->is_allocated(&gvn), "value type should be allocated");
578 assert(init_ctl != ctl || oop->is_Con() || oop->is_CheckCastPP() || oop->Opcode() == Op_ValueTypePtr ||
579 vt->is_loaded(&gvn) == oop, "value type should be loaded");
580 }
581
582 if (buffer_check && vk->is_bufferable()) {
583 // Check if oop is in heap bounds or if it points into the vtBuffer:
584 // base <= oop < (base + size) <=> (oop - base) <U size
585 // Discard buffer oops to avoid storing them into fields or arrays.
586 assert(!gvn.type(oop)->isa_narrowoop(), "should not be a narrow oop");
587 Node* heap_base = gvn.MakeConX((intptr_t)Universe::heap()->base());
588 Node* heap_size = gvn.MakeConX(Universe::heap()->max_capacity());
589 Node* sub = gvn.transform(new SubXNode(gvn.transform(new CastP2XNode(NULL, oop)), heap_base));
590 Node* chk = gvn.transform(new CmpUXNode(sub, heap_size));
591 Node* tst = gvn.transform(new BoolNode(chk, BoolTest::lt));
592 IfNode* iff = gvn.transform(new IfNode(ctl, tst, PROB_MAX, COUNT_UNKNOWN))->as_If();
593
594 Node* region = new RegionNode(3);
595 region->init_req(1, gvn.transform(new IfTrueNode(iff)));
596 region->init_req(2, gvn.transform(new IfFalseNode(iff)));
597 Node* new_oop = new PhiNode(region, vt->value_ptr());
598 new_oop->init_req(1, oop);
599 new_oop->init_req(2, gvn.zerocon(T_VALUETYPE));
600
601 gvn.hash_delete(vt);
602 vt->set_oop(gvn.transform(new_oop));
603 vt = gvn.transform(vt)->as_ValueType();
604 ctl = gvn.transform(region);
605 }
606
607 return vt;
608 }
609
610 // GraphKit wrapper for the 'make_from_oop' method
611 ValueTypeNode* ValueTypeNode::make_from_oop(GraphKit* kit, Node* oop, ciValueKlass* vk, bool null_check, bool buffer_check) {
612 Node* ctl = kit->control();
613 ValueTypeNode* vt = make_from_oop(kit->gvn(), ctl, kit->merged_memory(), oop, vk, null_check, buffer_check);
614 kit->set_control(ctl);
615 return vt;
616 }
617
648 if (field_count() == 0) {
649 assert(is_allocated(phase), "must be allocated");
650 return get_oop();
651 }
652 for (uint i = 0; i < field_count(); ++i) {
653 int offset = holder_offset + field_offset(i);
654 Node* value = field_value(i);
655 if (value->isa_DecodeN()) {
656 // Skip DecodeN
657 value = value->in(1);
658 }
659 if (value->isa_Load()) {
660 // Check if base and offset of field load matches value type layout
661 intptr_t loffset = 0;
662 Node* lbase = AddPNode::Ideal_base_and_offset(value->in(MemNode::Address), phase, loffset);
663 if (lbase == NULL || (lbase != base && base != NULL) || loffset != offset) {
664 return NULL;
665 } else if (base == NULL) {
666 // Set base and check if pointer type matches
667 base = lbase;
668 const TypeInstPtr* vtptr = phase->type(base)->isa_instptr();
669 if (vtptr == NULL || !vtptr->klass()->equals(vk)) {
670 return NULL;
671 }
672 }
673 } else if (value->isa_ValueType()) {
674 // Check value type field load recursively
675 ValueTypeNode* vt = value->as_ValueType();
676 base = vt->is_loaded(phase, vk, base, offset - vt->value_klass()->first_field_offset());
677 if (base == NULL) {
678 return NULL;
679 }
680 } else {
681 return NULL;
682 }
683 }
684 return base;
685 }
686
687 Node* ValueTypeNode::allocate_fields(GraphKit* kit) {
688 ValueTypeNode* vt = clone()->as_ValueType();
689 for (uint i = 0; i < field_count(); i++) {
899
900 ValueTypePtrNode* ValueTypePtrNode::make_from_value_type(PhaseGVN& gvn, ValueTypeNode* vt) {
901 ValueTypePtrNode* vtptr = new ValueTypePtrNode(vt->value_klass(), vt->get_oop());
902 for (uint i = Oop+1; i < vt->req(); i++) {
903 vtptr->init_req(i, vt->in(i));
904 }
905 return gvn.transform(vtptr)->as_ValueTypePtr();
906 }
907
908 ValueTypePtrNode* ValueTypePtrNode::make_from_call(GraphKit* kit, ciValueKlass* vk, CallNode* call) {
909 ValueTypePtrNode* vtptr = new ValueTypePtrNode(vk, kit->zerocon(T_VALUETYPE));
910 Node* ctl = kit->control();
911 vtptr->initialize(&kit->gvn(), ctl, kit->merged_memory(), call, vk);
912 kit->set_control(ctl);
913 return vtptr;
914 }
915
916 ValueTypePtrNode* ValueTypePtrNode::make_from_oop(PhaseGVN& gvn, Node*& ctl, Node* mem, Node* oop) {
917 // Create and initialize a ValueTypePtrNode by loading all field
918 // values from a heap-allocated version and also save the oop.
919 ciValueKlass* vk = gvn.type(oop)->value_klass();
920 ValueTypePtrNode* vtptr = new ValueTypePtrNode(vk, oop);
921 vtptr->load(gvn, ctl, mem, oop, oop, vk);
922 return vtptr;
923 }
|