298 }
299 }
300
301 if (field_is_flattenable(i)) {
302 // Non-flattened but flattenable value type
303 if (ft->as_value_klass()->is_scalarizable()) {
304 parm = ValueTypeNode::make_from_oop(kit, parm, ft->as_value_klass());
305 } else {
306 parm = kit->null2default(parm, ft->as_value_klass());
307 }
308 }
309
310 set_field_value(i, parm);
311 // Record all these guys for later GVN.
312 gvn.record_for_igvn(parm);
313 }
314 }
315 base_input += vk->value_arg_slots();
316 }
317
318 const TypePtr* ValueTypeBaseNode::field_adr_type(Node* base, int offset, ciInstanceKlass* holder, PhaseGVN& gvn) const {
319 const TypeAryPtr* ary_type = gvn.type(base)->isa_aryptr();
320 const TypePtr* adr_type = NULL;
321 bool is_array = ary_type != NULL;
322 if (is_array) {
323 // In the case of a flattened value type array, each field has its own slice
324 adr_type = ary_type->with_field_offset(offset)->add_offset(Type::OffsetBot);
325 } else {
326 ciField* field = holder->get_field_by_offset(offset, false);
327 assert(field != NULL, "field not found");
328 adr_type = gvn.C->alias_type(field)->adr_type();
329 }
330 return adr_type;
331 }
332
333 void ValueTypeBaseNode::load(GraphKit* kit, Node* base, Node* ptr, ciInstanceKlass* holder, int holder_offset) {
334 // Initialize the value type by loading its field values from
335 // memory and adding the values as input edges to the node.
336 for (uint i = 0; i < field_count(); ++i) {
337 int offset = holder_offset + field_offset(i);
338 Node* value = NULL;
339 ciType* ft = field_type(i);
340 if (field_is_flattened(i)) {
341 // Recursively load the flattened value type field
342 value = ValueTypeNode::make_from_flattened(kit, ft->as_value_klass(), base, ptr, holder, offset);
343 } else {
344 const TypeOopPtr* oop_ptr = kit->gvn().type(base)->isa_oopptr();
345 bool is_array = (oop_ptr->isa_aryptr() != NULL);
346 if (base->is_Con() && !is_array) {
347 // If the oop to the value type is constant (static final field), we can
348 // also treat the fields as constants because the value type is immutable.
349 ciObject* constant_oop = oop_ptr->const_oop();
350 ciField* field = holder->get_field_by_offset(offset, false);
351 assert(field != NULL, "field not found");
352 ciConstant constant = constant_oop->as_instance()->field_value(field);
353 const Type* con_type = Type::make_from_constant(constant, /*require_const=*/ true);
354 assert(con_type != NULL, "type not found");
355 value = kit->gvn().transform(kit->makecon(con_type));
356 } else {
357 // Load field value from memory
358 const TypePtr* adr_type = field_adr_type(base, offset, holder, kit->gvn());
359 Node* adr = kit->basic_plus_adr(base, ptr, offset);
360 BasicType bt = type2field[ft->basic_type()];
361 assert(is_java_primitive(bt) || adr->bottom_type()->is_ptr_to_narrowoop() == UseCompressedOops, "inconsistent");
362 const Type* val_type = Type::get_const_type(ft);
363 DecoratorSet decorators = IN_HEAP | MO_UNORDERED;
364 if (is_array) {
365 decorators |= IS_ARRAY;
366 }
367 value = kit->access_load_at(base, adr, adr_type, val_type, bt, decorators);
368 }
369 if (field_is_flattenable(i)) {
370 // Loading a non-flattened but flattenable value type from memory
371 if (ft->as_value_klass()->is_scalarizable()) {
372 value = ValueTypeNode::make_from_oop(kit, value, ft->as_value_klass());
373 } else {
374 value = kit->null2default(value, ft->as_value_klass());
375 }
376 }
377 }
378 set_field_value(i, value);
379 }
380 }
381
382 void ValueTypeBaseNode::store_flattened(GraphKit* kit, Node* base, Node* ptr, ciInstanceKlass* holder, int holder_offset) const {
383 // The value type is embedded into the object without an oop header. Subtract the
384 // offset of the first field to account for the missing header when storing the values.
385 if (holder == NULL) {
386 holder = value_klass();
387 }
388 holder_offset -= value_klass()->first_field_offset();
389 store(kit, base, ptr, holder, holder_offset);
390 }
391
392 void ValueTypeBaseNode::store(GraphKit* kit, Node* base, Node* ptr, ciInstanceKlass* holder, int holder_offset, bool deoptimize_on_exception) const {
393 // Write field values to memory
394 for (uint i = 0; i < field_count(); ++i) {
395 int offset = holder_offset + field_offset(i);
396 Node* value = field_value(i);
397 ciType* ft = field_type(i);
398 if (field_is_flattened(i)) {
399 // Recursively store the flattened value type field
400 if (!value->is_ValueType()) {
401 assert(!kit->gvn().type(value)->maybe_null(), "should never be null");
402 value = ValueTypeNode::make_from_oop(kit, value, ft->as_value_klass());
403 }
404 value->as_ValueType()->store_flattened(kit, base, ptr, holder, offset);
405 } else {
406 // Store field value to memory
407 const TypePtr* adr_type = field_adr_type(base, offset, holder, kit->gvn());
408 Node* adr = kit->basic_plus_adr(base, ptr, offset);
409 BasicType bt = type2field[ft->basic_type()];
410 assert(is_java_primitive(bt) || adr->bottom_type()->is_ptr_to_narrowoop() == UseCompressedOops, "inconsistent");
411 const Type* val_type = Type::get_const_type(ft);
412 const TypeAryPtr* ary_type = kit->gvn().type(base)->isa_aryptr();
413 DecoratorSet decorators = IN_HEAP | MO_UNORDERED;
414 if (ary_type != NULL) {
415 decorators |= IS_ARRAY;
416 }
417 kit->access_store_at(base, adr, adr_type, value, val_type, bt, decorators, deoptimize_on_exception);
418 }
419 }
420 }
421
422 ValueTypeBaseNode* ValueTypeBaseNode::allocate(GraphKit* kit, bool deoptimize_on_exception) {
423 // Check if value type is already allocated
424 Node* null_ctl = kit->top();
425 Node* not_null_oop = kit->null_check_oop(get_oop(), &null_ctl);
426 if (null_ctl->is_top()) {
427 // Value type is allocated
428 return this;
429 }
430 assert(!is_allocated(&kit->gvn()), "should not be allocated");
431 RegionNode* region = new RegionNode(3);
432
433 // Oop is non-NULL, use it
586 region->init_req(1, kit->control());
587 region->init_req(2, null_ctl);
588
589 vt = vt->clone_with_phis(&gvn, region)->as_ValueType();
590 vt->merge_with(&gvn, def, 2, true);
591 kit->set_control(gvn.transform(region));
592 }
593 } else {
594 // Oop can never be null
595 Node* init_ctl = kit->control();
596 vt->load(kit, oop, oop, vk, /* holder_offset */ 0);
597 assert(init_ctl != kit->control() || oop->is_Con() || oop->is_CheckCastPP() || oop->Opcode() == Op_ValueTypePtr ||
598 vt->is_loaded(&gvn) == oop, "value type should be loaded");
599 }
600
601 assert(vt->is_allocated(&gvn), "value type should be allocated");
602 return gvn.transform(vt)->as_ValueType();
603 }
604
605 // GraphKit wrapper for the 'make_from_flattened' method
606 ValueTypeNode* ValueTypeNode::make_from_flattened(GraphKit* kit, ciValueKlass* vk, Node* obj, Node* ptr, ciInstanceKlass* holder, int holder_offset) {
607 // Create and initialize a ValueTypeNode by loading all field values from
608 // a flattened value type field at 'holder_offset' or from a value type array.
609 ValueTypeNode* vt = make_uninitialized(kit->gvn(), vk);
610 // The value type is flattened into the object without an oop header. Subtract the
611 // offset of the first field to account for the missing header when loading the values.
612 holder_offset -= vk->first_field_offset();
613 vt->load(kit, obj, ptr, holder, holder_offset);
614 assert(vt->is_loaded(&kit->gvn()) != obj, "holder oop should not be used as flattened value type oop");
615 return kit->gvn().transform(vt)->as_ValueType();
616 }
617
618 ValueTypeNode* ValueTypeNode::make_from_multi(GraphKit* kit, MultiNode* multi, ciValueKlass* vk, uint& base_input, bool in) {
619 ValueTypeNode* vt = ValueTypeNode::make_uninitialized(kit->gvn(), vk);
620 vt->initialize(kit, multi, vk, 0, base_input, in);
621 return kit->gvn().transform(vt)->as_ValueType();
622 }
623
624 Node* ValueTypeNode::is_loaded(PhaseGVN* phase, ciValueKlass* vk, Node* base, int holder_offset) {
625 if (vk == NULL) {
626 vk = value_klass();
627 }
628 if (field_count() == 0) {
629 assert(is_allocated(phase), "must be allocated");
630 return get_oop();
631 }
632 for (uint i = 0; i < field_count(); ++i) {
633 int offset = holder_offset + field_offset(i);
634 Node* value = field_value(i);
635 if (value->is_ValueType()) {
636 ValueTypeNode* vt = value->as_ValueType();
637 if (field_is_flattened(i)) {
638 // Check value type field load recursively
639 base = vt->is_loaded(phase, vk, base, offset - vt->value_klass()->first_field_offset());
640 if (base == NULL) {
641 return NULL;
|
298 }
299 }
300
301 if (field_is_flattenable(i)) {
302 // Non-flattened but flattenable value type
303 if (ft->as_value_klass()->is_scalarizable()) {
304 parm = ValueTypeNode::make_from_oop(kit, parm, ft->as_value_klass());
305 } else {
306 parm = kit->null2default(parm, ft->as_value_klass());
307 }
308 }
309
310 set_field_value(i, parm);
311 // Record all these guys for later GVN.
312 gvn.record_for_igvn(parm);
313 }
314 }
315 base_input += vk->value_arg_slots();
316 }
317
318 const TypePtr* ValueTypeBaseNode::field_adr_type(Node* base, int offset, ciInstanceKlass* holder, DecoratorSet decorators, PhaseGVN& gvn) const {
319 const TypeAryPtr* ary_type = gvn.type(base)->isa_aryptr();
320 const TypePtr* adr_type = NULL;
321 bool is_array = ary_type != NULL;
322 if ((decorators & C2_MISMATCHED) != 0) {
323 adr_type = TypeRawPtr::BOTTOM;
324 } else if (is_array) {
325 // In the case of a flattened value type array, each field has its own slice
326 adr_type = ary_type->with_field_offset(offset)->add_offset(Type::OffsetBot);
327 } else {
328 ciField* field = holder->get_field_by_offset(offset, false);
329 assert(field != NULL, "field not found");
330 adr_type = gvn.C->alias_type(field)->adr_type();
331 }
332 return adr_type;
333 }
334
335 void ValueTypeBaseNode::load(GraphKit* kit, Node* base, Node* ptr, ciInstanceKlass* holder, int holder_offset, DecoratorSet decorators) {
336 // Initialize the value type by loading its field values from
337 // memory and adding the values as input edges to the node.
338 for (uint i = 0; i < field_count(); ++i) {
339 int offset = holder_offset + field_offset(i);
340 Node* value = NULL;
341 ciType* ft = field_type(i);
342 if (field_is_flattened(i)) {
343 // Recursively load the flattened value type field
344 value = ValueTypeNode::make_from_flattened(kit, ft->as_value_klass(), base, ptr, holder, offset, decorators);
345 } else {
346 const TypeOopPtr* oop_ptr = kit->gvn().type(base)->isa_oopptr();
347 bool is_array = (oop_ptr->isa_aryptr() != NULL);
348 if (base->is_Con() && !is_array) {
349 // If the oop to the value type is constant (static final field), we can
350 // also treat the fields as constants because the value type is immutable.
351 ciObject* constant_oop = oop_ptr->const_oop();
352 ciField* field = holder->get_field_by_offset(offset, false);
353 assert(field != NULL, "field not found");
354 ciConstant constant = constant_oop->as_instance()->field_value(field);
355 const Type* con_type = Type::make_from_constant(constant, /*require_const=*/ true);
356 assert(con_type != NULL, "type not found");
357 value = kit->gvn().transform(kit->makecon(con_type));
358 } else {
359 // Load field value from memory
360 const TypePtr* adr_type = field_adr_type(base, offset, holder, decorators, kit->gvn());
361 Node* adr = kit->basic_plus_adr(base, ptr, offset);
362 BasicType bt = type2field[ft->basic_type()];
363 assert(is_java_primitive(bt) || adr->bottom_type()->is_ptr_to_narrowoop() == UseCompressedOops, "inconsistent");
364 const Type* val_type = Type::get_const_type(ft);
365 if (is_array) {
366 decorators |= IS_ARRAY;
367 }
368 value = kit->access_load_at(base, adr, adr_type, val_type, bt, decorators);
369 }
370 if (field_is_flattenable(i)) {
371 // Loading a non-flattened but flattenable value type from memory
372 if (ft->as_value_klass()->is_scalarizable()) {
373 value = ValueTypeNode::make_from_oop(kit, value, ft->as_value_klass());
374 } else {
375 value = kit->null2default(value, ft->as_value_klass());
376 }
377 }
378 }
379 set_field_value(i, value);
380 }
381 }
382
383 void ValueTypeBaseNode::store_flattened(GraphKit* kit, Node* base, Node* ptr, ciInstanceKlass* holder, int holder_offset, DecoratorSet decorators) const {
384 // The value type is embedded into the object without an oop header. Subtract the
385 // offset of the first field to account for the missing header when storing the values.
386 if (holder == NULL) {
387 holder = value_klass();
388 }
389 holder_offset -= value_klass()->first_field_offset();
390 store(kit, base, ptr, holder, holder_offset, false, decorators);
391 }
392
393 void ValueTypeBaseNode::store(GraphKit* kit, Node* base, Node* ptr, ciInstanceKlass* holder, int holder_offset, bool deoptimize_on_exception, DecoratorSet decorators) const {
394 // Write field values to memory
395 for (uint i = 0; i < field_count(); ++i) {
396 int offset = holder_offset + field_offset(i);
397 Node* value = field_value(i);
398 ciType* ft = field_type(i);
399 if (field_is_flattened(i)) {
400 // Recursively store the flattened value type field
401 if (!value->is_ValueType()) {
402 assert(!kit->gvn().type(value)->maybe_null(), "should never be null");
403 value = ValueTypeNode::make_from_oop(kit, value, ft->as_value_klass());
404 }
405 value->as_ValueType()->store_flattened(kit, base, ptr, holder, offset, decorators);
406 } else {
407 // Store field value to memory
408 const TypePtr* adr_type = field_adr_type(base, offset, holder, decorators, kit->gvn());
409 Node* adr = kit->basic_plus_adr(base, ptr, offset);
410 BasicType bt = type2field[ft->basic_type()];
411 assert(is_java_primitive(bt) || adr->bottom_type()->is_ptr_to_narrowoop() == UseCompressedOops, "inconsistent");
412 const Type* val_type = Type::get_const_type(ft);
413 const TypeAryPtr* ary_type = kit->gvn().type(base)->isa_aryptr();
414 if (ary_type != NULL) {
415 decorators |= IS_ARRAY;
416 }
417 kit->access_store_at(base, adr, adr_type, value, val_type, bt, decorators, deoptimize_on_exception);
418 }
419 }
420 }
421
422 ValueTypeBaseNode* ValueTypeBaseNode::allocate(GraphKit* kit, bool deoptimize_on_exception) {
423 // Check if value type is already allocated
424 Node* null_ctl = kit->top();
425 Node* not_null_oop = kit->null_check_oop(get_oop(), &null_ctl);
426 if (null_ctl->is_top()) {
427 // Value type is allocated
428 return this;
429 }
430 assert(!is_allocated(&kit->gvn()), "should not be allocated");
431 RegionNode* region = new RegionNode(3);
432
433 // Oop is non-NULL, use it
586 region->init_req(1, kit->control());
587 region->init_req(2, null_ctl);
588
589 vt = vt->clone_with_phis(&gvn, region)->as_ValueType();
590 vt->merge_with(&gvn, def, 2, true);
591 kit->set_control(gvn.transform(region));
592 }
593 } else {
594 // Oop can never be null
595 Node* init_ctl = kit->control();
596 vt->load(kit, oop, oop, vk, /* holder_offset */ 0);
597 assert(init_ctl != kit->control() || oop->is_Con() || oop->is_CheckCastPP() || oop->Opcode() == Op_ValueTypePtr ||
598 vt->is_loaded(&gvn) == oop, "value type should be loaded");
599 }
600
601 assert(vt->is_allocated(&gvn), "value type should be allocated");
602 return gvn.transform(vt)->as_ValueType();
603 }
604
605 // GraphKit wrapper for the 'make_from_flattened' method
606 ValueTypeNode* ValueTypeNode::make_from_flattened(GraphKit* kit, ciValueKlass* vk, Node* obj, Node* ptr, ciInstanceKlass* holder, int holder_offset, DecoratorSet decorators) {
607 // Create and initialize a ValueTypeNode by loading all field values from
608 // a flattened value type field at 'holder_offset' or from a value type array.
609 ValueTypeNode* vt = make_uninitialized(kit->gvn(), vk);
610 // The value type is flattened into the object without an oop header. Subtract the
611 // offset of the first field to account for the missing header when loading the values.
612 holder_offset -= vk->first_field_offset();
613 vt->load(kit, obj, ptr, holder, holder_offset, decorators);
614 assert(vt->is_loaded(&kit->gvn()) != obj, "holder oop should not be used as flattened value type oop");
615 return kit->gvn().transform(vt)->as_ValueType();
616 }
617
618 ValueTypeNode* ValueTypeNode::make_from_multi(GraphKit* kit, MultiNode* multi, ciValueKlass* vk, uint& base_input, bool in) {
619 ValueTypeNode* vt = ValueTypeNode::make_uninitialized(kit->gvn(), vk);
620 vt->initialize(kit, multi, vk, 0, base_input, in);
621 return kit->gvn().transform(vt)->as_ValueType();
622 }
623
624 ValueTypeNode* ValueTypeNode::make_larval(GraphKit* kit, bool allocate) const {
625 ciValueKlass* vk = value_klass();
626 ValueTypeNode* res = clone()->as_ValueType();
627 if (allocate) {
628 Node* klass_node = kit->makecon(TypeKlassPtr::make(vk));
629 Node* alloc_oop = kit->new_instance(klass_node, NULL, NULL, false);
630 AllocateNode* alloc = AllocateNode::Ideal_allocation(alloc_oop, &kit->gvn());
631 alloc->_larval = true;
632
633 store(kit, alloc_oop, alloc_oop, vk, 0, false);
634 res->set_oop(alloc_oop);
635 }
636 res->set_type(TypeValueType::make(vk, true));
637 res = kit->gvn().transform(res)->as_ValueType();
638 return res;
639 }
640
641 ValueTypeNode* ValueTypeNode::finish_larval(GraphKit* kit) const {
642 Node* obj = get_oop();
643 Node* mark_addr = kit->basic_plus_adr(obj, oopDesc::mark_offset_in_bytes());
644 Node* mark = kit->make_load(NULL, mark_addr, TypeX_X, TypeX_X->basic_type(), MemNode::unordered);
645 mark = kit->gvn().transform(new AndXNode(mark, kit->MakeConX(~markOopDesc::larval_mask_in_place)));
646 kit->store_to_memory(kit->control(), mark_addr, mark, TypeX_X->basic_type(), kit->gvn().type(mark_addr)->is_ptr(), MemNode::unordered);
647
648 ciValueKlass* vk = value_klass();
649 ValueTypeNode* res = clone()->as_ValueType();
650 res->set_type(TypeValueType::make(vk, false));
651 res = kit->gvn().transform(res)->as_ValueType();
652 return res;
653 }
654
655 Node* ValueTypeNode::is_loaded(PhaseGVN* phase, ciValueKlass* vk, Node* base, int holder_offset) {
656 if (vk == NULL) {
657 vk = value_klass();
658 }
659 if (field_count() == 0) {
660 assert(is_allocated(phase), "must be allocated");
661 return get_oop();
662 }
663 for (uint i = 0; i < field_count(); ++i) {
664 int offset = holder_offset + field_offset(i);
665 Node* value = field_value(i);
666 if (value->is_ValueType()) {
667 ValueTypeNode* vt = value->as_ValueType();
668 if (field_is_flattened(i)) {
669 // Check value type field load recursively
670 base = vt->is_loaded(phase, vk, base, offset - vt->value_klass()->first_field_offset());
671 if (base == NULL) {
672 return NULL;
|