29 #include "opto/addnode.hpp"
30 #include "opto/arraycopynode.hpp"
31 #include "opto/callnode.hpp"
32 #include "opto/castnode.hpp"
33 #include "opto/cfgnode.hpp"
34 #include "opto/compile.hpp"
35 #include "opto/convertnode.hpp"
36 #include "opto/graphKit.hpp"
37 #include "opto/locknode.hpp"
38 #include "opto/loopnode.hpp"
39 #include "opto/macro.hpp"
40 #include "opto/memnode.hpp"
41 #include "opto/narrowptrnode.hpp"
42 #include "opto/node.hpp"
43 #include "opto/opaquenode.hpp"
44 #include "opto/phaseX.hpp"
45 #include "opto/rootnode.hpp"
46 #include "opto/runtime.hpp"
47 #include "opto/subnode.hpp"
48 #include "opto/type.hpp"
49 #include "runtime/sharedRuntime.hpp"
50 #include "utilities/macros.hpp"
51 #if INCLUDE_G1GC
52 #include "gc/g1/g1ThreadLocalData.hpp"
53 #endif // INCLUDE_G1GC
54 #if INCLUDE_SHENANDOAHGC
55 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
56 #endif
57
58
59 //
60 // Replace any references to "oldref" in inputs to "use" with "newref".
61 // Returns the number of replacements made.
62 //
63 int PhaseMacroExpand::replace_input(Node *use, Node *oldref, Node *newref) {
64 int nreplacements = 0;
65 uint req = use->req();
66 for (uint j = 0; j < use->len(); j++) {
67 Node *uin = use->in(j);
68 if (uin == oldref) {
69 if (j < req)
70 use->set_req(j, newref);
71 else
72 use->set_prec(j, newref);
73 nreplacements++;
74 } else if (j >= req && uin == NULL) {
75 break;
76 }
77 }
78 return nreplacements;
79 }
80
81 void PhaseMacroExpand::copy_call_debug_info(CallNode *oldcall, CallNode * newcall) {
82 // Copy debug information and adjust JVMState information
83 uint old_dbg_start = oldcall->tf()->domain()->cnt();
84 uint new_dbg_start = newcall->tf()->domain()->cnt();
85 int jvms_adj = new_dbg_start - old_dbg_start;
86 assert (new_dbg_start == newcall->req(), "argument count mismatch");
87
88 // SafePointScalarObject node could be referenced several times in debug info.
89 // Use Dict to record cloned nodes.
90 Dict* sosn_map = new Dict(cmpkey,hashkey);
91 for (uint i = old_dbg_start; i < oldcall->req(); i++) {
92 Node* old_in = oldcall->in(i);
93 // Clone old SafePointScalarObjectNodes, adjusting their field contents.
94 if (old_in != NULL && old_in->is_SafePointScalarObject()) {
95 SafePointScalarObjectNode* old_sosn = old_in->as_SafePointScalarObject();
96 uint old_unique = C->unique();
97 Node* new_in = old_sosn->clone(sosn_map);
98 if (old_unique != C->unique()) { // New node?
99 new_in->set_req(0, C->root()); // reset control edge
100 new_in = transform_later(new_in); // Register new node.
101 }
102 old_in = new_in;
103 }
104 newcall->add_req(old_in);
259 if (call->as_ArrayCopy()->modifies(offset, offset, phase, false)) {
260 return in;
261 }
262 }
263 mem = in->in(TypeFunc::Memory);
264 } else if (in->is_MemBar()) {
265 ArrayCopyNode* ac = NULL;
266 if (ArrayCopyNode::may_modify(tinst, in->as_MemBar(), phase, ac)) {
267 assert(ac != NULL && ac->is_clonebasic(), "Only basic clone is a non escaping clone");
268 return ac;
269 }
270 mem = in->in(TypeFunc::Memory);
271 } else {
272 assert(false, "unexpected projection");
273 }
274 } else if (mem->is_Store()) {
275 const TypePtr* atype = mem->as_Store()->adr_type();
276 int adr_idx = phase->C->get_alias_index(atype);
277 if (adr_idx == alias_idx) {
278 assert(atype->isa_oopptr(), "address type must be oopptr");
279 int adr_offset = atype->offset();
280 uint adr_iid = atype->is_oopptr()->instance_id();
281 // Array elements references have the same alias_idx
282 // but different offset and different instance_id.
283 if (adr_offset == offset && adr_iid == alloc->_idx)
284 return mem;
285 } else {
286 assert(adr_idx == Compile::AliasIdxRaw, "address must match or be raw");
287 }
288 mem = mem->in(MemNode::Memory);
289 } else if (mem->is_ClearArray()) {
290 if (!ClearArrayNode::step_through(&mem, alloc->_idx, phase)) {
291 // Can not bypass initialization of the instance
292 // we are looking.
293 debug_only(intptr_t offset;)
294 assert(alloc == AllocateNode::Ideal_allocation(mem->in(3), phase, offset), "sanity");
295 InitializeNode* init = alloc->as_Allocate()->initialization();
296 // We are looking for stored value, return Initialize node
297 // or memory edge from Allocate node.
298 if (init != NULL)
299 return init;
370 }
371 if (res != NULL) {
372 res = _igvn.transform(res);
373 if (ftype->isa_narrowoop()) {
374 // PhaseMacroExpand::scalar_replacement adds DecodeN nodes
375 res = _igvn.transform(new EncodePNode(res, ftype));
376 }
377 return res;
378 }
379 return NULL;
380 }
381
382 //
383 // Given a Memory Phi, compute a value Phi containing the values from stores
384 // on the input paths.
385 // Note: this function is recursive, its depth is limited by the "level" argument
386 // Returns the computed Phi, or NULL if it cannot compute it.
387 Node *PhaseMacroExpand::value_from_mem_phi(Node *mem, BasicType ft, const Type *phi_type, const TypeOopPtr *adr_t, AllocateNode *alloc, Node_Stack *value_phis, int level) {
388 assert(mem->is_Phi(), "sanity");
389 int alias_idx = C->get_alias_index(adr_t);
390 int offset = adr_t->offset();
391 int instance_id = adr_t->instance_id();
392
393 // Check if an appropriate value phi already exists.
394 Node* region = mem->in(0);
395 for (DUIterator_Fast kmax, k = region->fast_outs(kmax); k < kmax; k++) {
396 Node* phi = region->fast_out(k);
397 if (phi->is_Phi() && phi != mem &&
398 phi->as_Phi()->is_same_inst_field(phi_type, (int)mem->_idx, instance_id, alias_idx, offset)) {
399 return phi;
400 }
401 }
402 // Check if an appropriate new value phi already exists.
403 Node* new_phi = value_phis->find(mem->_idx);
404 if (new_phi != NULL)
405 return new_phi;
406
407 if (level <= 0) {
408 return NULL; // Give up: phi tree too deep
409 }
410 Node *start_mem = C->start()->proj_out_or_null(TypeFunc::Memory);
472 }
473 }
474 // Set Phi's inputs
475 for (uint j = 1; j < length; j++) {
476 if (values.at(j) == mem) {
477 phi->init_req(j, phi);
478 } else {
479 phi->init_req(j, values.at(j));
480 }
481 }
482 return phi;
483 }
484
485 // Search the last value stored into the object's field.
486 Node *PhaseMacroExpand::value_from_mem(Node *sfpt_mem, Node *sfpt_ctl, BasicType ft, const Type *ftype, const TypeOopPtr *adr_t, AllocateNode *alloc) {
487 assert(adr_t->is_known_instance_field(), "instance required");
488 int instance_id = adr_t->instance_id();
489 assert((uint)instance_id == alloc->_idx, "wrong allocation");
490
491 int alias_idx = C->get_alias_index(adr_t);
492 int offset = adr_t->offset();
493 Node *start_mem = C->start()->proj_out_or_null(TypeFunc::Memory);
494 Node *alloc_ctrl = alloc->in(TypeFunc::Control);
495 Node *alloc_mem = alloc->in(TypeFunc::Memory);
496 Arena *a = Thread::current()->resource_area();
497 VectorSet visited(a);
498
499
500 bool done = sfpt_mem == alloc_mem;
501 Node *mem = sfpt_mem;
502 while (!done) {
503 if (visited.test_set(mem->_idx)) {
504 return NULL; // found a loop, give up
505 }
506 mem = scan_mem_chain(mem, alias_idx, offset, start_mem, alloc, &_igvn);
507 if (mem == start_mem || mem == alloc_mem) {
508 done = true; // hit a sentinel, return appropriate 0 value
509 } else if (mem->is_Initialize()) {
510 mem = mem->as_Initialize()->find_captured_store(offset, type2aelembytes(ft), &_igvn);
511 if (mem == NULL) {
512 done = true; // Something go wrong.
513 } else if (mem->is_Store()) {
514 const TypePtr* atype = mem->as_Store()->adr_type();
515 assert(C->get_alias_index(atype) == Compile::AliasIdxRaw, "store is correct memory slice");
516 done = true;
517 }
518 } else if (mem->is_Store()) {
519 const TypeOopPtr* atype = mem->as_Store()->adr_type()->isa_oopptr();
520 assert(atype != NULL, "address type must be oopptr");
521 assert(C->get_alias_index(atype) == alias_idx &&
522 atype->is_known_instance_field() && atype->offset() == offset &&
523 atype->instance_id() == instance_id, "store is correct memory slice");
524 done = true;
525 } else if (mem->is_Phi()) {
526 // try to find a phi's unique input
527 Node *unique_input = NULL;
528 Node *top = C->top();
529 for (uint i = 1; i < mem->req(); i++) {
530 Node *n = scan_mem_chain(mem->in(i), alias_idx, offset, start_mem, alloc, &_igvn);
531 if (n == NULL || n == top || n == mem) {
532 continue;
533 } else if (unique_input == NULL) {
534 unique_input = n;
535 } else if (unique_input != n) {
536 unique_input = top;
537 break;
538 }
539 }
540 if (unique_input != NULL && unique_input != top) {
541 mem = unique_input;
542 } else {
543 done = true;
544 }
545 } else if (mem->is_ArrayCopy()) {
546 done = true;
547 } else {
548 assert(false, "unexpected node");
549 }
550 }
551 if (mem != NULL) {
552 if (mem == start_mem || mem == alloc_mem) {
553 // hit a sentinel, return appropriate 0 value
554 return _igvn.zerocon(ft);
555 } else if (mem->is_Store()) {
556 Node* n = mem->in(MemNode::ValueIn);
557 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
558 n = bs->step_over_gc_barrier(n);
559 return n;
560 } else if (mem->is_Phi()) {
561 // attempt to produce a Phi reflecting the values on the input paths of the Phi
562 Node_Stack value_phis(a, 8);
563 Node * phi = value_from_mem_phi(mem, ft, ftype, adr_t, alloc, &value_phis, ValueSearchLimit);
564 if (phi != NULL) {
565 return phi;
566 } else {
567 // Kill all new Phis
568 while(value_phis.is_nonempty()) {
569 Node* n = value_phis.node();
570 _igvn.replace_node(n, C->top());
571 value_phis.pop();
572 }
573 }
574 } else if (mem->is_ArrayCopy()) {
575 Node* ctl = mem->in(0);
576 Node* m = mem->in(TypeFunc::Memory);
577 if (sfpt_ctl->is_Proj() && sfpt_ctl->as_Proj()->is_uncommon_trap_proj(Deoptimization::Reason_none)) {
578 // pin the loads in the uncommon trap path
579 ctl = sfpt_ctl;
580 m = sfpt_mem;
581 }
582 return make_arraycopy_load(mem->as_ArrayCopy(), offset, ctl, m, ft, ftype, alloc);
583 }
584 }
585 // Something go wrong.
586 return NULL;
587 }
588
589 // Check the possibility of scalar replacement.
590 bool PhaseMacroExpand::can_eliminate_allocation(AllocateNode *alloc, GrowableArray <SafePointNode *>& safepoints) {
591 // Scan the uses of the allocation to check for anything that would
592 // prevent us from eliminating it.
593 NOT_PRODUCT( const char* fail_eliminate = NULL; )
594 DEBUG_ONLY( Node* disq_node = NULL; )
595 bool can_eliminate = true;
596
597 Node* res = alloc->result_cast();
598 const TypeOopPtr* res_type = NULL;
599 if (res == NULL) {
600 // All users were eliminated.
601 } else if (!res->is_CheckCastPP()) {
602 NOT_PRODUCT(fail_eliminate = "Allocation does not have unique CheckCastPP";)
603 can_eliminate = false;
604 } else {
605 res_type = _igvn.type(res)->isa_oopptr();
606 if (res_type == NULL) {
607 NOT_PRODUCT(fail_eliminate = "Neither instance or array allocation";)
608 can_eliminate = false;
624 const TypePtr* addp_type = _igvn.type(use)->is_ptr();
625 int offset = addp_type->offset();
626
627 if (offset == Type::OffsetTop || offset == Type::OffsetBot) {
628 NOT_PRODUCT(fail_eliminate = "Undefined field referrence";)
629 can_eliminate = false;
630 break;
631 }
632 for (DUIterator_Fast kmax, k = use->fast_outs(kmax);
633 k < kmax && can_eliminate; k++) {
634 Node* n = use->fast_out(k);
635 if (!n->is_Store() && n->Opcode() != Op_CastP2X &&
636 SHENANDOAHGC_ONLY((!UseShenandoahGC || !ShenandoahBarrierSetC2::is_shenandoah_wb_pre_call(n)) &&)
637 !(n->is_ArrayCopy() &&
638 n->as_ArrayCopy()->is_clonebasic() &&
639 n->in(ArrayCopyNode::Dest) == use)) {
640 DEBUG_ONLY(disq_node = n;)
641 if (n->is_Load() || n->is_LoadStore()) {
642 NOT_PRODUCT(fail_eliminate = "Field load";)
643 } else {
644 NOT_PRODUCT(fail_eliminate = "Not store field referrence";)
645 }
646 can_eliminate = false;
647 }
648 }
649 } else if (use->is_ArrayCopy() &&
650 (use->as_ArrayCopy()->is_arraycopy_validated() ||
651 use->as_ArrayCopy()->is_copyof_validated() ||
652 use->as_ArrayCopy()->is_copyofrange_validated()) &&
653 use->in(ArrayCopyNode::Dest) == res) {
654 // ok to eliminate
655 } else if (use->is_SafePoint()) {
656 SafePointNode* sfpt = use->as_SafePoint();
657 if (sfpt->is_Call() && sfpt->as_Call()->has_non_debug_use(res)) {
658 // Object is passed as argument.
659 DEBUG_ONLY(disq_node = use;)
660 NOT_PRODUCT(fail_eliminate = "Object is passed as argument";)
661 can_eliminate = false;
662 }
663 Node* sfptMem = sfpt->memory();
664 if (sfptMem == NULL || sfptMem->is_top()) {
665 DEBUG_ONLY(disq_node = use;)
666 NOT_PRODUCT(fail_eliminate = "NULL or TOP memory";)
667 can_eliminate = false;
668 } else {
669 safepoints.append_if_missing(sfpt);
670 }
671 } else if (use->Opcode() != Op_CastP2X) { // CastP2X is used by card mark
672 if (use->is_Phi()) {
673 if (use->outcnt() == 1 && use->unique_out()->Opcode() == Op_Return) {
674 NOT_PRODUCT(fail_eliminate = "Object is return value";)
675 } else {
676 NOT_PRODUCT(fail_eliminate = "Object is referenced by Phi";)
677 }
678 DEBUG_ONLY(disq_node = use;)
679 } else {
680 if (use->Opcode() == Op_Return) {
681 NOT_PRODUCT(fail_eliminate = "Object is return value";)
682 }else {
683 NOT_PRODUCT(fail_eliminate = "Object is referenced by node";)
684 }
685 DEBUG_ONLY(disq_node = use;)
686 }
687 can_eliminate = false;
688 }
689 }
690 }
691
692 #ifndef PRODUCT
693 if (PrintEliminateAllocations) {
694 if (can_eliminate) {
695 tty->print("Scalar ");
696 if (res == NULL)
697 alloc->dump();
698 else
699 res->dump();
700 } else if (alloc->_is_scalar_replaceable) {
701 tty->print("NotScalar (%s)", fail_eliminate);
702 if (res == NULL)
703 alloc->dump();
704 else
705 res->dump();
706 #ifdef ASSERT
707 if (disq_node != NULL) {
730 Node* res = alloc->result_cast();
731 assert(res == NULL || res->is_CheckCastPP(), "unexpected AllocateNode result");
732 const TypeOopPtr* res_type = NULL;
733 if (res != NULL) { // Could be NULL when there are no users
734 res_type = _igvn.type(res)->isa_oopptr();
735 }
736
737 if (res != NULL) {
738 klass = res_type->klass();
739 if (res_type->isa_instptr()) {
740 // find the fields of the class which will be needed for safepoint debug information
741 assert(klass->is_instance_klass(), "must be an instance klass.");
742 iklass = klass->as_instance_klass();
743 nfields = iklass->nof_nonstatic_fields();
744 } else {
745 // find the array's elements which will be needed for safepoint debug information
746 nfields = alloc->in(AllocateNode::ALength)->find_int_con(-1);
747 assert(klass->is_array_klass() && nfields >= 0, "must be an array klass.");
748 elem_type = klass->as_array_klass()->element_type();
749 basic_elem_type = elem_type->basic_type();
750 array_base = arrayOopDesc::base_offset_in_bytes(basic_elem_type);
751 element_size = type2aelembytes(basic_elem_type);
752 }
753 }
754 //
755 // Process the safepoint uses
756 //
757 while (safepoints.length() > 0) {
758 SafePointNode* sfpt = safepoints.pop();
759 Node* mem = sfpt->memory();
760 Node* ctl = sfpt->control();
761 assert(sfpt->jvms() != NULL, "missed JVMS");
762 // Fields of scalar objs are referenced only at the end
763 // of regular debuginfo at the last (youngest) JVMS.
764 // Record relative start index.
765 uint first_ind = (sfpt->req() - sfpt->jvms()->scloff());
766 SafePointScalarObjectNode* sobj = new SafePointScalarObjectNode(res_type,
767 #ifdef ASSERT
768 alloc,
769 #endif
770 first_ind, nfields);
771 sobj->init_req(0, C->root());
772 transform_later(sobj);
773
774 // Scan object's fields adding an input to the safepoint for each field.
775 for (int j = 0; j < nfields; j++) {
776 intptr_t offset;
777 ciField* field = NULL;
778 if (iklass != NULL) {
779 field = iklass->nonstatic_field_at(j);
780 offset = field->offset();
781 elem_type = field->type();
782 basic_elem_type = field->layout_type();
783 } else {
784 offset = array_base + j * (intptr_t)element_size;
785 }
786
787 const Type *field_type;
788 // The next code is taken from Parse::do_get_xxx().
789 if (basic_elem_type == T_OBJECT || basic_elem_type == T_ARRAY) {
790 if (!elem_type->is_loaded()) {
791 field_type = TypeInstPtr::BOTTOM;
792 } else if (field != NULL && field->is_static_constant()) {
793 // This can happen if the constant oop is non-perm.
794 ciObject* con = field->constant_value().as_object();
795 // Do not "join" in the previous type; it doesn't add value,
796 // and may yield a vacuous result if the field is of interface type.
797 field_type = TypeOopPtr::make_from_constant(con)->isa_oopptr();
798 assert(field_type != NULL, "field singleton type must be consistent");
799 } else {
800 field_type = TypeOopPtr::make_from_klass(elem_type->as_klass());
801 }
802 if (UseCompressedOops) {
803 field_type = field_type->make_narrowoop();
804 basic_elem_type = T_NARROWOOP;
805 }
806 } else {
807 field_type = Type::get_const_basic_type(basic_elem_type);
808 }
809
810 const TypeOopPtr *field_addr_type = res_type->add_offset(offset)->isa_oopptr();
811
812 Node *field_val = value_from_mem(mem, ctl, basic_elem_type, field_type, field_addr_type, alloc);
813 if (field_val == NULL) {
814 // We weren't able to find a value for this field,
815 // give up on eliminating this allocation.
816
817 // Remove any extra entries we added to the safepoint.
818 uint last = sfpt->req() - 1;
819 for (int k = 0; k < j; k++) {
820 sfpt->del_req(last--);
821 }
822 _igvn._worklist.push(sfpt);
823 // rollback processed safepoints
824 while (safepoints_done.length() > 0) {
825 SafePointNode* sfpt_done = safepoints_done.pop();
826 // remove any extra entries we added to the safepoint
827 last = sfpt_done->req() - 1;
828 for (int k = 0; k < nfields; k++) {
829 sfpt_done->del_req(last--);
830 }
831 JVMState *jvms = sfpt_done->jvms();
832 jvms->set_endoff(sfpt_done->req());
858 tty->print("=== At SafePoint node %d can't find value of array element [%d]",
859 sfpt->_idx, j);
860 }
861 tty->print(", which prevents elimination of: ");
862 if (res == NULL)
863 alloc->dump();
864 else
865 res->dump();
866 }
867 #endif
868 return false;
869 }
870 if (UseCompressedOops && field_type->isa_narrowoop()) {
871 // Enable "DecodeN(EncodeP(Allocate)) --> Allocate" transformation
872 // to be able scalar replace the allocation.
873 if (field_val->is_EncodeP()) {
874 field_val = field_val->in(1);
875 } else {
876 field_val = transform_later(new DecodeNNode(field_val, field_val->get_ptr_type()));
877 }
878 }
879 sfpt->add_req(field_val);
880 }
881 JVMState *jvms = sfpt->jvms();
882 jvms->set_endoff(sfpt->req());
883 // Now make a pass over the debug information replacing any references
884 // to the allocated object with "sobj"
885 int start = jvms->debug_start();
886 int end = jvms->debug_end();
887 sfpt->replace_edges_in_range(res, sobj, start, end);
888 _igvn._worklist.push(sfpt);
889 safepoints_done.append_if_missing(sfpt); // keep it for rollback
890 }
891 return true;
892 }
893
894 static void disconnect_projections(MultiNode* n, PhaseIterGVN& igvn) {
895 Node* ctl_proj = n->proj_out_or_null(TypeFunc::Control);
896 Node* mem_proj = n->proj_out_or_null(TypeFunc::Memory);
897 if (ctl_proj != NULL) {
898 igvn.replace_node(ctl_proj, n->in(0));
899 }
900 if (mem_proj != NULL) {
901 igvn.replace_node(mem_proj, n->in(TypeFunc::Memory));
902 }
903 }
904
905 // Process users of eliminated allocation.
906 void PhaseMacroExpand::process_users_of_allocation(CallNode *alloc) {
907 Node* res = alloc->result_cast();
908 if (res != NULL) {
909 for (DUIterator_Last jmin, j = res->last_outs(jmin); j >= jmin; ) {
910 Node *use = res->last_out(j);
935 Node* membar_after = ac->proj_out(TypeFunc::Control)->unique_ctrl_out();
936 disconnect_projections(ac, _igvn);
937 assert(alloc->in(0)->is_Proj() && alloc->in(0)->in(0)->Opcode() == Op_MemBarCPUOrder, "mem barrier expected before allocation");
938 Node* membar_before = alloc->in(0)->in(0);
939 disconnect_projections(membar_before->as_MemBar(), _igvn);
940 if (membar_after->is_MemBar()) {
941 disconnect_projections(membar_after->as_MemBar(), _igvn);
942 }
943 } else {
944 eliminate_gc_barrier(n);
945 }
946 k -= (oc2 - use->outcnt());
947 }
948 _igvn.remove_dead_node(use);
949 } else if (use->is_ArrayCopy()) {
950 // Disconnect ArrayCopy node
951 ArrayCopyNode* ac = use->as_ArrayCopy();
952 assert(ac->is_arraycopy_validated() ||
953 ac->is_copyof_validated() ||
954 ac->is_copyofrange_validated(), "unsupported");
955 CallProjections callprojs;
956 ac->extract_projections(&callprojs, true);
957
958 _igvn.replace_node(callprojs.fallthrough_ioproj, ac->in(TypeFunc::I_O));
959 _igvn.replace_node(callprojs.fallthrough_memproj, ac->in(TypeFunc::Memory));
960 _igvn.replace_node(callprojs.fallthrough_catchproj, ac->in(TypeFunc::Control));
961
962 // Set control to top. IGVN will remove the remaining projections
963 ac->set_req(0, top());
964 ac->replace_edge(res, top());
965
966 // Disconnect src right away: it can help find new
967 // opportunities for allocation elimination
968 Node* src = ac->in(ArrayCopyNode::Src);
969 ac->replace_edge(src, top());
970 // src can be top at this point if src and dest of the
971 // arraycopy were the same
972 if (src->outcnt() == 0 && !src->is_top()) {
973 _igvn.remove_dead_node(src);
974 }
975
976 _igvn._worklist.push(ac);
977 } else {
978 eliminate_gc_barrier(use);
979 }
980 j -= (oc1 - res->outcnt());
981 }
982 assert(res->outcnt() == 0, "all uses of allocated objects must be deleted");
983 _igvn.remove_dead_node(res);
984 }
985
986 //
987 // Process other users of allocation's projections
988 //
989 if (_resproj != NULL && _resproj->outcnt() != 0) {
990 // First disconnect stores captured by Initialize node.
991 // If Initialize node is eliminated first in the following code,
992 // it will kill such stores and DUIterator_Last will assert.
993 for (DUIterator_Fast jmax, j = _resproj->fast_outs(jmax); j < jmax; j++) {
994 Node *use = _resproj->fast_out(j);
995 if (use->is_AddP()) {
996 // raw memory addresses used only by the initialization
1108 if (alloc->is_AllocateArray())
1109 tty->print_cr("++++ Eliminated: %d AllocateArray", alloc->_idx);
1110 else
1111 tty->print_cr("++++ Eliminated: %d Allocate", alloc->_idx);
1112 }
1113 #endif
1114
1115 return true;
1116 }
1117
1118 bool PhaseMacroExpand::eliminate_boxing_node(CallStaticJavaNode *boxing) {
1119 // EA should remove all uses of non-escaping boxing node.
1120 if (!C->eliminate_boxing() || boxing->proj_out_or_null(TypeFunc::Parms) != NULL) {
1121 return false;
1122 }
1123
1124 assert(boxing->result_cast() == NULL, "unexpected boxing node result");
1125
1126 extract_call_projections(boxing);
1127
1128 const TypeTuple* r = boxing->tf()->range();
1129 assert(r->cnt() > TypeFunc::Parms, "sanity");
1130 const TypeInstPtr* t = r->field_at(TypeFunc::Parms)->isa_instptr();
1131 assert(t != NULL, "sanity");
1132
1133 CompileLog* log = C->log();
1134 if (log != NULL) {
1135 log->head("eliminate_boxing type='%d'",
1136 log->identify(t->klass()));
1137 JVMState* p = boxing->jvms();
1138 while (p != NULL) {
1139 log->elem("jvms bci='%d' method='%d'", p->bci(), log->identify(p->method()));
1140 p = p->caller();
1141 }
1142 log->tail("eliminate_boxing");
1143 }
1144
1145 process_users_of_allocation(boxing);
1146
1147 #ifndef PRODUCT
1148 if (PrintEliminateAllocations) {
1268 Node *result_phi_i_o = NULL;
1269
1270 // The initial slow comparison is a size check, the comparison
1271 // we want to do is a BoolTest::gt
1272 bool always_slow = false;
1273 int tv = _igvn.find_int_con(initial_slow_test, -1);
1274 if (tv >= 0) {
1275 always_slow = (tv == 1);
1276 initial_slow_test = NULL;
1277 } else {
1278 initial_slow_test = BoolNode::make_predicate(initial_slow_test, &_igvn);
1279 }
1280
1281 if (C->env()->dtrace_alloc_probes() ||
1282 (!UseTLAB && !Universe::heap()->supports_inline_contig_alloc())) {
1283 // Force slow-path allocation
1284 always_slow = true;
1285 initial_slow_test = NULL;
1286 }
1287
1288
1289 enum { too_big_or_final_path = 1, need_gc_path = 2 };
1290 Node *slow_region = NULL;
1291 Node *toobig_false = ctrl;
1292
1293 assert (initial_slow_test == NULL || !always_slow, "arguments must be consistent");
1294 // generate the initial test if necessary
1295 if (initial_slow_test != NULL ) {
1296 slow_region = new RegionNode(3);
1297
1298 // Now make the initial failure test. Usually a too-big test but
1299 // might be a TRUE for finalizers or a fancy class check for
1300 // newInstance0.
1301 IfNode *toobig_iff = new IfNode(ctrl, initial_slow_test, PROB_MIN, COUNT_UNKNOWN);
1302 transform_later(toobig_iff);
1303 // Plug the failing-too-big test into the slow-path region
1304 Node *toobig_true = new IfTrueNode( toobig_iff );
1305 transform_later(toobig_true);
1306 slow_region ->init_req( too_big_or_final_path, toobig_true );
1307 toobig_false = new IfFalseNode( toobig_iff );
1308 transform_later(toobig_false);
1309 } else { // No initial test, just fall into next case
1310 toobig_false = ctrl;
1311 debug_only(slow_region = NodeSentinel);
1312 }
1313
1314 Node *slow_mem = mem; // save the current memory state for slow path
1315 // generate the fast allocation code unless we know that the initial test will always go slow
1316 if (!always_slow) {
1317 // Fast path modifies only raw memory.
1318 if (mem->is_MergeMem()) {
1319 mem = mem->as_MergeMem()->memory_at(Compile::AliasIdxRaw);
1320 }
1321
1322 // allocate the Region and Phi nodes for the result
1323 result_region = new RegionNode(3);
1324 result_phi_rawmem = new PhiNode(result_region, Type::MEMORY, TypeRawPtr::BOTTOM);
1325 result_phi_rawoop = new PhiNode(result_region, TypeRawPtr::BOTTOM);
1326 result_phi_i_o = new PhiNode(result_region, Type::ABIO); // I/O is used for Prefetch
1327
1328 // Grab regular I/O before optional prefetch may change it.
1329 // Slow-path does no I/O so just set it to the original I/O.
1330 result_phi_i_o->init_req(slow_result_path, i_o);
1331
1332 Node* needgc_ctrl = NULL;
1333 // Name successful fast-path variables
1334 Node* fast_oop_ctrl;
1335 Node* fast_oop_rawmem;
1336
1337 intx prefetch_lines = length != NULL ? AllocatePrefetchLines : AllocateInstancePrefetchLines;
1338
1339 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
1340 Node* fast_oop = bs->obj_allocate(this, ctrl, mem, toobig_false, size_in_bytes, i_o, needgc_ctrl,
1341 fast_oop_ctrl, fast_oop_rawmem,
1342 prefetch_lines);
1343
1344 if (initial_slow_test) {
1345 slow_region->init_req(need_gc_path, needgc_ctrl);
1346 // This completes all paths into the slow merge point
1347 transform_later(slow_region);
1348 } else { // No initial slow path needed!
1349 // Just fall from the need-GC path straight into the VM call.
1350 slow_region = needgc_ctrl;
1351 }
1352
1353 InitializeNode* init = alloc->initialization();
1354 fast_oop_rawmem = initialize_object(alloc,
1355 fast_oop_ctrl, fast_oop_rawmem, fast_oop,
1356 klass_node, length, size_in_bytes);
1357
1358 // If initialization is performed by an array copy, any required
1359 // MemBarStoreStore was already added. If the object does not
1360 // escape no need for a MemBarStoreStore. If the object does not
1361 // escape in its initializer and memory barrier (MemBarStoreStore or
1362 // stronger) is already added at exit of initializer, also no need
1363 // for a MemBarStoreStore. Otherwise we need a MemBarStoreStore
1364 // so that stores that initialize this object can't be reordered
1365 // with a subsequent store that makes this object accessible by
1366 // other threads.
1367 // Other threads include java threads and JVM internal threads
1368 // (for example concurrent GC threads). Current concurrent GC
1595 } else {
1596 slow_result = _resproj->clone();
1597 transform_later(slow_result);
1598 _igvn.replace_node(_resproj, result_phi_rawoop);
1599 }
1600
1601 // Plug slow-path into result merge point
1602 result_region ->init_req( slow_result_path, ctrl );
1603 result_phi_rawoop->init_req( slow_result_path, slow_result);
1604 result_phi_rawmem->init_req( slow_result_path, _memproj_fallthrough );
1605 transform_later(result_region);
1606 transform_later(result_phi_rawoop);
1607 transform_later(result_phi_rawmem);
1608 transform_later(result_phi_i_o);
1609 // This completes all paths into the result merge point
1610 }
1611
1612
1613 // Helper for PhaseMacroExpand::expand_allocate_common.
1614 // Initializes the newly-allocated storage.
1615 Node*
1616 PhaseMacroExpand::initialize_object(AllocateNode* alloc,
1617 Node* control, Node* rawmem, Node* object,
1618 Node* klass_node, Node* length,
1619 Node* size_in_bytes) {
1620 InitializeNode* init = alloc->initialization();
1621 // Store the klass & mark bits
1622 Node* mark_node = NULL;
1623 // For now only enable fast locking for non-array types
1624 if (UseBiasedLocking && (length == NULL)) {
1625 mark_node = make_load(control, rawmem, klass_node, in_bytes(Klass::prototype_header_offset()), TypeRawPtr::BOTTOM, T_ADDRESS);
1626 } else {
1627 mark_node = makecon(TypeRawPtr::make((address)markOopDesc::prototype()));
1628 }
1629 rawmem = make_store(control, rawmem, object, oopDesc::mark_offset_in_bytes(), mark_node, T_ADDRESS);
1630
1631 rawmem = make_store(control, rawmem, object, oopDesc::klass_offset_in_bytes(), klass_node, T_METADATA);
1632 int header_size = alloc->minimum_header_size(); // conservatively small
1633
1634 // Array length
1635 if (length != NULL) { // Arrays need length field
1636 rawmem = make_store(control, rawmem, object, arrayOopDesc::length_offset_in_bytes(), length, T_INT);
1637 // conservatively small header size:
1638 header_size = arrayOopDesc::base_offset_in_bytes(T_BYTE);
1639 ciKlass* k = _igvn.type(klass_node)->is_klassptr()->klass();
1640 if (k->is_array_klass()) // we know the exact header size in most cases:
1641 header_size = Klass::layout_helper_header_size(k->layout_helper());
1642 }
1643
1644 // Clear the object body, if necessary.
1645 if (init == NULL) {
1646 // The init has somehow disappeared; be cautious and clear everything.
1647 //
1648 // This can happen if a node is allocated but an uncommon trap occurs
1649 // immediately. In this case, the Initialize gets associated with the
1650 // trap, and may be placed in a different (outer) loop, if the Allocate
1651 // is in a loop. If (this is rare) the inner loop gets unrolled, then
1652 // there can be two Allocates to one Initialize. The answer in all these
1653 // edge cases is safety first. It is always safe to clear immediately
1654 // within an Allocate, and then (maybe or maybe not) clear some more later.
1655 if (!(UseTLAB && ZeroTLAB)) {
1656 rawmem = ClearArrayNode::clear_memory(control, rawmem, object,
1657 header_size, size_in_bytes,
1658 &_igvn);
1659 }
1660 } else {
1661 if (!init->is_complete()) {
1662 // Try to win by zeroing only what the init does not store.
1663 // We can also try to do some peephole optimizations,
1664 // such as combining some adjacent subword stores.
1665 rawmem = init->complete_stores(control, rawmem, object,
1666 header_size, size_in_bytes, &_igvn);
1667 }
1668 // We have no more use for this link, since the AllocateNode goes away:
1669 init->set_req(InitializeNode::RawAddress, top());
1670 // (If we keep the link, it just confuses the register allocator,
1671 // who thinks he sees a real use of the address by the membar.)
1672 }
1673
1674 return rawmem;
1675 }
1676
2398 // No exceptions for unlocking
2399 // Capture slow path
2400 // disconnect fall-through projection from call and create a new one
2401 // hook up users of fall-through projection to region
2402 Node *slow_ctrl = _fallthroughproj->clone();
2403 transform_later(slow_ctrl);
2404 _igvn.hash_delete(_fallthroughproj);
2405 _fallthroughproj->disconnect_inputs(NULL, C);
2406 region->init_req(1, slow_ctrl);
2407 // region inputs are now complete
2408 transform_later(region);
2409 _igvn.replace_node(_fallthroughproj, region);
2410
2411 Node *memproj = transform_later(new ProjNode(call, TypeFunc::Memory) );
2412 mem_phi->init_req(1, memproj );
2413 mem_phi->init_req(2, mem);
2414 transform_later(mem_phi);
2415 _igvn.replace_node(_memproj_fallthrough, mem_phi);
2416 }
2417
2418 //---------------------------eliminate_macro_nodes----------------------
2419 // Eliminate scalar replaced allocations and associated locks.
2420 void PhaseMacroExpand::eliminate_macro_nodes() {
2421 if (C->macro_count() == 0)
2422 return;
2423
2424 // First, attempt to eliminate locks
2425 int cnt = C->macro_count();
2426 for (int i=0; i < cnt; i++) {
2427 Node *n = C->macro_node(i);
2428 if (n->is_AbstractLock()) { // Lock and Unlock nodes
2429 // Before elimination mark all associated (same box and obj)
2430 // lock and unlock nodes.
2431 mark_eliminated_locking_nodes(n->as_AbstractLock());
2432 }
2433 }
2434 bool progress = true;
2435 while (progress) {
2436 progress = false;
2437 for (int i = C->macro_count(); i > 0; i--) {
2442 success = eliminate_locking_node(n->as_AbstractLock());
2443 }
2444 assert(success == (C->macro_count() < old_macro_count), "elimination reduces macro count");
2445 progress = progress || success;
2446 }
2447 }
2448 // Next, attempt to eliminate allocations
2449 _has_locks = false;
2450 progress = true;
2451 while (progress) {
2452 progress = false;
2453 for (int i = C->macro_count(); i > 0; i--) {
2454 Node * n = C->macro_node(i-1);
2455 bool success = false;
2456 debug_only(int old_macro_count = C->macro_count(););
2457 switch (n->class_id()) {
2458 case Node::Class_Allocate:
2459 case Node::Class_AllocateArray:
2460 success = eliminate_allocate_node(n->as_Allocate());
2461 break;
2462 case Node::Class_CallStaticJava:
2463 success = eliminate_boxing_node(n->as_CallStaticJava());
2464 break;
2465 case Node::Class_Lock:
2466 case Node::Class_Unlock:
2467 assert(!n->as_AbstractLock()->is_eliminated(), "sanity");
2468 _has_locks = true;
2469 break;
2470 case Node::Class_ArrayCopy:
2471 break;
2472 case Node::Class_OuterStripMinedLoop:
2473 break;
2474 default:
2475 assert(n->Opcode() == Op_LoopLimit ||
2476 n->Opcode() == Op_Opaque1 ||
2477 n->Opcode() == Op_Opaque2 ||
2478 n->Opcode() == Op_Opaque3 ||
2479 BarrierSet::barrier_set()->barrier_set_c2()->is_gc_barrier_node(n),
2480 "unknown node type in macro list");
2481 }
2482 assert(success == (C->macro_count() < old_macro_count), "elimination reduces macro count");
2483 progress = progress || success;
2484 }
2494 // Make sure expansion will not cause node limit to be exceeded.
2495 // Worst case is a macro node gets expanded into about 200 nodes.
2496 // Allow 50% more for optimization.
2497 if (C->check_node_count(C->macro_count() * 300, "out of nodes before macro expansion" ) )
2498 return true;
2499
2500 // Eliminate Opaque and LoopLimit nodes. Do it after all loop optimizations.
2501 bool progress = true;
2502 while (progress) {
2503 progress = false;
2504 for (int i = C->macro_count(); i > 0; i--) {
2505 Node * n = C->macro_node(i-1);
2506 bool success = false;
2507 debug_only(int old_macro_count = C->macro_count(););
2508 if (n->Opcode() == Op_LoopLimit) {
2509 // Remove it from macro list and put on IGVN worklist to optimize.
2510 C->remove_macro_node(n);
2511 _igvn._worklist.push(n);
2512 success = true;
2513 } else if (n->Opcode() == Op_CallStaticJava) {
2514 // Remove it from macro list and put on IGVN worklist to optimize.
2515 C->remove_macro_node(n);
2516 _igvn._worklist.push(n);
2517 success = true;
2518 } else if (n->Opcode() == Op_Opaque1 || n->Opcode() == Op_Opaque2) {
2519 _igvn.replace_node(n, n->in(1));
2520 success = true;
2521 #if INCLUDE_RTM_OPT
2522 } else if ((n->Opcode() == Op_Opaque3) && ((Opaque3Node*)n)->rtm_opt()) {
2523 assert(C->profile_rtm(), "should be used only in rtm deoptimization code");
2524 assert((n->outcnt() == 1) && n->unique_out()->is_Cmp(), "");
2525 Node* cmp = n->unique_out();
2526 #ifdef ASSERT
2527 // Validate graph.
2528 assert((cmp->outcnt() == 1) && cmp->unique_out()->is_Bool(), "");
2529 BoolNode* bol = cmp->unique_out()->as_Bool();
2530 assert((bol->outcnt() == 1) && bol->unique_out()->is_If() &&
2531 (bol->_test._test == BoolTest::ne), "");
2532 IfNode* ifn = bol->unique_out()->as_If();
2533 assert((ifn->outcnt() == 2) &&
2534 ifn->proj_out(1)->is_uncommon_trap_proj(Deoptimization::Reason_rtm_state_change) != NULL, "");
2535 #endif
2536 Node* repl = n->in(1);
2537 if (!_has_locks) {
2577 int macro_count = C->macro_count();
2578 Node * n = C->macro_node(macro_count-1);
2579 assert(n->is_macro(), "only macro nodes expected here");
2580 if (_igvn.type(n) == Type::TOP || (n->in(0) != NULL && n->in(0)->is_top())) {
2581 // node is unreachable, so don't try to expand it
2582 C->remove_macro_node(n);
2583 continue;
2584 }
2585 switch (n->class_id()) {
2586 case Node::Class_Allocate:
2587 expand_allocate(n->as_Allocate());
2588 break;
2589 case Node::Class_AllocateArray:
2590 expand_allocate_array(n->as_AllocateArray());
2591 break;
2592 case Node::Class_Lock:
2593 expand_lock_node(n->as_Lock());
2594 break;
2595 case Node::Class_Unlock:
2596 expand_unlock_node(n->as_Unlock());
2597 break;
2598 default:
2599 assert(false, "unknown node type in macro list");
2600 }
2601 assert(C->macro_count() < macro_count, "must have deleted a node from macro list");
2602 if (C->failing()) return true;
2603 }
2604
2605 _igvn.set_delay_transform(false);
2606 _igvn.optimize();
2607 if (C->failing()) return true;
2608 return false;
2609 }
|
29 #include "opto/addnode.hpp"
30 #include "opto/arraycopynode.hpp"
31 #include "opto/callnode.hpp"
32 #include "opto/castnode.hpp"
33 #include "opto/cfgnode.hpp"
34 #include "opto/compile.hpp"
35 #include "opto/convertnode.hpp"
36 #include "opto/graphKit.hpp"
37 #include "opto/locknode.hpp"
38 #include "opto/loopnode.hpp"
39 #include "opto/macro.hpp"
40 #include "opto/memnode.hpp"
41 #include "opto/narrowptrnode.hpp"
42 #include "opto/node.hpp"
43 #include "opto/opaquenode.hpp"
44 #include "opto/phaseX.hpp"
45 #include "opto/rootnode.hpp"
46 #include "opto/runtime.hpp"
47 #include "opto/subnode.hpp"
48 #include "opto/type.hpp"
49 #include "opto/valuetypenode.hpp"
50 #include "runtime/sharedRuntime.hpp"
51 #include "utilities/macros.hpp"
52 #if INCLUDE_G1GC
53 #include "gc/g1/g1ThreadLocalData.hpp"
54 #endif // INCLUDE_G1GC
55 #if INCLUDE_SHENANDOAHGC
56 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
57 #endif
58
59
60 //
61 // Replace any references to "oldref" in inputs to "use" with "newref".
62 // Returns the number of replacements made.
63 //
64 int PhaseMacroExpand::replace_input(Node *use, Node *oldref, Node *newref) {
65 int nreplacements = 0;
66 uint req = use->req();
67 for (uint j = 0; j < use->len(); j++) {
68 Node *uin = use->in(j);
69 if (uin == oldref) {
70 if (j < req)
71 use->set_req(j, newref);
72 else
73 use->set_prec(j, newref);
74 nreplacements++;
75 } else if (j >= req && uin == NULL) {
76 break;
77 }
78 }
79 return nreplacements;
80 }
81
82 void PhaseMacroExpand::copy_call_debug_info(CallNode *oldcall, CallNode * newcall) {
83 // Copy debug information and adjust JVMState information
84 uint old_dbg_start = oldcall->tf()->domain_sig()->cnt();
85 uint new_dbg_start = newcall->tf()->domain_sig()->cnt();
86 int jvms_adj = new_dbg_start - old_dbg_start;
87 assert (new_dbg_start == newcall->req(), "argument count mismatch");
88
89 // SafePointScalarObject node could be referenced several times in debug info.
90 // Use Dict to record cloned nodes.
91 Dict* sosn_map = new Dict(cmpkey,hashkey);
92 for (uint i = old_dbg_start; i < oldcall->req(); i++) {
93 Node* old_in = oldcall->in(i);
94 // Clone old SafePointScalarObjectNodes, adjusting their field contents.
95 if (old_in != NULL && old_in->is_SafePointScalarObject()) {
96 SafePointScalarObjectNode* old_sosn = old_in->as_SafePointScalarObject();
97 uint old_unique = C->unique();
98 Node* new_in = old_sosn->clone(sosn_map);
99 if (old_unique != C->unique()) { // New node?
100 new_in->set_req(0, C->root()); // reset control edge
101 new_in = transform_later(new_in); // Register new node.
102 }
103 old_in = new_in;
104 }
105 newcall->add_req(old_in);
260 if (call->as_ArrayCopy()->modifies(offset, offset, phase, false)) {
261 return in;
262 }
263 }
264 mem = in->in(TypeFunc::Memory);
265 } else if (in->is_MemBar()) {
266 ArrayCopyNode* ac = NULL;
267 if (ArrayCopyNode::may_modify(tinst, in->as_MemBar(), phase, ac)) {
268 assert(ac != NULL && ac->is_clonebasic(), "Only basic clone is a non escaping clone");
269 return ac;
270 }
271 mem = in->in(TypeFunc::Memory);
272 } else {
273 assert(false, "unexpected projection");
274 }
275 } else if (mem->is_Store()) {
276 const TypePtr* atype = mem->as_Store()->adr_type();
277 int adr_idx = phase->C->get_alias_index(atype);
278 if (adr_idx == alias_idx) {
279 assert(atype->isa_oopptr(), "address type must be oopptr");
280 int adr_offset = atype->flattened_offset();
281 uint adr_iid = atype->is_oopptr()->instance_id();
282 // Array elements references have the same alias_idx
283 // but different offset and different instance_id.
284 if (adr_offset == offset && adr_iid == alloc->_idx)
285 return mem;
286 } else {
287 assert(adr_idx == Compile::AliasIdxRaw, "address must match or be raw");
288 }
289 mem = mem->in(MemNode::Memory);
290 } else if (mem->is_ClearArray()) {
291 if (!ClearArrayNode::step_through(&mem, alloc->_idx, phase)) {
292 // Can not bypass initialization of the instance
293 // we are looking.
294 debug_only(intptr_t offset;)
295 assert(alloc == AllocateNode::Ideal_allocation(mem->in(3), phase, offset), "sanity");
296 InitializeNode* init = alloc->as_Allocate()->initialization();
297 // We are looking for stored value, return Initialize node
298 // or memory edge from Allocate node.
299 if (init != NULL)
300 return init;
371 }
372 if (res != NULL) {
373 res = _igvn.transform(res);
374 if (ftype->isa_narrowoop()) {
375 // PhaseMacroExpand::scalar_replacement adds DecodeN nodes
376 res = _igvn.transform(new EncodePNode(res, ftype));
377 }
378 return res;
379 }
380 return NULL;
381 }
382
383 //
384 // Given a Memory Phi, compute a value Phi containing the values from stores
385 // on the input paths.
386 // Note: this function is recursive, its depth is limited by the "level" argument
387 // Returns the computed Phi, or NULL if it cannot compute it.
388 Node *PhaseMacroExpand::value_from_mem_phi(Node *mem, BasicType ft, const Type *phi_type, const TypeOopPtr *adr_t, AllocateNode *alloc, Node_Stack *value_phis, int level) {
389 assert(mem->is_Phi(), "sanity");
390 int alias_idx = C->get_alias_index(adr_t);
391 int offset = adr_t->flattened_offset();
392 int instance_id = adr_t->instance_id();
393
394 // Check if an appropriate value phi already exists.
395 Node* region = mem->in(0);
396 for (DUIterator_Fast kmax, k = region->fast_outs(kmax); k < kmax; k++) {
397 Node* phi = region->fast_out(k);
398 if (phi->is_Phi() && phi != mem &&
399 phi->as_Phi()->is_same_inst_field(phi_type, (int)mem->_idx, instance_id, alias_idx, offset)) {
400 return phi;
401 }
402 }
403 // Check if an appropriate new value phi already exists.
404 Node* new_phi = value_phis->find(mem->_idx);
405 if (new_phi != NULL)
406 return new_phi;
407
408 if (level <= 0) {
409 return NULL; // Give up: phi tree too deep
410 }
411 Node *start_mem = C->start()->proj_out_or_null(TypeFunc::Memory);
473 }
474 }
475 // Set Phi's inputs
476 for (uint j = 1; j < length; j++) {
477 if (values.at(j) == mem) {
478 phi->init_req(j, phi);
479 } else {
480 phi->init_req(j, values.at(j));
481 }
482 }
483 return phi;
484 }
485
486 // Search the last value stored into the object's field.
487 Node *PhaseMacroExpand::value_from_mem(Node *sfpt_mem, Node *sfpt_ctl, BasicType ft, const Type *ftype, const TypeOopPtr *adr_t, AllocateNode *alloc) {
488 assert(adr_t->is_known_instance_field(), "instance required");
489 int instance_id = adr_t->instance_id();
490 assert((uint)instance_id == alloc->_idx, "wrong allocation");
491
492 int alias_idx = C->get_alias_index(adr_t);
493 int offset = adr_t->flattened_offset();
494 Node *start_mem = C->start()->proj_out_or_null(TypeFunc::Memory);
495 Node *alloc_ctrl = alloc->in(TypeFunc::Control);
496 Node *alloc_mem = alloc->in(TypeFunc::Memory);
497 Arena *a = Thread::current()->resource_area();
498 VectorSet visited(a);
499
500 bool done = sfpt_mem == alloc_mem;
501 Node *mem = sfpt_mem;
502 while (!done) {
503 if (visited.test_set(mem->_idx)) {
504 return NULL; // found a loop, give up
505 }
506 mem = scan_mem_chain(mem, alias_idx, offset, start_mem, alloc, &_igvn);
507 if (mem == start_mem || mem == alloc_mem) {
508 done = true; // hit a sentinel, return appropriate 0 value
509 } else if (mem->is_Initialize()) {
510 mem = mem->as_Initialize()->find_captured_store(offset, type2aelembytes(ft), &_igvn);
511 if (mem == NULL) {
512 done = true; // Something went wrong.
513 } else if (mem->is_Store()) {
514 const TypePtr* atype = mem->as_Store()->adr_type();
515 assert(C->get_alias_index(atype) == Compile::AliasIdxRaw, "store is correct memory slice");
516 done = true;
517 }
518 } else if (mem->is_Store()) {
519 const TypeOopPtr* atype = mem->as_Store()->adr_type()->isa_oopptr();
520 assert(atype != NULL, "address type must be oopptr");
521 assert(C->get_alias_index(atype) == alias_idx &&
522 atype->is_known_instance_field() && atype->flattened_offset() == offset &&
523 atype->instance_id() == instance_id, "store is correct memory slice");
524 done = true;
525 } else if (mem->is_Phi()) {
526 // try to find a phi's unique input
527 Node *unique_input = NULL;
528 Node *top = C->top();
529 for (uint i = 1; i < mem->req(); i++) {
530 Node *n = scan_mem_chain(mem->in(i), alias_idx, offset, start_mem, alloc, &_igvn);
531 if (n == NULL || n == top || n == mem) {
532 continue;
533 } else if (unique_input == NULL) {
534 unique_input = n;
535 } else if (unique_input != n) {
536 unique_input = top;
537 break;
538 }
539 }
540 if (unique_input != NULL && unique_input != top) {
541 mem = unique_input;
542 } else {
543 done = true;
544 }
545 } else if (mem->is_ArrayCopy()) {
546 done = true;
547 } else {
548 assert(false, "unexpected node");
549 }
550 }
551 if (mem != NULL) {
552 if (mem == start_mem || mem == alloc_mem) {
553 // hit a sentinel, return appropriate 0 value
554 Node* default_value = alloc->in(AllocateNode::DefaultValue);
555 if (default_value != NULL) {
556 return default_value;
557 }
558 assert(alloc->in(AllocateNode::RawDefaultValue) == NULL, "default value may not be null");
559 return _igvn.zerocon(ft);
560 } else if (mem->is_Store()) {
561 Node* n = mem->in(MemNode::ValueIn);
562 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
563 n = bs->step_over_gc_barrier(n);
564 return n;
565 } else if (mem->is_Phi()) {
566 // attempt to produce a Phi reflecting the values on the input paths of the Phi
567 Node_Stack value_phis(a, 8);
568 Node * phi = value_from_mem_phi(mem, ft, ftype, adr_t, alloc, &value_phis, ValueSearchLimit);
569 if (phi != NULL) {
570 return phi;
571 } else {
572 // Kill all new Phis
573 while(value_phis.is_nonempty()) {
574 Node* n = value_phis.node();
575 _igvn.replace_node(n, C->top());
576 value_phis.pop();
577 }
578 }
579 } else if (mem->is_ArrayCopy()) {
580 Node* ctl = mem->in(0);
581 Node* m = mem->in(TypeFunc::Memory);
582 if (sfpt_ctl->is_Proj() && sfpt_ctl->as_Proj()->is_uncommon_trap_proj(Deoptimization::Reason_none)) {
583 // pin the loads in the uncommon trap path
584 ctl = sfpt_ctl;
585 m = sfpt_mem;
586 }
587 return make_arraycopy_load(mem->as_ArrayCopy(), offset, ctl, m, ft, ftype, alloc);
588 }
589 }
590 // Something went wrong.
591 return NULL;
592 }
593
594 // Search the last value stored into the value type's fields.
595 Node* PhaseMacroExpand::value_type_from_mem(Node* mem, Node* ctl, ciValueKlass* vk, const TypeAryPtr* adr_type, int offset, AllocateNode* alloc) {
596 // Subtract the offset of the first field to account for the missing oop header
597 offset -= vk->first_field_offset();
598 // Create a new ValueTypeNode and retrieve the field values from memory
599 ValueTypeNode* vt = ValueTypeNode::make_uninitialized(_igvn, vk)->as_ValueType();
600 for (int i = 0; i < vk->nof_declared_nonstatic_fields(); ++i) {
601 ciType* field_type = vt->field_type(i);
602 int field_offset = offset + vt->field_offset(i);
603 // Each value type field has its own memory slice
604 adr_type = adr_type->with_field_offset(field_offset);
605 Node* value = NULL;
606 if (vt->field_is_flattened(i)) {
607 value = value_type_from_mem(mem, ctl, field_type->as_value_klass(), adr_type, field_offset, alloc);
608 } else {
609 const Type* ft = Type::get_const_type(field_type);
610 BasicType bt = field_type->basic_type();
611 if (UseCompressedOops && !is_java_primitive(bt)) {
612 ft = ft->make_narrowoop();
613 bt = T_NARROWOOP;
614 }
615 value = value_from_mem(mem, ctl, bt, ft, adr_type, alloc);
616 if (value != NULL && ft->isa_narrowoop()) {
617 assert(UseCompressedOops, "unexpected narrow oop");
618 value = transform_later(new DecodeNNode(value, value->get_ptr_type()));
619 }
620 }
621 if (value != NULL) {
622 vt->set_field_value(i, value);
623 } else {
624 // We might have reached the TrackedInitializationLimit
625 return NULL;
626 }
627 }
628 return vt;
629 }
630
631 // Check the possibility of scalar replacement.
632 bool PhaseMacroExpand::can_eliminate_allocation(AllocateNode *alloc, GrowableArray <SafePointNode *>& safepoints) {
633 // Scan the uses of the allocation to check for anything that would
634 // prevent us from eliminating it.
635 NOT_PRODUCT( const char* fail_eliminate = NULL; )
636 DEBUG_ONLY( Node* disq_node = NULL; )
637 bool can_eliminate = true;
638
639 Node* res = alloc->result_cast();
640 const TypeOopPtr* res_type = NULL;
641 if (res == NULL) {
642 // All users were eliminated.
643 } else if (!res->is_CheckCastPP()) {
644 NOT_PRODUCT(fail_eliminate = "Allocation does not have unique CheckCastPP";)
645 can_eliminate = false;
646 } else {
647 res_type = _igvn.type(res)->isa_oopptr();
648 if (res_type == NULL) {
649 NOT_PRODUCT(fail_eliminate = "Neither instance or array allocation";)
650 can_eliminate = false;
666 const TypePtr* addp_type = _igvn.type(use)->is_ptr();
667 int offset = addp_type->offset();
668
669 if (offset == Type::OffsetTop || offset == Type::OffsetBot) {
670 NOT_PRODUCT(fail_eliminate = "Undefined field referrence";)
671 can_eliminate = false;
672 break;
673 }
674 for (DUIterator_Fast kmax, k = use->fast_outs(kmax);
675 k < kmax && can_eliminate; k++) {
676 Node* n = use->fast_out(k);
677 if (!n->is_Store() && n->Opcode() != Op_CastP2X &&
678 SHENANDOAHGC_ONLY((!UseShenandoahGC || !ShenandoahBarrierSetC2::is_shenandoah_wb_pre_call(n)) &&)
679 !(n->is_ArrayCopy() &&
680 n->as_ArrayCopy()->is_clonebasic() &&
681 n->in(ArrayCopyNode::Dest) == use)) {
682 DEBUG_ONLY(disq_node = n;)
683 if (n->is_Load() || n->is_LoadStore()) {
684 NOT_PRODUCT(fail_eliminate = "Field load";)
685 } else {
686 NOT_PRODUCT(fail_eliminate = "Not store field reference";)
687 }
688 can_eliminate = false;
689 }
690 }
691 } else if (use->is_ArrayCopy() &&
692 (use->as_ArrayCopy()->is_arraycopy_validated() ||
693 use->as_ArrayCopy()->is_copyof_validated() ||
694 use->as_ArrayCopy()->is_copyofrange_validated()) &&
695 use->in(ArrayCopyNode::Dest) == res) {
696 // ok to eliminate
697 } else if (use->is_SafePoint()) {
698 SafePointNode* sfpt = use->as_SafePoint();
699 if (sfpt->is_Call() && sfpt->as_Call()->has_non_debug_use(res)) {
700 // Object is passed as argument.
701 DEBUG_ONLY(disq_node = use;)
702 NOT_PRODUCT(fail_eliminate = "Object is passed as argument";)
703 can_eliminate = false;
704 }
705 Node* sfptMem = sfpt->memory();
706 if (sfptMem == NULL || sfptMem->is_top()) {
707 DEBUG_ONLY(disq_node = use;)
708 NOT_PRODUCT(fail_eliminate = "NULL or TOP memory";)
709 can_eliminate = false;
710 } else {
711 safepoints.append_if_missing(sfpt);
712 }
713 } else if (use->is_ValueType() && use->isa_ValueType()->get_oop() == res) {
714 // ok to eliminate
715 } else if (use->is_Store()) {
716 // store to mark work
717 } else if (use->Opcode() != Op_CastP2X) { // CastP2X is used by card mark
718 if (use->is_Phi()) {
719 if (use->outcnt() == 1 && use->unique_out()->Opcode() == Op_Return) {
720 NOT_PRODUCT(fail_eliminate = "Object is return value";)
721 } else {
722 NOT_PRODUCT(fail_eliminate = "Object is referenced by Phi";)
723 }
724 DEBUG_ONLY(disq_node = use;)
725 } else {
726 if (use->Opcode() == Op_Return) {
727 NOT_PRODUCT(fail_eliminate = "Object is return value";)
728 } else {
729 NOT_PRODUCT(fail_eliminate = "Object is referenced by node";)
730 }
731 DEBUG_ONLY(disq_node = use;)
732 }
733 can_eliminate = false;
734 } else {
735 assert(use->Opcode() == Op_CastP2X, "should be");
736 assert(!use->has_out_with(Op_OrL), "should have been removed because oop is never null");
737 }
738 }
739 }
740
741 #ifndef PRODUCT
742 if (PrintEliminateAllocations) {
743 if (can_eliminate) {
744 tty->print("Scalar ");
745 if (res == NULL)
746 alloc->dump();
747 else
748 res->dump();
749 } else if (alloc->_is_scalar_replaceable) {
750 tty->print("NotScalar (%s)", fail_eliminate);
751 if (res == NULL)
752 alloc->dump();
753 else
754 res->dump();
755 #ifdef ASSERT
756 if (disq_node != NULL) {
779 Node* res = alloc->result_cast();
780 assert(res == NULL || res->is_CheckCastPP(), "unexpected AllocateNode result");
781 const TypeOopPtr* res_type = NULL;
782 if (res != NULL) { // Could be NULL when there are no users
783 res_type = _igvn.type(res)->isa_oopptr();
784 }
785
786 if (res != NULL) {
787 klass = res_type->klass();
788 if (res_type->isa_instptr()) {
789 // find the fields of the class which will be needed for safepoint debug information
790 assert(klass->is_instance_klass(), "must be an instance klass.");
791 iklass = klass->as_instance_klass();
792 nfields = iklass->nof_nonstatic_fields();
793 } else {
794 // find the array's elements which will be needed for safepoint debug information
795 nfields = alloc->in(AllocateNode::ALength)->find_int_con(-1);
796 assert(klass->is_array_klass() && nfields >= 0, "must be an array klass.");
797 elem_type = klass->as_array_klass()->element_type();
798 basic_elem_type = elem_type->basic_type();
799 if (elem_type->is_valuetype()) {
800 ciValueKlass* vk = elem_type->as_value_klass();
801 if (!vk->flatten_array()) {
802 assert(basic_elem_type == T_VALUETYPE, "unexpected element basic type");
803 basic_elem_type = T_OBJECT;
804 }
805 }
806 array_base = arrayOopDesc::base_offset_in_bytes(basic_elem_type);
807 element_size = type2aelembytes(basic_elem_type);
808 if (klass->is_value_array_klass()) {
809 // Flattened value type array
810 element_size = klass->as_value_array_klass()->element_byte_size();
811 }
812 }
813 }
814 //
815 // Process the safepoint uses
816 //
817 Unique_Node_List value_worklist;
818 while (safepoints.length() > 0) {
819 SafePointNode* sfpt = safepoints.pop();
820 Node* mem = sfpt->memory();
821 Node* ctl = sfpt->control();
822 assert(sfpt->jvms() != NULL, "missed JVMS");
823 // Fields of scalar objs are referenced only at the end
824 // of regular debuginfo at the last (youngest) JVMS.
825 // Record relative start index.
826 uint first_ind = (sfpt->req() - sfpt->jvms()->scloff());
827 SafePointScalarObjectNode* sobj = new SafePointScalarObjectNode(res_type,
828 #ifdef ASSERT
829 alloc,
830 #endif
831 first_ind, nfields);
832 sobj->init_req(0, C->root());
833 transform_later(sobj);
834
835 // Scan object's fields adding an input to the safepoint for each field.
836 for (int j = 0; j < nfields; j++) {
837 intptr_t offset;
838 ciField* field = NULL;
839 if (iklass != NULL) {
840 field = iklass->nonstatic_field_at(j);
841 offset = field->offset();
842 elem_type = field->type();
843 basic_elem_type = field->layout_type();
844 assert(!field->is_flattened(), "flattened value type fields should not have safepoint uses");
845 } else {
846 offset = array_base + j * (intptr_t)element_size;
847 }
848
849 const Type *field_type;
850 // The next code is taken from Parse::do_get_xxx().
851 if (basic_elem_type == T_OBJECT || basic_elem_type == T_ARRAY) {
852 if (!elem_type->is_loaded()) {
853 field_type = TypeInstPtr::BOTTOM;
854 } else if (field != NULL && field->is_static_constant()) {
855 // This can happen if the constant oop is non-perm.
856 ciObject* con = field->constant_value().as_object();
857 // Do not "join" in the previous type; it doesn't add value,
858 // and may yield a vacuous result if the field is of interface type.
859 field_type = TypeOopPtr::make_from_constant(con)->isa_oopptr();
860 assert(field_type != NULL, "field singleton type must be consistent");
861 } else {
862 field_type = TypeOopPtr::make_from_klass(elem_type->as_klass());
863 }
864 if (UseCompressedOops) {
865 field_type = field_type->make_narrowoop();
866 basic_elem_type = T_NARROWOOP;
867 }
868 } else {
869 field_type = Type::get_const_basic_type(basic_elem_type);
870 }
871
872 Node* field_val = NULL;
873 const TypeOopPtr* field_addr_type = res_type->add_offset(offset)->isa_oopptr();
874 if (klass->is_value_array_klass()) {
875 ciValueKlass* vk = elem_type->as_value_klass();
876 assert(vk->flatten_array(), "must be flattened");
877 field_val = value_type_from_mem(mem, ctl, vk, field_addr_type->isa_aryptr(), 0, alloc);
878 } else {
879 field_val = value_from_mem(mem, ctl, basic_elem_type, field_type, field_addr_type, alloc);
880 }
881 if (field_val == NULL) {
882 // We weren't able to find a value for this field,
883 // give up on eliminating this allocation.
884
885 // Remove any extra entries we added to the safepoint.
886 uint last = sfpt->req() - 1;
887 for (int k = 0; k < j; k++) {
888 sfpt->del_req(last--);
889 }
890 _igvn._worklist.push(sfpt);
891 // rollback processed safepoints
892 while (safepoints_done.length() > 0) {
893 SafePointNode* sfpt_done = safepoints_done.pop();
894 // remove any extra entries we added to the safepoint
895 last = sfpt_done->req() - 1;
896 for (int k = 0; k < nfields; k++) {
897 sfpt_done->del_req(last--);
898 }
899 JVMState *jvms = sfpt_done->jvms();
900 jvms->set_endoff(sfpt_done->req());
926 tty->print("=== At SafePoint node %d can't find value of array element [%d]",
927 sfpt->_idx, j);
928 }
929 tty->print(", which prevents elimination of: ");
930 if (res == NULL)
931 alloc->dump();
932 else
933 res->dump();
934 }
935 #endif
936 return false;
937 }
938 if (UseCompressedOops && field_type->isa_narrowoop()) {
939 // Enable "DecodeN(EncodeP(Allocate)) --> Allocate" transformation
940 // to be able scalar replace the allocation.
941 if (field_val->is_EncodeP()) {
942 field_val = field_val->in(1);
943 } else {
944 field_val = transform_later(new DecodeNNode(field_val, field_val->get_ptr_type()));
945 }
946 } else if (field_val->is_ValueType()) {
947 // Keep track of value types to scalarize them later
948 value_worklist.push(field_val);
949 }
950 sfpt->add_req(field_val);
951 }
952 JVMState *jvms = sfpt->jvms();
953 jvms->set_endoff(sfpt->req());
954 // Now make a pass over the debug information replacing any references
955 // to the allocated object with "sobj"
956 int start = jvms->debug_start();
957 int end = jvms->debug_end();
958 sfpt->replace_edges_in_range(res, sobj, start, end);
959 _igvn._worklist.push(sfpt);
960 safepoints_done.append_if_missing(sfpt); // keep it for rollback
961 }
962 // Scalarize value types that were added to the safepoint
963 for (uint i = 0; i < value_worklist.size(); ++i) {
964 Node* vt = value_worklist.at(i);
965 vt->as_ValueType()->make_scalar_in_safepoints(&_igvn);
966 }
967 return true;
968 }
969
970 static void disconnect_projections(MultiNode* n, PhaseIterGVN& igvn) {
971 Node* ctl_proj = n->proj_out_or_null(TypeFunc::Control);
972 Node* mem_proj = n->proj_out_or_null(TypeFunc::Memory);
973 if (ctl_proj != NULL) {
974 igvn.replace_node(ctl_proj, n->in(0));
975 }
976 if (mem_proj != NULL) {
977 igvn.replace_node(mem_proj, n->in(TypeFunc::Memory));
978 }
979 }
980
981 // Process users of eliminated allocation.
982 void PhaseMacroExpand::process_users_of_allocation(CallNode *alloc) {
983 Node* res = alloc->result_cast();
984 if (res != NULL) {
985 for (DUIterator_Last jmin, j = res->last_outs(jmin); j >= jmin; ) {
986 Node *use = res->last_out(j);
1011 Node* membar_after = ac->proj_out(TypeFunc::Control)->unique_ctrl_out();
1012 disconnect_projections(ac, _igvn);
1013 assert(alloc->in(0)->is_Proj() && alloc->in(0)->in(0)->Opcode() == Op_MemBarCPUOrder, "mem barrier expected before allocation");
1014 Node* membar_before = alloc->in(0)->in(0);
1015 disconnect_projections(membar_before->as_MemBar(), _igvn);
1016 if (membar_after->is_MemBar()) {
1017 disconnect_projections(membar_after->as_MemBar(), _igvn);
1018 }
1019 } else {
1020 eliminate_gc_barrier(n);
1021 }
1022 k -= (oc2 - use->outcnt());
1023 }
1024 _igvn.remove_dead_node(use);
1025 } else if (use->is_ArrayCopy()) {
1026 // Disconnect ArrayCopy node
1027 ArrayCopyNode* ac = use->as_ArrayCopy();
1028 assert(ac->is_arraycopy_validated() ||
1029 ac->is_copyof_validated() ||
1030 ac->is_copyofrange_validated(), "unsupported");
1031 CallProjections* callprojs = ac->extract_projections(true);
1032
1033 _igvn.replace_node(callprojs->fallthrough_ioproj, ac->in(TypeFunc::I_O));
1034 _igvn.replace_node(callprojs->fallthrough_memproj, ac->in(TypeFunc::Memory));
1035 _igvn.replace_node(callprojs->fallthrough_catchproj, ac->in(TypeFunc::Control));
1036
1037 // Set control to top. IGVN will remove the remaining projections
1038 ac->set_req(0, top());
1039 ac->replace_edge(res, top());
1040
1041 // Disconnect src right away: it can help find new
1042 // opportunities for allocation elimination
1043 Node* src = ac->in(ArrayCopyNode::Src);
1044 ac->replace_edge(src, top());
1045 // src can be top at this point if src and dest of the
1046 // arraycopy were the same
1047 if (src->outcnt() == 0 && !src->is_top()) {
1048 _igvn.remove_dead_node(src);
1049 }
1050
1051 _igvn._worklist.push(ac);
1052 } else if (use->is_ValueType()) {
1053 assert(use->isa_ValueType()->get_oop() == res, "unexpected value type use");
1054 _igvn.rehash_node_delayed(use);
1055 use->isa_ValueType()->set_oop(_igvn.zerocon(T_VALUETYPE));
1056 } else if (use->is_Store()) {
1057 _igvn.replace_node(use, use->in(MemNode::Memory));
1058 } else {
1059 eliminate_gc_barrier(use);
1060 }
1061 j -= (oc1 - res->outcnt());
1062 }
1063 assert(res->outcnt() == 0, "all uses of allocated objects must be deleted");
1064 _igvn.remove_dead_node(res);
1065 }
1066
1067 //
1068 // Process other users of allocation's projections
1069 //
1070 if (_resproj != NULL && _resproj->outcnt() != 0) {
1071 // First disconnect stores captured by Initialize node.
1072 // If Initialize node is eliminated first in the following code,
1073 // it will kill such stores and DUIterator_Last will assert.
1074 for (DUIterator_Fast jmax, j = _resproj->fast_outs(jmax); j < jmax; j++) {
1075 Node *use = _resproj->fast_out(j);
1076 if (use->is_AddP()) {
1077 // raw memory addresses used only by the initialization
1189 if (alloc->is_AllocateArray())
1190 tty->print_cr("++++ Eliminated: %d AllocateArray", alloc->_idx);
1191 else
1192 tty->print_cr("++++ Eliminated: %d Allocate", alloc->_idx);
1193 }
1194 #endif
1195
1196 return true;
1197 }
1198
1199 bool PhaseMacroExpand::eliminate_boxing_node(CallStaticJavaNode *boxing) {
1200 // EA should remove all uses of non-escaping boxing node.
1201 if (!C->eliminate_boxing() || boxing->proj_out_or_null(TypeFunc::Parms) != NULL) {
1202 return false;
1203 }
1204
1205 assert(boxing->result_cast() == NULL, "unexpected boxing node result");
1206
1207 extract_call_projections(boxing);
1208
1209 const TypeTuple* r = boxing->tf()->range_sig();
1210 assert(r->cnt() > TypeFunc::Parms, "sanity");
1211 const TypeInstPtr* t = r->field_at(TypeFunc::Parms)->isa_instptr();
1212 assert(t != NULL, "sanity");
1213
1214 CompileLog* log = C->log();
1215 if (log != NULL) {
1216 log->head("eliminate_boxing type='%d'",
1217 log->identify(t->klass()));
1218 JVMState* p = boxing->jvms();
1219 while (p != NULL) {
1220 log->elem("jvms bci='%d' method='%d'", p->bci(), log->identify(p->method()));
1221 p = p->caller();
1222 }
1223 log->tail("eliminate_boxing");
1224 }
1225
1226 process_users_of_allocation(boxing);
1227
1228 #ifndef PRODUCT
1229 if (PrintEliminateAllocations) {
1349 Node *result_phi_i_o = NULL;
1350
1351 // The initial slow comparison is a size check, the comparison
1352 // we want to do is a BoolTest::gt
1353 bool always_slow = false;
1354 int tv = _igvn.find_int_con(initial_slow_test, -1);
1355 if (tv >= 0) {
1356 always_slow = (tv == 1);
1357 initial_slow_test = NULL;
1358 } else {
1359 initial_slow_test = BoolNode::make_predicate(initial_slow_test, &_igvn);
1360 }
1361
1362 if (C->env()->dtrace_alloc_probes() ||
1363 (!UseTLAB && !Universe::heap()->supports_inline_contig_alloc())) {
1364 // Force slow-path allocation
1365 always_slow = true;
1366 initial_slow_test = NULL;
1367 }
1368
1369 Node *slow_region = NULL;
1370 Node *toobig_false = ctrl;
1371
1372 assert (initial_slow_test == NULL || !always_slow, "arguments must be consistent");
1373 // generate the initial test if necessary
1374 if (initial_slow_test != NULL ) {
1375 if (slow_region == NULL) {
1376 slow_region = new RegionNode(1);
1377 }
1378 // Now make the initial failure test. Usually a too-big test but
1379 // might be a TRUE for finalizers or a fancy class check for
1380 // newInstance0.
1381 IfNode* toobig_iff = new IfNode(ctrl, initial_slow_test, PROB_MIN, COUNT_UNKNOWN);
1382 transform_later(toobig_iff);
1383 // Plug the failing-too-big test into the slow-path region
1384 Node* toobig_true = new IfTrueNode(toobig_iff);
1385 transform_later(toobig_true);
1386 slow_region ->add_req(toobig_true);
1387 toobig_false = new IfFalseNode(toobig_iff);
1388 transform_later(toobig_false);
1389 } else { // No initial test, just fall into next case
1390 toobig_false = ctrl;
1391 }
1392
1393 Node *slow_mem = mem; // save the current memory state for slow path
1394 // generate the fast allocation code unless we know that the initial test will always go slow
1395 if (!always_slow) {
1396 // Fast path modifies only raw memory.
1397 if (mem->is_MergeMem()) {
1398 mem = mem->as_MergeMem()->memory_at(Compile::AliasIdxRaw);
1399 }
1400
1401 // allocate the Region and Phi nodes for the result
1402 result_region = new RegionNode(3);
1403 result_phi_rawmem = new PhiNode(result_region, Type::MEMORY, TypeRawPtr::BOTTOM);
1404 result_phi_rawoop = new PhiNode(result_region, TypeRawPtr::BOTTOM);
1405 result_phi_i_o = new PhiNode(result_region, Type::ABIO); // I/O is used for Prefetch
1406
1407 // Grab regular I/O before optional prefetch may change it.
1408 // Slow-path does no I/O so just set it to the original I/O.
1409 result_phi_i_o->init_req(slow_result_path, i_o);
1410
1411 Node* needgc_ctrl = NULL;
1412 // Name successful fast-path variables
1413 Node* fast_oop_ctrl;
1414 Node* fast_oop_rawmem;
1415
1416 intx prefetch_lines = length != NULL ? AllocatePrefetchLines : AllocateInstancePrefetchLines;
1417
1418 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
1419 Node* fast_oop = bs->obj_allocate(this, ctrl, mem, toobig_false, size_in_bytes, i_o, needgc_ctrl,
1420 fast_oop_ctrl, fast_oop_rawmem,
1421 prefetch_lines);
1422
1423 if (slow_region != NULL) {
1424 slow_region->add_req(needgc_ctrl);
1425 // This completes all paths into the slow merge point
1426 transform_later(slow_region);
1427 } else {
1428 // Just fall from the need-GC path straight into the VM call.
1429 slow_region = needgc_ctrl;
1430 }
1431
1432 InitializeNode* init = alloc->initialization();
1433 fast_oop_rawmem = initialize_object(alloc,
1434 fast_oop_ctrl, fast_oop_rawmem, fast_oop,
1435 klass_node, length, size_in_bytes);
1436
1437 // If initialization is performed by an array copy, any required
1438 // MemBarStoreStore was already added. If the object does not
1439 // escape no need for a MemBarStoreStore. If the object does not
1440 // escape in its initializer and memory barrier (MemBarStoreStore or
1441 // stronger) is already added at exit of initializer, also no need
1442 // for a MemBarStoreStore. Otherwise we need a MemBarStoreStore
1443 // so that stores that initialize this object can't be reordered
1444 // with a subsequent store that makes this object accessible by
1445 // other threads.
1446 // Other threads include java threads and JVM internal threads
1447 // (for example concurrent GC threads). Current concurrent GC
1674 } else {
1675 slow_result = _resproj->clone();
1676 transform_later(slow_result);
1677 _igvn.replace_node(_resproj, result_phi_rawoop);
1678 }
1679
1680 // Plug slow-path into result merge point
1681 result_region ->init_req( slow_result_path, ctrl );
1682 result_phi_rawoop->init_req( slow_result_path, slow_result);
1683 result_phi_rawmem->init_req( slow_result_path, _memproj_fallthrough );
1684 transform_later(result_region);
1685 transform_later(result_phi_rawoop);
1686 transform_later(result_phi_rawmem);
1687 transform_later(result_phi_i_o);
1688 // This completes all paths into the result merge point
1689 }
1690
1691
1692 // Helper for PhaseMacroExpand::expand_allocate_common.
1693 // Initializes the newly-allocated storage.
1694 Node* PhaseMacroExpand::initialize_object(AllocateNode* alloc,
1695 Node* control, Node* rawmem, Node* object,
1696 Node* klass_node, Node* length,
1697 Node* size_in_bytes) {
1698 InitializeNode* init = alloc->initialization();
1699 // Store the klass & mark bits
1700 Node* mark_node = alloc->make_ideal_mark(&_igvn, object, control, rawmem, klass_node);
1701 if (!mark_node->is_Con()) {
1702 transform_later(mark_node);
1703 }
1704 rawmem = make_store(control, rawmem, object, oopDesc::mark_offset_in_bytes(), mark_node, TypeX_X->basic_type());
1705
1706 rawmem = make_store(control, rawmem, object, oopDesc::klass_offset_in_bytes(), klass_node, T_METADATA);
1707 int header_size = alloc->minimum_header_size(); // conservatively small
1708
1709 // Array length
1710 if (length != NULL) { // Arrays need length field
1711 rawmem = make_store(control, rawmem, object, arrayOopDesc::length_offset_in_bytes(), length, T_INT);
1712 // conservatively small header size:
1713 header_size = arrayOopDesc::base_offset_in_bytes(T_BYTE);
1714 ciKlass* k = _igvn.type(klass_node)->is_klassptr()->klass();
1715 if (k->is_array_klass()) // we know the exact header size in most cases:
1716 header_size = Klass::layout_helper_header_size(k->layout_helper());
1717 }
1718
1719 // Clear the object body, if necessary.
1720 if (init == NULL) {
1721 // The init has somehow disappeared; be cautious and clear everything.
1722 //
1723 // This can happen if a node is allocated but an uncommon trap occurs
1724 // immediately. In this case, the Initialize gets associated with the
1725 // trap, and may be placed in a different (outer) loop, if the Allocate
1726 // is in a loop. If (this is rare) the inner loop gets unrolled, then
1727 // there can be two Allocates to one Initialize. The answer in all these
1728 // edge cases is safety first. It is always safe to clear immediately
1729 // within an Allocate, and then (maybe or maybe not) clear some more later.
1730 if (!(UseTLAB && ZeroTLAB)) {
1731 rawmem = ClearArrayNode::clear_memory(control, rawmem, object,
1732 alloc->in(AllocateNode::DefaultValue),
1733 alloc->in(AllocateNode::RawDefaultValue),
1734 header_size, size_in_bytes,
1735 &_igvn);
1736 }
1737 } else {
1738 if (!init->is_complete()) {
1739 // Try to win by zeroing only what the init does not store.
1740 // We can also try to do some peephole optimizations,
1741 // such as combining some adjacent subword stores.
1742 rawmem = init->complete_stores(control, rawmem, object,
1743 header_size, size_in_bytes, &_igvn);
1744 }
1745 // We have no more use for this link, since the AllocateNode goes away:
1746 init->set_req(InitializeNode::RawAddress, top());
1747 // (If we keep the link, it just confuses the register allocator,
1748 // who thinks he sees a real use of the address by the membar.)
1749 }
1750
1751 return rawmem;
1752 }
1753
2475 // No exceptions for unlocking
2476 // Capture slow path
2477 // disconnect fall-through projection from call and create a new one
2478 // hook up users of fall-through projection to region
2479 Node *slow_ctrl = _fallthroughproj->clone();
2480 transform_later(slow_ctrl);
2481 _igvn.hash_delete(_fallthroughproj);
2482 _fallthroughproj->disconnect_inputs(NULL, C);
2483 region->init_req(1, slow_ctrl);
2484 // region inputs are now complete
2485 transform_later(region);
2486 _igvn.replace_node(_fallthroughproj, region);
2487
2488 Node *memproj = transform_later(new ProjNode(call, TypeFunc::Memory) );
2489 mem_phi->init_req(1, memproj );
2490 mem_phi->init_req(2, mem);
2491 transform_later(mem_phi);
2492 _igvn.replace_node(_memproj_fallthrough, mem_phi);
2493 }
2494
2495 // A value type might be returned from the call but we don't know its
2496 // type. Either we get a buffered value (and nothing needs to be done)
2497 // or one of the values being returned is the klass of the value type
2498 // and we need to allocate a value type instance of that type and
2499 // initialize it with other values being returned. In that case, we
2500 // first try a fast path allocation and initialize the value with the
2501 // value klass's pack handler or we fall back to a runtime call.
2502 void PhaseMacroExpand::expand_mh_intrinsic_return(CallStaticJavaNode* call) {
2503 assert(call->method()->is_method_handle_intrinsic(), "must be a method handle intrinsic call");
2504 Node* ret = call->proj_out_or_null(TypeFunc::Parms);
2505 if (ret == NULL) {
2506 return;
2507 }
2508 const TypeFunc* tf = call->_tf;
2509 const TypeTuple* domain = OptoRuntime::store_value_type_fields_Type()->domain_cc();
2510 const TypeFunc* new_tf = TypeFunc::make(tf->domain_sig(), tf->domain_cc(), tf->range_sig(), domain);
2511 call->_tf = new_tf;
2512 // Make sure the change of type is applied before projections are processed by igvn
2513 _igvn.set_type(call, call->Value(&_igvn));
2514 _igvn.set_type(ret, ret->Value(&_igvn));
2515
2516 // Before any new projection is added:
2517 CallProjections* projs = call->extract_projections(true, true);
2518
2519 Node* ctl = new Node(1);
2520 Node* mem = new Node(1);
2521 Node* io = new Node(1);
2522 Node* ex_ctl = new Node(1);
2523 Node* ex_mem = new Node(1);
2524 Node* ex_io = new Node(1);
2525 Node* res = new Node(1);
2526
2527 Node* cast = transform_later(new CastP2XNode(ctl, res));
2528 Node* mask = MakeConX(0x1);
2529 Node* masked = transform_later(new AndXNode(cast, mask));
2530 Node* cmp = transform_later(new CmpXNode(masked, mask));
2531 Node* bol = transform_later(new BoolNode(cmp, BoolTest::eq));
2532 IfNode* allocation_iff = new IfNode(ctl, bol, PROB_MAX, COUNT_UNKNOWN);
2533 transform_later(allocation_iff);
2534 Node* allocation_ctl = transform_later(new IfTrueNode(allocation_iff));
2535 Node* no_allocation_ctl = transform_later(new IfFalseNode(allocation_iff));
2536
2537 Node* no_allocation_res = transform_later(new CheckCastPPNode(no_allocation_ctl, res, TypeInstPtr::BOTTOM));
2538
2539 Node* mask2 = MakeConX(-2);
2540 Node* masked2 = transform_later(new AndXNode(cast, mask2));
2541 Node* rawklassptr = transform_later(new CastX2PNode(masked2));
2542 Node* klass_node = transform_later(new CheckCastPPNode(allocation_ctl, rawklassptr, TypeKlassPtr::OBJECT_OR_NULL));
2543
2544 Node* slowpath_bol = NULL;
2545 Node* top_adr = NULL;
2546 Node* old_top = NULL;
2547 Node* new_top = NULL;
2548 if (UseTLAB) {
2549 Node* end_adr = NULL;
2550 set_eden_pointers(top_adr, end_adr);
2551 Node* end = make_load(ctl, mem, end_adr, 0, TypeRawPtr::BOTTOM, T_ADDRESS);
2552 old_top = new LoadPNode(ctl, mem, top_adr, TypeRawPtr::BOTTOM, TypeRawPtr::BOTTOM, MemNode::unordered);
2553 transform_later(old_top);
2554 Node* layout_val = make_load(NULL, mem, klass_node, in_bytes(Klass::layout_helper_offset()), TypeInt::INT, T_INT);
2555 Node* size_in_bytes = ConvI2X(layout_val);
2556 new_top = new AddPNode(top(), old_top, size_in_bytes);
2557 transform_later(new_top);
2558 Node* slowpath_cmp = new CmpPNode(new_top, end);
2559 transform_later(slowpath_cmp);
2560 slowpath_bol = new BoolNode(slowpath_cmp, BoolTest::ge);
2561 transform_later(slowpath_bol);
2562 } else {
2563 slowpath_bol = intcon(1);
2564 top_adr = top();
2565 old_top = top();
2566 new_top = top();
2567 }
2568 IfNode* slowpath_iff = new IfNode(allocation_ctl, slowpath_bol, PROB_UNLIKELY_MAG(4), COUNT_UNKNOWN);
2569 transform_later(slowpath_iff);
2570
2571 Node* slowpath_true = new IfTrueNode(slowpath_iff);
2572 transform_later(slowpath_true);
2573
2574 CallStaticJavaNode* slow_call = new CallStaticJavaNode(OptoRuntime::store_value_type_fields_Type(),
2575 StubRoutines::store_value_type_fields_to_buf(),
2576 "store_value_type_fields",
2577 call->jvms()->bci(),
2578 TypePtr::BOTTOM);
2579 slow_call->init_req(TypeFunc::Control, slowpath_true);
2580 slow_call->init_req(TypeFunc::Memory, mem);
2581 slow_call->init_req(TypeFunc::I_O, io);
2582 slow_call->init_req(TypeFunc::FramePtr, call->in(TypeFunc::FramePtr));
2583 slow_call->init_req(TypeFunc::ReturnAdr, call->in(TypeFunc::ReturnAdr));
2584 slow_call->init_req(TypeFunc::Parms, res);
2585
2586 Node* slow_ctl = transform_later(new ProjNode(slow_call, TypeFunc::Control));
2587 Node* slow_mem = transform_later(new ProjNode(slow_call, TypeFunc::Memory));
2588 Node* slow_io = transform_later(new ProjNode(slow_call, TypeFunc::I_O));
2589 Node* slow_res = transform_later(new ProjNode(slow_call, TypeFunc::Parms));
2590 Node* slow_catc = transform_later(new CatchNode(slow_ctl, slow_io, 2));
2591 Node* slow_norm = transform_later(new CatchProjNode(slow_catc, CatchProjNode::fall_through_index, CatchProjNode::no_handler_bci));
2592 Node* slow_excp = transform_later(new CatchProjNode(slow_catc, CatchProjNode::catch_all_index, CatchProjNode::no_handler_bci));
2593
2594 Node* ex_r = new RegionNode(3);
2595 Node* ex_mem_phi = new PhiNode(ex_r, Type::MEMORY, TypePtr::BOTTOM);
2596 Node* ex_io_phi = new PhiNode(ex_r, Type::ABIO);
2597 ex_r->init_req(1, slow_excp);
2598 ex_mem_phi->init_req(1, slow_mem);
2599 ex_io_phi->init_req(1, slow_io);
2600 ex_r->init_req(2, ex_ctl);
2601 ex_mem_phi->init_req(2, ex_mem);
2602 ex_io_phi->init_req(2, ex_io);
2603
2604 transform_later(ex_r);
2605 transform_later(ex_mem_phi);
2606 transform_later(ex_io_phi);
2607
2608 Node* slowpath_false = new IfFalseNode(slowpath_iff);
2609 transform_later(slowpath_false);
2610 Node* rawmem = new StorePNode(slowpath_false, mem, top_adr, TypeRawPtr::BOTTOM, new_top, MemNode::unordered);
2611 transform_later(rawmem);
2612 Node* mark_node = makecon(TypeRawPtr::make((address)markOopDesc::always_locked_prototype()));
2613 rawmem = make_store(slowpath_false, rawmem, old_top, oopDesc::mark_offset_in_bytes(), mark_node, T_ADDRESS);
2614 rawmem = make_store(slowpath_false, rawmem, old_top, oopDesc::klass_offset_in_bytes(), klass_node, T_METADATA);
2615 if (UseCompressedClassPointers) {
2616 rawmem = make_store(slowpath_false, rawmem, old_top, oopDesc::klass_gap_offset_in_bytes(), intcon(0), T_INT);
2617 }
2618 Node* fixed_block = make_load(slowpath_false, rawmem, klass_node, in_bytes(InstanceKlass::adr_valueklass_fixed_block_offset()), TypeRawPtr::BOTTOM, T_ADDRESS);
2619 Node* pack_handler = make_load(slowpath_false, rawmem, fixed_block, in_bytes(ValueKlass::pack_handler_offset()), TypeRawPtr::BOTTOM, T_ADDRESS);
2620
2621 CallLeafNoFPNode* handler_call = new CallLeafNoFPNode(OptoRuntime::pack_value_type_Type(),
2622 NULL,
2623 "pack handler",
2624 TypeRawPtr::BOTTOM);
2625 handler_call->init_req(TypeFunc::Control, slowpath_false);
2626 handler_call->init_req(TypeFunc::Memory, rawmem);
2627 handler_call->init_req(TypeFunc::I_O, top());
2628 handler_call->init_req(TypeFunc::FramePtr, call->in(TypeFunc::FramePtr));
2629 handler_call->init_req(TypeFunc::ReturnAdr, top());
2630 handler_call->init_req(TypeFunc::Parms, pack_handler);
2631 handler_call->init_req(TypeFunc::Parms+1, old_top);
2632
2633 // We don't know how many values are returned. This assumes the
2634 // worst case, that all available registers are used.
2635 for (uint i = TypeFunc::Parms+1; i < domain->cnt(); i++) {
2636 if (domain->field_at(i) == Type::HALF) {
2637 slow_call->init_req(i, top());
2638 handler_call->init_req(i+1, top());
2639 continue;
2640 }
2641 Node* proj = transform_later(new ProjNode(call, i));
2642 slow_call->init_req(i, proj);
2643 handler_call->init_req(i+1, proj);
2644 }
2645
2646 // We can safepoint at that new call
2647 copy_call_debug_info(call, slow_call);
2648 transform_later(slow_call);
2649 transform_later(handler_call);
2650
2651 Node* handler_ctl = transform_later(new ProjNode(handler_call, TypeFunc::Control));
2652 rawmem = transform_later(new ProjNode(handler_call, TypeFunc::Memory));
2653 Node* slowpath_false_res = transform_later(new ProjNode(handler_call, TypeFunc::Parms));
2654
2655 MergeMemNode* slowpath_false_mem = MergeMemNode::make(mem);
2656 slowpath_false_mem->set_memory_at(Compile::AliasIdxRaw, rawmem);
2657 transform_later(slowpath_false_mem);
2658
2659 Node* r = new RegionNode(4);
2660 Node* mem_phi = new PhiNode(r, Type::MEMORY, TypePtr::BOTTOM);
2661 Node* io_phi = new PhiNode(r, Type::ABIO);
2662 Node* res_phi = new PhiNode(r, TypeInstPtr::BOTTOM);
2663
2664 r->init_req(1, no_allocation_ctl);
2665 mem_phi->init_req(1, mem);
2666 io_phi->init_req(1, io);
2667 res_phi->init_req(1, no_allocation_res);
2668 r->init_req(2, slow_norm);
2669 mem_phi->init_req(2, slow_mem);
2670 io_phi->init_req(2, slow_io);
2671 res_phi->init_req(2, slow_res);
2672 r->init_req(3, handler_ctl);
2673 mem_phi->init_req(3, slowpath_false_mem);
2674 io_phi->init_req(3, io);
2675 res_phi->init_req(3, slowpath_false_res);
2676
2677 transform_later(r);
2678 transform_later(mem_phi);
2679 transform_later(io_phi);
2680 transform_later(res_phi);
2681
2682 assert(projs->nb_resproj == 1, "unexpected number of results");
2683 _igvn.replace_in_uses(projs->fallthrough_catchproj, r);
2684 _igvn.replace_in_uses(projs->fallthrough_memproj, mem_phi);
2685 _igvn.replace_in_uses(projs->fallthrough_ioproj, io_phi);
2686 _igvn.replace_in_uses(projs->resproj[0], res_phi);
2687 _igvn.replace_in_uses(projs->catchall_catchproj, ex_r);
2688 _igvn.replace_in_uses(projs->catchall_memproj, ex_mem_phi);
2689 _igvn.replace_in_uses(projs->catchall_ioproj, ex_io_phi);
2690
2691 _igvn.replace_node(ctl, projs->fallthrough_catchproj);
2692 _igvn.replace_node(mem, projs->fallthrough_memproj);
2693 _igvn.replace_node(io, projs->fallthrough_ioproj);
2694 _igvn.replace_node(res, projs->resproj[0]);
2695 _igvn.replace_node(ex_ctl, projs->catchall_catchproj);
2696 _igvn.replace_node(ex_mem, projs->catchall_memproj);
2697 _igvn.replace_node(ex_io, projs->catchall_ioproj);
2698 }
2699
2700 //---------------------------eliminate_macro_nodes----------------------
2701 // Eliminate scalar replaced allocations and associated locks.
2702 void PhaseMacroExpand::eliminate_macro_nodes() {
2703 if (C->macro_count() == 0)
2704 return;
2705
2706 // First, attempt to eliminate locks
2707 int cnt = C->macro_count();
2708 for (int i=0; i < cnt; i++) {
2709 Node *n = C->macro_node(i);
2710 if (n->is_AbstractLock()) { // Lock and Unlock nodes
2711 // Before elimination mark all associated (same box and obj)
2712 // lock and unlock nodes.
2713 mark_eliminated_locking_nodes(n->as_AbstractLock());
2714 }
2715 }
2716 bool progress = true;
2717 while (progress) {
2718 progress = false;
2719 for (int i = C->macro_count(); i > 0; i--) {
2724 success = eliminate_locking_node(n->as_AbstractLock());
2725 }
2726 assert(success == (C->macro_count() < old_macro_count), "elimination reduces macro count");
2727 progress = progress || success;
2728 }
2729 }
2730 // Next, attempt to eliminate allocations
2731 _has_locks = false;
2732 progress = true;
2733 while (progress) {
2734 progress = false;
2735 for (int i = C->macro_count(); i > 0; i--) {
2736 Node * n = C->macro_node(i-1);
2737 bool success = false;
2738 debug_only(int old_macro_count = C->macro_count(););
2739 switch (n->class_id()) {
2740 case Node::Class_Allocate:
2741 case Node::Class_AllocateArray:
2742 success = eliminate_allocate_node(n->as_Allocate());
2743 break;
2744 case Node::Class_CallStaticJava: {
2745 CallStaticJavaNode* call = n->as_CallStaticJava();
2746 if (!call->method()->is_method_handle_intrinsic()) {
2747 success = eliminate_boxing_node(n->as_CallStaticJava());
2748 }
2749 break;
2750 }
2751 case Node::Class_Lock:
2752 case Node::Class_Unlock:
2753 assert(!n->as_AbstractLock()->is_eliminated(), "sanity");
2754 _has_locks = true;
2755 break;
2756 case Node::Class_ArrayCopy:
2757 break;
2758 case Node::Class_OuterStripMinedLoop:
2759 break;
2760 default:
2761 assert(n->Opcode() == Op_LoopLimit ||
2762 n->Opcode() == Op_Opaque1 ||
2763 n->Opcode() == Op_Opaque2 ||
2764 n->Opcode() == Op_Opaque3 ||
2765 BarrierSet::barrier_set()->barrier_set_c2()->is_gc_barrier_node(n),
2766 "unknown node type in macro list");
2767 }
2768 assert(success == (C->macro_count() < old_macro_count), "elimination reduces macro count");
2769 progress = progress || success;
2770 }
2780 // Make sure expansion will not cause node limit to be exceeded.
2781 // Worst case is a macro node gets expanded into about 200 nodes.
2782 // Allow 50% more for optimization.
2783 if (C->check_node_count(C->macro_count() * 300, "out of nodes before macro expansion" ) )
2784 return true;
2785
2786 // Eliminate Opaque and LoopLimit nodes. Do it after all loop optimizations.
2787 bool progress = true;
2788 while (progress) {
2789 progress = false;
2790 for (int i = C->macro_count(); i > 0; i--) {
2791 Node * n = C->macro_node(i-1);
2792 bool success = false;
2793 debug_only(int old_macro_count = C->macro_count(););
2794 if (n->Opcode() == Op_LoopLimit) {
2795 // Remove it from macro list and put on IGVN worklist to optimize.
2796 C->remove_macro_node(n);
2797 _igvn._worklist.push(n);
2798 success = true;
2799 } else if (n->Opcode() == Op_CallStaticJava) {
2800 CallStaticJavaNode* call = n->as_CallStaticJava();
2801 if (!call->method()->is_method_handle_intrinsic()) {
2802 // Remove it from macro list and put on IGVN worklist to optimize.
2803 C->remove_macro_node(n);
2804 _igvn._worklist.push(n);
2805 success = true;
2806 }
2807 } else if (n->Opcode() == Op_Opaque1 || n->Opcode() == Op_Opaque2) {
2808 _igvn.replace_node(n, n->in(1));
2809 success = true;
2810 #if INCLUDE_RTM_OPT
2811 } else if ((n->Opcode() == Op_Opaque3) && ((Opaque3Node*)n)->rtm_opt()) {
2812 assert(C->profile_rtm(), "should be used only in rtm deoptimization code");
2813 assert((n->outcnt() == 1) && n->unique_out()->is_Cmp(), "");
2814 Node* cmp = n->unique_out();
2815 #ifdef ASSERT
2816 // Validate graph.
2817 assert((cmp->outcnt() == 1) && cmp->unique_out()->is_Bool(), "");
2818 BoolNode* bol = cmp->unique_out()->as_Bool();
2819 assert((bol->outcnt() == 1) && bol->unique_out()->is_If() &&
2820 (bol->_test._test == BoolTest::ne), "");
2821 IfNode* ifn = bol->unique_out()->as_If();
2822 assert((ifn->outcnt() == 2) &&
2823 ifn->proj_out(1)->is_uncommon_trap_proj(Deoptimization::Reason_rtm_state_change) != NULL, "");
2824 #endif
2825 Node* repl = n->in(1);
2826 if (!_has_locks) {
2866 int macro_count = C->macro_count();
2867 Node * n = C->macro_node(macro_count-1);
2868 assert(n->is_macro(), "only macro nodes expected here");
2869 if (_igvn.type(n) == Type::TOP || (n->in(0) != NULL && n->in(0)->is_top())) {
2870 // node is unreachable, so don't try to expand it
2871 C->remove_macro_node(n);
2872 continue;
2873 }
2874 switch (n->class_id()) {
2875 case Node::Class_Allocate:
2876 expand_allocate(n->as_Allocate());
2877 break;
2878 case Node::Class_AllocateArray:
2879 expand_allocate_array(n->as_AllocateArray());
2880 break;
2881 case Node::Class_Lock:
2882 expand_lock_node(n->as_Lock());
2883 break;
2884 case Node::Class_Unlock:
2885 expand_unlock_node(n->as_Unlock());
2886 break;
2887 case Node::Class_CallStaticJava:
2888 expand_mh_intrinsic_return(n->as_CallStaticJava());
2889 C->remove_macro_node(n);
2890 break;
2891 default:
2892 assert(false, "unknown node type in macro list");
2893 }
2894 assert(C->macro_count() < macro_count, "must have deleted a node from macro list");
2895 if (C->failing()) return true;
2896 }
2897
2898 _igvn.set_delay_transform(false);
2899 _igvn.optimize();
2900 if (C->failing()) return true;
2901 return false;
2902 }
|