30 #include "opto/arraycopynode.hpp"
31 #include "opto/callnode.hpp"
32 #include "opto/castnode.hpp"
33 #include "opto/cfgnode.hpp"
34 #include "opto/compile.hpp"
35 #include "opto/convertnode.hpp"
36 #include "opto/graphKit.hpp"
37 #include "opto/locknode.hpp"
38 #include "opto/loopnode.hpp"
39 #include "opto/macro.hpp"
40 #include "opto/memnode.hpp"
41 #include "opto/narrowptrnode.hpp"
42 #include "opto/node.hpp"
43 #include "opto/opaquenode.hpp"
44 #include "opto/phaseX.hpp"
45 #include "opto/rootnode.hpp"
46 #include "opto/runtime.hpp"
47 #include "opto/subnode.hpp"
48 #include "opto/type.hpp"
49 #include "runtime/sharedRuntime.hpp"
50 #if INCLUDE_G1GC
51 #include "gc/g1/g1ThreadLocalData.hpp"
52 #endif // INCLUDE_G1GC
53
54
55 //
56 // Replace any references to "oldref" in inputs to "use" with "newref".
57 // Returns the number of replacements made.
58 //
59 int PhaseMacroExpand::replace_input(Node *use, Node *oldref, Node *newref) {
60 int nreplacements = 0;
61 uint req = use->req();
62 for (uint j = 0; j < use->len(); j++) {
63 Node *uin = use->in(j);
64 if (uin == oldref) {
65 if (j < req)
66 use->set_req(j, newref);
67 else
68 use->set_prec(j, newref);
69 nreplacements++;
70 } else if (j >= req && uin == NULL) {
71 break;
72 }
441 for (uint j = 1; j < length; j++) {
442 Node *in = mem->in(j);
443 if (in == NULL || in->is_top()) {
444 values.at_put(j, in);
445 } else {
446 Node *val = scan_mem_chain(in, alias_idx, offset, start_mem, alloc, &_igvn);
447 if (val == start_mem || val == alloc_mem) {
448 // hit a sentinel, return appropriate 0 value
449 values.at_put(j, _igvn.zerocon(ft));
450 continue;
451 }
452 if (val->is_Initialize()) {
453 val = val->as_Initialize()->find_captured_store(offset, type2aelembytes(ft), &_igvn);
454 }
455 if (val == NULL) {
456 return NULL; // can't find a value on this path
457 }
458 if (val == mem) {
459 values.at_put(j, mem);
460 } else if (val->is_Store()) {
461 values.at_put(j, val->in(MemNode::ValueIn));
462 } else if(val->is_Proj() && val->in(0) == alloc) {
463 values.at_put(j, _igvn.zerocon(ft));
464 } else if (val->is_Phi()) {
465 val = value_from_mem_phi(val, ft, phi_type, adr_t, alloc, value_phis, level-1);
466 if (val == NULL) {
467 return NULL;
468 }
469 values.at_put(j, val);
470 } else if (val->Opcode() == Op_SCMemProj) {
471 assert(val->in(0)->is_LoadStore() ||
472 val->in(0)->Opcode() == Op_EncodeISOArray ||
473 val->in(0)->Opcode() == Op_StrCompressedCopy, "sanity");
474 assert(false, "Object is not scalar replaceable if a LoadStore node accesses its field");
475 return NULL;
476 } else if (val->is_ArrayCopy()) {
477 Node* res = make_arraycopy_load(val->as_ArrayCopy(), offset, val->in(0), val->in(TypeFunc::Memory), ft, phi_type, alloc);
478 if (res == NULL) {
479 return NULL;
480 }
481 values.at_put(j, res);
552 unique_input = top;
553 break;
554 }
555 }
556 if (unique_input != NULL && unique_input != top) {
557 mem = unique_input;
558 } else {
559 done = true;
560 }
561 } else if (mem->is_ArrayCopy()) {
562 done = true;
563 } else {
564 assert(false, "unexpected node");
565 }
566 }
567 if (mem != NULL) {
568 if (mem == start_mem || mem == alloc_mem) {
569 // hit a sentinel, return appropriate 0 value
570 return _igvn.zerocon(ft);
571 } else if (mem->is_Store()) {
572 return mem->in(MemNode::ValueIn);
573 } else if (mem->is_Phi()) {
574 // attempt to produce a Phi reflecting the values on the input paths of the Phi
575 Node_Stack value_phis(a, 8);
576 Node * phi = value_from_mem_phi(mem, ft, ftype, adr_t, alloc, &value_phis, ValueSearchLimit);
577 if (phi != NULL) {
578 return phi;
579 } else {
580 // Kill all new Phis
581 while(value_phis.is_nonempty()) {
582 Node* n = value_phis.node();
583 _igvn.replace_node(n, C->top());
584 value_phis.pop();
585 }
586 }
587 } else if (mem->is_ArrayCopy()) {
588 Node* ctl = mem->in(0);
589 Node* m = mem->in(TypeFunc::Memory);
590 if (sfpt_ctl->is_Proj() && sfpt_ctl->as_Proj()->is_uncommon_trap_proj(Deoptimization::Reason_none)) {
591 // pin the loads in the uncommon trap path
592 ctl = sfpt_ctl;
629 }
630
631 if (can_eliminate && res != NULL) {
632 for (DUIterator_Fast jmax, j = res->fast_outs(jmax);
633 j < jmax && can_eliminate; j++) {
634 Node* use = res->fast_out(j);
635
636 if (use->is_AddP()) {
637 const TypePtr* addp_type = _igvn.type(use)->is_ptr();
638 int offset = addp_type->offset();
639
640 if (offset == Type::OffsetTop || offset == Type::OffsetBot) {
641 NOT_PRODUCT(fail_eliminate = "Undefined field referrence";)
642 can_eliminate = false;
643 break;
644 }
645 for (DUIterator_Fast kmax, k = use->fast_outs(kmax);
646 k < kmax && can_eliminate; k++) {
647 Node* n = use->fast_out(k);
648 if (!n->is_Store() && n->Opcode() != Op_CastP2X &&
649 !(n->is_ArrayCopy() &&
650 n->as_ArrayCopy()->is_clonebasic() &&
651 n->in(ArrayCopyNode::Dest) == use)) {
652 DEBUG_ONLY(disq_node = n;)
653 if (n->is_Load() || n->is_LoadStore()) {
654 NOT_PRODUCT(fail_eliminate = "Field load";)
655 } else {
656 NOT_PRODUCT(fail_eliminate = "Not store field referrence";)
657 }
658 can_eliminate = false;
659 }
660 }
661 } else if (use->is_ArrayCopy() &&
662 (use->as_ArrayCopy()->is_arraycopy_validated() ||
663 use->as_ArrayCopy()->is_copyof_validated() ||
664 use->as_ArrayCopy()->is_copyofrange_validated()) &&
665 use->in(ArrayCopyNode::Dest) == res) {
666 // ok to eliminate
667 } else if (use->is_SafePoint()) {
668 SafePointNode* sfpt = use->as_SafePoint();
|
30 #include "opto/arraycopynode.hpp"
31 #include "opto/callnode.hpp"
32 #include "opto/castnode.hpp"
33 #include "opto/cfgnode.hpp"
34 #include "opto/compile.hpp"
35 #include "opto/convertnode.hpp"
36 #include "opto/graphKit.hpp"
37 #include "opto/locknode.hpp"
38 #include "opto/loopnode.hpp"
39 #include "opto/macro.hpp"
40 #include "opto/memnode.hpp"
41 #include "opto/narrowptrnode.hpp"
42 #include "opto/node.hpp"
43 #include "opto/opaquenode.hpp"
44 #include "opto/phaseX.hpp"
45 #include "opto/rootnode.hpp"
46 #include "opto/runtime.hpp"
47 #include "opto/subnode.hpp"
48 #include "opto/type.hpp"
49 #include "runtime/sharedRuntime.hpp"
50 #include "utilities/macros.hpp"
51 #if INCLUDE_G1GC
52 #include "gc/g1/g1ThreadLocalData.hpp"
53 #endif // INCLUDE_G1GC
54 #if INCLUDE_SHENANDOAHGC
55 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
56 #endif
57
58
59 //
60 // Replace any references to "oldref" in inputs to "use" with "newref".
61 // Returns the number of replacements made.
62 //
63 int PhaseMacroExpand::replace_input(Node *use, Node *oldref, Node *newref) {
64 int nreplacements = 0;
65 uint req = use->req();
66 for (uint j = 0; j < use->len(); j++) {
67 Node *uin = use->in(j);
68 if (uin == oldref) {
69 if (j < req)
70 use->set_req(j, newref);
71 else
72 use->set_prec(j, newref);
73 nreplacements++;
74 } else if (j >= req && uin == NULL) {
75 break;
76 }
445 for (uint j = 1; j < length; j++) {
446 Node *in = mem->in(j);
447 if (in == NULL || in->is_top()) {
448 values.at_put(j, in);
449 } else {
450 Node *val = scan_mem_chain(in, alias_idx, offset, start_mem, alloc, &_igvn);
451 if (val == start_mem || val == alloc_mem) {
452 // hit a sentinel, return appropriate 0 value
453 values.at_put(j, _igvn.zerocon(ft));
454 continue;
455 }
456 if (val->is_Initialize()) {
457 val = val->as_Initialize()->find_captured_store(offset, type2aelembytes(ft), &_igvn);
458 }
459 if (val == NULL) {
460 return NULL; // can't find a value on this path
461 }
462 if (val == mem) {
463 values.at_put(j, mem);
464 } else if (val->is_Store()) {
465 Node* n = val->in(MemNode::ValueIn);
466 #if INCLUDE_SHENANDOAHGC
467 if (UseShenandoahGC) {
468 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
469 n = bs->step_over_gc_barrier(n);
470 }
471 #endif
472 values.at_put(j, n);
473 } else if(val->is_Proj() && val->in(0) == alloc) {
474 values.at_put(j, _igvn.zerocon(ft));
475 } else if (val->is_Phi()) {
476 val = value_from_mem_phi(val, ft, phi_type, adr_t, alloc, value_phis, level-1);
477 if (val == NULL) {
478 return NULL;
479 }
480 values.at_put(j, val);
481 } else if (val->Opcode() == Op_SCMemProj) {
482 assert(val->in(0)->is_LoadStore() ||
483 val->in(0)->Opcode() == Op_EncodeISOArray ||
484 val->in(0)->Opcode() == Op_StrCompressedCopy, "sanity");
485 assert(false, "Object is not scalar replaceable if a LoadStore node accesses its field");
486 return NULL;
487 } else if (val->is_ArrayCopy()) {
488 Node* res = make_arraycopy_load(val->as_ArrayCopy(), offset, val->in(0), val->in(TypeFunc::Memory), ft, phi_type, alloc);
489 if (res == NULL) {
490 return NULL;
491 }
492 values.at_put(j, res);
563 unique_input = top;
564 break;
565 }
566 }
567 if (unique_input != NULL && unique_input != top) {
568 mem = unique_input;
569 } else {
570 done = true;
571 }
572 } else if (mem->is_ArrayCopy()) {
573 done = true;
574 } else {
575 assert(false, "unexpected node");
576 }
577 }
578 if (mem != NULL) {
579 if (mem == start_mem || mem == alloc_mem) {
580 // hit a sentinel, return appropriate 0 value
581 return _igvn.zerocon(ft);
582 } else if (mem->is_Store()) {
583 Node* n = mem->in(MemNode::ValueIn);
584 #if INCLUDE_SHENANDOAHGC
585 if (UseShenandoahGC) {
586 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
587 n = bs->step_over_gc_barrier(n);
588 }
589 #endif
590 return n;
591 } else if (mem->is_Phi()) {
592 // attempt to produce a Phi reflecting the values on the input paths of the Phi
593 Node_Stack value_phis(a, 8);
594 Node * phi = value_from_mem_phi(mem, ft, ftype, adr_t, alloc, &value_phis, ValueSearchLimit);
595 if (phi != NULL) {
596 return phi;
597 } else {
598 // Kill all new Phis
599 while(value_phis.is_nonempty()) {
600 Node* n = value_phis.node();
601 _igvn.replace_node(n, C->top());
602 value_phis.pop();
603 }
604 }
605 } else if (mem->is_ArrayCopy()) {
606 Node* ctl = mem->in(0);
607 Node* m = mem->in(TypeFunc::Memory);
608 if (sfpt_ctl->is_Proj() && sfpt_ctl->as_Proj()->is_uncommon_trap_proj(Deoptimization::Reason_none)) {
609 // pin the loads in the uncommon trap path
610 ctl = sfpt_ctl;
647 }
648
649 if (can_eliminate && res != NULL) {
650 for (DUIterator_Fast jmax, j = res->fast_outs(jmax);
651 j < jmax && can_eliminate; j++) {
652 Node* use = res->fast_out(j);
653
654 if (use->is_AddP()) {
655 const TypePtr* addp_type = _igvn.type(use)->is_ptr();
656 int offset = addp_type->offset();
657
658 if (offset == Type::OffsetTop || offset == Type::OffsetBot) {
659 NOT_PRODUCT(fail_eliminate = "Undefined field referrence";)
660 can_eliminate = false;
661 break;
662 }
663 for (DUIterator_Fast kmax, k = use->fast_outs(kmax);
664 k < kmax && can_eliminate; k++) {
665 Node* n = use->fast_out(k);
666 if (!n->is_Store() && n->Opcode() != Op_CastP2X &&
667 SHENANDOAHGC_ONLY((!UseShenandoahGC || !ShenandoahBarrierSetC2::is_shenandoah_wb_pre_call(n)) &&)
668 !(n->is_ArrayCopy() &&
669 n->as_ArrayCopy()->is_clonebasic() &&
670 n->in(ArrayCopyNode::Dest) == use)) {
671 DEBUG_ONLY(disq_node = n;)
672 if (n->is_Load() || n->is_LoadStore()) {
673 NOT_PRODUCT(fail_eliminate = "Field load";)
674 } else {
675 NOT_PRODUCT(fail_eliminate = "Not store field referrence";)
676 }
677 can_eliminate = false;
678 }
679 }
680 } else if (use->is_ArrayCopy() &&
681 (use->as_ArrayCopy()->is_arraycopy_validated() ||
682 use->as_ArrayCopy()->is_copyof_validated() ||
683 use->as_ArrayCopy()->is_copyofrange_validated()) &&
684 use->in(ArrayCopyNode::Dest) == res) {
685 // ok to eliminate
686 } else if (use->is_SafePoint()) {
687 SafePointNode* sfpt = use->as_SafePoint();
|