25 #include "precompiled.hpp"
26 #include "compiler/compileLog.hpp"
27 #include "libadt/vectset.hpp"
28 #include "opto/addnode.hpp"
29 #include "opto/arraycopynode.hpp"
30 #include "opto/callnode.hpp"
31 #include "opto/castnode.hpp"
32 #include "opto/cfgnode.hpp"
33 #include "opto/compile.hpp"
34 #include "opto/convertnode.hpp"
35 #include "opto/locknode.hpp"
36 #include "opto/loopnode.hpp"
37 #include "opto/macro.hpp"
38 #include "opto/memnode.hpp"
39 #include "opto/narrowptrnode.hpp"
40 #include "opto/node.hpp"
41 #include "opto/opaquenode.hpp"
42 #include "opto/phaseX.hpp"
43 #include "opto/rootnode.hpp"
44 #include "opto/runtime.hpp"
45 #include "opto/subnode.hpp"
46 #include "opto/type.hpp"
47 #include "runtime/sharedRuntime.hpp"
48
49
50 //
51 // Replace any references to "oldref" in inputs to "use" with "newref".
52 // Returns the number of replacements made.
53 //
54 int PhaseMacroExpand::replace_input(Node *use, Node *oldref, Node *newref) {
55 int nreplacements = 0;
56 uint req = use->req();
57 for (uint j = 0; j < use->len(); j++) {
58 Node *uin = use->in(j);
59 if (uin == oldref) {
60 if (j < req)
61 use->set_req(j, newref);
62 else
63 use->set_prec(j, newref);
64 nreplacements++;
916 sfpt->_idx, j);
917 }
918 tty->print(", which prevents elimination of: ");
919 if (res == NULL)
920 alloc->dump();
921 else
922 res->dump();
923 }
924 #endif
925 return false;
926 }
927 if (UseCompressedOops && field_type->isa_narrowoop()) {
928 // Enable "DecodeN(EncodeP(Allocate)) --> Allocate" transformation
929 // to be able scalar replace the allocation.
930 if (field_val->is_EncodeP()) {
931 field_val = field_val->in(1);
932 } else {
933 field_val = transform_later(new DecodeNNode(field_val, field_val->get_ptr_type()));
934 }
935 }
936 sfpt->add_req(field_val);
937 }
938 JVMState *jvms = sfpt->jvms();
939 jvms->set_endoff(sfpt->req());
940 // Now make a pass over the debug information replacing any references
941 // to the allocated object with "sobj"
942 int start = jvms->debug_start();
943 int end = jvms->debug_end();
944 sfpt->replace_edges_in_range(res, sobj, start, end);
945 _igvn._worklist.push(sfpt);
946 safepoints_done.append_if_missing(sfpt); // keep it for rollback
947 }
948 return true;
949 }
950
951 // Process users of eliminated allocation.
952 void PhaseMacroExpand::process_users_of_allocation(CallNode *alloc) {
953 Node* res = alloc->result_cast();
954 if (res != NULL) {
955 for (DUIterator_Last jmin, j = res->last_outs(jmin); j >= jmin; ) {
1394 contended_phi_rawmem = mem;
1395 } else {
1396 contended_region = new RegionNode(3);
1397 contended_phi_rawmem = new PhiNode(contended_region, Type::MEMORY, TypeRawPtr::BOTTOM);
1398 // Now handle the passing-too-big test. We fall into the contended
1399 // loop-back merge point.
1400 contended_region ->init_req(fall_in_path, toobig_false);
1401 contended_phi_rawmem->init_req(fall_in_path, mem);
1402 transform_later(contended_region);
1403 transform_later(contended_phi_rawmem);
1404 }
1405
1406 // Load(-locked) the heap top.
1407 // See note above concerning the control input when using a TLAB
1408 Node *old_eden_top = UseTLAB
1409 ? new LoadPNode (ctrl, contended_phi_rawmem, eden_top_adr, TypeRawPtr::BOTTOM, TypeRawPtr::BOTTOM, MemNode::unordered)
1410 : new LoadPLockedNode(contended_region, contended_phi_rawmem, eden_top_adr, MemNode::acquire);
1411
1412 transform_later(old_eden_top);
1413 // Add to heap top to get a new heap top
1414 Node *new_eden_top = new AddPNode(top(), old_eden_top, size_in_bytes);
1415 transform_later(new_eden_top);
1416 // Check for needing a GC; compare against heap end
1417 Node *needgc_cmp = new CmpPNode(new_eden_top, eden_end);
1418 transform_later(needgc_cmp);
1419 Node *needgc_bol = new BoolNode(needgc_cmp, BoolTest::ge);
1420 transform_later(needgc_bol);
1421 IfNode *needgc_iff = new IfNode(contended_region, needgc_bol, PROB_UNLIKELY_MAG(4), COUNT_UNKNOWN);
1422 transform_later(needgc_iff);
1423
1424 // Plug the failing-heap-space-need-gc test into the slow-path region
1425 Node *needgc_true = new IfTrueNode(needgc_iff);
1426 transform_later(needgc_true);
1427 if (initial_slow_test) {
1428 slow_region->init_req(need_gc_path, needgc_true);
1429 // This completes all paths into the slow merge point
1430 transform_later(slow_region);
1431 } else { // No initial slow path needed!
1432 // Just fall from the need-GC path straight into the VM call.
1433 slow_region = needgc_true;
1484
1485 // Bump total allocated bytes for this thread
1486 Node* thread = new ThreadLocalNode();
1487 transform_later(thread);
1488 Node* alloc_bytes_adr = basic_plus_adr(top()/*not oop*/, thread,
1489 in_bytes(JavaThread::allocated_bytes_offset()));
1490 Node* alloc_bytes = make_load(fast_oop_ctrl, store_eden_top, alloc_bytes_adr,
1491 0, TypeLong::LONG, T_LONG);
1492 #ifdef _LP64
1493 Node* alloc_size = size_in_bytes;
1494 #else
1495 Node* alloc_size = new ConvI2LNode(size_in_bytes);
1496 transform_later(alloc_size);
1497 #endif
1498 Node* new_alloc_bytes = new AddLNode(alloc_bytes, alloc_size);
1499 transform_later(new_alloc_bytes);
1500 fast_oop_rawmem = make_store(fast_oop_ctrl, store_eden_top, alloc_bytes_adr,
1501 0, new_alloc_bytes, T_LONG);
1502 }
1503
1504 InitializeNode* init = alloc->initialization();
1505 fast_oop_rawmem = initialize_object(alloc,
1506 fast_oop_ctrl, fast_oop_rawmem, fast_oop,
1507 klass_node, length, size_in_bytes);
1508
1509 // If initialization is performed by an array copy, any required
1510 // MemBarStoreStore was already added. If the object does not
1511 // escape no need for a MemBarStoreStore. Otherwise we need a
1512 // MemBarStoreStore so that stores that initialize this object
1513 // can't be reordered with a subsequent store that makes this
1514 // object accessible by other threads.
1515 if (init == NULL || (!init->is_complete_with_arraycopy() && !init->does_not_escape())) {
1516 if (init == NULL || init->req() < InitializeNode::RawStores) {
1517 // No InitializeNode or no stores captured by zeroing
1518 // elimination. Simply add the MemBarStoreStore after object
1519 // initialization.
1520 MemBarNode* mb = MemBarNode::make(C, Op_MemBarStoreStore, Compile::AliasIdxBot);
1521 transform_later(mb);
1522
1523 mb->init_req(TypeFunc::Memory, fast_oop_rawmem);
1759 Node* mark_node = NULL;
1760 // For now only enable fast locking for non-array types
1761 if (UseBiasedLocking && (length == NULL)) {
1762 mark_node = make_load(control, rawmem, klass_node, in_bytes(Klass::prototype_header_offset()), TypeRawPtr::BOTTOM, T_ADDRESS);
1763 } else {
1764 mark_node = makecon(TypeRawPtr::make((address)markOopDesc::prototype()));
1765 }
1766 rawmem = make_store(control, rawmem, object, oopDesc::mark_offset_in_bytes(), mark_node, T_ADDRESS);
1767
1768 rawmem = make_store(control, rawmem, object, oopDesc::klass_offset_in_bytes(), klass_node, T_METADATA);
1769 int header_size = alloc->minimum_header_size(); // conservatively small
1770
1771 // Array length
1772 if (length != NULL) { // Arrays need length field
1773 rawmem = make_store(control, rawmem, object, arrayOopDesc::length_offset_in_bytes(), length, T_INT);
1774 // conservatively small header size:
1775 header_size = arrayOopDesc::base_offset_in_bytes(T_BYTE);
1776 ciKlass* k = _igvn.type(klass_node)->is_klassptr()->klass();
1777 if (k->is_array_klass()) // we know the exact header size in most cases:
1778 header_size = Klass::layout_helper_header_size(k->layout_helper());
1779 }
1780
1781 // Clear the object body, if necessary.
1782 if (init == NULL) {
1783 // The init has somehow disappeared; be cautious and clear everything.
1784 //
1785 // This can happen if a node is allocated but an uncommon trap occurs
1786 // immediately. In this case, the Initialize gets associated with the
1787 // trap, and may be placed in a different (outer) loop, if the Allocate
1788 // is in a loop. If (this is rare) the inner loop gets unrolled, then
1789 // there can be two Allocates to one Initialize. The answer in all these
1790 // edge cases is safety first. It is always safe to clear immediately
1791 // within an Allocate, and then (maybe or maybe not) clear some more later.
1792 if (!ZeroTLAB)
1793 rawmem = ClearArrayNode::clear_memory(control, rawmem, object,
1794 header_size, size_in_bytes,
1795 &_igvn);
1796 } else {
1797 if (!init->is_complete()) {
1798 // Try to win by zeroing only what the init does not store.
|
25 #include "precompiled.hpp"
26 #include "compiler/compileLog.hpp"
27 #include "libadt/vectset.hpp"
28 #include "opto/addnode.hpp"
29 #include "opto/arraycopynode.hpp"
30 #include "opto/callnode.hpp"
31 #include "opto/castnode.hpp"
32 #include "opto/cfgnode.hpp"
33 #include "opto/compile.hpp"
34 #include "opto/convertnode.hpp"
35 #include "opto/locknode.hpp"
36 #include "opto/loopnode.hpp"
37 #include "opto/macro.hpp"
38 #include "opto/memnode.hpp"
39 #include "opto/narrowptrnode.hpp"
40 #include "opto/node.hpp"
41 #include "opto/opaquenode.hpp"
42 #include "opto/phaseX.hpp"
43 #include "opto/rootnode.hpp"
44 #include "opto/runtime.hpp"
45 #include "opto/shenandoahSupport.hpp"
46 #include "opto/subnode.hpp"
47 #include "opto/type.hpp"
48 #include "runtime/sharedRuntime.hpp"
49
50
51 //
52 // Replace any references to "oldref" in inputs to "use" with "newref".
53 // Returns the number of replacements made.
54 //
55 int PhaseMacroExpand::replace_input(Node *use, Node *oldref, Node *newref) {
56 int nreplacements = 0;
57 uint req = use->req();
58 for (uint j = 0; j < use->len(); j++) {
59 Node *uin = use->in(j);
60 if (uin == oldref) {
61 if (j < req)
62 use->set_req(j, newref);
63 else
64 use->set_prec(j, newref);
65 nreplacements++;
917 sfpt->_idx, j);
918 }
919 tty->print(", which prevents elimination of: ");
920 if (res == NULL)
921 alloc->dump();
922 else
923 res->dump();
924 }
925 #endif
926 return false;
927 }
928 if (UseCompressedOops && field_type->isa_narrowoop()) {
929 // Enable "DecodeN(EncodeP(Allocate)) --> Allocate" transformation
930 // to be able scalar replace the allocation.
931 if (field_val->is_EncodeP()) {
932 field_val = field_val->in(1);
933 } else {
934 field_val = transform_later(new DecodeNNode(field_val, field_val->get_ptr_type()));
935 }
936 }
937 if (field_val->isa_ShenandoahBarrier()) {
938 field_val = field_val->in(ShenandoahBarrierNode::ValueIn);
939 }
940 sfpt->add_req(field_val);
941 }
942 JVMState *jvms = sfpt->jvms();
943 jvms->set_endoff(sfpt->req());
944 // Now make a pass over the debug information replacing any references
945 // to the allocated object with "sobj"
946 int start = jvms->debug_start();
947 int end = jvms->debug_end();
948 sfpt->replace_edges_in_range(res, sobj, start, end);
949 _igvn._worklist.push(sfpt);
950 safepoints_done.append_if_missing(sfpt); // keep it for rollback
951 }
952 return true;
953 }
954
955 // Process users of eliminated allocation.
956 void PhaseMacroExpand::process_users_of_allocation(CallNode *alloc) {
957 Node* res = alloc->result_cast();
958 if (res != NULL) {
959 for (DUIterator_Last jmin, j = res->last_outs(jmin); j >= jmin; ) {
1398 contended_phi_rawmem = mem;
1399 } else {
1400 contended_region = new RegionNode(3);
1401 contended_phi_rawmem = new PhiNode(contended_region, Type::MEMORY, TypeRawPtr::BOTTOM);
1402 // Now handle the passing-too-big test. We fall into the contended
1403 // loop-back merge point.
1404 contended_region ->init_req(fall_in_path, toobig_false);
1405 contended_phi_rawmem->init_req(fall_in_path, mem);
1406 transform_later(contended_region);
1407 transform_later(contended_phi_rawmem);
1408 }
1409
1410 // Load(-locked) the heap top.
1411 // See note above concerning the control input when using a TLAB
1412 Node *old_eden_top = UseTLAB
1413 ? new LoadPNode (ctrl, contended_phi_rawmem, eden_top_adr, TypeRawPtr::BOTTOM, TypeRawPtr::BOTTOM, MemNode::unordered)
1414 : new LoadPLockedNode(contended_region, contended_phi_rawmem, eden_top_adr, MemNode::acquire);
1415
1416 transform_later(old_eden_top);
1417 // Add to heap top to get a new heap top
1418
1419 if (UseShenandoahGC) {
1420 // Allocate one word more for the Shenandoah brooks pointer.
1421 size_in_bytes = new AddLNode(size_in_bytes, _igvn.MakeConX(8));
1422 transform_later(size_in_bytes);
1423 }
1424
1425 Node *new_eden_top = new AddPNode(top(), old_eden_top, size_in_bytes);
1426 transform_later(new_eden_top);
1427 // Check for needing a GC; compare against heap end
1428 Node *needgc_cmp = new CmpPNode(new_eden_top, eden_end);
1429 transform_later(needgc_cmp);
1430 Node *needgc_bol = new BoolNode(needgc_cmp, BoolTest::ge);
1431 transform_later(needgc_bol);
1432 IfNode *needgc_iff = new IfNode(contended_region, needgc_bol, PROB_UNLIKELY_MAG(4), COUNT_UNKNOWN);
1433 transform_later(needgc_iff);
1434
1435 // Plug the failing-heap-space-need-gc test into the slow-path region
1436 Node *needgc_true = new IfTrueNode(needgc_iff);
1437 transform_later(needgc_true);
1438 if (initial_slow_test) {
1439 slow_region->init_req(need_gc_path, needgc_true);
1440 // This completes all paths into the slow merge point
1441 transform_later(slow_region);
1442 } else { // No initial slow path needed!
1443 // Just fall from the need-GC path straight into the VM call.
1444 slow_region = needgc_true;
1495
1496 // Bump total allocated bytes for this thread
1497 Node* thread = new ThreadLocalNode();
1498 transform_later(thread);
1499 Node* alloc_bytes_adr = basic_plus_adr(top()/*not oop*/, thread,
1500 in_bytes(JavaThread::allocated_bytes_offset()));
1501 Node* alloc_bytes = make_load(fast_oop_ctrl, store_eden_top, alloc_bytes_adr,
1502 0, TypeLong::LONG, T_LONG);
1503 #ifdef _LP64
1504 Node* alloc_size = size_in_bytes;
1505 #else
1506 Node* alloc_size = new ConvI2LNode(size_in_bytes);
1507 transform_later(alloc_size);
1508 #endif
1509 Node* new_alloc_bytes = new AddLNode(alloc_bytes, alloc_size);
1510 transform_later(new_alloc_bytes);
1511 fast_oop_rawmem = make_store(fast_oop_ctrl, store_eden_top, alloc_bytes_adr,
1512 0, new_alloc_bytes, T_LONG);
1513 }
1514
1515 if (UseShenandoahGC) {
1516 // Bump up object by one word. The preceding word is used for
1517 // the Shenandoah brooks pointer.
1518 fast_oop = new AddPNode(top(), fast_oop, _igvn.MakeConX(8));
1519 transform_later(fast_oop);
1520 }
1521
1522 InitializeNode* init = alloc->initialization();
1523 fast_oop_rawmem = initialize_object(alloc,
1524 fast_oop_ctrl, fast_oop_rawmem, fast_oop,
1525 klass_node, length, size_in_bytes);
1526
1527 // If initialization is performed by an array copy, any required
1528 // MemBarStoreStore was already added. If the object does not
1529 // escape no need for a MemBarStoreStore. Otherwise we need a
1530 // MemBarStoreStore so that stores that initialize this object
1531 // can't be reordered with a subsequent store that makes this
1532 // object accessible by other threads.
1533 if (init == NULL || (!init->is_complete_with_arraycopy() && !init->does_not_escape())) {
1534 if (init == NULL || init->req() < InitializeNode::RawStores) {
1535 // No InitializeNode or no stores captured by zeroing
1536 // elimination. Simply add the MemBarStoreStore after object
1537 // initialization.
1538 MemBarNode* mb = MemBarNode::make(C, Op_MemBarStoreStore, Compile::AliasIdxBot);
1539 transform_later(mb);
1540
1541 mb->init_req(TypeFunc::Memory, fast_oop_rawmem);
1777 Node* mark_node = NULL;
1778 // For now only enable fast locking for non-array types
1779 if (UseBiasedLocking && (length == NULL)) {
1780 mark_node = make_load(control, rawmem, klass_node, in_bytes(Klass::prototype_header_offset()), TypeRawPtr::BOTTOM, T_ADDRESS);
1781 } else {
1782 mark_node = makecon(TypeRawPtr::make((address)markOopDesc::prototype()));
1783 }
1784 rawmem = make_store(control, rawmem, object, oopDesc::mark_offset_in_bytes(), mark_node, T_ADDRESS);
1785
1786 rawmem = make_store(control, rawmem, object, oopDesc::klass_offset_in_bytes(), klass_node, T_METADATA);
1787 int header_size = alloc->minimum_header_size(); // conservatively small
1788
1789 // Array length
1790 if (length != NULL) { // Arrays need length field
1791 rawmem = make_store(control, rawmem, object, arrayOopDesc::length_offset_in_bytes(), length, T_INT);
1792 // conservatively small header size:
1793 header_size = arrayOopDesc::base_offset_in_bytes(T_BYTE);
1794 ciKlass* k = _igvn.type(klass_node)->is_klassptr()->klass();
1795 if (k->is_array_klass()) // we know the exact header size in most cases:
1796 header_size = Klass::layout_helper_header_size(k->layout_helper());
1797 }
1798
1799 if (UseShenandoahGC) {
1800 // Initialize Shenandoah brooks pointer to point to the object itself.
1801 rawmem = make_store(control, rawmem, object, -8, object, T_OBJECT);
1802 }
1803
1804 // Clear the object body, if necessary.
1805 if (init == NULL) {
1806 // The init has somehow disappeared; be cautious and clear everything.
1807 //
1808 // This can happen if a node is allocated but an uncommon trap occurs
1809 // immediately. In this case, the Initialize gets associated with the
1810 // trap, and may be placed in a different (outer) loop, if the Allocate
1811 // is in a loop. If (this is rare) the inner loop gets unrolled, then
1812 // there can be two Allocates to one Initialize. The answer in all these
1813 // edge cases is safety first. It is always safe to clear immediately
1814 // within an Allocate, and then (maybe or maybe not) clear some more later.
1815 if (!ZeroTLAB)
1816 rawmem = ClearArrayNode::clear_memory(control, rawmem, object,
1817 header_size, size_in_bytes,
1818 &_igvn);
1819 } else {
1820 if (!init->is_complete()) {
1821 // Try to win by zeroing only what the init does not store.
|