7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "compiler/compileLog.hpp"
27 #include "libadt/vectset.hpp"
28 #include "opto/addnode.hpp"
29 #include "opto/arraycopynode.hpp"
30 #include "opto/callnode.hpp"
31 #include "opto/castnode.hpp"
32 #include "opto/cfgnode.hpp"
33 #include "opto/compile.hpp"
34 #include "opto/convertnode.hpp"
35 #include "opto/locknode.hpp"
36 #include "opto/loopnode.hpp"
37 #include "opto/macro.hpp"
38 #include "opto/memnode.hpp"
39 #include "opto/narrowptrnode.hpp"
40 #include "opto/node.hpp"
41 #include "opto/opaquenode.hpp"
42 #include "opto/phaseX.hpp"
43 #include "opto/rootnode.hpp"
44 #include "opto/runtime.hpp"
45 #include "opto/subnode.hpp"
46 #include "opto/type.hpp"
47 #include "runtime/sharedRuntime.hpp"
48
49
50 //
51 // Replace any references to "oldref" in inputs to "use" with "newref".
52 // Returns the number of replacements made.
53 //
54 int PhaseMacroExpand::replace_input(Node *use, Node *oldref, Node *newref) {
55 int nreplacements = 0;
56 uint req = use->req();
57 for (uint j = 0; j < use->len(); j++) {
58 Node *uin = use->in(j);
59 if (uin == oldref) {
60 if (j < req)
61 use->set_req(j, newref);
62 else
63 use->set_prec(j, newref);
64 nreplacements++;
916 sfpt->_idx, j);
917 }
918 tty->print(", which prevents elimination of: ");
919 if (res == NULL)
920 alloc->dump();
921 else
922 res->dump();
923 }
924 #endif
925 return false;
926 }
927 if (UseCompressedOops && field_type->isa_narrowoop()) {
928 // Enable "DecodeN(EncodeP(Allocate)) --> Allocate" transformation
929 // to be able scalar replace the allocation.
930 if (field_val->is_EncodeP()) {
931 field_val = field_val->in(1);
932 } else {
933 field_val = transform_later(new DecodeNNode(field_val, field_val->get_ptr_type()));
934 }
935 }
936 sfpt->add_req(field_val);
937 }
938 JVMState *jvms = sfpt->jvms();
939 jvms->set_endoff(sfpt->req());
940 // Now make a pass over the debug information replacing any references
941 // to the allocated object with "sobj"
942 int start = jvms->debug_start();
943 int end = jvms->debug_end();
944 sfpt->replace_edges_in_range(res, sobj, start, end);
945 _igvn._worklist.push(sfpt);
946 safepoints_done.append_if_missing(sfpt); // keep it for rollback
947 }
948 return true;
949 }
950
951 // Process users of eliminated allocation.
952 void PhaseMacroExpand::process_users_of_allocation(CallNode *alloc) {
953 Node* res = alloc->result_cast();
954 if (res != NULL) {
955 for (DUIterator_Last jmin, j = res->last_outs(jmin); j >= jmin; ) {
1394 contended_phi_rawmem = mem;
1395 } else {
1396 contended_region = new RegionNode(3);
1397 contended_phi_rawmem = new PhiNode(contended_region, Type::MEMORY, TypeRawPtr::BOTTOM);
1398 // Now handle the passing-too-big test. We fall into the contended
1399 // loop-back merge point.
1400 contended_region ->init_req(fall_in_path, toobig_false);
1401 contended_phi_rawmem->init_req(fall_in_path, mem);
1402 transform_later(contended_region);
1403 transform_later(contended_phi_rawmem);
1404 }
1405
1406 // Load(-locked) the heap top.
1407 // See note above concerning the control input when using a TLAB
1408 Node *old_eden_top = UseTLAB
1409 ? new LoadPNode (ctrl, contended_phi_rawmem, eden_top_adr, TypeRawPtr::BOTTOM, TypeRawPtr::BOTTOM, MemNode::unordered)
1410 : new LoadPLockedNode(contended_region, contended_phi_rawmem, eden_top_adr, MemNode::acquire);
1411
1412 transform_later(old_eden_top);
1413 // Add to heap top to get a new heap top
1414 Node *new_eden_top = new AddPNode(top(), old_eden_top, size_in_bytes);
1415 transform_later(new_eden_top);
1416 // Check for needing a GC; compare against heap end
1417 Node *needgc_cmp = new CmpPNode(new_eden_top, eden_end);
1418 transform_later(needgc_cmp);
1419 Node *needgc_bol = new BoolNode(needgc_cmp, BoolTest::ge);
1420 transform_later(needgc_bol);
1421 IfNode *needgc_iff = new IfNode(contended_region, needgc_bol, PROB_UNLIKELY_MAG(4), COUNT_UNKNOWN);
1422 transform_later(needgc_iff);
1423
1424 // Plug the failing-heap-space-need-gc test into the slow-path region
1425 Node *needgc_true = new IfTrueNode(needgc_iff);
1426 transform_later(needgc_true);
1427 if (initial_slow_test) {
1428 slow_region->init_req(need_gc_path, needgc_true);
1429 // This completes all paths into the slow merge point
1430 transform_later(slow_region);
1431 } else { // No initial slow path needed!
1432 // Just fall from the need-GC path straight into the VM call.
1433 slow_region = needgc_true;
1484
1485 // Bump total allocated bytes for this thread
1486 Node* thread = new ThreadLocalNode();
1487 transform_later(thread);
1488 Node* alloc_bytes_adr = basic_plus_adr(top()/*not oop*/, thread,
1489 in_bytes(JavaThread::allocated_bytes_offset()));
1490 Node* alloc_bytes = make_load(fast_oop_ctrl, store_eden_top, alloc_bytes_adr,
1491 0, TypeLong::LONG, T_LONG);
1492 #ifdef _LP64
1493 Node* alloc_size = size_in_bytes;
1494 #else
1495 Node* alloc_size = new ConvI2LNode(size_in_bytes);
1496 transform_later(alloc_size);
1497 #endif
1498 Node* new_alloc_bytes = new AddLNode(alloc_bytes, alloc_size);
1499 transform_later(new_alloc_bytes);
1500 fast_oop_rawmem = make_store(fast_oop_ctrl, store_eden_top, alloc_bytes_adr,
1501 0, new_alloc_bytes, T_LONG);
1502 }
1503
1504 InitializeNode* init = alloc->initialization();
1505 fast_oop_rawmem = initialize_object(alloc,
1506 fast_oop_ctrl, fast_oop_rawmem, fast_oop,
1507 klass_node, length, size_in_bytes);
1508
1509 // If initialization is performed by an array copy, any required
1510 // MemBarStoreStore was already added. If the object does not
1511 // escape no need for a MemBarStoreStore. Otherwise we need a
1512 // MemBarStoreStore so that stores that initialize this object
1513 // can't be reordered with a subsequent store that makes this
1514 // object accessible by other threads.
1515 if (init == NULL || (!init->is_complete_with_arraycopy() && !init->does_not_escape())) {
1516 if (init == NULL || init->req() < InitializeNode::RawStores) {
1517 // No InitializeNode or no stores captured by zeroing
1518 // elimination. Simply add the MemBarStoreStore after object
1519 // initialization.
1520 MemBarNode* mb = MemBarNode::make(C, Op_MemBarStoreStore, Compile::AliasIdxBot);
1521 transform_later(mb);
1522
1523 mb->init_req(TypeFunc::Memory, fast_oop_rawmem);
1759 Node* mark_node = NULL;
1760 // For now only enable fast locking for non-array types
1761 if (UseBiasedLocking && (length == NULL)) {
1762 mark_node = make_load(control, rawmem, klass_node, in_bytes(Klass::prototype_header_offset()), TypeRawPtr::BOTTOM, T_ADDRESS);
1763 } else {
1764 mark_node = makecon(TypeRawPtr::make((address)markOopDesc::prototype()));
1765 }
1766 rawmem = make_store(control, rawmem, object, oopDesc::mark_offset_in_bytes(), mark_node, T_ADDRESS);
1767
1768 rawmem = make_store(control, rawmem, object, oopDesc::klass_offset_in_bytes(), klass_node, T_METADATA);
1769 int header_size = alloc->minimum_header_size(); // conservatively small
1770
1771 // Array length
1772 if (length != NULL) { // Arrays need length field
1773 rawmem = make_store(control, rawmem, object, arrayOopDesc::length_offset_in_bytes(), length, T_INT);
1774 // conservatively small header size:
1775 header_size = arrayOopDesc::base_offset_in_bytes(T_BYTE);
1776 ciKlass* k = _igvn.type(klass_node)->is_klassptr()->klass();
1777 if (k->is_array_klass()) // we know the exact header size in most cases:
1778 header_size = Klass::layout_helper_header_size(k->layout_helper());
1779 }
1780
1781 // Clear the object body, if necessary.
1782 if (init == NULL) {
1783 // The init has somehow disappeared; be cautious and clear everything.
1784 //
1785 // This can happen if a node is allocated but an uncommon trap occurs
1786 // immediately. In this case, the Initialize gets associated with the
1787 // trap, and may be placed in a different (outer) loop, if the Allocate
1788 // is in a loop. If (this is rare) the inner loop gets unrolled, then
1789 // there can be two Allocates to one Initialize. The answer in all these
1790 // edge cases is safety first. It is always safe to clear immediately
1791 // within an Allocate, and then (maybe or maybe not) clear some more later.
1792 if (!ZeroTLAB)
1793 rawmem = ClearArrayNode::clear_memory(control, rawmem, object,
1794 header_size, size_in_bytes,
1795 &_igvn);
1796 } else {
1797 if (!init->is_complete()) {
1798 // Try to win by zeroing only what the init does not store.
|
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "compiler/compileLog.hpp"
27 #include "gc/shenandoah/brooksPointer.hpp"
28 #include "libadt/vectset.hpp"
29 #include "opto/addnode.hpp"
30 #include "opto/arraycopynode.hpp"
31 #include "opto/callnode.hpp"
32 #include "opto/castnode.hpp"
33 #include "opto/cfgnode.hpp"
34 #include "opto/compile.hpp"
35 #include "opto/convertnode.hpp"
36 #include "opto/locknode.hpp"
37 #include "opto/loopnode.hpp"
38 #include "opto/macro.hpp"
39 #include "opto/memnode.hpp"
40 #include "opto/narrowptrnode.hpp"
41 #include "opto/node.hpp"
42 #include "opto/opaquenode.hpp"
43 #include "opto/phaseX.hpp"
44 #include "opto/rootnode.hpp"
45 #include "opto/runtime.hpp"
46 #include "opto/shenandoahSupport.hpp"
47 #include "opto/subnode.hpp"
48 #include "opto/type.hpp"
49 #include "runtime/sharedRuntime.hpp"
50
51
52 //
53 // Replace any references to "oldref" in inputs to "use" with "newref".
54 // Returns the number of replacements made.
55 //
56 int PhaseMacroExpand::replace_input(Node *use, Node *oldref, Node *newref) {
57 int nreplacements = 0;
58 uint req = use->req();
59 for (uint j = 0; j < use->len(); j++) {
60 Node *uin = use->in(j);
61 if (uin == oldref) {
62 if (j < req)
63 use->set_req(j, newref);
64 else
65 use->set_prec(j, newref);
66 nreplacements++;
918 sfpt->_idx, j);
919 }
920 tty->print(", which prevents elimination of: ");
921 if (res == NULL)
922 alloc->dump();
923 else
924 res->dump();
925 }
926 #endif
927 return false;
928 }
929 if (UseCompressedOops && field_type->isa_narrowoop()) {
930 // Enable "DecodeN(EncodeP(Allocate)) --> Allocate" transformation
931 // to be able scalar replace the allocation.
932 if (field_val->is_EncodeP()) {
933 field_val = field_val->in(1);
934 } else {
935 field_val = transform_later(new DecodeNNode(field_val, field_val->get_ptr_type()));
936 }
937 }
938 if (field_val->isa_ShenandoahBarrier()) {
939 field_val = field_val->in(ShenandoahBarrierNode::ValueIn);
940 }
941 sfpt->add_req(field_val);
942 }
943 JVMState *jvms = sfpt->jvms();
944 jvms->set_endoff(sfpt->req());
945 // Now make a pass over the debug information replacing any references
946 // to the allocated object with "sobj"
947 int start = jvms->debug_start();
948 int end = jvms->debug_end();
949 sfpt->replace_edges_in_range(res, sobj, start, end);
950 _igvn._worklist.push(sfpt);
951 safepoints_done.append_if_missing(sfpt); // keep it for rollback
952 }
953 return true;
954 }
955
956 // Process users of eliminated allocation.
957 void PhaseMacroExpand::process_users_of_allocation(CallNode *alloc) {
958 Node* res = alloc->result_cast();
959 if (res != NULL) {
960 for (DUIterator_Last jmin, j = res->last_outs(jmin); j >= jmin; ) {
1399 contended_phi_rawmem = mem;
1400 } else {
1401 contended_region = new RegionNode(3);
1402 contended_phi_rawmem = new PhiNode(contended_region, Type::MEMORY, TypeRawPtr::BOTTOM);
1403 // Now handle the passing-too-big test. We fall into the contended
1404 // loop-back merge point.
1405 contended_region ->init_req(fall_in_path, toobig_false);
1406 contended_phi_rawmem->init_req(fall_in_path, mem);
1407 transform_later(contended_region);
1408 transform_later(contended_phi_rawmem);
1409 }
1410
1411 // Load(-locked) the heap top.
1412 // See note above concerning the control input when using a TLAB
1413 Node *old_eden_top = UseTLAB
1414 ? new LoadPNode (ctrl, contended_phi_rawmem, eden_top_adr, TypeRawPtr::BOTTOM, TypeRawPtr::BOTTOM, MemNode::unordered)
1415 : new LoadPLockedNode(contended_region, contended_phi_rawmem, eden_top_adr, MemNode::acquire);
1416
1417 transform_later(old_eden_top);
1418 // Add to heap top to get a new heap top
1419
1420 if (UseShenandoahGC) {
1421 // Allocate one word more for the Shenandoah brooks pointer.
1422 size_in_bytes = new AddLNode(size_in_bytes, _igvn.MakeConX(8));
1423 transform_later(size_in_bytes);
1424 }
1425
1426 Node *new_eden_top = new AddPNode(top(), old_eden_top, size_in_bytes);
1427 transform_later(new_eden_top);
1428 // Check for needing a GC; compare against heap end
1429 Node *needgc_cmp = new CmpPNode(new_eden_top, eden_end);
1430 transform_later(needgc_cmp);
1431 Node *needgc_bol = new BoolNode(needgc_cmp, BoolTest::ge);
1432 transform_later(needgc_bol);
1433 IfNode *needgc_iff = new IfNode(contended_region, needgc_bol, PROB_UNLIKELY_MAG(4), COUNT_UNKNOWN);
1434 transform_later(needgc_iff);
1435
1436 // Plug the failing-heap-space-need-gc test into the slow-path region
1437 Node *needgc_true = new IfTrueNode(needgc_iff);
1438 transform_later(needgc_true);
1439 if (initial_slow_test) {
1440 slow_region->init_req(need_gc_path, needgc_true);
1441 // This completes all paths into the slow merge point
1442 transform_later(slow_region);
1443 } else { // No initial slow path needed!
1444 // Just fall from the need-GC path straight into the VM call.
1445 slow_region = needgc_true;
1496
1497 // Bump total allocated bytes for this thread
1498 Node* thread = new ThreadLocalNode();
1499 transform_later(thread);
1500 Node* alloc_bytes_adr = basic_plus_adr(top()/*not oop*/, thread,
1501 in_bytes(JavaThread::allocated_bytes_offset()));
1502 Node* alloc_bytes = make_load(fast_oop_ctrl, store_eden_top, alloc_bytes_adr,
1503 0, TypeLong::LONG, T_LONG);
1504 #ifdef _LP64
1505 Node* alloc_size = size_in_bytes;
1506 #else
1507 Node* alloc_size = new ConvI2LNode(size_in_bytes);
1508 transform_later(alloc_size);
1509 #endif
1510 Node* new_alloc_bytes = new AddLNode(alloc_bytes, alloc_size);
1511 transform_later(new_alloc_bytes);
1512 fast_oop_rawmem = make_store(fast_oop_ctrl, store_eden_top, alloc_bytes_adr,
1513 0, new_alloc_bytes, T_LONG);
1514 }
1515
1516 if (UseShenandoahGC) {
1517 // Bump up object by one word. The preceding word is used for
1518 // the Shenandoah brooks pointer.
1519 fast_oop = new AddPNode(top(), fast_oop, _igvn.MakeConX(8));
1520 transform_later(fast_oop);
1521 }
1522
1523 InitializeNode* init = alloc->initialization();
1524 fast_oop_rawmem = initialize_object(alloc,
1525 fast_oop_ctrl, fast_oop_rawmem, fast_oop,
1526 klass_node, length, size_in_bytes);
1527
1528 // If initialization is performed by an array copy, any required
1529 // MemBarStoreStore was already added. If the object does not
1530 // escape no need for a MemBarStoreStore. Otherwise we need a
1531 // MemBarStoreStore so that stores that initialize this object
1532 // can't be reordered with a subsequent store that makes this
1533 // object accessible by other threads.
1534 if (init == NULL || (!init->is_complete_with_arraycopy() && !init->does_not_escape())) {
1535 if (init == NULL || init->req() < InitializeNode::RawStores) {
1536 // No InitializeNode or no stores captured by zeroing
1537 // elimination. Simply add the MemBarStoreStore after object
1538 // initialization.
1539 MemBarNode* mb = MemBarNode::make(C, Op_MemBarStoreStore, Compile::AliasIdxBot);
1540 transform_later(mb);
1541
1542 mb->init_req(TypeFunc::Memory, fast_oop_rawmem);
1778 Node* mark_node = NULL;
1779 // For now only enable fast locking for non-array types
1780 if (UseBiasedLocking && (length == NULL)) {
1781 mark_node = make_load(control, rawmem, klass_node, in_bytes(Klass::prototype_header_offset()), TypeRawPtr::BOTTOM, T_ADDRESS);
1782 } else {
1783 mark_node = makecon(TypeRawPtr::make((address)markOopDesc::prototype()));
1784 }
1785 rawmem = make_store(control, rawmem, object, oopDesc::mark_offset_in_bytes(), mark_node, T_ADDRESS);
1786
1787 rawmem = make_store(control, rawmem, object, oopDesc::klass_offset_in_bytes(), klass_node, T_METADATA);
1788 int header_size = alloc->minimum_header_size(); // conservatively small
1789
1790 // Array length
1791 if (length != NULL) { // Arrays need length field
1792 rawmem = make_store(control, rawmem, object, arrayOopDesc::length_offset_in_bytes(), length, T_INT);
1793 // conservatively small header size:
1794 header_size = arrayOopDesc::base_offset_in_bytes(T_BYTE);
1795 ciKlass* k = _igvn.type(klass_node)->is_klassptr()->klass();
1796 if (k->is_array_klass()) // we know the exact header size in most cases:
1797 header_size = Klass::layout_helper_header_size(k->layout_helper());
1798 }
1799
1800 if (UseShenandoahGC) {
1801 // Initialize Shenandoah brooks pointer to point to the object itself.
1802 rawmem = make_store(control, rawmem, object, BrooksPointer::BYTE_OFFSET, object, T_OBJECT);
1803 }
1804
1805 // Clear the object body, if necessary.
1806 if (init == NULL) {
1807 // The init has somehow disappeared; be cautious and clear everything.
1808 //
1809 // This can happen if a node is allocated but an uncommon trap occurs
1810 // immediately. In this case, the Initialize gets associated with the
1811 // trap, and may be placed in a different (outer) loop, if the Allocate
1812 // is in a loop. If (this is rare) the inner loop gets unrolled, then
1813 // there can be two Allocates to one Initialize. The answer in all these
1814 // edge cases is safety first. It is always safe to clear immediately
1815 // within an Allocate, and then (maybe or maybe not) clear some more later.
1816 if (!ZeroTLAB)
1817 rawmem = ClearArrayNode::clear_memory(control, rawmem, object,
1818 header_size, size_in_bytes,
1819 &_igvn);
1820 } else {
1821 if (!init->is_complete()) {
1822 // Try to win by zeroing only what the init does not store.
|