9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 * version 2 for more details (a copy is included in the LICENSE file that
12 * accompanied this code).
13 *
14 * You should have received a copy of the GNU General Public License version
15 * 2 along with this work; if not, write to the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 #include "precompiled.hpp"
25
26 #include "gc/shenandoah/c2/shenandoahSupport.hpp"
27 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
28 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
29 #include "gc/shenandoah/shenandoahForwarding.hpp"
30 #include "gc/shenandoah/shenandoahHeap.hpp"
31 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
32 #include "gc/shenandoah/shenandoahRuntime.hpp"
33 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
34 #include "opto/arraycopynode.hpp"
35 #include "opto/block.hpp"
36 #include "opto/callnode.hpp"
37 #include "opto/castnode.hpp"
38 #include "opto/movenode.hpp"
39 #include "opto/phaseX.hpp"
40 #include "opto/rootnode.hpp"
41 #include "opto/runtime.hpp"
42 #include "opto/subnode.hpp"
43
44 bool ShenandoahBarrierC2Support::expand(Compile* C, PhaseIterGVN& igvn) {
45 ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state();
46 if ((state->enqueue_barriers_count() +
47 state->load_reference_barriers_count()) > 0) {
48 bool attempt_more_loopopts = ShenandoahLoopOptsAfterExpansion;
49 C->clear_major_progress();
1441 Node* unc_ctrl = NULL;
1442 if (unc != NULL) {
1443 if (val->in(ShenandoahLoadReferenceBarrierNode::Control) != ctrl) {
1444 unc = NULL;
1445 } else {
1446 unc_ctrl = val->in(ShenandoahLoadReferenceBarrierNode::Control);
1447 }
1448 }
1449
1450 Node* uncasted_val = val;
1451 if (unc != NULL) {
1452 uncasted_val = val->in(1);
1453 }
1454
1455 Node* heap_stable_ctrl = NULL;
1456 Node* null_ctrl = NULL;
1457
1458 assert(val->bottom_type()->make_oopptr(), "need oop");
1459 assert(val->bottom_type()->make_oopptr()->const_oop() == NULL, "expect non-constant");
1460
1461 enum { _heap_stable = 1, _not_cset, _not_equal, _evac_path, _null_path, PATH_LIMIT };
1462 Node* region = new RegionNode(PATH_LIMIT);
1463 Node* val_phi = new PhiNode(region, uncasted_val->bottom_type()->is_oopptr());
1464 Node* raw_mem_phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);
1465
1466 // Stable path.
1467 test_heap_stable(ctrl, raw_mem, heap_stable_ctrl, phase);
1468 IfNode* heap_stable_iff = heap_stable_ctrl->in(0)->as_If();
1469
1470 // Heap stable case
1471 region->init_req(_heap_stable, heap_stable_ctrl);
1472 val_phi->init_req(_heap_stable, uncasted_val);
1473 raw_mem_phi->init_req(_heap_stable, raw_mem);
1474
1475 Node* reg2_ctrl = NULL;
1476 // Null case
1477 test_null(ctrl, val, null_ctrl, phase);
1478 if (null_ctrl != NULL) {
1479 reg2_ctrl = null_ctrl->in(0);
1480 region->init_req(_null_path, null_ctrl);
1481 val_phi->init_req(_null_path, uncasted_val);
1491 Node* not_cset_ctrl = NULL;
1492 in_cset_fast_test(ctrl, not_cset_ctrl, uncasted_val, raw_mem, phase);
1493 if (not_cset_ctrl != NULL) {
1494 if (reg2_ctrl == NULL) reg2_ctrl = not_cset_ctrl->in(0);
1495 region->init_req(_not_cset, not_cset_ctrl);
1496 val_phi->init_req(_not_cset, uncasted_val);
1497 raw_mem_phi->init_req(_not_cset, raw_mem);
1498 }
1499
1500 // Resolve object when orig-value is in cset.
1501 // Make the unconditional resolve for fwdptr.
1502 Node* new_val = uncasted_val;
1503 if (unc_ctrl != NULL) {
1504 // Clone the null check in this branch to allow implicit null check
1505 new_val = clone_null_check(ctrl, val, unc_ctrl, phase);
1506 fix_null_check(unc, unc_ctrl, ctrl->in(0)->as_If()->proj_out(0), uses, phase);
1507
1508 IfNode* iff = unc_ctrl->in(0)->as_If();
1509 phase->igvn().replace_input_of(iff, 1, phase->igvn().intcon(1));
1510 }
1511 Node* addr = new AddPNode(new_val, uncasted_val, phase->igvn().MakeConX(ShenandoahForwarding::byte_offset()));
1512 phase->register_new_node(addr, ctrl);
1513 assert(val->bottom_type()->isa_oopptr(), "what else?");
1514 const TypePtr* obj_type = val->bottom_type()->is_oopptr();
1515 const TypePtr* adr_type = TypeRawPtr::BOTTOM;
1516 Node* fwd = new LoadPNode(ctrl, raw_mem, addr, adr_type, obj_type, MemNode::unordered);
1517 phase->register_new_node(fwd, ctrl);
1518
1519 // Only branch to LRB stub if object is not forwarded; otherwise reply with fwd ptr
1520 Node* cmp = new CmpPNode(fwd, new_val);
1521 phase->register_new_node(cmp, ctrl);
1522 Node* bol = new BoolNode(cmp, BoolTest::eq);
1523 phase->register_new_node(bol, ctrl);
1524
1525 IfNode* iff = new IfNode(ctrl, bol, PROB_UNLIKELY(0.999), COUNT_UNKNOWN);
1526 if (reg2_ctrl == NULL) reg2_ctrl = iff;
1527 phase->register_control(iff, loop, ctrl);
1528 Node* if_not_eq = new IfFalseNode(iff);
1529 phase->register_control(if_not_eq, loop, iff);
1530 Node* if_eq = new IfTrueNode(iff);
1531 phase->register_control(if_eq, loop, iff);
1532
1533 // Wire up not-equal-path in slots 3.
1534 region->init_req(_not_equal, if_not_eq);
1535 val_phi->init_req(_not_equal, fwd);
1536 raw_mem_phi->init_req(_not_equal, raw_mem);
1537
1538 // Call wb-stub and wire up that path in slots 4
1539 Node* result_mem = NULL;
1540 ctrl = if_eq;
1541 call_lrb_stub(ctrl, fwd, result_mem, raw_mem, phase);
1542 region->init_req(_evac_path, ctrl);
1543 val_phi->init_req(_evac_path, fwd);
1544 raw_mem_phi->init_req(_evac_path, result_mem);
1545
1546 phase->register_control(region, loop, heap_stable_iff);
1547 Node* out_val = val_phi;
1548 phase->register_new_node(val_phi, region);
1549 phase->register_new_node(raw_mem_phi, region);
1550
1551 fix_ctrl(lrb, region, fixer, uses, uses_to_ignore, last, phase);
1552
1553 ctrl = orig_ctrl;
1554
1555 if (unc != NULL) {
1556 for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) {
1557 Node* u = val->fast_out(i);
1558 Node* c = phase->ctrl_or_self(u);
1559 if (u != lrb && (c != ctrl || is_dominator_same_ctrl(c, lrb, u, phase))) {
1560 phase->igvn().rehash_node_delayed(u);
3165 case Op_GetAndAddS:
3166 case Op_ShenandoahEnqueueBarrier:
3167 case Op_FastLock:
3168 case Op_FastUnlock:
3169 case Op_Rethrow:
3170 case Op_Return:
3171 case Op_StoreB:
3172 case Op_StoreC:
3173 case Op_StoreD:
3174 case Op_StoreF:
3175 case Op_StoreL:
3176 case Op_StoreLConditional:
3177 case Op_StoreI:
3178 case Op_StoreVector:
3179 case Op_StrInflatedCopy:
3180 case Op_StrCompressedCopy:
3181 case Op_EncodeP:
3182 case Op_CastP2X:
3183 case Op_SafePoint:
3184 case Op_EncodeISOArray:
3185 strength = STRONG;
3186 break;
3187 case Op_LoadB:
3188 case Op_LoadUB:
3189 case Op_LoadUS:
3190 case Op_LoadD:
3191 case Op_LoadF:
3192 case Op_LoadL:
3193 case Op_LoadI:
3194 case Op_LoadS:
3195 case Op_LoadN:
3196 case Op_LoadP:
3197 case Op_LoadVector: {
3198 const TypePtr* adr_type = n->adr_type();
3199 int alias_idx = Compile::current()->get_alias_index(adr_type);
3200 Compile::AliasType* alias_type = Compile::current()->alias_type(alias_idx);
3201 ciField* field = alias_type->field();
3202 bool is_static = field != NULL && field->is_static();
3203 bool is_final = field != NULL && field->is_final();
3204 bool is_stable = field != NULL && field->is_stable();
3216 case Op_AryEq: {
3217 Node* n1 = n->in(2);
3218 Node* n2 = n->in(3);
3219 if (!ShenandoahOptimizeStableFinals ||
3220 !n1->bottom_type()->isa_aryptr() || !n1->bottom_type()->isa_aryptr()->is_stable() ||
3221 !n2->bottom_type()->isa_aryptr() || !n2->bottom_type()->isa_aryptr()->is_stable()) {
3222 strength = WEAK;
3223 }
3224 break;
3225 }
3226 case Op_StrEquals:
3227 case Op_StrComp:
3228 case Op_StrIndexOf:
3229 case Op_StrIndexOfChar:
3230 if (!ShenandoahOptimizeStableFinals) {
3231 strength = WEAK;
3232 }
3233 break;
3234 case Op_Conv2B:
3235 case Op_LoadRange:
3236 case Op_LoadKlass:
3237 case Op_LoadNKlass:
3238 // NONE, i.e. leave current strength as is
3239 break;
3240 case Op_AddP:
3241 case Op_CheckCastPP:
3242 case Op_CastPP:
3243 case Op_CMoveP:
3244 case Op_Phi:
3245 case Op_ShenandoahLoadReferenceBarrier:
3246 visit_users = true;
3247 break;
3248 default: {
3249 #ifdef ASSERT
3250 tty->print_cr("Unknown node in get_barrier_strength:");
3251 n->dump(1);
3252 ShouldNotReachHere();
3253 #else
3254 strength = STRONG;
3255 #endif
3256 }
3257 }
|
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 * version 2 for more details (a copy is included in the LICENSE file that
12 * accompanied this code).
13 *
14 * You should have received a copy of the GNU General Public License version
15 * 2 along with this work; if not, write to the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 #include "precompiled.hpp"
25
26 #include "gc/shenandoah/c2/shenandoahSupport.hpp"
27 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
28 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
29 #include "gc/shenandoah/shenandoahHeap.hpp"
30 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
31 #include "gc/shenandoah/shenandoahRuntime.hpp"
32 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
33 #include "opto/arraycopynode.hpp"
34 #include "opto/block.hpp"
35 #include "opto/callnode.hpp"
36 #include "opto/castnode.hpp"
37 #include "opto/movenode.hpp"
38 #include "opto/phaseX.hpp"
39 #include "opto/rootnode.hpp"
40 #include "opto/runtime.hpp"
41 #include "opto/subnode.hpp"
42
43 bool ShenandoahBarrierC2Support::expand(Compile* C, PhaseIterGVN& igvn) {
44 ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state();
45 if ((state->enqueue_barriers_count() +
46 state->load_reference_barriers_count()) > 0) {
47 bool attempt_more_loopopts = ShenandoahLoopOptsAfterExpansion;
48 C->clear_major_progress();
1440 Node* unc_ctrl = NULL;
1441 if (unc != NULL) {
1442 if (val->in(ShenandoahLoadReferenceBarrierNode::Control) != ctrl) {
1443 unc = NULL;
1444 } else {
1445 unc_ctrl = val->in(ShenandoahLoadReferenceBarrierNode::Control);
1446 }
1447 }
1448
1449 Node* uncasted_val = val;
1450 if (unc != NULL) {
1451 uncasted_val = val->in(1);
1452 }
1453
1454 Node* heap_stable_ctrl = NULL;
1455 Node* null_ctrl = NULL;
1456
1457 assert(val->bottom_type()->make_oopptr(), "need oop");
1458 assert(val->bottom_type()->make_oopptr()->const_oop() == NULL, "expect non-constant");
1459
1460 enum { _heap_stable = 1, _not_cset, _fwded, _evac_path, _null_path, PATH_LIMIT };
1461 Node* region = new RegionNode(PATH_LIMIT);
1462 Node* val_phi = new PhiNode(region, uncasted_val->bottom_type()->is_oopptr());
1463 Node* raw_mem_phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);
1464
1465 // Stable path.
1466 test_heap_stable(ctrl, raw_mem, heap_stable_ctrl, phase);
1467 IfNode* heap_stable_iff = heap_stable_ctrl->in(0)->as_If();
1468
1469 // Heap stable case
1470 region->init_req(_heap_stable, heap_stable_ctrl);
1471 val_phi->init_req(_heap_stable, uncasted_val);
1472 raw_mem_phi->init_req(_heap_stable, raw_mem);
1473
1474 Node* reg2_ctrl = NULL;
1475 // Null case
1476 test_null(ctrl, val, null_ctrl, phase);
1477 if (null_ctrl != NULL) {
1478 reg2_ctrl = null_ctrl->in(0);
1479 region->init_req(_null_path, null_ctrl);
1480 val_phi->init_req(_null_path, uncasted_val);
1490 Node* not_cset_ctrl = NULL;
1491 in_cset_fast_test(ctrl, not_cset_ctrl, uncasted_val, raw_mem, phase);
1492 if (not_cset_ctrl != NULL) {
1493 if (reg2_ctrl == NULL) reg2_ctrl = not_cset_ctrl->in(0);
1494 region->init_req(_not_cset, not_cset_ctrl);
1495 val_phi->init_req(_not_cset, uncasted_val);
1496 raw_mem_phi->init_req(_not_cset, raw_mem);
1497 }
1498
1499 // Resolve object when orig-value is in cset.
1500 // Make the unconditional resolve for fwdptr.
1501 Node* new_val = uncasted_val;
1502 if (unc_ctrl != NULL) {
1503 // Clone the null check in this branch to allow implicit null check
1504 new_val = clone_null_check(ctrl, val, unc_ctrl, phase);
1505 fix_null_check(unc, unc_ctrl, ctrl->in(0)->as_If()->proj_out(0), uses, phase);
1506
1507 IfNode* iff = unc_ctrl->in(0)->as_If();
1508 phase->igvn().replace_input_of(iff, 1, phase->igvn().intcon(1));
1509 }
1510 Node* addr = new AddPNode(new_val, new_val, phase->igvn().MakeConX(oopDesc::mark_offset_in_bytes()));
1511 phase->register_new_node(addr, ctrl);
1512 assert(new_val->bottom_type()->isa_oopptr(), "what else?");
1513 Node* markword = new LoadXNode(ctrl, raw_mem, addr, TypeRawPtr::BOTTOM, TypeX_X, MemNode::unordered);
1514 phase->register_new_node(markword, ctrl);
1515
1516 // Test if object is forwarded. This is the case if lowest two bits are set.
1517 Node* masked = new AndXNode(markword, phase->igvn().MakeConX(markOopDesc::lock_mask_in_place));
1518 phase->register_new_node(masked, ctrl);
1519 Node* cmp = new CmpXNode(masked, phase->igvn().MakeConX(markOopDesc::marked_value));
1520 phase->register_new_node(cmp, ctrl);
1521
1522 // Only branch to LRB stub if object is not forwarded; otherwise reply with fwd ptr
1523 Node* bol = new BoolNode(cmp, BoolTest::eq); // Equals 3 means it's forwarded
1524 phase->register_new_node(bol, ctrl);
1525
1526 IfNode* iff = new IfNode(ctrl, bol, PROB_UNLIKELY(0.999), COUNT_UNKNOWN);
1527 phase->register_control(iff, loop, ctrl);
1528 Node* if_fwd = new IfTrueNode(iff);
1529 phase->register_control(if_fwd, loop, iff);
1530 Node* if_not_fwd = new IfFalseNode(iff);
1531 phase->register_control(if_not_fwd, loop, iff);
1532
1533 // Decode forward pointer.
1534 Node* masked2 = new AndXNode(markword, phase->igvn().MakeConX(~markOopDesc::lock_mask_in_place));
1535 phase->register_new_node(masked2, if_fwd);
1536 Node* fwdraw = new CastX2PNode(masked2);
1537 fwdraw->init_req(0, if_fwd);
1538 phase->register_new_node(fwdraw, if_fwd);
1539 Node* fwd = new CheckCastPPNode(NULL, fwdraw, val->bottom_type());
1540 phase->register_new_node(fwd, if_fwd);
1541
1542 // Wire up not-equal-path in slots 3.
1543 region->init_req(_fwded, if_fwd);
1544 val_phi->init_req(_fwded, fwd);
1545 raw_mem_phi->init_req(_fwded, raw_mem);
1546
1547 // Call wb-stub and wire up that path in slots 4
1548 Node* result_mem = NULL;
1549 ctrl = if_not_fwd;
1550 fwd = new_val;
1551 call_lrb_stub(ctrl, fwd, result_mem, raw_mem, phase);
1552 region->init_req(_evac_path, ctrl);
1553 val_phi->init_req(_evac_path, fwd);
1554 raw_mem_phi->init_req(_evac_path, result_mem);
1555
1556 phase->register_control(region, loop, heap_stable_iff);
1557 Node* out_val = val_phi;
1558 phase->register_new_node(val_phi, region);
1559 phase->register_new_node(raw_mem_phi, region);
1560
1561 fix_ctrl(lrb, region, fixer, uses, uses_to_ignore, last, phase);
1562
1563 ctrl = orig_ctrl;
1564
1565 if (unc != NULL) {
1566 for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) {
1567 Node* u = val->fast_out(i);
1568 Node* c = phase->ctrl_or_self(u);
1569 if (u != lrb && (c != ctrl || is_dominator_same_ctrl(c, lrb, u, phase))) {
1570 phase->igvn().rehash_node_delayed(u);
3175 case Op_GetAndAddS:
3176 case Op_ShenandoahEnqueueBarrier:
3177 case Op_FastLock:
3178 case Op_FastUnlock:
3179 case Op_Rethrow:
3180 case Op_Return:
3181 case Op_StoreB:
3182 case Op_StoreC:
3183 case Op_StoreD:
3184 case Op_StoreF:
3185 case Op_StoreL:
3186 case Op_StoreLConditional:
3187 case Op_StoreI:
3188 case Op_StoreVector:
3189 case Op_StrInflatedCopy:
3190 case Op_StrCompressedCopy:
3191 case Op_EncodeP:
3192 case Op_CastP2X:
3193 case Op_SafePoint:
3194 case Op_EncodeISOArray:
3195 case Op_LoadKlass:
3196 case Op_LoadNKlass:
3197 strength = STRONG;
3198 break;
3199 case Op_LoadB:
3200 case Op_LoadUB:
3201 case Op_LoadUS:
3202 case Op_LoadD:
3203 case Op_LoadF:
3204 case Op_LoadL:
3205 case Op_LoadI:
3206 case Op_LoadS:
3207 case Op_LoadN:
3208 case Op_LoadP:
3209 case Op_LoadVector: {
3210 const TypePtr* adr_type = n->adr_type();
3211 int alias_idx = Compile::current()->get_alias_index(adr_type);
3212 Compile::AliasType* alias_type = Compile::current()->alias_type(alias_idx);
3213 ciField* field = alias_type->field();
3214 bool is_static = field != NULL && field->is_static();
3215 bool is_final = field != NULL && field->is_final();
3216 bool is_stable = field != NULL && field->is_stable();
3228 case Op_AryEq: {
3229 Node* n1 = n->in(2);
3230 Node* n2 = n->in(3);
3231 if (!ShenandoahOptimizeStableFinals ||
3232 !n1->bottom_type()->isa_aryptr() || !n1->bottom_type()->isa_aryptr()->is_stable() ||
3233 !n2->bottom_type()->isa_aryptr() || !n2->bottom_type()->isa_aryptr()->is_stable()) {
3234 strength = WEAK;
3235 }
3236 break;
3237 }
3238 case Op_StrEquals:
3239 case Op_StrComp:
3240 case Op_StrIndexOf:
3241 case Op_StrIndexOfChar:
3242 if (!ShenandoahOptimizeStableFinals) {
3243 strength = WEAK;
3244 }
3245 break;
3246 case Op_Conv2B:
3247 case Op_LoadRange:
3248 // NONE, i.e. leave current strength as is
3249 break;
3250 case Op_AddP:
3251 case Op_CheckCastPP:
3252 case Op_CastPP:
3253 case Op_CMoveP:
3254 case Op_Phi:
3255 case Op_ShenandoahLoadReferenceBarrier:
3256 visit_users = true;
3257 break;
3258 default: {
3259 #ifdef ASSERT
3260 tty->print_cr("Unknown node in get_barrier_strength:");
3261 n->dump(1);
3262 ShouldNotReachHere();
3263 #else
3264 strength = STRONG;
3265 #endif
3266 }
3267 }
|