1 /* 2 * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 */ 23 24 #include "precompiled.hpp" 25 #include "opto/compile.hpp" 26 #include "opto/castnode.hpp" 27 #include "opto/escape.hpp" 28 #include "opto/graphKit.hpp" 29 #include "opto/idealKit.hpp" 30 #include "opto/loopnode.hpp" 31 #include "opto/macro.hpp" 32 #include "opto/node.hpp" 33 #include "opto/type.hpp" 34 #include "utilities/macros.hpp" 35 #include "gc/z/zBarrierSet.hpp" 36 #include "gc/z/c2/zBarrierSetC2.hpp" 37 #include "gc/z/zThreadLocalData.hpp" 38 #include "gc/z/zBarrierSetRuntime.hpp" 39 40 ZBarrierSetC2State::ZBarrierSetC2State(Arena* comp_arena) 41 : _load_barrier_nodes(new (comp_arena) GrowableArray<LoadBarrierNode*>(comp_arena, 8, 0, NULL)) {} 42 43 int ZBarrierSetC2State::load_barrier_count() const { 44 return _load_barrier_nodes->length(); 45 } 46 47 void ZBarrierSetC2State::add_load_barrier_node(LoadBarrierNode * n) { 48 assert(!_load_barrier_nodes->contains(n), " duplicate entry in expand list"); 49 _load_barrier_nodes->append(n); 50 } 51 52 void ZBarrierSetC2State::remove_load_barrier_node(LoadBarrierNode * n) { 53 // this function may be called twice for a node so check 54 // that the node is in the array before attempting to remove it 55 if (_load_barrier_nodes->contains(n)) { 56 _load_barrier_nodes->remove(n); 57 } 58 } 59 60 LoadBarrierNode* ZBarrierSetC2State::load_barrier_node(int idx) const { 61 return _load_barrier_nodes->at(idx); 62 } 63 64 void* ZBarrierSetC2::create_barrier_state(Arena* comp_arena) const { 65 return new(comp_arena) ZBarrierSetC2State(comp_arena); 66 } 67 68 ZBarrierSetC2State* ZBarrierSetC2::state() const { 69 return reinterpret_cast<ZBarrierSetC2State*>(Compile::current()->barrier_set_state()); 70 } 71 72 bool ZBarrierSetC2::is_gc_barrier_node(Node* node) const { 73 // 1. This step follows potential oop projections of a load barrier before expansion 74 if (node->is_Proj()) { 75 node = node->in(0); 76 } 77 78 // 2. This step checks for unexpanded load barriers 79 if (node->is_LoadBarrier()) { 80 return true; 81 } 82 83 // 3. This step checks for the phi corresponding to an optimized load barrier expansion 84 if (node->is_Phi()) { 85 PhiNode* phi = node->as_Phi(); 86 Node* n = phi->in(1); 87 if (n != NULL && (n->is_LoadBarrierSlowReg() || n->is_LoadBarrierWeakSlowReg())) { 88 return true; 89 } 90 } 91 92 return false; 93 } 94 95 void ZBarrierSetC2::register_potential_barrier_node(Node* node) const { 96 if (node->is_LoadBarrier()) { 97 state()->add_load_barrier_node(node->as_LoadBarrier()); 98 } 99 } 100 101 void ZBarrierSetC2::unregister_potential_barrier_node(Node* node) const { 102 if (node->is_LoadBarrier()) { 103 state()->remove_load_barrier_node(node->as_LoadBarrier()); 104 } 105 } 106 107 void ZBarrierSetC2::eliminate_useless_gc_barriers(Unique_Node_List &useful, Compile* C) const { 108 // Remove useless LoadBarrier nodes 109 ZBarrierSetC2State* s = state(); 110 for (int i = s->load_barrier_count()-1; i >= 0; i--) { 111 LoadBarrierNode* n = s->load_barrier_node(i); 112 if (!useful.member(n)) { 113 unregister_potential_barrier_node(n); 114 } 115 } 116 } 117 118 void ZBarrierSetC2::enqueue_useful_gc_barrier(PhaseIterGVN* igvn, Node* node) const { 119 if (node->is_LoadBarrier() && !node->as_LoadBarrier()->has_true_uses()) { 120 igvn->_worklist.push(node); 121 } 122 } 123 124 void ZBarrierSetC2::find_dominating_barriers(PhaseIterGVN& igvn) { 125 // Look for dominating barriers on the same address only once all 126 // other loop opts are over: loop opts may cause a safepoint to be 127 // inserted between a barrier and its dominating barrier. 128 Compile* C = Compile::current(); 129 ZBarrierSetC2* bs = (ZBarrierSetC2*)BarrierSet::barrier_set()->barrier_set_c2(); 130 ZBarrierSetC2State* s = bs->state(); 131 if (s->load_barrier_count() >= 2) { 132 Compile::TracePhase tp("idealLoop", &C->timers[Phase::_t_idealLoop]); 133 PhaseIdealLoop ideal_loop(igvn, LoopOptsLastRound); 134 if (C->major_progress()) C->print_method(PHASE_PHASEIDEALLOOP_ITERATIONS, 2); 135 } 136 } 137 138 void ZBarrierSetC2::add_users_to_worklist(Unique_Node_List* worklist) const { 139 // Permanent temporary workaround 140 // Loadbarriers may have non-obvious dead uses keeping them alive during parsing. The use is 141 // removed by RemoveUseless (after parsing, before optimize) but the barriers won't be added to 142 // the worklist. Unless we add them explicitly they are not guaranteed to end up there. 143 ZBarrierSetC2State* s = state(); 144 145 for (int i = 0; i < s->load_barrier_count(); i++) { 146 LoadBarrierNode* n = s->load_barrier_node(i); 147 worklist->push(n); 148 } 149 } 150 151 const TypeFunc* ZBarrierSetC2::load_barrier_Type() const { 152 const Type** fields; 153 154 // Create input types (domain) 155 fields = TypeTuple::fields(2); 156 fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; 157 fields[TypeFunc::Parms+1] = TypeOopPtr::BOTTOM; 158 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2, fields); 159 160 // Create result type (range) 161 fields = TypeTuple::fields(1); 162 fields[TypeFunc::Parms+0] = TypeInstPtr::BOTTOM; 163 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+1, fields); 164 165 return TypeFunc::make(domain, range); 166 } 167 168 // == LoadBarrierNode == 169 170 LoadBarrierNode::LoadBarrierNode(Compile* C, 171 Node* c, 172 Node* mem, 173 Node* val, 174 Node* adr, 175 bool weak, 176 bool writeback, 177 bool oop_reload_allowed) : 178 MultiNode(Number_of_Inputs), 179 _weak(weak), 180 _writeback(writeback), 181 _oop_reload_allowed(oop_reload_allowed) { 182 init_req(Control, c); 183 init_req(Memory, mem); 184 init_req(Oop, val); 185 init_req(Address, adr); 186 init_req(Similar, C->top()); 187 188 init_class_id(Class_LoadBarrier); 189 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); 190 bs->register_potential_barrier_node(this); 191 } 192 193 uint LoadBarrierNode::size_of() const { 194 return sizeof(*this); 195 } 196 197 uint LoadBarrierNode::cmp(const Node& n) const { 198 ShouldNotReachHere(); 199 return 0; 200 } 201 202 const Type *LoadBarrierNode::bottom_type() const { 203 const Type** floadbarrier = (const Type **)(Compile::current()->type_arena()->Amalloc_4((Number_of_Outputs)*sizeof(Type*))); 204 Node* in_oop = in(Oop); 205 floadbarrier[Control] = Type::CONTROL; 206 floadbarrier[Memory] = Type::MEMORY; 207 floadbarrier[Oop] = in_oop == NULL ? Type::TOP : in_oop->bottom_type(); 208 return TypeTuple::make(Number_of_Outputs, floadbarrier); 209 } 210 211 const TypePtr* LoadBarrierNode::adr_type() const { 212 ShouldNotReachHere(); 213 return NULL; 214 } 215 216 const Type *LoadBarrierNode::Value(PhaseGVN *phase) const { 217 const Type** floadbarrier = (const Type **)(phase->C->type_arena()->Amalloc_4((Number_of_Outputs)*sizeof(Type*))); 218 const Type* val_t = phase->type(in(Oop)); 219 floadbarrier[Control] = Type::CONTROL; 220 floadbarrier[Memory] = Type::MEMORY; 221 floadbarrier[Oop] = val_t; 222 return TypeTuple::make(Number_of_Outputs, floadbarrier); 223 } 224 225 bool LoadBarrierNode::is_dominator(PhaseIdealLoop* phase, bool linear_only, Node *d, Node *n) { 226 if (phase != NULL) { 227 return phase->is_dominator(d, n); 228 } 229 230 for (int i = 0; i < 10 && n != NULL; i++) { 231 n = IfNode::up_one_dom(n, linear_only); 232 if (n == d) { 233 return true; 234 } 235 } 236 237 return false; 238 } 239 240 LoadBarrierNode* LoadBarrierNode::has_dominating_barrier(PhaseIdealLoop* phase, bool linear_only, bool look_for_similar) { 241 Node* val = in(LoadBarrierNode::Oop); 242 if (in(Similar)->is_Proj() && in(Similar)->in(0)->is_LoadBarrier()) { 243 LoadBarrierNode* lb = in(Similar)->in(0)->as_LoadBarrier(); 244 assert(lb->in(Address) == in(Address), ""); 245 // Load barrier on Similar edge dominates so if it now has the Oop field it can replace this barrier. 246 if (lb->in(Oop) == in(Oop)) { 247 return lb; 248 } 249 // Follow chain of load barrier through Similar edges 250 while (!lb->in(Similar)->is_top()) { 251 lb = lb->in(Similar)->in(0)->as_LoadBarrier(); 252 assert(lb->in(Address) == in(Address), ""); 253 } 254 if (lb != in(Similar)->in(0)) { 255 return lb; 256 } 257 } 258 for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) { 259 Node* u = val->fast_out(i); 260 if (u != this && u->is_LoadBarrier() && u->in(Oop) == val && u->as_LoadBarrier()->has_true_uses()) { 261 Node* this_ctrl = in(LoadBarrierNode::Control); 262 Node* other_ctrl = u->in(LoadBarrierNode::Control); 263 if (is_dominator(phase, linear_only, other_ctrl, this_ctrl)) { 264 return u->as_LoadBarrier(); 265 } 266 } 267 } 268 269 if (ZVerifyLoadBarriers || can_be_eliminated()) { 270 return NULL; 271 } 272 273 if (!look_for_similar) { 274 return NULL; 275 } 276 277 Node* addr = in(LoadBarrierNode::Address); 278 for (DUIterator_Fast imax, i = addr->fast_outs(imax); i < imax; i++) { 279 Node* u = addr->fast_out(i); 280 if (u != this && u->is_LoadBarrier() && u->as_LoadBarrier()->has_true_uses()) { 281 Node* this_ctrl = in(LoadBarrierNode::Control); 282 Node* other_ctrl = u->in(LoadBarrierNode::Control); 283 if (is_dominator(phase, linear_only, other_ctrl, this_ctrl)) { 284 ResourceMark rm; 285 Unique_Node_List wq; 286 wq.push(in(LoadBarrierNode::Control)); 287 bool ok = true; 288 bool dom_found = false; 289 for (uint next = 0; next < wq.size(); ++next) { 290 Node *n = wq.at(next); 291 if (n->is_top()) { 292 return NULL; 293 } 294 assert(n->is_CFG(), ""); 295 if (n->is_SafePoint()) { 296 ok = false; 297 break; 298 } 299 if (n == u) { 300 dom_found = true; 301 continue; 302 } 303 if (n->is_Region()) { 304 for (uint i = 1; i < n->req(); i++) { 305 Node* m = n->in(i); 306 if (m != NULL) { 307 wq.push(m); 308 } 309 } 310 } else { 311 Node* m = n->in(0); 312 if (m != NULL) { 313 wq.push(m); 314 } 315 } 316 } 317 if (ok) { 318 assert(dom_found, ""); 319 return u->as_LoadBarrier();; 320 } 321 break; 322 } 323 } 324 } 325 326 return NULL; 327 } 328 329 void LoadBarrierNode::push_dominated_barriers(PhaseIterGVN* igvn) const { 330 // Change to that barrier may affect a dominated barrier so re-push those 331 Node* val = in(LoadBarrierNode::Oop); 332 333 for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) { 334 Node* u = val->fast_out(i); 335 if (u != this && u->is_LoadBarrier() && u->in(Oop) == val) { 336 Node* this_ctrl = in(Control); 337 Node* other_ctrl = u->in(Control); 338 if (is_dominator(NULL, false, this_ctrl, other_ctrl)) { 339 igvn->_worklist.push(u); 340 } 341 } 342 343 Node* addr = in(LoadBarrierNode::Address); 344 for (DUIterator_Fast imax, i = addr->fast_outs(imax); i < imax; i++) { 345 Node* u = addr->fast_out(i); 346 if (u != this && u->is_LoadBarrier() && u->in(Similar)->is_top()) { 347 Node* this_ctrl = in(Control); 348 Node* other_ctrl = u->in(Control); 349 if (is_dominator(NULL, false, this_ctrl, other_ctrl)) { 350 igvn->_worklist.push(u); 351 } 352 } 353 } 354 } 355 } 356 357 Node *LoadBarrierNode::Identity(PhaseGVN *phase) { 358 if (!phase->C->directive()->ZOptimizeLoadBarriersOption) { 359 return this; 360 } 361 362 bool redundant_addr = false; 363 LoadBarrierNode* dominating_barrier = has_dominating_barrier(NULL, true, false); 364 if (dominating_barrier != NULL) { 365 assert(dominating_barrier->in(Oop) == in(Oop), ""); 366 return dominating_barrier; 367 } 368 369 return this; 370 } 371 372 Node *LoadBarrierNode::Ideal(PhaseGVN *phase, bool can_reshape) { 373 if (remove_dead_region(phase, can_reshape)) { 374 return this; 375 } 376 377 Node* val = in(Oop); 378 Node* mem = in(Memory); 379 Node* ctrl = in(Control); 380 Node* adr = in(Address); 381 assert(val->Opcode() != Op_LoadN, ""); 382 383 if (mem->is_MergeMem()) { 384 Node* new_mem = mem->as_MergeMem()->memory_at(Compile::AliasIdxRaw); 385 set_req(Memory, new_mem); 386 if (mem->outcnt() == 0 && can_reshape) { 387 phase->is_IterGVN()->_worklist.push(mem); 388 } 389 390 return this; 391 } 392 393 bool optimizeLoadBarriers = phase->C->directive()->ZOptimizeLoadBarriersOption; 394 LoadBarrierNode* dominating_barrier = optimizeLoadBarriers ? has_dominating_barrier(NULL, !can_reshape, !phase->C->major_progress()) : NULL; 395 if (dominating_barrier != NULL && dominating_barrier->in(Oop) != in(Oop)) { 396 assert(in(Address) == dominating_barrier->in(Address), ""); 397 set_req(Similar, dominating_barrier->proj_out(Oop)); 398 return this; 399 } 400 401 bool eliminate = (optimizeLoadBarriers && !(val->is_Phi() || val->Opcode() == Op_LoadP || val->Opcode() == Op_GetAndSetP || val->is_DecodeN())) || 402 (can_reshape && (dominating_barrier != NULL || !has_true_uses())); 403 404 if (eliminate) { 405 if (can_reshape) { 406 PhaseIterGVN* igvn = phase->is_IterGVN(); 407 Node* out_ctrl = proj_out_or_null(Control); 408 Node* out_res = proj_out_or_null(Oop); 409 410 if (out_ctrl != NULL) { 411 igvn->replace_node(out_ctrl, ctrl); 412 } 413 414 // That transformation may cause the Similar edge on the load barrier to be invalid 415 fix_similar_in_uses(igvn); 416 if (out_res != NULL) { 417 if (dominating_barrier != NULL) { 418 igvn->replace_node(out_res, dominating_barrier->proj_out(Oop)); 419 } else { 420 igvn->replace_node(out_res, val); 421 } 422 } 423 } 424 425 return new ConINode(TypeInt::ZERO); 426 } 427 428 // If the Similar edge is no longer a load barrier, clear it 429 Node* similar = in(Similar); 430 if (!similar->is_top() && !(similar->is_Proj() && similar->in(0)->is_LoadBarrier())) { 431 set_req(Similar, phase->C->top()); 432 return this; 433 } 434 435 if (can_reshape) { 436 // If this barrier is linked through the Similar edge by a 437 // dominated barrier and both barriers have the same Oop field, 438 // the dominated barrier can go away, so push it for reprocessing. 439 // We also want to avoid a barrier to depend on another dominating 440 // barrier through its Similar edge that itself depend on another 441 // barrier through its Similar edge and rather have the first 442 // depend on the third. 443 PhaseIterGVN* igvn = phase->is_IterGVN(); 444 Node* out_res = proj_out(Oop); 445 for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) { 446 Node* u = out_res->fast_out(i); 447 if (u->is_LoadBarrier() && u->in(Similar) == out_res && 448 (u->in(Oop) == val || !u->in(Similar)->is_top())) { 449 igvn->_worklist.push(u); 450 } 451 } 452 453 push_dominated_barriers(igvn); 454 } 455 456 return NULL; 457 } 458 459 uint LoadBarrierNode::match_edge(uint idx) const { 460 ShouldNotReachHere(); 461 return 0; 462 } 463 464 void LoadBarrierNode::fix_similar_in_uses(PhaseIterGVN* igvn) { 465 Node* out_res = proj_out_or_null(Oop); 466 if (out_res == NULL) { 467 return; 468 } 469 470 for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) { 471 Node* u = out_res->fast_out(i); 472 if (u->is_LoadBarrier() && u->in(Similar) == out_res) { 473 igvn->replace_input_of(u, Similar, igvn->C->top()); 474 --i; 475 --imax; 476 } 477 } 478 } 479 480 bool LoadBarrierNode::has_true_uses() const { 481 Node* out_res = proj_out_or_null(Oop); 482 if (out_res == NULL) { 483 return false; 484 } 485 486 for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) { 487 Node* u = out_res->fast_out(i); 488 if (!u->is_LoadBarrier() || u->in(Similar) != out_res) { 489 return true; 490 } 491 } 492 493 return false; 494 } 495 496 // == Accesses == 497 498 Node* ZBarrierSetC2::make_cas_loadbarrier(C2AtomicParseAccess& access) const { 499 assert(!UseCompressedOops, "Not allowed"); 500 CompareAndSwapNode* cas = (CompareAndSwapNode*)access.raw_access(); 501 PhaseGVN& gvn = access.gvn(); 502 Compile* C = Compile::current(); 503 GraphKit* kit = access.kit(); 504 505 Node* in_ctrl = cas->in(MemNode::Control); 506 Node* in_mem = cas->in(MemNode::Memory); 507 Node* in_adr = cas->in(MemNode::Address); 508 Node* in_val = cas->in(MemNode::ValueIn); 509 Node* in_expected = cas->in(LoadStoreConditionalNode::ExpectedIn); 510 511 float likely = PROB_LIKELY(0.999); 512 513 const TypePtr *adr_type = gvn.type(in_adr)->isa_ptr(); 514 Compile::AliasType* alias_type = C->alias_type(adr_type); 515 int alias_idx = C->get_alias_index(adr_type); 516 517 // Outer check - true: continue, false: load and check 518 Node* region = new RegionNode(3); 519 Node* phi = new PhiNode(region, TypeInt::BOOL); 520 Node* phi_mem = new PhiNode(region, Type::MEMORY, adr_type); 521 522 // Inner check - is the healed ref equal to the expected 523 Node* region2 = new RegionNode(3); 524 Node* phi2 = new PhiNode(region2, TypeInt::BOOL); 525 Node* phi_mem2 = new PhiNode(region2, Type::MEMORY, adr_type); 526 527 // CAS node returns 0 or 1 528 Node* cmp = gvn.transform(new CmpINode(cas, kit->intcon(0))); 529 Node* bol = gvn.transform(new BoolNode(cmp, BoolTest::ne))->as_Bool(); 530 IfNode* iff = gvn.transform(new IfNode(in_ctrl, bol, likely, COUNT_UNKNOWN))->as_If(); 531 Node* then = gvn.transform(new IfTrueNode(iff)); 532 Node* elsen = gvn.transform(new IfFalseNode(iff)); 533 534 Node* scmemproj1 = gvn.transform(new SCMemProjNode(cas)); 535 536 kit->set_memory(scmemproj1, alias_idx); 537 phi_mem->init_req(1, scmemproj1); 538 phi_mem2->init_req(2, scmemproj1); 539 540 // CAS fail - reload and heal oop 541 Node* reload = kit->make_load(elsen, in_adr, TypeOopPtr::BOTTOM, T_OBJECT, MemNode::unordered); 542 Node* barrier = gvn.transform(new LoadBarrierNode(C, elsen, scmemproj1, reload, in_adr, false, true, false)); 543 Node* barrierctrl = gvn.transform(new ProjNode(barrier, LoadBarrierNode::Control)); 544 Node* barrierdata = gvn.transform(new ProjNode(barrier, LoadBarrierNode::Oop)); 545 546 // Check load 547 Node* tmpX = gvn.transform(new CastP2XNode(NULL, barrierdata)); 548 Node* in_expX = gvn.transform(new CastP2XNode(NULL, in_expected)); 549 Node* cmp2 = gvn.transform(new CmpXNode(tmpX, in_expX)); 550 Node *bol2 = gvn.transform(new BoolNode(cmp2, BoolTest::ne))->as_Bool(); 551 IfNode* iff2 = gvn.transform(new IfNode(barrierctrl, bol2, likely, COUNT_UNKNOWN))->as_If(); 552 Node* then2 = gvn.transform(new IfTrueNode(iff2)); 553 Node* elsen2 = gvn.transform(new IfFalseNode(iff2)); 554 555 // redo CAS 556 Node* cas2 = gvn.transform(new CompareAndSwapPNode(elsen2, kit->memory(alias_idx), in_adr, in_val, in_expected, cas->order())); 557 Node* scmemproj2 = gvn.transform(new SCMemProjNode(cas2)); 558 kit->set_control(elsen2); 559 kit->set_memory(scmemproj2, alias_idx); 560 561 // Merge inner flow - check if healed oop was equal too expected. 562 region2->set_req(1, kit->control()); 563 region2->set_req(2, then2); 564 phi2->set_req(1, cas2); 565 phi2->set_req(2, kit->intcon(0)); 566 phi_mem2->init_req(1, scmemproj2); 567 kit->set_memory(phi_mem2, alias_idx); 568 569 // Merge outer flow - then check if first CAS succeeded 570 region->set_req(1, then); 571 region->set_req(2, region2); 572 phi->set_req(1, kit->intcon(1)); 573 phi->set_req(2, phi2); 574 phi_mem->init_req(2, phi_mem2); 575 kit->set_memory(phi_mem, alias_idx); 576 577 gvn.transform(region2); 578 gvn.transform(phi2); 579 gvn.transform(phi_mem2); 580 gvn.transform(region); 581 gvn.transform(phi); 582 gvn.transform(phi_mem); 583 584 kit->set_control(region); 585 kit->insert_mem_bar(Op_MemBarCPUOrder); 586 587 return phi; 588 } 589 590 Node* ZBarrierSetC2::make_cmpx_loadbarrier(C2AtomicParseAccess& access) const { 591 CompareAndExchangePNode* cmpx = (CompareAndExchangePNode*)access.raw_access(); 592 GraphKit* kit = access.kit(); 593 PhaseGVN& gvn = kit->gvn(); 594 Compile* C = Compile::current(); 595 596 Node* in_ctrl = cmpx->in(MemNode::Control); 597 Node* in_mem = cmpx->in(MemNode::Memory); 598 Node* in_adr = cmpx->in(MemNode::Address); 599 Node* in_val = cmpx->in(MemNode::ValueIn); 600 Node* in_expected = cmpx->in(LoadStoreConditionalNode::ExpectedIn); 601 602 float likely = PROB_LIKELY(0.999); 603 604 const TypePtr *adr_type = cmpx->get_ptr_type(); 605 Compile::AliasType* alias_type = C->alias_type(adr_type); 606 int alias_idx = C->get_alias_index(adr_type); 607 608 // Outer check - true: continue, false: load and check 609 Node* region = new RegionNode(3); 610 Node* phi = new PhiNode(region, adr_type); 611 612 // Inner check - is the healed ref equal to the expected 613 Node* region2 = new RegionNode(3); 614 Node* phi2 = new PhiNode(region2, adr_type); 615 616 // Check if cmpx succeeded 617 Node* cmp = gvn.transform(new CmpPNode(cmpx, in_expected)); 618 Node* bol = gvn.transform(new BoolNode(cmp, BoolTest::eq))->as_Bool(); 619 IfNode* iff = gvn.transform(new IfNode(in_ctrl, bol, likely, COUNT_UNKNOWN))->as_If(); 620 Node* then = gvn.transform(new IfTrueNode(iff)); 621 Node* elsen = gvn.transform(new IfFalseNode(iff)); 622 623 Node* scmemproj1 = gvn.transform(new SCMemProjNode(cmpx)); 624 kit->set_memory(scmemproj1, alias_idx); 625 626 // CAS fail - reload and heal oop 627 Node* reload = kit->make_load(elsen, in_adr, TypeOopPtr::BOTTOM, T_OBJECT, MemNode::unordered); 628 Node* barrier = gvn.transform(new LoadBarrierNode(C, elsen, scmemproj1, reload, in_adr, false, true, false)); 629 Node* barrierctrl = gvn.transform(new ProjNode(barrier, LoadBarrierNode::Control)); 630 Node* barrierdata = gvn.transform(new ProjNode(barrier, LoadBarrierNode::Oop)); 631 632 // Check load 633 Node* tmpX = gvn.transform(new CastP2XNode(NULL, barrierdata)); 634 Node* in_expX = gvn.transform(new CastP2XNode(NULL, in_expected)); 635 Node* cmp2 = gvn.transform(new CmpXNode(tmpX, in_expX)); 636 Node *bol2 = gvn.transform(new BoolNode(cmp2, BoolTest::ne))->as_Bool(); 637 IfNode* iff2 = gvn.transform(new IfNode(barrierctrl, bol2, likely, COUNT_UNKNOWN))->as_If(); 638 Node* then2 = gvn.transform(new IfTrueNode(iff2)); 639 Node* elsen2 = gvn.transform(new IfFalseNode(iff2)); 640 641 // Redo CAS 642 Node* cmpx2 = gvn.transform(new CompareAndExchangePNode(elsen2, kit->memory(alias_idx), in_adr, in_val, in_expected, adr_type, cmpx->get_ptr_type(), cmpx->order())); 643 Node* scmemproj2 = gvn.transform(new SCMemProjNode(cmpx2)); 644 kit->set_control(elsen2); 645 kit->set_memory(scmemproj2, alias_idx); 646 647 // Merge inner flow - check if healed oop was equal too expected. 648 region2->set_req(1, kit->control()); 649 region2->set_req(2, then2); 650 phi2->set_req(1, cmpx2); 651 phi2->set_req(2, barrierdata); 652 653 // Merge outer flow - then check if first cas succeeded 654 region->set_req(1, then); 655 region->set_req(2, region2); 656 phi->set_req(1, cmpx); 657 phi->set_req(2, phi2); 658 659 gvn.transform(region2); 660 gvn.transform(phi2); 661 gvn.transform(region); 662 gvn.transform(phi); 663 664 kit->set_control(region); 665 kit->set_memory(in_mem, alias_idx); 666 kit->insert_mem_bar(Op_MemBarCPUOrder); 667 668 return phi; 669 } 670 671 Node* ZBarrierSetC2::load_barrier(GraphKit* kit, Node* val, Node* adr, bool weak, bool writeback, bool oop_reload_allowed) const { 672 PhaseGVN& gvn = kit->gvn(); 673 Node* barrier = new LoadBarrierNode(Compile::current(), kit->control(), kit->memory(TypeRawPtr::BOTTOM), val, adr, weak, writeback, oop_reload_allowed); 674 Node* transformed_barrier = gvn.transform(barrier); 675 676 if (transformed_barrier->is_LoadBarrier()) { 677 if (barrier == transformed_barrier) { 678 kit->set_control(gvn.transform(new ProjNode(barrier, LoadBarrierNode::Control))); 679 } 680 Node* result = gvn.transform(new ProjNode(transformed_barrier, LoadBarrierNode::Oop)); 681 return result; 682 } else { 683 return val; 684 } 685 } 686 687 static bool barrier_needed(C2Access& access) { 688 return ZBarrierSet::barrier_needed(access.decorators(), access.type()); 689 } 690 691 Node* ZBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const { 692 Node* p = BarrierSetC2::load_at_resolved(access, val_type); 693 if (!barrier_needed(access)) { 694 return p; 695 } 696 697 bool weak = (access.decorators() & ON_WEAK_OOP_REF) != 0; 698 699 assert(access.is_parse_access(), "entry not supported at optimization time"); 700 C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access); 701 GraphKit* kit = parse_access.kit(); 702 PhaseGVN& gvn = kit->gvn(); 703 Node* adr = access.addr().node(); 704 Node* heap_base_oop = access.base(); 705 bool unsafe = (access.decorators() & C2_UNSAFE_ACCESS) != 0; 706 if (unsafe) { 707 if (!ZVerifyLoadBarriers) { 708 p = load_barrier(kit, p, adr); 709 } else { 710 if (!TypePtr::NULL_PTR->higher_equal(gvn.type(heap_base_oop))) { 711 p = load_barrier(kit, p, adr); 712 } else { 713 IdealKit ideal(kit); 714 IdealVariable res(ideal); 715 #define __ ideal. 716 __ declarations_done(); 717 __ set(res, p); 718 __ if_then(heap_base_oop, BoolTest::ne, kit->null(), PROB_UNLIKELY(0.999)); { 719 kit->sync_kit(ideal); 720 p = load_barrier(kit, p, adr); 721 __ set(res, p); 722 __ sync_kit(kit); 723 } __ end_if(); 724 kit->final_sync(ideal); 725 p = __ value(res); 726 #undef __ 727 } 728 } 729 return p; 730 } else { 731 return load_barrier(parse_access.kit(), p, access.addr().node(), weak, true, true); 732 } 733 } 734 735 Node* ZBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val, 736 Node* new_val, const Type* val_type) const { 737 Node* result = BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, val_type); 738 if (!barrier_needed(access)) { 739 return result; 740 } 741 742 access.set_needs_pinning(false); 743 return make_cmpx_loadbarrier(access); 744 } 745 746 Node* ZBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val, 747 Node* new_val, const Type* value_type) const { 748 Node* result = BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type); 749 if (!barrier_needed(access)) { 750 return result; 751 } 752 753 Node* load_store = access.raw_access(); 754 bool weak_cas = (access.decorators() & C2_WEAK_CMPXCHG) != 0; 755 bool expected_is_null = (expected_val->get_ptr_type() == TypePtr::NULL_PTR); 756 757 if (!expected_is_null) { 758 if (weak_cas) { 759 access.set_needs_pinning(false); 760 load_store = make_cas_loadbarrier(access); 761 } else { 762 access.set_needs_pinning(false); 763 load_store = make_cas_loadbarrier(access); 764 } 765 } 766 767 return load_store; 768 } 769 770 Node* ZBarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* val_type) const { 771 Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, new_val, val_type); 772 if (!barrier_needed(access)) { 773 return result; 774 } 775 776 Node* load_store = access.raw_access(); 777 Node* adr = access.addr().node(); 778 779 assert(access.is_parse_access(), "entry not supported at optimization time"); 780 C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access); 781 return load_barrier(parse_access.kit(), load_store, adr, false, false, false); 782 } 783 784 // == Macro Expansion == 785 786 void ZBarrierSetC2::expand_loadbarrier_node(PhaseMacroExpand* phase, LoadBarrierNode* barrier) const { 787 Node* in_ctrl = barrier->in(LoadBarrierNode::Control); 788 Node* in_mem = barrier->in(LoadBarrierNode::Memory); 789 Node* in_val = barrier->in(LoadBarrierNode::Oop); 790 Node* in_adr = barrier->in(LoadBarrierNode::Address); 791 792 Node* out_ctrl = barrier->proj_out(LoadBarrierNode::Control); 793 Node* out_res = barrier->proj_out(LoadBarrierNode::Oop); 794 795 PhaseIterGVN &igvn = phase->igvn(); 796 797 if (ZVerifyLoadBarriers) { 798 igvn.replace_node(out_res, in_val); 799 igvn.replace_node(out_ctrl, in_ctrl); 800 return; 801 } 802 803 if (barrier->can_be_eliminated()) { 804 // Clone and pin the load for this barrier below the dominating 805 // barrier: the load cannot be allowed to float above the 806 // dominating barrier 807 Node* load = in_val; 808 809 if (load->is_Load()) { 810 Node* new_load = load->clone(); 811 Node* addp = new_load->in(MemNode::Address); 812 assert(addp->is_AddP() || addp->is_Phi() || addp->is_Load(), "bad address"); 813 Node* cast = new CastPPNode(addp, igvn.type(addp), true); 814 Node* ctrl = NULL; 815 Node* similar = barrier->in(LoadBarrierNode::Similar); 816 if (similar->is_Phi()) { 817 // already expanded 818 ctrl = similar->in(0); 819 } else { 820 assert(similar->is_Proj() && similar->in(0)->is_LoadBarrier(), "unexpected graph shape"); 821 ctrl = similar->in(0)->as_LoadBarrier()->proj_out(LoadBarrierNode::Control); 822 } 823 assert(ctrl != NULL, "bad control"); 824 cast->set_req(0, ctrl); 825 igvn.transform(cast); 826 new_load->set_req(MemNode::Address, cast); 827 igvn.transform(new_load); 828 829 igvn.replace_node(out_res, new_load); 830 igvn.replace_node(out_ctrl, in_ctrl); 831 return; 832 } 833 // cannot eliminate 834 } 835 836 // There are two cases that require the basic loadbarrier 837 // 1) When the writeback of a healed oop must be avoided (swap) 838 // 2) When we must guarantee that no reload of is done (swap, cas, cmpx) 839 if (!barrier->is_writeback()) { 840 assert(!barrier->oop_reload_allowed(), "writeback barriers should be marked as requires oop"); 841 } 842 843 if (!barrier->oop_reload_allowed()) { 844 expand_loadbarrier_basic(phase, barrier); 845 } else { 846 expand_loadbarrier_optimized(phase, barrier); 847 } 848 } 849 850 // Basic loadbarrier using conventional argument passing 851 void ZBarrierSetC2::expand_loadbarrier_basic(PhaseMacroExpand* phase, LoadBarrierNode *barrier) const { 852 PhaseIterGVN &igvn = phase->igvn(); 853 854 Node* in_ctrl = barrier->in(LoadBarrierNode::Control); 855 Node* in_mem = barrier->in(LoadBarrierNode::Memory); 856 Node* in_val = barrier->in(LoadBarrierNode::Oop); 857 Node* in_adr = barrier->in(LoadBarrierNode::Address); 858 859 Node* out_ctrl = barrier->proj_out(LoadBarrierNode::Control); 860 Node* out_res = barrier->proj_out(LoadBarrierNode::Oop); 861 862 float unlikely = PROB_UNLIKELY(0.999); 863 const Type* in_val_maybe_null_t = igvn.type(in_val); 864 865 Node* jthread = igvn.transform(new ThreadLocalNode()); 866 Node* adr = phase->basic_plus_adr(jthread, in_bytes(ZThreadLocalData::address_bad_mask_offset())); 867 Node* bad_mask = igvn.transform(LoadNode::make(igvn, in_ctrl, in_mem, adr, TypeRawPtr::BOTTOM, TypeX_X, TypeX_X->basic_type(), MemNode::unordered)); 868 Node* cast = igvn.transform(new CastP2XNode(in_ctrl, in_val)); 869 Node* obj_masked = igvn.transform(new AndXNode(cast, bad_mask)); 870 Node* cmp = igvn.transform(new CmpXNode(obj_masked, igvn.zerocon(TypeX_X->basic_type()))); 871 Node *bol = igvn.transform(new BoolNode(cmp, BoolTest::ne))->as_Bool(); 872 IfNode* iff = igvn.transform(new IfNode(in_ctrl, bol, unlikely, COUNT_UNKNOWN))->as_If(); 873 Node* then = igvn.transform(new IfTrueNode(iff)); 874 Node* elsen = igvn.transform(new IfFalseNode(iff)); 875 876 Node* result_region; 877 Node* result_val; 878 879 result_region = new RegionNode(3); 880 result_val = new PhiNode(result_region, TypeInstPtr::BOTTOM); 881 882 result_region->set_req(1, elsen); 883 Node* res = igvn.transform(new CastPPNode(in_val, in_val_maybe_null_t)); 884 res->init_req(0, elsen); 885 result_val->set_req(1, res); 886 887 const TypeFunc *tf = load_barrier_Type(); 888 Node* call; 889 if (barrier->is_weak()) { 890 call = new CallLeafNode(tf, 891 ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded_addr(), 892 "ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded", 893 TypeRawPtr::BOTTOM); 894 } else { 895 call = new CallLeafNode(tf, 896 ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(), 897 "ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded", 898 TypeRawPtr::BOTTOM); 899 } 900 901 call->init_req(TypeFunc::Control, then); 902 call->init_req(TypeFunc::I_O , phase->top()); 903 call->init_req(TypeFunc::Memory , in_mem); 904 call->init_req(TypeFunc::FramePtr, phase->top()); 905 call->init_req(TypeFunc::ReturnAdr, phase->top()); 906 call->init_req(TypeFunc::Parms+0, in_val); 907 if (barrier->is_writeback()) { 908 call->init_req(TypeFunc::Parms+1, in_adr); 909 } else { 910 // When slow path is called with a null address, the healed oop will not be written back 911 call->init_req(TypeFunc::Parms+1, igvn.zerocon(T_OBJECT)); 912 } 913 call = igvn.transform(call); 914 915 Node* ctrl = igvn.transform(new ProjNode(call, TypeFunc::Control)); 916 res = igvn.transform(new ProjNode(call, TypeFunc::Parms)); 917 res = igvn.transform(new CheckCastPPNode(ctrl, res, in_val_maybe_null_t)); 918 919 result_region->set_req(2, ctrl); 920 result_val->set_req(2, res); 921 922 result_region = igvn.transform(result_region); 923 result_val = igvn.transform(result_val); 924 925 if (out_ctrl != NULL) { // Added if cond 926 igvn.replace_node(out_ctrl, result_region); 927 } 928 igvn.replace_node(out_res, result_val); 929 } 930 931 // Optimized, low spill, loadbarrier variant using stub specialized on register used 932 void ZBarrierSetC2::expand_loadbarrier_optimized(PhaseMacroExpand* phase, LoadBarrierNode *barrier) const { 933 PhaseIterGVN &igvn = phase->igvn(); 934 #ifdef PRINT_NODE_TRAVERSALS 935 Node* preceding_barrier_node = barrier->in(LoadBarrierNode::Oop); 936 #endif 937 938 Node* in_ctrl = barrier->in(LoadBarrierNode::Control); 939 Node* in_mem = barrier->in(LoadBarrierNode::Memory); 940 Node* in_val = barrier->in(LoadBarrierNode::Oop); 941 Node* in_adr = barrier->in(LoadBarrierNode::Address); 942 943 Node* out_ctrl = barrier->proj_out(LoadBarrierNode::Control); 944 Node* out_res = barrier->proj_out(LoadBarrierNode::Oop); 945 946 assert(barrier->in(LoadBarrierNode::Oop) != NULL, "oop to loadbarrier node cannot be null"); 947 948 #ifdef PRINT_NODE_TRAVERSALS 949 tty->print("\n\n\nBefore barrier optimization:\n"); 950 traverse(barrier, out_ctrl, out_res, -1); 951 952 tty->print("\nBefore barrier optimization: preceding_barrier_node\n"); 953 traverse(preceding_barrier_node, out_ctrl, out_res, -1); 954 #endif 955 956 float unlikely = PROB_UNLIKELY(0.999); 957 958 Node* jthread = igvn.transform(new ThreadLocalNode()); 959 Node* adr = phase->basic_plus_adr(jthread, in_bytes(ZThreadLocalData::address_bad_mask_offset())); 960 Node* bad_mask = igvn.transform(LoadNode::make(igvn, in_ctrl, in_mem, adr, 961 TypeRawPtr::BOTTOM, TypeX_X, TypeX_X->basic_type(), 962 MemNode::unordered)); 963 Node* cast = igvn.transform(new CastP2XNode(in_ctrl, in_val)); 964 Node* obj_masked = igvn.transform(new AndXNode(cast, bad_mask)); 965 Node* cmp = igvn.transform(new CmpXNode(obj_masked, igvn.zerocon(TypeX_X->basic_type()))); 966 Node *bol = igvn.transform(new BoolNode(cmp, BoolTest::ne))->as_Bool(); 967 IfNode* iff = igvn.transform(new IfNode(in_ctrl, bol, unlikely, COUNT_UNKNOWN))->as_If(); 968 Node* then = igvn.transform(new IfTrueNode(iff)); 969 Node* elsen = igvn.transform(new IfFalseNode(iff)); 970 971 Node* slow_path_surrogate; 972 if (!barrier->is_weak()) { 973 slow_path_surrogate = igvn.transform(new LoadBarrierSlowRegNode(then, in_mem, in_adr, in_val->adr_type(), 974 (const TypePtr*) in_val->bottom_type(), MemNode::unordered)); 975 } else { 976 slow_path_surrogate = igvn.transform(new LoadBarrierWeakSlowRegNode(then, in_mem, in_adr, in_val->adr_type(), 977 (const TypePtr*) in_val->bottom_type(), MemNode::unordered)); 978 } 979 980 Node *new_loadp; 981 new_loadp = slow_path_surrogate; 982 // Create the final region/phi pair to converge cntl/data paths to downstream code 983 Node* result_region = igvn.transform(new RegionNode(3)); 984 result_region->set_req(1, then); 985 result_region->set_req(2, elsen); 986 987 Node* result_phi = igvn.transform(new PhiNode(result_region, TypeInstPtr::BOTTOM)); 988 result_phi->set_req(1, new_loadp); 989 result_phi->set_req(2, barrier->in(LoadBarrierNode::Oop)); 990 991 // Finally, connect the original outputs to the barrier region and phi to complete the expansion/substitution 992 // igvn.replace_node(out_ctrl, result_region); 993 if (out_ctrl != NULL) { // added if cond 994 igvn.replace_node(out_ctrl, result_region); 995 } 996 igvn.replace_node(out_res, result_phi); 997 998 assert(barrier->outcnt() == 0,"LoadBarrier macro node has non-null outputs after expansion!"); 999 1000 #ifdef PRINT_NODE_TRAVERSALS 1001 tty->print("\nAfter barrier optimization: old out_ctrl\n"); 1002 traverse(out_ctrl, out_ctrl, out_res, -1); 1003 tty->print("\nAfter barrier optimization: old out_res\n"); 1004 traverse(out_res, out_ctrl, out_res, -1); 1005 tty->print("\nAfter barrier optimization: old barrier\n"); 1006 traverse(barrier, out_ctrl, out_res, -1); 1007 tty->print("\nAfter barrier optimization: preceding_barrier_node\n"); 1008 traverse(preceding_barrier_node, result_region, result_phi, -1); 1009 #endif 1010 1011 assert(is_gc_barrier_node(result_phi), "sanity"); 1012 assert(step_over_gc_barrier(result_phi) == in_val, "sanity"); 1013 1014 return; 1015 } 1016 1017 bool ZBarrierSetC2::expand_barriers(Compile* C, PhaseIterGVN& igvn) const { 1018 ZBarrierSetC2State* s = state(); 1019 if (s->load_barrier_count() > 0) { 1020 PhaseMacroExpand macro(igvn); 1021 #ifdef ASSERT 1022 verify_gc_barriers(false); 1023 #endif 1024 int skipped = 0; 1025 while (s->load_barrier_count() > skipped) { 1026 int load_barrier_count = s->load_barrier_count(); 1027 LoadBarrierNode * n = s->load_barrier_node(load_barrier_count-1-skipped); 1028 if (igvn.type(n) == Type::TOP || (n->in(0) != NULL && n->in(0)->is_top())) { 1029 // Node is unreachable, so don't try to expand it 1030 s->remove_load_barrier_node(n); 1031 continue; 1032 } 1033 if (!n->can_be_eliminated()) { 1034 skipped++; 1035 continue; 1036 } 1037 expand_loadbarrier_node(¯o, n); 1038 assert(s->load_barrier_count() < load_barrier_count, "must have deleted a node from load barrier list"); 1039 if (C->failing()) return true; 1040 } 1041 while (s->load_barrier_count() > 0) { 1042 int load_barrier_count = s->load_barrier_count(); 1043 LoadBarrierNode* n = s->load_barrier_node(load_barrier_count - 1); 1044 assert(!(igvn.type(n) == Type::TOP || (n->in(0) != NULL && n->in(0)->is_top())), "should have been processed already"); 1045 assert(!n->can_be_eliminated(), "should have been processed already"); 1046 expand_loadbarrier_node(¯o, n); 1047 assert(s->load_barrier_count() < load_barrier_count, "must have deleted a node from load barrier list"); 1048 if (C->failing()) return true; 1049 } 1050 igvn.set_delay_transform(false); 1051 igvn.optimize(); 1052 if (C->failing()) return true; 1053 } 1054 return false; 1055 } 1056 1057 // == Loop optimization == 1058 1059 static bool replace_with_dominating_barrier(PhaseIdealLoop* phase, LoadBarrierNode* lb, bool last_round) { 1060 PhaseIterGVN &igvn = phase->igvn(); 1061 Compile* C = Compile::current(); 1062 1063 LoadBarrierNode* lb2 = lb->has_dominating_barrier(phase, false, last_round); 1064 if (lb2 != NULL) { 1065 if (lb->in(LoadBarrierNode::Oop) != lb2->in(LoadBarrierNode::Oop)) { 1066 assert(lb->in(LoadBarrierNode::Address) == lb2->in(LoadBarrierNode::Address), ""); 1067 igvn.replace_input_of(lb, LoadBarrierNode::Similar, lb2->proj_out(LoadBarrierNode::Oop)); 1068 C->set_major_progress(); 1069 } else { 1070 // That transformation may cause the Similar edge on dominated load barriers to be invalid 1071 lb->fix_similar_in_uses(&igvn); 1072 1073 Node* val = lb->proj_out(LoadBarrierNode::Oop); 1074 assert(lb2->has_true_uses(), ""); 1075 assert(lb2->in(LoadBarrierNode::Oop) == lb->in(LoadBarrierNode::Oop), ""); 1076 1077 phase->lazy_update(lb, lb->in(LoadBarrierNode::Control)); 1078 phase->lazy_replace(lb->proj_out(LoadBarrierNode::Control), lb->in(LoadBarrierNode::Control)); 1079 igvn.replace_node(val, lb2->proj_out(LoadBarrierNode::Oop)); 1080 1081 return true; 1082 } 1083 } 1084 return false; 1085 } 1086 1087 static Node* find_dominating_memory(PhaseIdealLoop* phase, Node* mem, Node* dom, int i) { 1088 assert(dom->is_Region() || i == -1, ""); 1089 Node* m = mem; 1090 while(phase->is_dominator(dom, phase->has_ctrl(m) ? phase->get_ctrl(m) : m->in(0))) { 1091 if (m->is_Mem()) { 1092 assert(m->as_Mem()->adr_type() == TypeRawPtr::BOTTOM, ""); 1093 m = m->in(MemNode::Memory); 1094 } else if (m->is_MergeMem()) { 1095 m = m->as_MergeMem()->memory_at(Compile::AliasIdxRaw); 1096 } else if (m->is_Phi()) { 1097 if (m->in(0) == dom && i != -1) { 1098 m = m->in(i); 1099 break; 1100 } else { 1101 m = m->in(LoopNode::EntryControl); 1102 } 1103 } else if (m->is_Proj()) { 1104 m = m->in(0); 1105 } else if (m->is_SafePoint() || m->is_MemBar()) { 1106 m = m->in(TypeFunc::Memory); 1107 } else { 1108 #ifdef ASSERT 1109 m->dump(); 1110 #endif 1111 ShouldNotReachHere(); 1112 } 1113 } 1114 return m; 1115 } 1116 1117 static LoadBarrierNode* clone_load_barrier(PhaseIdealLoop* phase, LoadBarrierNode* lb, Node* ctl, Node* mem, Node* oop_in) { 1118 PhaseIterGVN &igvn = phase->igvn(); 1119 Compile* C = Compile::current(); 1120 Node* the_clone = lb->clone(); 1121 the_clone->set_req(LoadBarrierNode::Control, ctl); 1122 the_clone->set_req(LoadBarrierNode::Memory, mem); 1123 if (oop_in != NULL) { 1124 the_clone->set_req(LoadBarrierNode::Oop, oop_in); 1125 } 1126 1127 LoadBarrierNode* new_lb = the_clone->as_LoadBarrier(); 1128 igvn.register_new_node_with_optimizer(new_lb); 1129 IdealLoopTree *loop = phase->get_loop(new_lb->in(0)); 1130 phase->set_ctrl(new_lb, new_lb->in(0)); 1131 phase->set_loop(new_lb, loop); 1132 phase->set_idom(new_lb, new_lb->in(0), phase->dom_depth(new_lb->in(0))+1); 1133 if (!loop->_child) { 1134 loop->_body.push(new_lb); 1135 } 1136 1137 Node* proj_ctl = new ProjNode(new_lb, LoadBarrierNode::Control); 1138 igvn.register_new_node_with_optimizer(proj_ctl); 1139 phase->set_ctrl(proj_ctl, proj_ctl->in(0)); 1140 phase->set_loop(proj_ctl, loop); 1141 phase->set_idom(proj_ctl, new_lb, phase->dom_depth(new_lb)+1); 1142 if (!loop->_child) { 1143 loop->_body.push(proj_ctl); 1144 } 1145 1146 Node* proj_oop = new ProjNode(new_lb, LoadBarrierNode::Oop); 1147 phase->register_new_node(proj_oop, new_lb); 1148 1149 if (!new_lb->in(LoadBarrierNode::Similar)->is_top()) { 1150 LoadBarrierNode* similar = new_lb->in(LoadBarrierNode::Similar)->in(0)->as_LoadBarrier(); 1151 if (!phase->is_dominator(similar, ctl)) { 1152 igvn.replace_input_of(new_lb, LoadBarrierNode::Similar, C->top()); 1153 } 1154 } 1155 1156 return new_lb; 1157 } 1158 1159 static void replace_barrier(PhaseIdealLoop* phase, LoadBarrierNode* lb, Node* new_val) { 1160 PhaseIterGVN &igvn = phase->igvn(); 1161 Node* val = lb->proj_out(LoadBarrierNode::Oop); 1162 igvn.replace_node(val, new_val); 1163 phase->lazy_update(lb, lb->in(LoadBarrierNode::Control)); 1164 phase->lazy_replace(lb->proj_out(LoadBarrierNode::Control), lb->in(LoadBarrierNode::Control)); 1165 } 1166 1167 static bool split_barrier_thru_phi(PhaseIdealLoop* phase, LoadBarrierNode* lb) { 1168 PhaseIterGVN &igvn = phase->igvn(); 1169 Compile* C = Compile::current(); 1170 1171 if (lb->in(LoadBarrierNode::Oop)->is_Phi()) { 1172 Node* oop_phi = lb->in(LoadBarrierNode::Oop); 1173 1174 if ((oop_phi->req() != 3) || (oop_phi->in(2) == oop_phi)) { 1175 // Ignore phis with only one input 1176 return false; 1177 } 1178 1179 if (phase->is_dominator(phase->get_ctrl(lb->in(LoadBarrierNode::Address)), 1180 oop_phi->in(0)) && phase->get_ctrl(lb->in(LoadBarrierNode::Address)) != oop_phi->in(0)) { 1181 // That transformation may cause the Similar edge on dominated load barriers to be invalid 1182 lb->fix_similar_in_uses(&igvn); 1183 1184 RegionNode* region = oop_phi->in(0)->as_Region(); 1185 1186 int backedge = LoopNode::LoopBackControl; 1187 if (region->is_Loop() && region->in(backedge)->is_Proj() && region->in(backedge)->in(0)->is_If()) { 1188 Node* c = region->in(backedge)->in(0)->in(0); 1189 assert(c->unique_ctrl_out() == region->in(backedge)->in(0), ""); 1190 Node* oop = lb->in(LoadBarrierNode::Oop)->in(backedge); 1191 Node* oop_c = phase->has_ctrl(oop) ? phase->get_ctrl(oop) : oop; 1192 if (!phase->is_dominator(oop_c, c)) { 1193 return false; 1194 } 1195 } 1196 1197 // If the node on the backedge above the phi is the node itself - we have a self loop. 1198 // Don't clone - this will be folded later. 1199 if (oop_phi->in(LoopNode::LoopBackControl) == lb->proj_out(LoadBarrierNode::Oop)) { 1200 return false; 1201 } 1202 1203 bool is_strip_mined = region->is_CountedLoop() && region->as_CountedLoop()->is_strip_mined(); 1204 Node *phi = oop_phi->clone(); 1205 1206 for (uint i = 1; i < region->req(); i++) { 1207 Node* ctrl = region->in(i); 1208 if (ctrl != C->top()) { 1209 assert(!phase->is_dominator(ctrl, region) || region->is_Loop(), ""); 1210 1211 Node* mem = lb->in(LoadBarrierNode::Memory); 1212 Node* m = find_dominating_memory(phase, mem, region, i); 1213 1214 if (region->is_Loop() && i == LoopNode::LoopBackControl && ctrl->is_Proj() && ctrl->in(0)->is_If()) { 1215 ctrl = ctrl->in(0)->in(0); 1216 } else if (region->is_Loop() && is_strip_mined) { 1217 // If this is a strip mined loop, control must move above OuterStripMinedLoop 1218 assert(i == LoopNode::EntryControl, "check"); 1219 assert(ctrl->is_OuterStripMinedLoop(), "sanity"); 1220 ctrl = ctrl->as_OuterStripMinedLoop()->in(LoopNode::EntryControl); 1221 } 1222 1223 LoadBarrierNode* new_lb = clone_load_barrier(phase, lb, ctrl, m, lb->in(LoadBarrierNode::Oop)->in(i)); 1224 Node* out_ctrl = new_lb->proj_out(LoadBarrierNode::Control); 1225 1226 if (is_strip_mined && (i == LoopNode::EntryControl)) { 1227 assert(region->in(i)->is_OuterStripMinedLoop(), ""); 1228 igvn.replace_input_of(region->in(i), i, out_ctrl); 1229 phase->set_idom(region->in(i), out_ctrl, phase->dom_depth(out_ctrl)); 1230 } else if (ctrl == region->in(i)) { 1231 igvn.replace_input_of(region, i, out_ctrl); 1232 // Only update the idom if is the loop entry we are updating 1233 // - A loop backedge doesn't change the idom 1234 if (region->is_Loop() && i == LoopNode::EntryControl) { 1235 phase->set_idom(region, out_ctrl, phase->dom_depth(out_ctrl)); 1236 } 1237 } else { 1238 Node* iff = region->in(i)->in(0); 1239 igvn.replace_input_of(iff, 0, out_ctrl); 1240 phase->set_idom(iff, out_ctrl, phase->dom_depth(out_ctrl)+1); 1241 } 1242 phi->set_req(i, new_lb->proj_out(LoadBarrierNode::Oop)); 1243 } 1244 } 1245 phase->register_new_node(phi, region); 1246 replace_barrier(phase, lb, phi); 1247 1248 if (region->is_Loop()) { 1249 // Load barrier moved to the back edge of the Loop may now 1250 // have a safepoint on the path to the barrier on the Similar 1251 // edge 1252 igvn.replace_input_of(phi->in(LoopNode::LoopBackControl)->in(0), LoadBarrierNode::Similar, C->top()); 1253 Node* head = region->in(LoopNode::EntryControl); 1254 phase->set_idom(region, head, phase->dom_depth(head)+1); 1255 phase->recompute_dom_depth(); 1256 if (head->is_CountedLoop() && head->as_CountedLoop()->is_main_loop()) { 1257 head->as_CountedLoop()->set_normal_loop(); 1258 } 1259 } 1260 1261 return true; 1262 } 1263 } 1264 1265 return false; 1266 } 1267 1268 static bool move_out_of_loop(PhaseIdealLoop* phase, LoadBarrierNode* lb) { 1269 PhaseIterGVN &igvn = phase->igvn(); 1270 IdealLoopTree *lb_loop = phase->get_loop(lb->in(0)); 1271 if (lb_loop != phase->ltree_root() && !lb_loop->_irreducible) { 1272 Node* oop_ctrl = phase->get_ctrl(lb->in(LoadBarrierNode::Oop)); 1273 IdealLoopTree *oop_loop = phase->get_loop(oop_ctrl); 1274 IdealLoopTree* adr_loop = phase->get_loop(phase->get_ctrl(lb->in(LoadBarrierNode::Address))); 1275 if (!lb_loop->is_member(oop_loop) && !lb_loop->is_member(adr_loop)) { 1276 // That transformation may cause the Similar edge on dominated load barriers to be invalid 1277 lb->fix_similar_in_uses(&igvn); 1278 1279 Node* head = lb_loop->_head; 1280 assert(head->is_Loop(), ""); 1281 1282 if (phase->is_dominator(head, oop_ctrl)) { 1283 assert(oop_ctrl->Opcode() == Op_CProj && oop_ctrl->in(0)->Opcode() == Op_NeverBranch, ""); 1284 assert(lb_loop->is_member(phase->get_loop(oop_ctrl->in(0)->in(0))), ""); 1285 return false; 1286 } 1287 1288 if (head->is_CountedLoop()) { 1289 CountedLoopNode* cloop = head->as_CountedLoop(); 1290 if (cloop->is_main_loop()) { 1291 cloop->set_normal_loop(); 1292 } 1293 // When we are moving barrier out of a counted loop, 1294 // make sure we move it all the way out of the strip mined outer loop. 1295 if (cloop->is_strip_mined()) { 1296 head = cloop->outer_loop(); 1297 } 1298 } 1299 1300 Node* mem = lb->in(LoadBarrierNode::Memory); 1301 Node* m = find_dominating_memory(phase, mem, head, -1); 1302 1303 LoadBarrierNode* new_lb = clone_load_barrier(phase, lb, head->in(LoopNode::EntryControl), m, NULL); 1304 1305 assert(phase->idom(head) == head->in(LoopNode::EntryControl), ""); 1306 Node* proj_ctl = new_lb->proj_out(LoadBarrierNode::Control); 1307 igvn.replace_input_of(head, LoopNode::EntryControl, proj_ctl); 1308 phase->set_idom(head, proj_ctl, phase->dom_depth(proj_ctl) + 1); 1309 1310 replace_barrier(phase, lb, new_lb->proj_out(LoadBarrierNode::Oop)); 1311 1312 phase->recompute_dom_depth(); 1313 1314 return true; 1315 } 1316 } 1317 1318 return false; 1319 } 1320 1321 static bool common_barriers(PhaseIdealLoop* phase, LoadBarrierNode* lb) { 1322 PhaseIterGVN &igvn = phase->igvn(); 1323 Node* in_val = lb->in(LoadBarrierNode::Oop); 1324 for (DUIterator_Fast imax, i = in_val->fast_outs(imax); i < imax; i++) { 1325 Node* u = in_val->fast_out(i); 1326 if (u != lb && u->is_LoadBarrier() && u->as_LoadBarrier()->has_true_uses()) { 1327 Node* this_ctrl = lb->in(LoadBarrierNode::Control); 1328 Node* other_ctrl = u->in(LoadBarrierNode::Control); 1329 1330 Node* lca = phase->dom_lca(this_ctrl, other_ctrl); 1331 bool ok = true; 1332 1333 Node* proj1 = NULL; 1334 Node* proj2 = NULL; 1335 1336 while (this_ctrl != lca && ok) { 1337 if (this_ctrl->in(0) != NULL && 1338 this_ctrl->in(0)->is_MultiBranch()) { 1339 if (this_ctrl->in(0)->in(0) == lca) { 1340 assert(proj1 == NULL, ""); 1341 assert(this_ctrl->is_Proj(), ""); 1342 proj1 = this_ctrl; 1343 } else if (!(this_ctrl->in(0)->is_If() && this_ctrl->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none))) { 1344 ok = false; 1345 } 1346 } 1347 this_ctrl = phase->idom(this_ctrl); 1348 } 1349 while (other_ctrl != lca && ok) { 1350 if (other_ctrl->in(0) != NULL && 1351 other_ctrl->in(0)->is_MultiBranch()) { 1352 if (other_ctrl->in(0)->in(0) == lca) { 1353 assert(other_ctrl->is_Proj(), ""); 1354 assert(proj2 == NULL, ""); 1355 proj2 = other_ctrl; 1356 } else if (!(other_ctrl->in(0)->is_If() && other_ctrl->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none))) { 1357 ok = false; 1358 } 1359 } 1360 other_ctrl = phase->idom(other_ctrl); 1361 } 1362 assert(proj1 == NULL || proj2 == NULL || proj1->in(0) == proj2->in(0), ""); 1363 if (ok && proj1 && proj2 && proj1 != proj2 && proj1->in(0)->is_If()) { 1364 // That transformation may cause the Similar edge on dominated load barriers to be invalid 1365 lb->fix_similar_in_uses(&igvn); 1366 u->as_LoadBarrier()->fix_similar_in_uses(&igvn); 1367 1368 Node* split = lca->unique_ctrl_out(); 1369 assert(split->in(0) == lca, ""); 1370 1371 Node* mem = lb->in(LoadBarrierNode::Memory); 1372 Node* m = find_dominating_memory(phase, mem, split, -1); 1373 LoadBarrierNode* new_lb = clone_load_barrier(phase, lb, lca, m, NULL); 1374 1375 Node* proj_ctl = new_lb->proj_out(LoadBarrierNode::Control); 1376 igvn.replace_input_of(split, 0, new_lb->proj_out(LoadBarrierNode::Control)); 1377 phase->set_idom(split, proj_ctl, phase->dom_depth(proj_ctl)+1); 1378 1379 Node* proj_oop = new_lb->proj_out(LoadBarrierNode::Oop); 1380 replace_barrier(phase, lb, proj_oop); 1381 replace_barrier(phase, u->as_LoadBarrier(), proj_oop); 1382 1383 phase->recompute_dom_depth(); 1384 1385 return true; 1386 } 1387 } 1388 } 1389 1390 return false; 1391 } 1392 1393 static void optimize_load_barrier(PhaseIdealLoop* phase, LoadBarrierNode* lb, bool last_round) { 1394 Compile* C = Compile::current(); 1395 1396 if (!C->directive()->ZOptimizeLoadBarriersOption) { 1397 return; 1398 } 1399 1400 if (lb->has_true_uses()) { 1401 if (replace_with_dominating_barrier(phase, lb, last_round)) { 1402 return; 1403 } 1404 1405 if (split_barrier_thru_phi(phase, lb)) { 1406 return; 1407 } 1408 1409 if (move_out_of_loop(phase, lb)) { 1410 return; 1411 } 1412 1413 if (common_barriers(phase, lb)) { 1414 return; 1415 } 1416 } 1417 } 1418 1419 void ZBarrierSetC2::loop_optimize_gc_barrier(PhaseIdealLoop* phase, Node* node, bool last_round) { 1420 if (node->is_LoadBarrier()) { 1421 optimize_load_barrier(phase, node->as_LoadBarrier(), last_round); 1422 } 1423 } 1424 1425 Node* ZBarrierSetC2::step_over_gc_barrier(Node* c) const { 1426 Node* node = c; 1427 1428 // 1. This step follows potential oop projections of a load barrier before expansion 1429 if (node->is_Proj()) { 1430 node = node->in(0); 1431 } 1432 1433 // 2. This step checks for unexpanded load barriers 1434 if (node->is_LoadBarrier()) { 1435 return node->in(LoadBarrierNode::Oop); 1436 } 1437 1438 // 3. This step checks for the phi corresponding to an optimized load barrier expansion 1439 if (node->is_Phi()) { 1440 PhiNode* phi = node->as_Phi(); 1441 Node* n = phi->in(1); 1442 if (n != NULL && (n->is_LoadBarrierSlowReg() || n->is_LoadBarrierWeakSlowReg())) { 1443 assert(c == node, "projections from step 1 should only be seen before macro expansion"); 1444 return phi->in(2); 1445 } 1446 } 1447 1448 return c; 1449 } 1450 1451 bool ZBarrierSetC2::array_copy_requires_gc_barriers(bool tightly_coupled_alloc, BasicType type, bool is_clone, ArrayCopyPhase phase) const { 1452 return type == T_OBJECT || type == T_ARRAY; 1453 } 1454 1455 bool ZBarrierSetC2::final_graph_reshaping(Compile* compile, Node* n, uint opcode) const { 1456 bool handled; 1457 switch (opcode) { 1458 case Op_LoadBarrierSlowReg: 1459 case Op_LoadBarrierWeakSlowReg: 1460 #ifdef ASSERT 1461 if (VerifyOptoOopOffsets) { 1462 MemNode* mem = n->as_Mem(); 1463 // Check to see if address types have grounded out somehow. 1464 const TypeInstPtr* tp = mem->in(MemNode::Address)->bottom_type()->isa_instptr(); 1465 ciInstanceKlass* k = tp->klass()->as_instance_klass(); 1466 bool oop_offset_is_sane = k->contains_field_offset(tp->offset()); 1467 assert(!tp || oop_offset_is_sane, ""); 1468 } 1469 #endif 1470 handled = true; 1471 break; 1472 default: 1473 handled = false; 1474 } 1475 return handled; 1476 } 1477 1478 bool ZBarrierSetC2::matcher_find_shared_visit(Matcher* matcher, Matcher::MStack& mstack, Node* n, uint opcode, bool& mem_op, int& mem_addr_idx) const { 1479 if (opcode == Op_CallLeaf && 1480 (n->as_Call()->entry_point() == ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr() || 1481 n->as_Call()->entry_point() == ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded_addr())) { 1482 mem_op = true; 1483 mem_addr_idx = TypeFunc::Parms + 1; 1484 return true; 1485 } 1486 return false; 1487 } 1488 1489 // == Verification == 1490 1491 #ifdef ASSERT 1492 1493 static bool look_for_barrier(Node* n, bool post_parse, VectorSet& visited) { 1494 if (visited.test_set(n->_idx)) { 1495 return true; 1496 } 1497 1498 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { 1499 Node* u = n->fast_out(i); 1500 if (u->is_LoadBarrier()) { 1501 } else if ((u->is_Phi() || u->is_CMove()) && !post_parse) { 1502 if (!look_for_barrier(u, post_parse, visited)) { 1503 return false; 1504 } 1505 } else if (u->Opcode() == Op_EncodeP || u->Opcode() == Op_DecodeN) { 1506 if (!look_for_barrier(u, post_parse, visited)) { 1507 return false; 1508 } 1509 } else if (u->Opcode() != Op_SCMemProj) { 1510 tty->print("bad use"); u->dump(); 1511 return false; 1512 } 1513 } 1514 1515 return true; 1516 } 1517 1518 void ZBarrierSetC2::verify_gc_barriers(Compile* compile, CompilePhase phase) const { 1519 if (phase == BarrierSetC2::BeforeCodeGen) return; 1520 bool post_parse = phase == BarrierSetC2::BeforeOptimize; 1521 verify_gc_barriers(post_parse); 1522 } 1523 1524 void ZBarrierSetC2::verify_gc_barriers(bool post_parse) const { 1525 ZBarrierSetC2State* s = state(); 1526 Compile* C = Compile::current(); 1527 ResourceMark rm; 1528 VectorSet visited(Thread::current()->resource_area()); 1529 for (int i = 0; i < s->load_barrier_count(); i++) { 1530 LoadBarrierNode* n = s->load_barrier_node(i); 1531 1532 // The dominating barrier on the same address if it exists and 1533 // this barrier must not be applied on the value from the same 1534 // load otherwise the value is not reloaded before it's used the 1535 // second time. 1536 assert(n->in(LoadBarrierNode::Similar)->is_top() || 1537 (n->in(LoadBarrierNode::Similar)->in(0)->is_LoadBarrier() && 1538 n->in(LoadBarrierNode::Similar)->in(0)->in(LoadBarrierNode::Address) == n->in(LoadBarrierNode::Address) && 1539 n->in(LoadBarrierNode::Similar)->in(0)->in(LoadBarrierNode::Oop) != n->in(LoadBarrierNode::Oop)), 1540 "broken similar edge"); 1541 1542 assert(post_parse || n->as_LoadBarrier()->has_true_uses(), 1543 "found unneeded load barrier"); 1544 1545 // Several load barrier nodes chained through their Similar edge 1546 // break the code that remove the barriers in final graph reshape. 1547 assert(n->in(LoadBarrierNode::Similar)->is_top() || 1548 (n->in(LoadBarrierNode::Similar)->in(0)->is_LoadBarrier() && 1549 n->in(LoadBarrierNode::Similar)->in(0)->in(LoadBarrierNode::Similar)->is_top()), 1550 "chain of Similar load barriers"); 1551 1552 if (!n->in(LoadBarrierNode::Similar)->is_top()) { 1553 ResourceMark rm; 1554 Unique_Node_List wq; 1555 Node* other = n->in(LoadBarrierNode::Similar)->in(0); 1556 wq.push(n); 1557 bool ok = true; 1558 bool dom_found = false; 1559 for (uint next = 0; next < wq.size(); ++next) { 1560 Node *n = wq.at(next); 1561 assert(n->is_CFG(), ""); 1562 assert(!n->is_SafePoint(), ""); 1563 1564 if (n == other) { 1565 continue; 1566 } 1567 1568 if (n->is_Region()) { 1569 for (uint i = 1; i < n->req(); i++) { 1570 Node* m = n->in(i); 1571 if (m != NULL) { 1572 wq.push(m); 1573 } 1574 } 1575 } else { 1576 Node* m = n->in(0); 1577 if (m != NULL) { 1578 wq.push(m); 1579 } 1580 } 1581 } 1582 } 1583 1584 if (ZVerifyLoadBarriers) { 1585 if ((n->is_Load() || n->is_LoadStore()) && n->bottom_type()->make_oopptr() != NULL) { 1586 visited.Clear(); 1587 bool found = look_for_barrier(n, post_parse, visited); 1588 if (!found) { 1589 n->dump(1); 1590 n->dump(-3); 1591 stringStream ss; 1592 C->method()->print_short_name(&ss); 1593 tty->print_cr("-%s-", ss.as_string()); 1594 assert(found, ""); 1595 } 1596 } 1597 } 1598 } 1599 } 1600 1601 #endif 1602 1603 bool ZBarrierSetC2::escape_add_to_con_graph(ConnectionGraph* conn_graph, PhaseGVN* gvn, Unique_Node_List* delayed_worklist, Node* n, uint opcode) const { 1604 switch (opcode) { 1605 case Op_LoadBarrierSlowReg: 1606 case Op_LoadBarrierWeakSlowReg: 1607 conn_graph->add_objload_to_connection_graph(n, delayed_worklist); 1608 return true; 1609 case Op_Proj: 1610 if (n->as_Proj()->_con == LoadBarrierNode::Oop && n->in(0)->is_LoadBarrier()) { 1611 conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(0)->in(LoadBarrierNode::Oop), 1612 delayed_worklist); 1613 return true; 1614 } 1615 default: 1616 break; 1617 } 1618 return false; 1619 } 1620 1621 bool ZBarrierSetC2::escape_add_final_edges(ConnectionGraph* conn_graph, PhaseGVN* gvn, Node* n, uint opcode) const { 1622 switch (opcode) { 1623 case Op_LoadBarrierSlowReg: 1624 case Op_LoadBarrierWeakSlowReg: { 1625 const Type *t = gvn->type(n); 1626 if (t->make_ptr() != NULL) { 1627 Node *adr = n->in(MemNode::Address); 1628 conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, adr, NULL); 1629 return true; 1630 } 1631 } 1632 case Op_Proj: { 1633 if (n->as_Proj()->_con == LoadBarrierNode::Oop && n->in(0)->is_LoadBarrier()) { 1634 conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(0)->in(LoadBarrierNode::Oop), NULL); 1635 return true; 1636 } 1637 } 1638 default: 1639 break; 1640 } 1641 return false; 1642 }