1 /* 2 * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 */ 23 24 #include "precompiled.hpp" 25 #include "opto/compile.hpp" 26 #include "opto/castnode.hpp" 27 #include "opto/graphKit.hpp" 28 #include "opto/idealKit.hpp" 29 #include "opto/loopnode.hpp" 30 #include "opto/macro.hpp" 31 #include "opto/node.hpp" 32 #include "opto/type.hpp" 33 #include "utilities/macros.hpp" 34 #include "gc/z/zBarrierSet.hpp" 35 #include "gc/z/c2/zBarrierSetC2.hpp" 36 #include "gc/z/zThreadLocalData.hpp" 37 #include "gc/z/zBarrierSetRuntime.hpp" 38 39 ZBarrierSetC2State::ZBarrierSetC2State(Arena* comp_arena) 40 : _load_barrier_nodes(new (comp_arena) GrowableArray<LoadBarrierNode*>(comp_arena, 8, 0, NULL)) {} 41 42 int ZBarrierSetC2State::load_barrier_count() const { 43 return _load_barrier_nodes->length(); 44 } 45 46 void ZBarrierSetC2State::add_load_barrier_node(LoadBarrierNode * n) { 47 assert(!_load_barrier_nodes->contains(n), " duplicate entry in expand list"); 48 _load_barrier_nodes->append(n); 49 } 50 51 void ZBarrierSetC2State::remove_load_barrier_node(LoadBarrierNode * n) { 52 // this function may be called twice for a node so check 53 // that the node is in the array before attempting to remove it 54 if (_load_barrier_nodes->contains(n)) { 55 _load_barrier_nodes->remove(n); 56 } 57 } 58 59 LoadBarrierNode* ZBarrierSetC2State::load_barrier_node(int idx) const { 60 return _load_barrier_nodes->at(idx); 61 } 62 63 void* ZBarrierSetC2::create_barrier_state(Arena* comp_arena) const { 64 return new(comp_arena) ZBarrierSetC2State(comp_arena); 65 } 66 67 ZBarrierSetC2State* ZBarrierSetC2::state() const { 68 return reinterpret_cast<ZBarrierSetC2State*>(Compile::current()->barrier_set_state()); 69 } 70 71 bool ZBarrierSetC2::is_gc_barrier_node(Node* node) const { 72 // 1. This step follows potential oop projections of a load barrier before expansion 73 if (node->is_Proj()) { 74 node = node->in(0); 75 } 76 77 // 2. This step checks for unexpanded load barriers 78 if (node->is_LoadBarrier()) { 79 return true; 80 } 81 82 // 3. This step checks for the phi corresponding to an optimized load barrier expansion 83 if (node->is_Phi()) { 84 PhiNode* phi = node->as_Phi(); 85 Node* n = phi->in(1); 86 if (n != NULL && (n->is_LoadBarrierSlowReg() || n->is_LoadBarrierWeakSlowReg())) { 87 return true; 88 } 89 } 90 91 return false; 92 } 93 94 void ZBarrierSetC2::register_potential_barrier_node(Node* node) const { 95 if (node->is_LoadBarrier()) { 96 state()->add_load_barrier_node(node->as_LoadBarrier()); 97 } 98 } 99 100 void ZBarrierSetC2::unregister_potential_barrier_node(Node* node) const { 101 if (node->is_LoadBarrier()) { 102 state()->remove_load_barrier_node(node->as_LoadBarrier()); 103 } 104 } 105 106 void ZBarrierSetC2::eliminate_useless_gc_barriers(Unique_Node_List &useful) const { 107 // Remove useless LoadBarrier nodes 108 ZBarrierSetC2State* s = state(); 109 for (int i = s->load_barrier_count()-1; i >= 0; i--) { 110 LoadBarrierNode* n = s->load_barrier_node(i); 111 if (!useful.member(n)) { 112 unregister_potential_barrier_node(n); 113 } 114 } 115 } 116 117 void ZBarrierSetC2::enqueue_useful_gc_barrier(Unique_Node_List &worklist, Node* node) const { 118 if (node->is_LoadBarrier() && !node->as_LoadBarrier()->has_true_uses()) { 119 worklist.push(node); 120 } 121 } 122 123 void ZBarrierSetC2::find_dominating_barriers(PhaseIterGVN& igvn) { 124 // Look for dominating barriers on the same address only once all 125 // other loop opts are over: loop opts may cause a safepoint to be 126 // inserted between a barrier and its dominating barrier. 127 Compile* C = Compile::current(); 128 ZBarrierSetC2* bs = (ZBarrierSetC2*)BarrierSet::barrier_set()->barrier_set_c2(); 129 ZBarrierSetC2State* s = bs->state(); 130 if (s->load_barrier_count() >= 2) { 131 Compile::TracePhase tp("idealLoop", &C->timers[Phase::_t_idealLoop]); 132 PhaseIdealLoop ideal_loop(igvn, LoopOptsLastRound); 133 if (C->major_progress()) C->print_method(PHASE_PHASEIDEALLOOP_ITERATIONS, 2); 134 } 135 } 136 137 void ZBarrierSetC2::add_users_to_worklist(Unique_Node_List* worklist) const { 138 // Permanent temporary workaround 139 // Loadbarriers may have non-obvious dead uses keeping them alive during parsing. The use is 140 // removed by RemoveUseless (after parsing, before optimize) but the barriers won't be added to 141 // the worklist. Unless we add them explicitly they are not guaranteed to end up there. 142 ZBarrierSetC2State* s = state(); 143 144 for (int i = 0; i < s->load_barrier_count(); i++) { 145 LoadBarrierNode* n = s->load_barrier_node(i); 146 worklist->push(n); 147 } 148 } 149 150 const TypeFunc* ZBarrierSetC2::load_barrier_Type() const { 151 const Type** fields; 152 153 // Create input types (domain) 154 fields = TypeTuple::fields(2); 155 fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; 156 fields[TypeFunc::Parms+1] = TypeOopPtr::BOTTOM; 157 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2, fields); 158 159 // Create result type (range) 160 fields = TypeTuple::fields(1); 161 fields[TypeFunc::Parms+0] = TypeInstPtr::BOTTOM; 162 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+1, fields); 163 164 return TypeFunc::make(domain, range); 165 } 166 167 // == LoadBarrierNode == 168 169 LoadBarrierNode::LoadBarrierNode(Compile* C, 170 Node* c, 171 Node* mem, 172 Node* val, 173 Node* adr, 174 bool weak, 175 bool writeback, 176 bool oop_reload_allowed) : 177 MultiNode(Number_of_Inputs), 178 _weak(weak), 179 _writeback(writeback), 180 _oop_reload_allowed(oop_reload_allowed) { 181 init_req(Control, c); 182 init_req(Memory, mem); 183 init_req(Oop, val); 184 init_req(Address, adr); 185 init_req(Similar, C->top()); 186 187 init_class_id(Class_LoadBarrier); 188 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); 189 bs->register_potential_barrier_node(this); 190 } 191 192 uint LoadBarrierNode::size_of() const { 193 return sizeof(*this); 194 } 195 196 uint LoadBarrierNode::cmp(const Node& n) const { 197 ShouldNotReachHere(); 198 return 0; 199 } 200 201 const Type *LoadBarrierNode::bottom_type() const { 202 const Type** floadbarrier = (const Type **)(Compile::current()->type_arena()->Amalloc_4((Number_of_Outputs)*sizeof(Type*))); 203 Node* in_oop = in(Oop); 204 floadbarrier[Control] = Type::CONTROL; 205 floadbarrier[Memory] = Type::MEMORY; 206 floadbarrier[Oop] = in_oop == NULL ? Type::TOP : in_oop->bottom_type(); 207 return TypeTuple::make(Number_of_Outputs, floadbarrier); 208 } 209 210 const TypePtr* LoadBarrierNode::adr_type() const { 211 return TypeRawPtr::BOTTOM; 212 } 213 214 const Type *LoadBarrierNode::Value(PhaseGVN *phase) const { 215 const Type** floadbarrier = (const Type **)(phase->C->type_arena()->Amalloc_4((Number_of_Outputs)*sizeof(Type*))); 216 const Type* val_t = phase->type(in(Oop)); 217 floadbarrier[Control] = Type::CONTROL; 218 floadbarrier[Memory] = Type::MEMORY; 219 floadbarrier[Oop] = val_t; 220 return TypeTuple::make(Number_of_Outputs, floadbarrier); 221 } 222 223 bool LoadBarrierNode::is_dominator(PhaseIdealLoop* phase, bool linear_only, Node *d, Node *n) { 224 if (phase != NULL) { 225 return phase->is_dominator(d, n); 226 } 227 228 for (int i = 0; i < 10 && n != NULL; i++) { 229 n = IfNode::up_one_dom(n, linear_only); 230 if (n == d) { 231 return true; 232 } 233 } 234 235 return false; 236 } 237 238 LoadBarrierNode* LoadBarrierNode::has_dominating_barrier(PhaseIdealLoop* phase, bool linear_only, bool look_for_similar) { 239 Node* val = in(LoadBarrierNode::Oop); 240 if (in(Similar)->is_Proj() && in(Similar)->in(0)->is_LoadBarrier()) { 241 LoadBarrierNode* lb = in(Similar)->in(0)->as_LoadBarrier(); 242 assert(lb->in(Address) == in(Address), ""); 243 // Load barrier on Similar edge dominates so if it now has the Oop field it can replace this barrier. 244 if (lb->in(Oop) == in(Oop)) { 245 return lb; 246 } 247 // Follow chain of load barrier through Similar edges 248 while (!lb->in(Similar)->is_top()) { 249 lb = lb->in(Similar)->in(0)->as_LoadBarrier(); 250 assert(lb->in(Address) == in(Address), ""); 251 } 252 if (lb != in(Similar)->in(0)) { 253 return lb; 254 } 255 } 256 for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) { 257 Node* u = val->fast_out(i); 258 if (u != this && u->is_LoadBarrier() && u->in(Oop) == val && u->as_LoadBarrier()->has_true_uses()) { 259 Node* this_ctrl = in(LoadBarrierNode::Control); 260 Node* other_ctrl = u->in(LoadBarrierNode::Control); 261 if (is_dominator(phase, linear_only, other_ctrl, this_ctrl)) { 262 return u->as_LoadBarrier(); 263 } 264 } 265 } 266 267 if (ZVerifyLoadBarriers || can_be_eliminated()) { 268 return NULL; 269 } 270 271 if (!look_for_similar) { 272 return NULL; 273 } 274 275 Node* addr = in(LoadBarrierNode::Address); 276 for (DUIterator_Fast imax, i = addr->fast_outs(imax); i < imax; i++) { 277 Node* u = addr->fast_out(i); 278 if (u != this && u->is_LoadBarrier() && u->as_LoadBarrier()->has_true_uses()) { 279 Node* this_ctrl = in(LoadBarrierNode::Control); 280 Node* other_ctrl = u->in(LoadBarrierNode::Control); 281 if (is_dominator(phase, linear_only, other_ctrl, this_ctrl)) { 282 ResourceMark rm; 283 Unique_Node_List wq; 284 wq.push(in(LoadBarrierNode::Control)); 285 bool ok = true; 286 bool dom_found = false; 287 for (uint next = 0; next < wq.size(); ++next) { 288 Node *n = wq.at(next); 289 if (n->is_top()) { 290 return NULL; 291 } 292 assert(n->is_CFG(), ""); 293 if (n->is_SafePoint()) { 294 ok = false; 295 break; 296 } 297 if (n == u) { 298 dom_found = true; 299 continue; 300 } 301 if (n->is_Region()) { 302 for (uint i = 1; i < n->req(); i++) { 303 Node* m = n->in(i); 304 if (m != NULL) { 305 wq.push(m); 306 } 307 } 308 } else { 309 Node* m = n->in(0); 310 if (m != NULL) { 311 wq.push(m); 312 } 313 } 314 } 315 if (ok) { 316 assert(dom_found, ""); 317 return u->as_LoadBarrier();; 318 } 319 break; 320 } 321 } 322 } 323 324 return NULL; 325 } 326 327 void LoadBarrierNode::push_dominated_barriers(PhaseIterGVN* igvn) const { 328 // Change to that barrier may affect a dominated barrier so re-push those 329 Node* val = in(LoadBarrierNode::Oop); 330 331 for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) { 332 Node* u = val->fast_out(i); 333 if (u != this && u->is_LoadBarrier() && u->in(Oop) == val) { 334 Node* this_ctrl = in(Control); 335 Node* other_ctrl = u->in(Control); 336 if (is_dominator(NULL, false, this_ctrl, other_ctrl)) { 337 igvn->_worklist.push(u); 338 } 339 } 340 341 Node* addr = in(LoadBarrierNode::Address); 342 for (DUIterator_Fast imax, i = addr->fast_outs(imax); i < imax; i++) { 343 Node* u = addr->fast_out(i); 344 if (u != this && u->is_LoadBarrier() && u->in(Similar)->is_top()) { 345 Node* this_ctrl = in(Control); 346 Node* other_ctrl = u->in(Control); 347 if (is_dominator(NULL, false, this_ctrl, other_ctrl)) { 348 igvn->_worklist.push(u); 349 } 350 } 351 } 352 } 353 } 354 355 Node *LoadBarrierNode::Identity(PhaseGVN *phase) { 356 if (!phase->C->directive()->ZOptimizeLoadBarriersOption) { 357 return this; 358 } 359 360 bool redundant_addr = false; 361 LoadBarrierNode* dominating_barrier = has_dominating_barrier(NULL, true, false); 362 if (dominating_barrier != NULL) { 363 assert(dominating_barrier->in(Oop) == in(Oop), ""); 364 return dominating_barrier; 365 } 366 367 return this; 368 } 369 370 Node *LoadBarrierNode::Ideal(PhaseGVN *phase, bool can_reshape) { 371 if (remove_dead_region(phase, can_reshape)) { 372 return this; 373 } 374 375 Node* val = in(Oop); 376 Node* mem = in(Memory); 377 Node* ctrl = in(Control); 378 Node* adr = in(Address); 379 assert(val->Opcode() != Op_LoadN, ""); 380 381 if (mem->is_MergeMem()) { 382 Node* new_mem = mem->as_MergeMem()->memory_at(Compile::AliasIdxRaw); 383 set_req(Memory, new_mem); 384 if (mem->outcnt() == 0 && can_reshape) { 385 phase->is_IterGVN()->_worklist.push(mem); 386 } 387 388 return this; 389 } 390 391 bool optimizeLoadBarriers = phase->C->directive()->ZOptimizeLoadBarriersOption; 392 LoadBarrierNode* dominating_barrier = optimizeLoadBarriers ? has_dominating_barrier(NULL, !can_reshape, !phase->C->major_progress()) : NULL; 393 if (dominating_barrier != NULL && dominating_barrier->in(Oop) != in(Oop)) { 394 assert(in(Address) == dominating_barrier->in(Address), ""); 395 set_req(Similar, dominating_barrier->proj_out(Oop)); 396 return this; 397 } 398 399 bool eliminate = (optimizeLoadBarriers && !(val->is_Phi() || val->Opcode() == Op_LoadP || val->Opcode() == Op_GetAndSetP || val->is_DecodeN())) || 400 (can_reshape && (dominating_barrier != NULL || !has_true_uses())); 401 402 if (eliminate) { 403 if (can_reshape) { 404 PhaseIterGVN* igvn = phase->is_IterGVN(); 405 Node* out_ctrl = proj_out_or_null(Control); 406 Node* out_res = proj_out_or_null(Oop); 407 408 if (out_ctrl != NULL) { 409 igvn->replace_node(out_ctrl, ctrl); 410 } 411 412 // That transformation may cause the Similar edge on the load barrier to be invalid 413 fix_similar_in_uses(igvn); 414 if (out_res != NULL) { 415 if (dominating_barrier != NULL) { 416 igvn->replace_node(out_res, dominating_barrier->proj_out(Oop)); 417 } else { 418 igvn->replace_node(out_res, val); 419 } 420 } 421 } 422 423 return new ConINode(TypeInt::ZERO); 424 } 425 426 // If the Similar edge is no longer a load barrier, clear it 427 Node* similar = in(Similar); 428 if (!similar->is_top() && !(similar->is_Proj() && similar->in(0)->is_LoadBarrier())) { 429 set_req(Similar, phase->C->top()); 430 return this; 431 } 432 433 if (can_reshape) { 434 // If this barrier is linked through the Similar edge by a 435 // dominated barrier and both barriers have the same Oop field, 436 // the dominated barrier can go away, so push it for reprocessing. 437 // We also want to avoid a barrier to depend on another dominating 438 // barrier through its Similar edge that itself depend on another 439 // barrier through its Similar edge and rather have the first 440 // depend on the third. 441 PhaseIterGVN* igvn = phase->is_IterGVN(); 442 Node* out_res = proj_out(Oop); 443 for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) { 444 Node* u = out_res->fast_out(i); 445 if (u->is_LoadBarrier() && u->in(Similar) == out_res && 446 (u->in(Oop) == val || !u->in(Similar)->is_top())) { 447 igvn->_worklist.push(u); 448 } 449 } 450 451 push_dominated_barriers(igvn); 452 } 453 454 return NULL; 455 } 456 457 uint LoadBarrierNode::match_edge(uint idx) const { 458 ShouldNotReachHere(); 459 return 0; 460 } 461 462 void LoadBarrierNode::fix_similar_in_uses(PhaseIterGVN* igvn) { 463 Node* out_res = proj_out_or_null(Oop); 464 if (out_res == NULL) { 465 return; 466 } 467 468 for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) { 469 Node* u = out_res->fast_out(i); 470 if (u->is_LoadBarrier() && u->in(Similar) == out_res) { 471 igvn->replace_input_of(u, Similar, igvn->C->top()); 472 --i; 473 --imax; 474 } 475 } 476 } 477 478 bool LoadBarrierNode::has_true_uses() const { 479 Node* out_res = proj_out_or_null(Oop); 480 if (out_res == NULL) { 481 return false; 482 } 483 484 for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) { 485 Node* u = out_res->fast_out(i); 486 if (!u->is_LoadBarrier() || u->in(Similar) != out_res) { 487 return true; 488 } 489 } 490 491 return false; 492 } 493 494 // == Accesses == 495 496 Node* ZBarrierSetC2::make_cas_loadbarrier(C2AtomicAccess& access) const { 497 assert(!UseCompressedOops, "Not allowed"); 498 CompareAndSwapNode* cas = (CompareAndSwapNode*)access.raw_access(); 499 PhaseGVN& gvn = access.kit()->gvn(); 500 Compile* C = Compile::current(); 501 GraphKit* kit = access.kit(); 502 503 Node* in_ctrl = cas->in(MemNode::Control); 504 Node* in_mem = cas->in(MemNode::Memory); 505 Node* in_adr = cas->in(MemNode::Address); 506 Node* in_val = cas->in(MemNode::ValueIn); 507 Node* in_expected = cas->in(LoadStoreConditionalNode::ExpectedIn); 508 509 float likely = PROB_LIKELY(0.999); 510 511 const TypePtr *adr_type = gvn.type(in_adr)->isa_ptr(); 512 Compile::AliasType* alias_type = C->alias_type(adr_type); 513 int alias_idx = C->get_alias_index(adr_type); 514 515 // Outer check - true: continue, false: load and check 516 Node* region = new RegionNode(3); 517 Node* phi = new PhiNode(region, TypeInt::BOOL); 518 Node* phi_mem = new PhiNode(region, Type::MEMORY, adr_type); 519 520 // Inner check - is the healed ref equal to the expected 521 Node* region2 = new RegionNode(3); 522 Node* phi2 = new PhiNode(region2, TypeInt::BOOL); 523 Node* phi_mem2 = new PhiNode(region2, Type::MEMORY, adr_type); 524 525 // CAS node returns 0 or 1 526 Node* cmp = gvn.transform(new CmpINode(cas, kit->intcon(0))); 527 Node* bol = gvn.transform(new BoolNode(cmp, BoolTest::ne))->as_Bool(); 528 IfNode* iff = gvn.transform(new IfNode(in_ctrl, bol, likely, COUNT_UNKNOWN))->as_If(); 529 Node* then = gvn.transform(new IfTrueNode(iff)); 530 Node* elsen = gvn.transform(new IfFalseNode(iff)); 531 532 Node* scmemproj1 = gvn.transform(new SCMemProjNode(cas)); 533 534 kit->set_memory(scmemproj1, alias_idx); 535 phi_mem->init_req(1, scmemproj1); 536 phi_mem2->init_req(2, scmemproj1); 537 538 // CAS fail - reload and heal oop 539 Node* reload = kit->make_load(elsen, in_adr, TypeOopPtr::BOTTOM, T_OBJECT, MemNode::unordered); 540 Node* barrier = gvn.transform(new LoadBarrierNode(C, elsen, scmemproj1, reload, in_adr, false, true, false)); 541 Node* barrierctrl = gvn.transform(new ProjNode(barrier, LoadBarrierNode::Control)); 542 Node* barrierdata = gvn.transform(new ProjNode(barrier, LoadBarrierNode::Oop)); 543 544 // Check load 545 Node* tmpX = gvn.transform(new CastP2XNode(NULL, barrierdata)); 546 Node* in_expX = gvn.transform(new CastP2XNode(NULL, in_expected)); 547 Node* cmp2 = gvn.transform(new CmpXNode(tmpX, in_expX)); 548 Node *bol2 = gvn.transform(new BoolNode(cmp2, BoolTest::ne))->as_Bool(); 549 IfNode* iff2 = gvn.transform(new IfNode(barrierctrl, bol2, likely, COUNT_UNKNOWN))->as_If(); 550 Node* then2 = gvn.transform(new IfTrueNode(iff2)); 551 Node* elsen2 = gvn.transform(new IfFalseNode(iff2)); 552 553 // redo CAS 554 Node* cas2 = gvn.transform(new CompareAndSwapPNode(elsen2, kit->memory(alias_idx), in_adr, in_val, in_expected, cas->order())); 555 Node* scmemproj2 = gvn.transform(new SCMemProjNode(cas2)); 556 kit->set_control(elsen2); 557 kit->set_memory(scmemproj2, alias_idx); 558 559 // Merge inner flow - check if healed oop was equal too expected. 560 region2->set_req(1, kit->control()); 561 region2->set_req(2, then2); 562 phi2->set_req(1, cas2); 563 phi2->set_req(2, kit->intcon(0)); 564 phi_mem2->init_req(1, scmemproj2); 565 kit->set_memory(phi_mem2, alias_idx); 566 567 // Merge outer flow - then check if first CAS succeeded 568 region->set_req(1, then); 569 region->set_req(2, region2); 570 phi->set_req(1, kit->intcon(1)); 571 phi->set_req(2, phi2); 572 phi_mem->init_req(2, phi_mem2); 573 kit->set_memory(phi_mem, alias_idx); 574 575 gvn.transform(region2); 576 gvn.transform(phi2); 577 gvn.transform(phi_mem2); 578 gvn.transform(region); 579 gvn.transform(phi); 580 gvn.transform(phi_mem); 581 582 kit->set_control(region); 583 kit->insert_mem_bar(Op_MemBarCPUOrder); 584 585 return phi; 586 } 587 588 Node* ZBarrierSetC2::make_cmpx_loadbarrier(C2AtomicAccess& access) const { 589 CompareAndExchangePNode* cmpx = (CompareAndExchangePNode*)access.raw_access(); 590 GraphKit* kit = access.kit(); 591 PhaseGVN& gvn = kit->gvn(); 592 Compile* C = Compile::current(); 593 594 Node* in_ctrl = cmpx->in(MemNode::Control); 595 Node* in_mem = cmpx->in(MemNode::Memory); 596 Node* in_adr = cmpx->in(MemNode::Address); 597 Node* in_val = cmpx->in(MemNode::ValueIn); 598 Node* in_expected = cmpx->in(LoadStoreConditionalNode::ExpectedIn); 599 600 float likely = PROB_LIKELY(0.999); 601 602 const TypePtr *adr_type = cmpx->get_ptr_type(); 603 Compile::AliasType* alias_type = C->alias_type(adr_type); 604 int alias_idx = C->get_alias_index(adr_type); 605 606 // Outer check - true: continue, false: load and check 607 Node* region = new RegionNode(3); 608 Node* phi = new PhiNode(region, adr_type); 609 610 // Inner check - is the healed ref equal to the expected 611 Node* region2 = new RegionNode(3); 612 Node* phi2 = new PhiNode(region2, adr_type); 613 614 // Check if cmpx succeeded 615 Node* cmp = gvn.transform(new CmpPNode(cmpx, in_expected)); 616 Node* bol = gvn.transform(new BoolNode(cmp, BoolTest::eq))->as_Bool(); 617 IfNode* iff = gvn.transform(new IfNode(in_ctrl, bol, likely, COUNT_UNKNOWN))->as_If(); 618 Node* then = gvn.transform(new IfTrueNode(iff)); 619 Node* elsen = gvn.transform(new IfFalseNode(iff)); 620 621 Node* scmemproj1 = gvn.transform(new SCMemProjNode(cmpx)); 622 kit->set_memory(scmemproj1, alias_idx); 623 624 // CAS fail - reload and heal oop 625 Node* reload = kit->make_load(elsen, in_adr, TypeOopPtr::BOTTOM, T_OBJECT, MemNode::unordered); 626 Node* barrier = gvn.transform(new LoadBarrierNode(C, elsen, scmemproj1, reload, in_adr, false, true, false)); 627 Node* barrierctrl = gvn.transform(new ProjNode(barrier, LoadBarrierNode::Control)); 628 Node* barrierdata = gvn.transform(new ProjNode(barrier, LoadBarrierNode::Oop)); 629 630 // Check load 631 Node* tmpX = gvn.transform(new CastP2XNode(NULL, barrierdata)); 632 Node* in_expX = gvn.transform(new CastP2XNode(NULL, in_expected)); 633 Node* cmp2 = gvn.transform(new CmpXNode(tmpX, in_expX)); 634 Node *bol2 = gvn.transform(new BoolNode(cmp2, BoolTest::ne))->as_Bool(); 635 IfNode* iff2 = gvn.transform(new IfNode(barrierctrl, bol2, likely, COUNT_UNKNOWN))->as_If(); 636 Node* then2 = gvn.transform(new IfTrueNode(iff2)); 637 Node* elsen2 = gvn.transform(new IfFalseNode(iff2)); 638 639 // Redo CAS 640 Node* cmpx2 = gvn.transform(new CompareAndExchangePNode(elsen2, kit->memory(alias_idx), in_adr, in_val, in_expected, adr_type, cmpx->get_ptr_type(), cmpx->order())); 641 Node* scmemproj2 = gvn.transform(new SCMemProjNode(cmpx2)); 642 kit->set_control(elsen2); 643 kit->set_memory(scmemproj2, alias_idx); 644 645 // Merge inner flow - check if healed oop was equal too expected. 646 region2->set_req(1, kit->control()); 647 region2->set_req(2, then2); 648 phi2->set_req(1, cmpx2); 649 phi2->set_req(2, barrierdata); 650 651 // Merge outer flow - then check if first cas succeeded 652 region->set_req(1, then); 653 region->set_req(2, region2); 654 phi->set_req(1, cmpx); 655 phi->set_req(2, phi2); 656 657 gvn.transform(region2); 658 gvn.transform(phi2); 659 gvn.transform(region); 660 gvn.transform(phi); 661 662 kit->set_control(region); 663 kit->set_memory(in_mem, alias_idx); 664 kit->insert_mem_bar(Op_MemBarCPUOrder); 665 666 return phi; 667 } 668 669 Node* ZBarrierSetC2::load_barrier(GraphKit* kit, Node* val, Node* adr, bool weak, bool writeback, bool oop_reload_allowed) const { 670 PhaseGVN& gvn = kit->gvn(); 671 Node* barrier = new LoadBarrierNode(Compile::current(), kit->control(), kit->memory(TypeRawPtr::BOTTOM), val, adr, weak, writeback, oop_reload_allowed); 672 Node* transformed_barrier = gvn.transform(barrier); 673 674 if (transformed_barrier->is_LoadBarrier()) { 675 if (barrier == transformed_barrier) { 676 kit->set_control(gvn.transform(new ProjNode(barrier, LoadBarrierNode::Control))); 677 } 678 Node* result = gvn.transform(new ProjNode(transformed_barrier, LoadBarrierNode::Oop)); 679 assert(is_gc_barrier_node(result), "sanity"); 680 assert(step_over_gc_barrier(result) == val, "sanity"); 681 return result; 682 } else { 683 return val; 684 } 685 } 686 687 static bool barrier_needed(C2Access access) { 688 return ZBarrierSet::barrier_needed(access.decorators(), access.type()); 689 } 690 691 Node* ZBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const { 692 Node* p = BarrierSetC2::load_at_resolved(access, val_type); 693 if (!barrier_needed(access)) { 694 return p; 695 } 696 697 bool weak = (access.decorators() & ON_WEAK_OOP_REF) != 0; 698 699 GraphKit* kit = access.kit(); 700 PhaseGVN& gvn = kit->gvn(); 701 Node* adr = access.addr().node(); 702 Node* heap_base_oop = access.base(); 703 bool unsafe = (access.decorators() & C2_UNSAFE_ACCESS) != 0; 704 if (unsafe) { 705 if (!ZVerifyLoadBarriers) { 706 p = load_barrier(kit, p, adr); 707 } else { 708 if (!TypePtr::NULL_PTR->higher_equal(gvn.type(heap_base_oop))) { 709 p = load_barrier(kit, p, adr); 710 } else { 711 IdealKit ideal(kit); 712 IdealVariable res(ideal); 713 #define __ ideal. 714 __ declarations_done(); 715 __ set(res, p); 716 __ if_then(heap_base_oop, BoolTest::ne, kit->null(), PROB_UNLIKELY(0.999)); { 717 kit->sync_kit(ideal); 718 p = load_barrier(kit, p, adr); 719 __ set(res, p); 720 __ sync_kit(kit); 721 } __ end_if(); 722 kit->final_sync(ideal); 723 p = __ value(res); 724 #undef __ 725 } 726 } 727 return p; 728 } else { 729 return load_barrier(access.kit(), p, access.addr().node(), weak, true, true); 730 } 731 } 732 733 Node* ZBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicAccess& access, Node* expected_val, 734 Node* new_val, const Type* val_type) const { 735 Node* result = BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, val_type); 736 if (!barrier_needed(access)) { 737 return result; 738 } 739 740 access.set_needs_pinning(false); 741 return make_cmpx_loadbarrier(access); 742 } 743 744 Node* ZBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicAccess& access, Node* expected_val, 745 Node* new_val, const Type* value_type) const { 746 Node* result = BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type); 747 if (!barrier_needed(access)) { 748 return result; 749 } 750 751 Node* load_store = access.raw_access(); 752 bool weak_cas = (access.decorators() & C2_WEAK_CMPXCHG) != 0; 753 bool expected_is_null = (expected_val->get_ptr_type() == TypePtr::NULL_PTR); 754 755 if (!expected_is_null) { 756 if (weak_cas) { 757 access.set_needs_pinning(false); 758 load_store = make_cas_loadbarrier(access); 759 } else { 760 access.set_needs_pinning(false); 761 load_store = make_cas_loadbarrier(access); 762 } 763 } 764 765 return load_store; 766 } 767 768 Node* ZBarrierSetC2::atomic_xchg_at_resolved(C2AtomicAccess& access, Node* new_val, const Type* val_type) const { 769 Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, new_val, val_type); 770 if (!barrier_needed(access)) { 771 return result; 772 } 773 774 Node* load_store = access.raw_access(); 775 Node* adr = access.addr().node(); 776 777 return load_barrier(access.kit(), load_store, adr, false, false, false); 778 } 779 780 // == Macro Expansion == 781 782 void ZBarrierSetC2::expand_loadbarrier_node(PhaseMacroExpand* phase, LoadBarrierNode* barrier) const { 783 Node* in_ctrl = barrier->in(LoadBarrierNode::Control); 784 Node* in_mem = barrier->in(LoadBarrierNode::Memory); 785 Node* in_val = barrier->in(LoadBarrierNode::Oop); 786 Node* in_adr = barrier->in(LoadBarrierNode::Address); 787 788 Node* out_ctrl = barrier->proj_out(LoadBarrierNode::Control); 789 Node* out_res = barrier->proj_out(LoadBarrierNode::Oop); 790 791 PhaseIterGVN &igvn = phase->igvn(); 792 793 if (ZVerifyLoadBarriers) { 794 igvn.replace_node(out_res, in_val); 795 igvn.replace_node(out_ctrl, in_ctrl); 796 return; 797 } 798 799 if (barrier->can_be_eliminated()) { 800 // Clone and pin the load for this barrier below the dominating 801 // barrier: the load cannot be allowed to float above the 802 // dominating barrier 803 Node* load = in_val; 804 805 if (load->is_Load()) { 806 Node* new_load = load->clone(); 807 Node* addp = new_load->in(MemNode::Address); 808 assert(addp->is_AddP() || addp->is_Phi() || addp->is_Load(), "bad address"); 809 Node* cast = new CastPPNode(addp, igvn.type(addp), true); 810 Node* ctrl = NULL; 811 Node* similar = barrier->in(LoadBarrierNode::Similar); 812 if (similar->is_Phi()) { 813 // already expanded 814 ctrl = similar->in(0); 815 } else { 816 assert(similar->is_Proj() && similar->in(0)->is_LoadBarrier(), "unexpected graph shape"); 817 ctrl = similar->in(0)->as_LoadBarrier()->proj_out(LoadBarrierNode::Control); 818 } 819 assert(ctrl != NULL, "bad control"); 820 cast->set_req(0, ctrl); 821 igvn.transform(cast); 822 new_load->set_req(MemNode::Address, cast); 823 igvn.transform(new_load); 824 825 igvn.replace_node(out_res, new_load); 826 igvn.replace_node(out_ctrl, in_ctrl); 827 return; 828 } 829 // cannot eliminate 830 } 831 832 // There are two cases that require the basic loadbarrier 833 // 1) When the writeback of a healed oop must be avoided (swap) 834 // 2) When we must guarantee that no reload of is done (swap, cas, cmpx) 835 if (!barrier->is_writeback()) { 836 assert(!barrier->oop_reload_allowed(), "writeback barriers should be marked as requires oop"); 837 } 838 839 if (!barrier->oop_reload_allowed()) { 840 expand_loadbarrier_basic(phase, barrier); 841 } else { 842 expand_loadbarrier_optimized(phase, barrier); 843 } 844 } 845 846 // Basic loadbarrier using conventional argument passing 847 void ZBarrierSetC2::expand_loadbarrier_basic(PhaseMacroExpand* phase, LoadBarrierNode *barrier) const { 848 PhaseIterGVN &igvn = phase->igvn(); 849 850 Node* in_ctrl = barrier->in(LoadBarrierNode::Control); 851 Node* in_mem = barrier->in(LoadBarrierNode::Memory); 852 Node* in_val = barrier->in(LoadBarrierNode::Oop); 853 Node* in_adr = barrier->in(LoadBarrierNode::Address); 854 855 Node* out_ctrl = barrier->proj_out(LoadBarrierNode::Control); 856 Node* out_res = barrier->proj_out(LoadBarrierNode::Oop); 857 858 float unlikely = PROB_UNLIKELY(0.999); 859 const Type* in_val_maybe_null_t = igvn.type(in_val); 860 861 Node* jthread = igvn.transform(new ThreadLocalNode()); 862 Node* adr = phase->basic_plus_adr(jthread, in_bytes(ZThreadLocalData::address_bad_mask_offset())); 863 Node* bad_mask = igvn.transform(LoadNode::make(igvn, in_ctrl, in_mem, adr, TypeRawPtr::BOTTOM, TypeX_X, TypeX_X->basic_type(), MemNode::unordered)); 864 Node* cast = igvn.transform(new CastP2XNode(in_ctrl, in_val)); 865 Node* obj_masked = igvn.transform(new AndXNode(cast, bad_mask)); 866 Node* cmp = igvn.transform(new CmpXNode(obj_masked, igvn.zerocon(TypeX_X->basic_type()))); 867 Node *bol = igvn.transform(new BoolNode(cmp, BoolTest::ne))->as_Bool(); 868 IfNode* iff = igvn.transform(new IfNode(in_ctrl, bol, unlikely, COUNT_UNKNOWN))->as_If(); 869 Node* then = igvn.transform(new IfTrueNode(iff)); 870 Node* elsen = igvn.transform(new IfFalseNode(iff)); 871 872 Node* result_region; 873 Node* result_val; 874 875 result_region = new RegionNode(3); 876 result_val = new PhiNode(result_region, TypeInstPtr::BOTTOM); 877 878 result_region->set_req(1, elsen); 879 Node* res = igvn.transform(new CastPPNode(in_val, in_val_maybe_null_t)); 880 res->init_req(0, elsen); 881 result_val->set_req(1, res); 882 883 const TypeFunc *tf = load_barrier_Type(); 884 Node* call; 885 if (barrier->is_weak()) { 886 call = new CallLeafNode(tf, 887 ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded_addr(), 888 "ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded", 889 TypeRawPtr::BOTTOM); 890 } else { 891 call = new CallLeafNode(tf, 892 ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(), 893 "ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded", 894 TypeRawPtr::BOTTOM); 895 } 896 897 call->init_req(TypeFunc::Control, then); 898 call->init_req(TypeFunc::I_O , phase->top()); 899 call->init_req(TypeFunc::Memory , in_mem); 900 call->init_req(TypeFunc::FramePtr, phase->top()); 901 call->init_req(TypeFunc::ReturnAdr, phase->top()); 902 call->init_req(TypeFunc::Parms+0, in_val); 903 if (barrier->is_writeback()) { 904 call->init_req(TypeFunc::Parms+1, in_adr); 905 } else { 906 // When slow path is called with a null address, the healed oop will not be written back 907 call->init_req(TypeFunc::Parms+1, igvn.zerocon(T_OBJECT)); 908 } 909 call = igvn.transform(call); 910 911 Node* ctrl = igvn.transform(new ProjNode(call, TypeFunc::Control)); 912 res = igvn.transform(new ProjNode(call, TypeFunc::Parms)); 913 res = igvn.transform(new CheckCastPPNode(ctrl, res, in_val_maybe_null_t)); 914 915 result_region->set_req(2, ctrl); 916 result_val->set_req(2, res); 917 918 result_region = igvn.transform(result_region); 919 result_val = igvn.transform(result_val); 920 921 if (out_ctrl != NULL) { // Added if cond 922 igvn.replace_node(out_ctrl, result_region); 923 } 924 igvn.replace_node(out_res, result_val); 925 } 926 927 // Optimized, low spill, loadbarrier variant using stub specialized on register used 928 void ZBarrierSetC2::expand_loadbarrier_optimized(PhaseMacroExpand* phase, LoadBarrierNode *barrier) const { 929 PhaseIterGVN &igvn = phase->igvn(); 930 #ifdef PRINT_NODE_TRAVERSALS 931 Node* preceding_barrier_node = barrier->in(LoadBarrierNode::Oop); 932 #endif 933 934 Node* in_ctrl = barrier->in(LoadBarrierNode::Control); 935 Node* in_mem = barrier->in(LoadBarrierNode::Memory); 936 Node* in_val = barrier->in(LoadBarrierNode::Oop); 937 Node* in_adr = barrier->in(LoadBarrierNode::Address); 938 939 Node* out_ctrl = barrier->proj_out(LoadBarrierNode::Control); 940 Node* out_res = barrier->proj_out(LoadBarrierNode::Oop); 941 942 assert(barrier->in(LoadBarrierNode::Oop) != NULL, "oop to loadbarrier node cannot be null"); 943 944 #ifdef PRINT_NODE_TRAVERSALS 945 tty->print("\n\n\nBefore barrier optimization:\n"); 946 traverse(barrier, out_ctrl, out_res, -1); 947 948 tty->print("\nBefore barrier optimization: preceding_barrier_node\n"); 949 traverse(preceding_barrier_node, out_ctrl, out_res, -1); 950 #endif 951 952 float unlikely = PROB_UNLIKELY(0.999); 953 954 Node* jthread = igvn.transform(new ThreadLocalNode()); 955 Node* adr = phase->basic_plus_adr(jthread, in_bytes(ZThreadLocalData::address_bad_mask_offset())); 956 Node* bad_mask = igvn.transform(LoadNode::make(igvn, in_ctrl, in_mem, adr, 957 TypeRawPtr::BOTTOM, TypeX_X, TypeX_X->basic_type(), 958 MemNode::unordered)); 959 Node* cast = igvn.transform(new CastP2XNode(in_ctrl, in_val)); 960 Node* obj_masked = igvn.transform(new AndXNode(cast, bad_mask)); 961 Node* cmp = igvn.transform(new CmpXNode(obj_masked, igvn.zerocon(TypeX_X->basic_type()))); 962 Node *bol = igvn.transform(new BoolNode(cmp, BoolTest::ne))->as_Bool(); 963 IfNode* iff = igvn.transform(new IfNode(in_ctrl, bol, unlikely, COUNT_UNKNOWN))->as_If(); 964 Node* then = igvn.transform(new IfTrueNode(iff)); 965 Node* elsen = igvn.transform(new IfFalseNode(iff)); 966 967 Node* slow_path_surrogate; 968 if (!barrier->is_weak()) { 969 slow_path_surrogate = igvn.transform(new LoadBarrierSlowRegNode(then, in_mem, in_adr, in_val->adr_type(), 970 (const TypePtr*) in_val->bottom_type(), MemNode::unordered)); 971 } else { 972 slow_path_surrogate = igvn.transform(new LoadBarrierWeakSlowRegNode(then, in_mem, in_adr, in_val->adr_type(), 973 (const TypePtr*) in_val->bottom_type(), MemNode::unordered)); 974 } 975 976 Node *new_loadp; 977 new_loadp = slow_path_surrogate; 978 // Create the final region/phi pair to converge cntl/data paths to downstream code 979 Node* result_region = igvn.transform(new RegionNode(3)); 980 result_region->set_req(1, then); 981 result_region->set_req(2, elsen); 982 983 Node* result_phi = igvn.transform(new PhiNode(result_region, TypeInstPtr::BOTTOM)); 984 result_phi->set_req(1, new_loadp); 985 result_phi->set_req(2, barrier->in(LoadBarrierNode::Oop)); 986 987 // Finally, connect the original outputs to the barrier region and phi to complete the expansion/substitution 988 // igvn.replace_node(out_ctrl, result_region); 989 if (out_ctrl != NULL) { // added if cond 990 igvn.replace_node(out_ctrl, result_region); 991 } 992 igvn.replace_node(out_res, result_phi); 993 994 assert(barrier->outcnt() == 0,"LoadBarrier macro node has non-null outputs after expansion!"); 995 996 #ifdef PRINT_NODE_TRAVERSALS 997 tty->print("\nAfter barrier optimization: old out_ctrl\n"); 998 traverse(out_ctrl, out_ctrl, out_res, -1); 999 tty->print("\nAfter barrier optimization: old out_res\n"); 1000 traverse(out_res, out_ctrl, out_res, -1); 1001 tty->print("\nAfter barrier optimization: old barrier\n"); 1002 traverse(barrier, out_ctrl, out_res, -1); 1003 tty->print("\nAfter barrier optimization: preceding_barrier_node\n"); 1004 traverse(preceding_barrier_node, result_region, result_phi, -1); 1005 #endif 1006 1007 assert(is_gc_barrier_node(result_phi), "sanity"); 1008 assert(step_over_gc_barrier(result_phi) == in_val, "sanity"); 1009 1010 return; 1011 } 1012 1013 bool ZBarrierSetC2::expand_macro_nodes(PhaseMacroExpand* macro) const { 1014 Compile* C = Compile::current(); 1015 PhaseIterGVN &igvn = macro->igvn(); 1016 ZBarrierSetC2State* s = state(); 1017 if (s->load_barrier_count() > 0) { 1018 #ifdef ASSERT 1019 verify_gc_barriers(false); 1020 #endif 1021 igvn.set_delay_transform(true); 1022 int skipped = 0; 1023 while (s->load_barrier_count() > skipped) { 1024 int load_barrier_count = s->load_barrier_count(); 1025 LoadBarrierNode * n = s->load_barrier_node(load_barrier_count-1-skipped); 1026 if (igvn.type(n) == Type::TOP || (n->in(0) != NULL && n->in(0)->is_top())) { 1027 // Node is unreachable, so don't try to expand it 1028 s->remove_load_barrier_node(n); 1029 continue; 1030 } 1031 if (!n->can_be_eliminated()) { 1032 skipped++; 1033 continue; 1034 } 1035 expand_loadbarrier_node(macro, n); 1036 assert(s->load_barrier_count() < load_barrier_count, "must have deleted a node from load barrier list"); 1037 if (C->failing()) return true; 1038 } 1039 while (s->load_barrier_count() > 0) { 1040 int load_barrier_count = s->load_barrier_count(); 1041 LoadBarrierNode* n = s->load_barrier_node(load_barrier_count - 1); 1042 assert(!(igvn.type(n) == Type::TOP || (n->in(0) != NULL && n->in(0)->is_top())), "should have been processed already"); 1043 assert(!n->can_be_eliminated(), "should have been processed already"); 1044 expand_loadbarrier_node(macro, n); 1045 assert(s->load_barrier_count() < load_barrier_count, "must have deleted a node from load barrier list"); 1046 if (C->failing()) return true; 1047 } 1048 igvn.set_delay_transform(false); 1049 igvn.optimize(); 1050 if (C->failing()) return true; 1051 } 1052 return false; 1053 } 1054 1055 // == Loop optimization == 1056 1057 static bool replace_with_dominating_barrier(PhaseIdealLoop* phase, LoadBarrierNode* lb, bool last_round) { 1058 PhaseIterGVN &igvn = phase->igvn(); 1059 Compile* C = Compile::current(); 1060 1061 LoadBarrierNode* lb2 = lb->has_dominating_barrier(phase, false, last_round); 1062 if (lb2 != NULL) { 1063 if (lb->in(LoadBarrierNode::Oop) != lb2->in(LoadBarrierNode::Oop)) { 1064 assert(lb->in(LoadBarrierNode::Address) == lb2->in(LoadBarrierNode::Address), ""); 1065 igvn.replace_input_of(lb, LoadBarrierNode::Similar, lb2->proj_out(LoadBarrierNode::Oop)); 1066 C->set_major_progress(); 1067 } else { 1068 // That transformation may cause the Similar edge on dominated load barriers to be invalid 1069 lb->fix_similar_in_uses(&igvn); 1070 1071 Node* val = lb->proj_out(LoadBarrierNode::Oop); 1072 assert(lb2->has_true_uses(), ""); 1073 assert(lb2->in(LoadBarrierNode::Oop) == lb->in(LoadBarrierNode::Oop), ""); 1074 1075 phase->lazy_update(lb, lb->in(LoadBarrierNode::Control)); 1076 phase->lazy_replace(lb->proj_out(LoadBarrierNode::Control), lb->in(LoadBarrierNode::Control)); 1077 igvn.replace_node(val, lb2->proj_out(LoadBarrierNode::Oop)); 1078 1079 return true; 1080 } 1081 } 1082 return false; 1083 } 1084 1085 static Node* find_dominating_memory(PhaseIdealLoop* phase, Node* mem, Node* dom, int i) { 1086 assert(dom->is_Region() || i == -1, ""); 1087 Node* m = mem; 1088 while(phase->is_dominator(dom, phase->has_ctrl(m) ? phase->get_ctrl(m) : m->in(0))) { 1089 if (m->is_Mem()) { 1090 assert(m->as_Mem()->adr_type() == TypeRawPtr::BOTTOM, ""); 1091 m = m->in(MemNode::Memory); 1092 } else if (m->is_MergeMem()) { 1093 m = m->as_MergeMem()->memory_at(Compile::AliasIdxRaw); 1094 } else if (m->is_Phi()) { 1095 if (m->in(0) == dom && i != -1) { 1096 m = m->in(i); 1097 break; 1098 } else { 1099 m = m->in(LoopNode::EntryControl); 1100 } 1101 } else if (m->is_Proj()) { 1102 m = m->in(0); 1103 } else if (m->is_SafePoint() || m->is_MemBar()) { 1104 m = m->in(TypeFunc::Memory); 1105 } else { 1106 #ifdef ASSERT 1107 m->dump(); 1108 #endif 1109 ShouldNotReachHere(); 1110 } 1111 } 1112 return m; 1113 } 1114 1115 static LoadBarrierNode* clone_load_barrier(PhaseIdealLoop* phase, LoadBarrierNode* lb, Node* ctl, Node* mem, Node* oop_in) { 1116 PhaseIterGVN &igvn = phase->igvn(); 1117 Compile* C = Compile::current(); 1118 Node* the_clone = lb->clone(); 1119 the_clone->set_req(LoadBarrierNode::Control, ctl); 1120 the_clone->set_req(LoadBarrierNode::Memory, mem); 1121 if (oop_in != NULL) { 1122 the_clone->set_req(LoadBarrierNode::Oop, oop_in); 1123 } 1124 1125 LoadBarrierNode* new_lb = the_clone->as_LoadBarrier(); 1126 igvn.register_new_node_with_optimizer(new_lb); 1127 IdealLoopTree *loop = phase->get_loop(new_lb->in(0)); 1128 phase->set_ctrl(new_lb, new_lb->in(0)); 1129 phase->set_loop(new_lb, loop); 1130 phase->set_idom(new_lb, new_lb->in(0), phase->dom_depth(new_lb->in(0))+1); 1131 if (!loop->_child) { 1132 loop->_body.push(new_lb); 1133 } 1134 1135 Node* proj_ctl = new ProjNode(new_lb, LoadBarrierNode::Control); 1136 igvn.register_new_node_with_optimizer(proj_ctl); 1137 phase->set_ctrl(proj_ctl, proj_ctl->in(0)); 1138 phase->set_loop(proj_ctl, loop); 1139 phase->set_idom(proj_ctl, new_lb, phase->dom_depth(new_lb)+1); 1140 if (!loop->_child) { 1141 loop->_body.push(proj_ctl); 1142 } 1143 1144 Node* proj_oop = new ProjNode(new_lb, LoadBarrierNode::Oop); 1145 phase->register_new_node(proj_oop, new_lb); 1146 1147 if (!new_lb->in(LoadBarrierNode::Similar)->is_top()) { 1148 LoadBarrierNode* similar = new_lb->in(LoadBarrierNode::Similar)->in(0)->as_LoadBarrier(); 1149 if (!phase->is_dominator(similar, ctl)) { 1150 igvn.replace_input_of(new_lb, LoadBarrierNode::Similar, C->top()); 1151 } 1152 } 1153 1154 return new_lb; 1155 } 1156 1157 static void replace_barrier(PhaseIdealLoop* phase, LoadBarrierNode* lb, Node* new_val) { 1158 PhaseIterGVN &igvn = phase->igvn(); 1159 Node* val = lb->proj_out(LoadBarrierNode::Oop); 1160 igvn.replace_node(val, new_val); 1161 phase->lazy_update(lb, lb->in(LoadBarrierNode::Control)); 1162 phase->lazy_replace(lb->proj_out(LoadBarrierNode::Control), lb->in(LoadBarrierNode::Control)); 1163 } 1164 1165 static bool split_barrier_thru_phi(PhaseIdealLoop* phase, LoadBarrierNode* lb) { 1166 PhaseIterGVN &igvn = phase->igvn(); 1167 Compile* C = Compile::current(); 1168 1169 if (lb->in(LoadBarrierNode::Oop)->is_Phi()) { 1170 Node* oop_phi = lb->in(LoadBarrierNode::Oop); 1171 1172 if ((oop_phi->req() != 3) || (oop_phi->in(2) == oop_phi)) { 1173 // Ignore phis with only one input 1174 return false; 1175 } 1176 1177 if (phase->is_dominator(phase->get_ctrl(lb->in(LoadBarrierNode::Address)), 1178 oop_phi->in(0)) && phase->get_ctrl(lb->in(LoadBarrierNode::Address)) != oop_phi->in(0)) { 1179 // That transformation may cause the Similar edge on dominated load barriers to be invalid 1180 lb->fix_similar_in_uses(&igvn); 1181 1182 RegionNode* region = oop_phi->in(0)->as_Region(); 1183 1184 int backedge = LoopNode::LoopBackControl; 1185 if (region->is_Loop() && region->in(backedge)->is_Proj() && region->in(backedge)->in(0)->is_If()) { 1186 Node* c = region->in(backedge)->in(0)->in(0); 1187 assert(c->unique_ctrl_out() == region->in(backedge)->in(0), ""); 1188 Node* oop = lb->in(LoadBarrierNode::Oop)->in(backedge); 1189 Node* oop_c = phase->has_ctrl(oop) ? phase->get_ctrl(oop) : oop; 1190 if (!phase->is_dominator(oop_c, c)) { 1191 return false; 1192 } 1193 } 1194 1195 // If the node on the backedge above the phi is the node itself - we have a self loop. 1196 // Don't clone - this will be folded later. 1197 if (oop_phi->in(LoopNode::LoopBackControl) == lb->proj_out(LoadBarrierNode::Oop)) { 1198 return false; 1199 } 1200 1201 bool is_strip_mined = region->is_CountedLoop() && region->as_CountedLoop()->is_strip_mined(); 1202 Node *phi = oop_phi->clone(); 1203 1204 for (uint i = 1; i < region->req(); i++) { 1205 Node* ctrl = region->in(i); 1206 if (ctrl != C->top()) { 1207 assert(!phase->is_dominator(ctrl, region) || region->is_Loop(), ""); 1208 1209 Node* mem = lb->in(LoadBarrierNode::Memory); 1210 Node* m = find_dominating_memory(phase, mem, region, i); 1211 1212 if (region->is_Loop() && i == LoopNode::LoopBackControl && ctrl->is_Proj() && ctrl->in(0)->is_If()) { 1213 ctrl = ctrl->in(0)->in(0); 1214 } else if (region->is_Loop() && is_strip_mined) { 1215 // If this is a strip mined loop, control must move above OuterStripMinedLoop 1216 assert(i == LoopNode::EntryControl, "check"); 1217 assert(ctrl->is_OuterStripMinedLoop(), "sanity"); 1218 ctrl = ctrl->as_OuterStripMinedLoop()->in(LoopNode::EntryControl); 1219 } 1220 1221 LoadBarrierNode* new_lb = clone_load_barrier(phase, lb, ctrl, m, lb->in(LoadBarrierNode::Oop)->in(i)); 1222 Node* out_ctrl = new_lb->proj_out(LoadBarrierNode::Control); 1223 1224 if (is_strip_mined && (i == LoopNode::EntryControl)) { 1225 assert(region->in(i)->is_OuterStripMinedLoop(), ""); 1226 igvn.replace_input_of(region->in(i), i, out_ctrl); 1227 phase->set_idom(region->in(i), out_ctrl, phase->dom_depth(out_ctrl)); 1228 } else if (ctrl == region->in(i)) { 1229 igvn.replace_input_of(region, i, out_ctrl); 1230 // Only update the idom if is the loop entry we are updating 1231 // - A loop backedge doesn't change the idom 1232 if (region->is_Loop() && i == LoopNode::EntryControl) { 1233 phase->set_idom(region, out_ctrl, phase->dom_depth(out_ctrl)); 1234 } 1235 } else { 1236 Node* iff = region->in(i)->in(0); 1237 igvn.replace_input_of(iff, 0, out_ctrl); 1238 phase->set_idom(iff, out_ctrl, phase->dom_depth(out_ctrl)+1); 1239 } 1240 phi->set_req(i, new_lb->proj_out(LoadBarrierNode::Oop)); 1241 } 1242 } 1243 phase->register_new_node(phi, region); 1244 replace_barrier(phase, lb, phi); 1245 1246 if (region->is_Loop()) { 1247 // Load barrier moved to the back edge of the Loop may now 1248 // have a safepoint on the path to the barrier on the Similar 1249 // edge 1250 igvn.replace_input_of(phi->in(LoopNode::LoopBackControl)->in(0), LoadBarrierNode::Similar, C->top()); 1251 Node* head = region->in(LoopNode::EntryControl); 1252 phase->set_idom(region, head, phase->dom_depth(head)+1); 1253 phase->recompute_dom_depth(); 1254 if (head->is_CountedLoop() && head->as_CountedLoop()->is_main_loop()) { 1255 head->as_CountedLoop()->set_normal_loop(); 1256 } 1257 } 1258 1259 return true; 1260 } 1261 } 1262 1263 return false; 1264 } 1265 1266 static bool move_out_of_loop(PhaseIdealLoop* phase, LoadBarrierNode* lb) { 1267 PhaseIterGVN &igvn = phase->igvn(); 1268 IdealLoopTree *lb_loop = phase->get_loop(lb->in(0)); 1269 if (lb_loop != phase->ltree_root() && !lb_loop->_irreducible) { 1270 Node* oop_ctrl = phase->get_ctrl(lb->in(LoadBarrierNode::Oop)); 1271 IdealLoopTree *oop_loop = phase->get_loop(oop_ctrl); 1272 IdealLoopTree* adr_loop = phase->get_loop(phase->get_ctrl(lb->in(LoadBarrierNode::Address))); 1273 if (!lb_loop->is_member(oop_loop) && !lb_loop->is_member(adr_loop)) { 1274 // That transformation may cause the Similar edge on dominated load barriers to be invalid 1275 lb->fix_similar_in_uses(&igvn); 1276 1277 Node* head = lb_loop->_head; 1278 assert(head->is_Loop(), ""); 1279 1280 if (phase->is_dominator(head, oop_ctrl)) { 1281 assert(oop_ctrl->Opcode() == Op_CProj && oop_ctrl->in(0)->Opcode() == Op_NeverBranch, ""); 1282 assert(lb_loop->is_member(phase->get_loop(oop_ctrl->in(0)->in(0))), ""); 1283 return false; 1284 } 1285 1286 if (head->is_CountedLoop()) { 1287 CountedLoopNode* cloop = head->as_CountedLoop(); 1288 if (cloop->is_main_loop()) { 1289 cloop->set_normal_loop(); 1290 } 1291 // When we are moving barrier out of a counted loop, 1292 // make sure we move it all the way out of the strip mined outer loop. 1293 if (cloop->is_strip_mined()) { 1294 head = cloop->outer_loop(); 1295 } 1296 } 1297 1298 Node* mem = lb->in(LoadBarrierNode::Memory); 1299 Node* m = find_dominating_memory(phase, mem, head, -1); 1300 1301 LoadBarrierNode* new_lb = clone_load_barrier(phase, lb, head->in(LoopNode::EntryControl), m, NULL); 1302 1303 assert(phase->idom(head) == head->in(LoopNode::EntryControl), ""); 1304 Node* proj_ctl = new_lb->proj_out(LoadBarrierNode::Control); 1305 igvn.replace_input_of(head, LoopNode::EntryControl, proj_ctl); 1306 phase->set_idom(head, proj_ctl, phase->dom_depth(proj_ctl) + 1); 1307 1308 replace_barrier(phase, lb, new_lb->proj_out(LoadBarrierNode::Oop)); 1309 1310 phase->recompute_dom_depth(); 1311 1312 return true; 1313 } 1314 } 1315 1316 return false; 1317 } 1318 1319 static bool common_barriers(PhaseIdealLoop* phase, LoadBarrierNode* lb) { 1320 PhaseIterGVN &igvn = phase->igvn(); 1321 Node* in_val = lb->in(LoadBarrierNode::Oop); 1322 for (DUIterator_Fast imax, i = in_val->fast_outs(imax); i < imax; i++) { 1323 Node* u = in_val->fast_out(i); 1324 if (u != lb && u->is_LoadBarrier() && u->as_LoadBarrier()->has_true_uses()) { 1325 Node* this_ctrl = lb->in(LoadBarrierNode::Control); 1326 Node* other_ctrl = u->in(LoadBarrierNode::Control); 1327 1328 Node* lca = phase->dom_lca(this_ctrl, other_ctrl); 1329 Node* proj1 = NULL; 1330 Node* proj2 = NULL; 1331 bool ok = (lb->in(LoadBarrierNode::Address) == u->in(LoadBarrierNode::Address)); 1332 1333 while (this_ctrl != lca && ok) { 1334 if (this_ctrl->in(0) != NULL && 1335 this_ctrl->in(0)->is_MultiBranch()) { 1336 if (this_ctrl->in(0)->in(0) == lca) { 1337 assert(proj1 == NULL, ""); 1338 assert(this_ctrl->is_Proj(), ""); 1339 proj1 = this_ctrl; 1340 } else if (!(this_ctrl->in(0)->is_If() && this_ctrl->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none))) { 1341 ok = false; 1342 } 1343 } 1344 this_ctrl = phase->idom(this_ctrl); 1345 } 1346 while (other_ctrl != lca && ok) { 1347 if (other_ctrl->in(0) != NULL && 1348 other_ctrl->in(0)->is_MultiBranch()) { 1349 if (other_ctrl->in(0)->in(0) == lca) { 1350 assert(other_ctrl->is_Proj(), ""); 1351 assert(proj2 == NULL, ""); 1352 proj2 = other_ctrl; 1353 } else if (!(other_ctrl->in(0)->is_If() && other_ctrl->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none))) { 1354 ok = false; 1355 } 1356 } 1357 other_ctrl = phase->idom(other_ctrl); 1358 } 1359 assert(proj1 == NULL || proj2 == NULL || proj1->in(0) == proj2->in(0), ""); 1360 if (ok && proj1 && proj2 && proj1 != proj2 && proj1->in(0)->is_If()) { 1361 // That transformation may cause the Similar edge on dominated load barriers to be invalid 1362 lb->fix_similar_in_uses(&igvn); 1363 u->as_LoadBarrier()->fix_similar_in_uses(&igvn); 1364 1365 Node* split = lca->unique_ctrl_out(); 1366 assert(split->in(0) == lca, ""); 1367 1368 Node* mem = lb->in(LoadBarrierNode::Memory); 1369 Node* m = find_dominating_memory(phase, mem, split, -1); 1370 LoadBarrierNode* new_lb = clone_load_barrier(phase, lb, lca, m, NULL); 1371 1372 Node* proj_ctl = new_lb->proj_out(LoadBarrierNode::Control); 1373 igvn.replace_input_of(split, 0, new_lb->proj_out(LoadBarrierNode::Control)); 1374 phase->set_idom(split, proj_ctl, phase->dom_depth(proj_ctl)+1); 1375 1376 Node* proj_oop = new_lb->proj_out(LoadBarrierNode::Oop); 1377 replace_barrier(phase, lb, proj_oop); 1378 replace_barrier(phase, u->as_LoadBarrier(), proj_oop); 1379 1380 phase->recompute_dom_depth(); 1381 1382 return true; 1383 } 1384 } 1385 } 1386 1387 return false; 1388 } 1389 1390 static void optimize_load_barrier(PhaseIdealLoop* phase, LoadBarrierNode* lb, bool last_round) { 1391 Compile* C = Compile::current(); 1392 1393 if (!C->directive()->ZOptimizeLoadBarriersOption) { 1394 return; 1395 } 1396 1397 if (lb->has_true_uses()) { 1398 if (replace_with_dominating_barrier(phase, lb, last_round)) { 1399 return; 1400 } 1401 1402 if (split_barrier_thru_phi(phase, lb)) { 1403 return; 1404 } 1405 1406 if (move_out_of_loop(phase, lb)) { 1407 return; 1408 } 1409 1410 if (common_barriers(phase, lb)) { 1411 return; 1412 } 1413 } 1414 } 1415 1416 void ZBarrierSetC2::loop_optimize_gc_barrier(PhaseIdealLoop* phase, Node* node, bool last_round) { 1417 if (node->is_LoadBarrier()) { 1418 optimize_load_barrier(phase, node->as_LoadBarrier(), last_round); 1419 } 1420 } 1421 1422 Node* ZBarrierSetC2::step_over_gc_barrier(Node* c) const { 1423 Node* node = c; 1424 1425 // 1. This step follows potential oop projections of a load barrier before expansion 1426 if (node->is_Proj()) { 1427 node = node->in(0); 1428 } 1429 1430 // 2. This step checks for unexpanded load barriers 1431 if (node->is_LoadBarrier()) { 1432 return node->in(LoadBarrierNode::Oop); 1433 } 1434 1435 // 3. This step checks for the phi corresponding to an optimized load barrier expansion 1436 if (node->is_Phi()) { 1437 PhiNode* phi = node->as_Phi(); 1438 Node* n = phi->in(1); 1439 if (n != NULL && (n->is_LoadBarrierSlowReg() || n->is_LoadBarrierWeakSlowReg())) { 1440 assert(c == node, "projections from step 1 should only be seen before macro expansion"); 1441 return phi->in(2); 1442 } 1443 } 1444 1445 return c; 1446 } 1447 1448 // == Verification == 1449 1450 #ifdef ASSERT 1451 1452 static bool look_for_barrier(Node* n, bool post_parse, VectorSet& visited) { 1453 if (visited.test_set(n->_idx)) { 1454 return true; 1455 } 1456 1457 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { 1458 Node* u = n->fast_out(i); 1459 if (u->is_LoadBarrier()) { 1460 } else if ((u->is_Phi() || u->is_CMove()) && !post_parse) { 1461 if (!look_for_barrier(u, post_parse, visited)) { 1462 return false; 1463 } 1464 } else if (u->Opcode() == Op_EncodeP || u->Opcode() == Op_DecodeN) { 1465 if (!look_for_barrier(u, post_parse, visited)) { 1466 return false; 1467 } 1468 } else if (u->Opcode() != Op_SCMemProj) { 1469 tty->print("bad use"); u->dump(); 1470 return false; 1471 } 1472 } 1473 1474 return true; 1475 } 1476 1477 void ZBarrierSetC2::verify_gc_barriers(bool post_parse) const { 1478 ZBarrierSetC2State* s = state(); 1479 Compile* C = Compile::current(); 1480 ResourceMark rm; 1481 VectorSet visited(Thread::current()->resource_area()); 1482 for (int i = 0; i < s->load_barrier_count(); i++) { 1483 LoadBarrierNode* n = s->load_barrier_node(i); 1484 1485 // The dominating barrier on the same address if it exists and 1486 // this barrier must not be applied on the value from the same 1487 // load otherwise the value is not reloaded before it's used the 1488 // second time. 1489 assert(n->in(LoadBarrierNode::Similar)->is_top() || 1490 (n->in(LoadBarrierNode::Similar)->in(0)->is_LoadBarrier() && 1491 n->in(LoadBarrierNode::Similar)->in(0)->in(LoadBarrierNode::Address) == n->in(LoadBarrierNode::Address) && 1492 n->in(LoadBarrierNode::Similar)->in(0)->in(LoadBarrierNode::Oop) != n->in(LoadBarrierNode::Oop)), 1493 "broken similar edge"); 1494 1495 assert(post_parse || n->as_LoadBarrier()->has_true_uses(), 1496 "found unneeded load barrier"); 1497 1498 // Several load barrier nodes chained through their Similar edge 1499 // break the code that remove the barriers in final graph reshape. 1500 assert(n->in(LoadBarrierNode::Similar)->is_top() || 1501 (n->in(LoadBarrierNode::Similar)->in(0)->is_LoadBarrier() && 1502 n->in(LoadBarrierNode::Similar)->in(0)->in(LoadBarrierNode::Similar)->is_top()), 1503 "chain of Similar load barriers"); 1504 1505 if (!n->in(LoadBarrierNode::Similar)->is_top()) { 1506 ResourceMark rm; 1507 Unique_Node_List wq; 1508 Node* other = n->in(LoadBarrierNode::Similar)->in(0); 1509 wq.push(n); 1510 bool ok = true; 1511 bool dom_found = false; 1512 for (uint next = 0; next < wq.size(); ++next) { 1513 Node *n = wq.at(next); 1514 assert(n->is_CFG(), ""); 1515 assert(!n->is_SafePoint(), ""); 1516 1517 if (n == other) { 1518 continue; 1519 } 1520 1521 if (n->is_Region()) { 1522 for (uint i = 1; i < n->req(); i++) { 1523 Node* m = n->in(i); 1524 if (m != NULL) { 1525 wq.push(m); 1526 } 1527 } 1528 } else { 1529 Node* m = n->in(0); 1530 if (m != NULL) { 1531 wq.push(m); 1532 } 1533 } 1534 } 1535 } 1536 1537 if (ZVerifyLoadBarriers) { 1538 if ((n->is_Load() || n->is_LoadStore()) && n->bottom_type()->make_oopptr() != NULL) { 1539 visited.Clear(); 1540 bool found = look_for_barrier(n, post_parse, visited); 1541 if (!found) { 1542 n->dump(1); 1543 n->dump(-3); 1544 stringStream ss; 1545 C->method()->print_short_name(&ss); 1546 tty->print_cr("-%s-", ss.as_string()); 1547 assert(found, ""); 1548 } 1549 } 1550 } 1551 } 1552 } 1553 1554 #endif