1 /* 2 * Copyright (c) 2015, 2019, Red Hat, Inc. All rights reserved. 3 * 4 * This code is free software; you can redistribute it and/or modify it 5 * under the terms of the GNU General Public License version 2 only, as 6 * published by the Free Software Foundation. 7 * 8 * This code is distributed in the hope that it will be useful, but WITHOUT 9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 11 * version 2 for more details (a copy is included in the LICENSE file that 12 * accompanied this code). 13 * 14 * You should have received a copy of the GNU General Public License version 15 * 2 along with this work; if not, write to the Free Software Foundation, 16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 17 * 18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 19 * or visit www.oracle.com if you need additional information or have any 20 * questions. 21 * 22 */ 23 24 #include "precompiled.hpp" 25 26 #include "gc/shenandoah/c2/shenandoahSupport.hpp" 27 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp" 28 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp" 29 #include "gc/shenandoah/shenandoahBrooksPointer.hpp" 30 #include "gc/shenandoah/shenandoahHeap.hpp" 31 #include "gc/shenandoah/shenandoahHeapRegion.hpp" 32 #include "gc/shenandoah/shenandoahRuntime.hpp" 33 #include "gc/shenandoah/shenandoahThreadLocalData.hpp" 34 #include "opto/arraycopynode.hpp" 35 #include "opto/block.hpp" 36 #include "opto/callnode.hpp" 37 #include "opto/castnode.hpp" 38 #include "opto/movenode.hpp" 39 #include "opto/phaseX.hpp" 40 #include "opto/rootnode.hpp" 41 #include "opto/runtime.hpp" 42 #include "opto/subnode.hpp" 43 44 Node* ShenandoahBarrierNode::skip_through_barrier(Node* n) { 45 if (n == NULL) { 46 return NULL; 47 } 48 if (n->Opcode() == Op_ShenandoahEnqueueBarrier) { 49 n = n->in(1); 50 } 51 52 if (n->is_ShenandoahBarrier()) { 53 return n->in(ValueIn); 54 } else if (n->is_Phi() && 55 n->req() == 3 && 56 n->in(1) != NULL && 57 n->in(1)->is_ShenandoahBarrier() && 58 n->in(2) != NULL && 59 n->in(2)->bottom_type() == TypePtr::NULL_PTR && 60 n->in(0) != NULL && 61 n->in(0)->in(1) != NULL && 62 n->in(0)->in(1)->is_IfProj() && 63 n->in(0)->in(2) != NULL && 64 n->in(0)->in(2)->is_IfProj() && 65 n->in(0)->in(1)->in(0) != NULL && 66 n->in(0)->in(1)->in(0) == n->in(0)->in(2)->in(0) && 67 n->in(1)->in(ValueIn)->Opcode() == Op_CastPP) { 68 Node* iff = n->in(0)->in(1)->in(0); 69 Node* res = n->in(1)->in(ValueIn)->in(1); 70 if (iff->is_If() && 71 iff->in(1) != NULL && 72 iff->in(1)->is_Bool() && 73 iff->in(1)->as_Bool()->_test._test == BoolTest::ne && 74 iff->in(1)->in(1) != NULL && 75 iff->in(1)->in(1)->Opcode() == Op_CmpP && 76 iff->in(1)->in(1)->in(1) != NULL && 77 iff->in(1)->in(1)->in(1) == res && 78 iff->in(1)->in(1)->in(2) != NULL && 79 iff->in(1)->in(1)->in(2)->bottom_type() == TypePtr::NULL_PTR) { 80 return res; 81 } 82 } 83 return n; 84 } 85 86 bool ShenandoahBarrierNode::needs_barrier(PhaseGVN* phase, ShenandoahBarrierNode* orig, Node* n, Node* rb_mem, bool allow_fromspace) { 87 Unique_Node_List visited; 88 return needs_barrier_impl(phase, orig, n, rb_mem, allow_fromspace, visited); 89 } 90 91 bool ShenandoahBarrierNode::needs_barrier_impl(PhaseGVN* phase, ShenandoahBarrierNode* orig, Node* n, Node* rb_mem, bool allow_fromspace, Unique_Node_List &visited) { 92 if (visited.member(n)) { 93 return false; // Been there. 94 } 95 visited.push(n); 96 97 if (n->is_Allocate()) { 98 return false; 99 } 100 101 if (n->is_Call()) { 102 return true; 103 } 104 105 const Type* type = phase->type(n); 106 if (type == Type::TOP) { 107 return false; 108 } 109 if (type->make_ptr()->higher_equal(TypePtr::NULL_PTR)) { 110 return false; 111 } 112 if (type->make_oopptr() && type->make_oopptr()->const_oop() != NULL) { 113 return false; 114 } 115 116 if (ShenandoahOptimizeStableFinals) { 117 const TypeAryPtr* ary = type->isa_aryptr(); 118 if (ary && ary->is_stable() && allow_fromspace) { 119 return false; 120 } 121 } 122 123 if (n->is_CheckCastPP() || n->is_ConstraintCast() || n->Opcode() == Op_ShenandoahEnqueueBarrier) { 124 return needs_barrier_impl(phase, orig, n->in(1), rb_mem, allow_fromspace, visited); 125 } 126 if (n->is_Parm()) { 127 return true; 128 } 129 if (n->is_Proj()) { 130 return needs_barrier_impl(phase, orig, n->in(0), rb_mem, allow_fromspace, visited); 131 } 132 133 if (n->Opcode() == Op_ShenandoahWBMemProj) { 134 return needs_barrier_impl(phase, orig, n->in(ShenandoahWBMemProjNode::WriteBarrier), rb_mem, allow_fromspace, visited); 135 } 136 if (n->is_Phi()) { 137 bool need_barrier = false; 138 for (uint i = 1; i < n->req() && ! need_barrier; i++) { 139 Node* input = n->in(i); 140 if (input == NULL) { 141 need_barrier = true; // Phi not complete yet? 142 } else if (needs_barrier_impl(phase, orig, input, rb_mem, allow_fromspace, visited)) { 143 need_barrier = true; 144 } 145 } 146 return need_barrier; 147 } 148 if (n->is_CMove()) { 149 return needs_barrier_impl(phase, orig, n->in(CMoveNode::IfFalse), rb_mem, allow_fromspace, visited) || 150 needs_barrier_impl(phase, orig, n->in(CMoveNode::IfTrue ), rb_mem, allow_fromspace, visited); 151 } 152 if (n->Opcode() == Op_CreateEx) { 153 return true; 154 } 155 if (n->Opcode() == Op_ShenandoahWriteBarrier) { 156 return false; 157 } 158 if (n->Opcode() == Op_ShenandoahReadBarrier) { 159 if (rb_mem == n->in(Memory)) { 160 return false; 161 } else { 162 return true; 163 } 164 } 165 166 if (n->Opcode() == Op_LoadP || 167 n->Opcode() == Op_LoadN || 168 n->Opcode() == Op_GetAndSetP || 169 n->Opcode() == Op_CompareAndExchangeP || 170 n->Opcode() == Op_ShenandoahCompareAndExchangeP || 171 n->Opcode() == Op_GetAndSetN || 172 n->Opcode() == Op_CompareAndExchangeN || 173 n->Opcode() == Op_ShenandoahCompareAndExchangeN) { 174 return true; 175 } 176 if (n->Opcode() == Op_DecodeN || 177 n->Opcode() == Op_EncodeP) { 178 return needs_barrier_impl(phase, orig, n->in(1), rb_mem, allow_fromspace, visited); 179 } 180 181 #ifdef ASSERT 182 tty->print("need barrier on?: "); n->dump(); 183 ShouldNotReachHere(); 184 #endif 185 return true; 186 } 187 188 bool ShenandoahReadBarrierNode::dominates_memory_rb_impl(PhaseGVN* phase, 189 Node* b1, 190 Node* b2, 191 Node* current, 192 bool linear) { 193 ResourceMark rm; 194 VectorSet visited(Thread::current()->resource_area()); 195 Node_Stack phis(0); 196 197 for(int i = 0; i < 10; i++) { 198 if (current == NULL) { 199 return false; 200 } else if (visited.test_set(current->_idx) || current->is_top() || current == b1) { 201 current = NULL; 202 while (phis.is_nonempty() && current == NULL) { 203 uint idx = phis.index(); 204 Node* phi = phis.node(); 205 if (idx >= phi->req()) { 206 phis.pop(); 207 } else { 208 current = phi->in(idx); 209 phis.set_index(idx+1); 210 } 211 } 212 if (current == NULL) { 213 return true; 214 } 215 } else if (current == phase->C->immutable_memory()) { 216 return false; 217 } else if (current->isa_Phi()) { 218 if (!linear) { 219 return false; 220 } 221 phis.push(current, 2); 222 current = current->in(1); 223 } else if (current->Opcode() == Op_ShenandoahWriteBarrier) { 224 const Type* in_type = current->bottom_type(); 225 const Type* this_type = b2->bottom_type(); 226 if (is_independent(in_type, this_type)) { 227 current = current->in(Memory); 228 } else { 229 return false; 230 } 231 } else if (current->Opcode() == Op_ShenandoahWBMemProj) { 232 current = current->in(ShenandoahWBMemProjNode::WriteBarrier); 233 } else if (current->is_Proj()) { 234 current = current->in(0); 235 } else if (current->is_Call()) { 236 return false; // TODO: Maybe improve by looking at the call's memory effects? 237 } else if (current->is_MemBar()) { 238 return false; // TODO: Do we need to stop at *any* membar? 239 } else if (current->is_MergeMem()) { 240 const TypePtr* adr_type = brooks_pointer_type(phase->type(b2)); 241 uint alias_idx = phase->C->get_alias_index(adr_type); 242 current = current->as_MergeMem()->memory_at(alias_idx); 243 } else { 244 #ifdef ASSERT 245 current->dump(); 246 #endif 247 ShouldNotReachHere(); 248 return false; 249 } 250 } 251 return false; 252 } 253 254 bool ShenandoahReadBarrierNode::is_independent(Node* mem) { 255 if (mem->is_Phi() || mem->is_Proj() || mem->is_MergeMem()) { 256 return true; 257 } else if (mem->Opcode() == Op_ShenandoahWBMemProj) { 258 return true; 259 } else if (mem->Opcode() == Op_ShenandoahWriteBarrier) { 260 const Type* mem_type = mem->bottom_type(); 261 const Type* this_type = bottom_type(); 262 if (is_independent(mem_type, this_type)) { 263 return true; 264 } else { 265 return false; 266 } 267 } else if (mem->is_Call() || mem->is_MemBar()) { 268 return false; 269 } 270 #ifdef ASSERT 271 mem->dump(); 272 #endif 273 ShouldNotReachHere(); 274 return true; 275 } 276 277 bool ShenandoahReadBarrierNode::dominates_memory_rb(PhaseGVN* phase, Node* b1, Node* b2, bool linear) { 278 return dominates_memory_rb_impl(phase, b1->in(Memory), b2, b2->in(Memory), linear); 279 } 280 281 bool ShenandoahReadBarrierNode::is_independent(const Type* in_type, const Type* this_type) { 282 assert(in_type->isa_oopptr(), "expect oop ptr"); 283 assert(this_type->isa_oopptr(), "expect oop ptr"); 284 285 ciKlass* in_kls = in_type->is_oopptr()->klass(); 286 ciKlass* this_kls = this_type->is_oopptr()->klass(); 287 if (in_kls != NULL && this_kls != NULL && 288 in_kls->is_loaded() && this_kls->is_loaded() && 289 (!in_kls->is_subclass_of(this_kls)) && 290 (!this_kls->is_subclass_of(in_kls))) { 291 return true; 292 } 293 return false; 294 } 295 296 Node* ShenandoahReadBarrierNode::Ideal(PhaseGVN *phase, bool can_reshape) { 297 if (! can_reshape) { 298 return NULL; 299 } 300 301 if (in(Memory) == phase->C->immutable_memory()) return NULL; 302 303 // If memory input is a MergeMem, take the appropriate slice out of it. 304 Node* mem_in = in(Memory); 305 if (mem_in->isa_MergeMem()) { 306 const TypePtr* adr_type = brooks_pointer_type(bottom_type()); 307 uint alias_idx = phase->C->get_alias_index(adr_type); 308 mem_in = mem_in->as_MergeMem()->memory_at(alias_idx); 309 set_req(Memory, mem_in); 310 return this; 311 } 312 313 Node* input = in(Memory); 314 if (input->Opcode() == Op_ShenandoahWBMemProj) { 315 ResourceMark rm; 316 VectorSet seen(Thread::current()->resource_area()); 317 Node* n = in(Memory); 318 while (n->Opcode() == Op_ShenandoahWBMemProj && 319 n->in(ShenandoahWBMemProjNode::WriteBarrier) != NULL && 320 n->in(ShenandoahWBMemProjNode::WriteBarrier)->Opcode() == Op_ShenandoahWriteBarrier && 321 n->in(ShenandoahWBMemProjNode::WriteBarrier)->in(Memory) != NULL) { 322 if (seen.test_set(n->_idx)) { 323 return NULL; // loop 324 } 325 n = n->in(ShenandoahWBMemProjNode::WriteBarrier)->in(Memory); 326 } 327 328 Node* wb = input->in(ShenandoahWBMemProjNode::WriteBarrier); 329 const Type* in_type = phase->type(wb); 330 // is_top() test not sufficient here: we can come here after CCP 331 // in a dead branch of the graph that has not yet been removed. 332 if (in_type == Type::TOP) return NULL; // Dead path. 333 assert(wb->Opcode() == Op_ShenandoahWriteBarrier, "expect write barrier"); 334 if (is_independent(in_type, _type)) { 335 phase->igvn_rehash_node_delayed(wb); 336 set_req(Memory, wb->in(Memory)); 337 if (can_reshape && input->outcnt() == 0) { 338 phase->is_IterGVN()->_worklist.push(input); 339 } 340 return this; 341 } 342 } 343 return NULL; 344 } 345 346 ShenandoahWriteBarrierNode::ShenandoahWriteBarrierNode(Compile* C, Node* ctrl, Node* mem, Node* obj) 347 : ShenandoahBarrierNode(ctrl, mem, obj, false) { 348 assert(UseShenandoahGC && ShenandoahWriteBarrier, "should be enabled"); 349 ShenandoahBarrierSetC2::bsc2()->state()->add_shenandoah_barrier(this); 350 } 351 352 Node* ShenandoahWriteBarrierNode::Identity(PhaseGVN* phase) { 353 assert(in(0) != NULL, "should have control"); 354 PhaseIterGVN* igvn = phase->is_IterGVN(); 355 Node* mem_in = in(Memory); 356 Node* mem_proj = NULL; 357 358 if (igvn != NULL) { 359 mem_proj = find_out_with(Op_ShenandoahWBMemProj); 360 if (mem_in == mem_proj) { 361 return this; 362 } 363 } 364 365 Node* replacement = Identity_impl(phase); 366 if (igvn != NULL) { 367 if (replacement != NULL && replacement != this && mem_proj != NULL) { 368 igvn->replace_node(mem_proj, mem_in); 369 } 370 } 371 return replacement; 372 } 373 374 Node* ShenandoahWriteBarrierNode::Ideal(PhaseGVN *phase, bool can_reshape) { 375 assert(in(0) != NULL, "should have control"); 376 if (!can_reshape) { 377 return NULL; 378 } 379 380 Node* mem_in = in(Memory); 381 382 if (mem_in->isa_MergeMem()) { 383 const TypePtr* adr_type = brooks_pointer_type(bottom_type()); 384 uint alias_idx = phase->C->get_alias_index(adr_type); 385 mem_in = mem_in->as_MergeMem()->memory_at(alias_idx); 386 set_req(Memory, mem_in); 387 return this; 388 } 389 390 Node* val = in(ValueIn); 391 if (val->is_ShenandoahBarrier()) { 392 set_req(ValueIn, val->in(ValueIn)); 393 return this; 394 } 395 396 return NULL; 397 } 398 399 bool ShenandoahWriteBarrierNode::expand(Compile* C, PhaseIterGVN& igvn) { 400 if (UseShenandoahGC) { 401 if (ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count() > 0 || (!ShenandoahWriteBarrier && ShenandoahStoreValEnqueueBarrier)) { 402 bool attempt_more_loopopts = ShenandoahLoopOptsAfterExpansion; 403 C->clear_major_progress(); 404 PhaseIdealLoop ideal_loop(igvn, LoopOptsShenandoahExpand); 405 if (C->failing()) return false; 406 PhaseIdealLoop::verify(igvn); 407 DEBUG_ONLY(ShenandoahBarrierNode::verify_raw_mem(C->root());) 408 if (attempt_more_loopopts) { 409 C->set_major_progress(); 410 if (!C->optimize_loops(igvn, LoopOptsShenandoahPostExpand)) { 411 return false; 412 } 413 C->clear_major_progress(); 414 } 415 } 416 } 417 return true; 418 } 419 420 bool ShenandoahWriteBarrierNode::is_heap_state_test(Node* iff, int mask) { 421 if (!UseShenandoahGC) { 422 return false; 423 } 424 assert(iff->is_If(), "bad input"); 425 if (iff->Opcode() != Op_If) { 426 return false; 427 } 428 Node* bol = iff->in(1); 429 if (!bol->is_Bool() || bol->as_Bool()->_test._test != BoolTest::ne) { 430 return false; 431 } 432 Node* cmp = bol->in(1); 433 if (cmp->Opcode() != Op_CmpI) { 434 return false; 435 } 436 Node* in1 = cmp->in(1); 437 Node* in2 = cmp->in(2); 438 if (in2->find_int_con(-1) != 0) { 439 return false; 440 } 441 if (in1->Opcode() != Op_AndI) { 442 return false; 443 } 444 in2 = in1->in(2); 445 if (in2->find_int_con(-1) != mask) { 446 return false; 447 } 448 in1 = in1->in(1); 449 450 return is_gc_state_load(in1); 451 } 452 453 bool ShenandoahWriteBarrierNode::is_heap_stable_test(Node* iff) { 454 return is_heap_state_test(iff, ShenandoahHeap::HAS_FORWARDED); 455 } 456 457 bool ShenandoahWriteBarrierNode::is_gc_state_load(Node *n) { 458 if (!UseShenandoahGC) { 459 return false; 460 } 461 if (n->Opcode() != Op_LoadB && n->Opcode() != Op_LoadUB) { 462 return false; 463 } 464 Node* addp = n->in(MemNode::Address); 465 if (!addp->is_AddP()) { 466 return false; 467 } 468 Node* base = addp->in(AddPNode::Address); 469 Node* off = addp->in(AddPNode::Offset); 470 if (base->Opcode() != Op_ThreadLocal) { 471 return false; 472 } 473 if (off->find_intptr_t_con(-1) != in_bytes(ShenandoahThreadLocalData::gc_state_offset())) { 474 return false; 475 } 476 return true; 477 } 478 479 bool ShenandoahWriteBarrierNode::has_safepoint_between(Node* start, Node* stop, PhaseIdealLoop *phase) { 480 assert(phase->is_dominator(stop, start), "bad inputs"); 481 ResourceMark rm; 482 Unique_Node_List wq; 483 wq.push(start); 484 for (uint next = 0; next < wq.size(); next++) { 485 Node *m = wq.at(next); 486 if (m == stop) { 487 continue; 488 } 489 if (m->is_SafePoint() && !m->is_CallLeaf()) { 490 return true; 491 } 492 if (m->is_Region()) { 493 for (uint i = 1; i < m->req(); i++) { 494 wq.push(m->in(i)); 495 } 496 } else { 497 wq.push(m->in(0)); 498 } 499 } 500 return false; 501 } 502 503 bool ShenandoahWriteBarrierNode::try_common_gc_state_load(Node *n, PhaseIdealLoop *phase) { 504 assert(is_gc_state_load(n), "inconsistent"); 505 Node* addp = n->in(MemNode::Address); 506 Node* dominator = NULL; 507 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) { 508 Node* u = addp->fast_out(i); 509 assert(is_gc_state_load(u), "inconsistent"); 510 if (u != n && phase->is_dominator(u->in(0), n->in(0))) { 511 if (dominator == NULL) { 512 dominator = u; 513 } else { 514 if (phase->dom_depth(u->in(0)) < phase->dom_depth(dominator->in(0))) { 515 dominator = u; 516 } 517 } 518 } 519 } 520 if (dominator == NULL || has_safepoint_between(n->in(0), dominator->in(0), phase)) { 521 return false; 522 } 523 phase->igvn().replace_node(n, dominator); 524 525 return true; 526 } 527 528 bool ShenandoahBarrierNode::dominates_memory_impl(PhaseGVN* phase, 529 Node* b1, 530 Node* b2, 531 Node* current, 532 bool linear) { 533 ResourceMark rm; 534 VectorSet visited(Thread::current()->resource_area()); 535 Node_Stack phis(0); 536 537 for(int i = 0; i < 10; i++) { 538 if (current == NULL) { 539 return false; 540 } else if (visited.test_set(current->_idx) || current->is_top() || current == b1) { 541 current = NULL; 542 while (phis.is_nonempty() && current == NULL) { 543 uint idx = phis.index(); 544 Node* phi = phis.node(); 545 if (idx >= phi->req()) { 546 phis.pop(); 547 } else { 548 current = phi->in(idx); 549 phis.set_index(idx+1); 550 } 551 } 552 if (current == NULL) { 553 return true; 554 } 555 } else if (current == b2) { 556 return false; 557 } else if (current == phase->C->immutable_memory()) { 558 return false; 559 } else if (current->isa_Phi()) { 560 if (!linear) { 561 return false; 562 } 563 phis.push(current, 2); 564 current = current->in(1); 565 } else if (current->Opcode() == Op_ShenandoahWriteBarrier) { 566 current = current->in(Memory); 567 } else if (current->Opcode() == Op_ShenandoahWBMemProj) { 568 current = current->in(ShenandoahWBMemProjNode::WriteBarrier); 569 } else if (current->is_Proj()) { 570 current = current->in(0); 571 } else if (current->is_Call()) { 572 current = current->in(TypeFunc::Memory); 573 } else if (current->is_MemBar()) { 574 current = current->in(TypeFunc::Memory); 575 } else if (current->is_MergeMem()) { 576 const TypePtr* adr_type = brooks_pointer_type(phase->type(b2)); 577 uint alias_idx = phase->C->get_alias_index(adr_type); 578 current = current->as_MergeMem()->memory_at(alias_idx); 579 } else { 580 #ifdef ASSERT 581 current->dump(); 582 #endif 583 ShouldNotReachHere(); 584 return false; 585 } 586 } 587 return false; 588 } 589 590 /** 591 * Determines if b1 dominates b2 through memory inputs. It returns true if: 592 * - b1 can be reached by following each branch in b2's memory input (through phis, etc) 593 * - or we get back to b2 (i.e. through a loop) without seeing b1 594 * In all other cases, (in particular, if we reach immutable_memory without having seen b1) 595 * we return false. 596 */ 597 bool ShenandoahBarrierNode::dominates_memory(PhaseGVN* phase, Node* b1, Node* b2, bool linear) { 598 return dominates_memory_impl(phase, b1, b2, b2->in(Memory), linear); 599 } 600 601 Node* ShenandoahBarrierNode::Identity_impl(PhaseGVN* phase) { 602 Node* n = in(ValueIn); 603 604 Node* rb_mem = Opcode() == Op_ShenandoahReadBarrier ? in(Memory) : NULL; 605 if (! needs_barrier(phase, this, n, rb_mem, _allow_fromspace)) { 606 return n; 607 } 608 609 // Try to find a write barrier sibling with identical inputs that we can fold into. 610 for (DUIterator i = n->outs(); n->has_out(i); i++) { 611 Node* sibling = n->out(i); 612 if (sibling == this) { 613 continue; 614 } 615 if (sibling->Opcode() != Op_ShenandoahWriteBarrier) { 616 continue; 617 } 618 619 assert(sibling->in(ValueIn) == in(ValueIn), "sanity"); 620 assert(sibling->Opcode() == Op_ShenandoahWriteBarrier, "sanity"); 621 622 if (dominates_memory(phase, sibling, this, phase->is_IterGVN() == NULL)) { 623 return sibling; 624 } 625 } 626 return this; 627 } 628 629 #ifndef PRODUCT 630 void ShenandoahBarrierNode::dump_spec(outputStream *st) const { 631 const TypePtr* adr = adr_type(); 632 if (adr == NULL) { 633 return; 634 } 635 st->print(" @"); 636 adr->dump_on(st); 637 st->print(" ("); 638 Compile::current()->alias_type(adr)->adr_type()->dump_on(st); 639 st->print(") "); 640 } 641 #endif 642 643 Node* ShenandoahReadBarrierNode::Identity(PhaseGVN* phase) { 644 Node* id = Identity_impl(phase); 645 646 if (id == this && phase->is_IterGVN()) { 647 Node* n = in(ValueIn); 648 // No success in super call. Try to combine identical read barriers. 649 for (DUIterator i = n->outs(); n->has_out(i); i++) { 650 Node* sibling = n->out(i); 651 if (sibling == this || sibling->Opcode() != Op_ShenandoahReadBarrier) { 652 continue; 653 } 654 assert(sibling->in(ValueIn) == in(ValueIn), "sanity"); 655 if (phase->is_IterGVN()->hash_find(sibling) && 656 sibling->bottom_type() == bottom_type() && 657 sibling->in(Control) == in(Control) && 658 dominates_memory_rb(phase, sibling, this, phase->is_IterGVN() == NULL)) { 659 return sibling; 660 } 661 } 662 } 663 return id; 664 } 665 666 const Type* ShenandoahBarrierNode::Value(PhaseGVN* phase) const { 667 // Either input is TOP ==> the result is TOP 668 const Type *t1 = phase->type(in(Memory)); 669 if (t1 == Type::TOP) return Type::TOP; 670 const Type *t2 = phase->type(in(ValueIn)); 671 if( t2 == Type::TOP ) return Type::TOP; 672 673 if (t2 == TypePtr::NULL_PTR) { 674 return _type; 675 } 676 677 const Type* type = t2->is_oopptr()->cast_to_nonconst(); 678 return type; 679 } 680 681 uint ShenandoahBarrierNode::hash() const { 682 return TypeNode::hash() + _allow_fromspace; 683 } 684 685 uint ShenandoahBarrierNode::cmp(const Node& n) const { 686 return _allow_fromspace == ((ShenandoahBarrierNode&) n)._allow_fromspace 687 && TypeNode::cmp(n); 688 } 689 690 uint ShenandoahBarrierNode::size_of() const { 691 return sizeof(*this); 692 } 693 694 Node* ShenandoahWBMemProjNode::Identity(PhaseGVN* phase) { 695 Node* wb = in(WriteBarrier); 696 if (wb->is_top()) return phase->C->top(); // Dead path. 697 698 assert(wb->Opcode() == Op_ShenandoahWriteBarrier, "expect write barrier"); 699 PhaseIterGVN* igvn = phase->is_IterGVN(); 700 // We can't do the below unless the graph is fully constructed. 701 if (igvn == NULL) { 702 return this; 703 } 704 705 // If the mem projection has no barrier users, it's not needed anymore. 706 if (wb->outcnt() == 1) { 707 return wb->in(ShenandoahBarrierNode::Memory); 708 } 709 710 return this; 711 } 712 713 #ifdef ASSERT 714 bool ShenandoahBarrierNode::verify_helper(Node* in, Node_Stack& phis, VectorSet& visited, verify_type t, bool trace, Unique_Node_List& barriers_used) { 715 assert(phis.size() == 0, ""); 716 717 while (true) { 718 if (in->bottom_type() == TypePtr::NULL_PTR) { 719 if (trace) {tty->print_cr("NULL");} 720 } else if (!in->bottom_type()->make_ptr()->make_oopptr()) { 721 if (trace) {tty->print_cr("Non oop");} 722 } else if (t == ShenandoahLoad && ShenandoahOptimizeStableFinals && 723 in->bottom_type()->make_ptr()->isa_aryptr() && 724 in->bottom_type()->make_ptr()->is_aryptr()->is_stable()) { 725 if (trace) {tty->print_cr("Stable array load");} 726 } else { 727 if (in->is_ConstraintCast()) { 728 in = in->in(1); 729 continue; 730 } else if (in->is_AddP()) { 731 assert(!in->in(AddPNode::Address)->is_top(), "no raw memory access"); 732 in = in->in(AddPNode::Address); 733 continue; 734 } else if (in->is_Con()) { 735 if (trace) {tty->print("Found constant"); in->dump();} 736 } else if (in->is_ShenandoahBarrier()) { 737 if (t == ShenandoahOopStore) { 738 if (in->Opcode() != Op_ShenandoahWriteBarrier) { 739 return false; 740 } 741 uint i = 0; 742 for (; i < phis.size(); i++) { 743 Node* n = phis.node_at(i); 744 if (n->Opcode() == Op_ShenandoahEnqueueBarrier) { 745 break; 746 } 747 } 748 if (i == phis.size()) { 749 return false; 750 } 751 } else if (t == ShenandoahStore && in->Opcode() != Op_ShenandoahWriteBarrier) { 752 return false; 753 } 754 barriers_used.push(in); 755 if (trace) {tty->print("Found barrier"); in->dump();} 756 } else if (in->Opcode() == Op_ShenandoahEnqueueBarrier) { 757 if (t != ShenandoahOopStore) { 758 in = in->in(1); 759 continue; 760 } 761 if (trace) {tty->print("Found enqueue barrier"); in->dump();} 762 phis.push(in, in->req()); 763 in = in->in(1); 764 continue; 765 } else if (in->is_Proj() && in->in(0)->is_Allocate()) { 766 if (trace) {tty->print("Found alloc"); in->in(0)->dump();} 767 } else if (in->is_Phi()) { 768 if (!visited.test_set(in->_idx)) { 769 if (trace) {tty->print("Pushed phi:"); in->dump();} 770 phis.push(in, 2); 771 in = in->in(1); 772 continue; 773 } 774 if (trace) {tty->print("Already seen phi:"); in->dump();} 775 } else if (in->Opcode() == Op_CMoveP || in->Opcode() == Op_CMoveN) { 776 if (!visited.test_set(in->_idx)) { 777 if (trace) {tty->print("Pushed cmovep:"); in->dump();} 778 phis.push(in, CMoveNode::IfTrue); 779 in = in->in(CMoveNode::IfFalse); 780 continue; 781 } 782 if (trace) {tty->print("Already seen cmovep:"); in->dump();} 783 } else if (in->Opcode() == Op_EncodeP || in->Opcode() == Op_DecodeN) { 784 in = in->in(1); 785 continue; 786 } else { 787 return false; 788 } 789 } 790 bool cont = false; 791 while (phis.is_nonempty()) { 792 uint idx = phis.index(); 793 Node* phi = phis.node(); 794 if (idx >= phi->req()) { 795 if (trace) {tty->print("Popped phi:"); phi->dump();} 796 phis.pop(); 797 continue; 798 } 799 if (trace) {tty->print("Next entry(%d) for phi:", idx); phi->dump();} 800 in = phi->in(idx); 801 phis.set_index(idx+1); 802 cont = true; 803 break; 804 } 805 if (!cont) { 806 break; 807 } 808 } 809 return true; 810 } 811 812 void ShenandoahBarrierNode::report_verify_failure(const char *msg, Node *n1, Node *n2) { 813 if (n1 != NULL) { 814 n1->dump(+10); 815 } 816 if (n2 != NULL) { 817 n2->dump(+10); 818 } 819 fatal("%s", msg); 820 } 821 822 void ShenandoahBarrierNode::verify(RootNode* root) { 823 ResourceMark rm; 824 Unique_Node_List wq; 825 GrowableArray<Node*> barriers; 826 Unique_Node_List barriers_used; 827 Node_Stack phis(0); 828 VectorSet visited(Thread::current()->resource_area()); 829 const bool trace = false; 830 const bool verify_no_useless_barrier = false; 831 832 wq.push(root); 833 for (uint next = 0; next < wq.size(); next++) { 834 Node *n = wq.at(next); 835 if (n->is_Load()) { 836 const bool trace = false; 837 if (trace) {tty->print("Verifying"); n->dump();} 838 if (n->Opcode() == Op_LoadRange || n->Opcode() == Op_LoadKlass || n->Opcode() == Op_LoadNKlass) { 839 if (trace) {tty->print_cr("Load range/klass");} 840 } else { 841 const TypePtr* adr_type = n->as_Load()->adr_type(); 842 843 if (adr_type->isa_oopptr() && adr_type->is_oopptr()->offset() == oopDesc::mark_offset_in_bytes()) { 844 if (trace) {tty->print_cr("Mark load");} 845 } else if (adr_type->isa_instptr() && 846 adr_type->is_instptr()->klass()->is_subtype_of(Compile::current()->env()->Reference_klass()) && 847 adr_type->is_instptr()->offset() == java_lang_ref_Reference::referent_offset) { 848 if (trace) {tty->print_cr("Reference.get()");} 849 } else { 850 bool verify = true; 851 if (adr_type->isa_instptr()) { 852 const TypeInstPtr* tinst = adr_type->is_instptr(); 853 ciKlass* k = tinst->klass(); 854 assert(k->is_instance_klass(), ""); 855 ciInstanceKlass* ik = (ciInstanceKlass*)k; 856 int offset = adr_type->offset(); 857 858 if ((ik->debug_final_field_at(offset) && ShenandoahOptimizeInstanceFinals) || 859 (ik->debug_stable_field_at(offset) && ShenandoahOptimizeStableFinals)) { 860 if (trace) {tty->print_cr("Final/stable");} 861 verify = false; 862 } else if (k == ciEnv::current()->Class_klass() && 863 tinst->const_oop() != NULL && 864 tinst->offset() >= (ik->size_helper() * wordSize)) { 865 ciInstanceKlass* k = tinst->const_oop()->as_instance()->java_lang_Class_klass()->as_instance_klass(); 866 ciField* field = k->get_field_by_offset(tinst->offset(), true); 867 if ((ShenandoahOptimizeStaticFinals && field->is_final()) || 868 (ShenandoahOptimizeStableFinals && field->is_stable())) { 869 verify = false; 870 } 871 } 872 } 873 874 if (verify && !ShenandoahBarrierNode::verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahLoad, trace, barriers_used)) { 875 report_verify_failure("Shenandoah verification: Load should have barriers", n); 876 } 877 } 878 } 879 } else if (n->is_Store()) { 880 const bool trace = false; 881 882 if (trace) {tty->print("Verifying"); n->dump();} 883 if (n->in(MemNode::ValueIn)->bottom_type()->make_oopptr()) { 884 Node* adr = n->in(MemNode::Address); 885 bool verify = true; 886 887 if (adr->is_AddP() && adr->in(AddPNode::Base)->is_top()) { 888 adr = adr->in(AddPNode::Address); 889 if (adr->is_AddP()) { 890 assert(adr->in(AddPNode::Base)->is_top(), ""); 891 adr = adr->in(AddPNode::Address); 892 if (adr->Opcode() == Op_LoadP && 893 adr->in(MemNode::Address)->in(AddPNode::Base)->is_top() && 894 adr->in(MemNode::Address)->in(AddPNode::Address)->Opcode() == Op_ThreadLocal && 895 adr->in(MemNode::Address)->in(AddPNode::Offset)->find_intptr_t_con(-1) == in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset())) { 896 if (trace) {tty->print_cr("SATB prebarrier");} 897 verify = false; 898 } 899 } 900 } 901 902 if (verify && !ShenandoahBarrierNode::verify_helper(n->in(MemNode::ValueIn), phis, visited, ShenandoahStoreValEnqueueBarrier ? ShenandoahOopStore : ShenandoahValue, trace, barriers_used)) { 903 report_verify_failure("Shenandoah verification: Store should have barriers", n); 904 } 905 } 906 if (!ShenandoahBarrierNode::verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahStore, trace, barriers_used)) { 907 report_verify_failure("Shenandoah verification: Store (address) should have barriers", n); 908 } 909 } else if (n->Opcode() == Op_CmpP) { 910 const bool trace = false; 911 912 Node* in1 = n->in(1); 913 Node* in2 = n->in(2); 914 if (in1->bottom_type()->isa_oopptr()) { 915 if (trace) {tty->print("Verifying"); n->dump();} 916 917 bool mark_inputs = false; 918 if (in1->bottom_type() == TypePtr::NULL_PTR || in2->bottom_type() == TypePtr::NULL_PTR || 919 (in1->is_Con() || in2->is_Con())) { 920 if (trace) {tty->print_cr("Comparison against a constant");} 921 mark_inputs = true; 922 } else if ((in1->is_CheckCastPP() && in1->in(1)->is_Proj() && in1->in(1)->in(0)->is_Allocate()) || 923 (in2->is_CheckCastPP() && in2->in(1)->is_Proj() && in2->in(1)->in(0)->is_Allocate())) { 924 if (trace) {tty->print_cr("Comparison with newly alloc'ed object");} 925 mark_inputs = true; 926 } else { 927 assert(in2->bottom_type()->isa_oopptr(), ""); 928 929 if (!ShenandoahBarrierNode::verify_helper(in1, phis, visited, ShenandoahStore, trace, barriers_used) || 930 !ShenandoahBarrierNode::verify_helper(in2, phis, visited, ShenandoahStore, trace, barriers_used)) { 931 report_verify_failure("Shenandoah verification: Cmp should have barriers", n); 932 } 933 } 934 if (verify_no_useless_barrier && 935 mark_inputs && 936 (!ShenandoahBarrierNode::verify_helper(in1, phis, visited, ShenandoahValue, trace, barriers_used) || 937 !ShenandoahBarrierNode::verify_helper(in2, phis, visited, ShenandoahValue, trace, barriers_used))) { 938 phis.clear(); 939 visited.Reset(); 940 } 941 } 942 } else if (n->is_LoadStore()) { 943 if (n->in(MemNode::ValueIn)->bottom_type()->make_ptr() && 944 !ShenandoahBarrierNode::verify_helper(n->in(MemNode::ValueIn), phis, visited, ShenandoahStoreValEnqueueBarrier ? ShenandoahOopStore : ShenandoahValue, trace, barriers_used)) { 945 report_verify_failure("Shenandoah verification: LoadStore (value) should have barriers", n); 946 } 947 948 if (n->in(MemNode::Address)->bottom_type()->make_oopptr() && !ShenandoahBarrierNode::verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahStore, trace, barriers_used)) { 949 report_verify_failure("Shenandoah verification: LoadStore (address) should have barriers", n); 950 } 951 } else if (n->Opcode() == Op_CallLeafNoFP || n->Opcode() == Op_CallLeaf) { 952 CallNode* call = n->as_Call(); 953 954 static struct { 955 const char* name; 956 struct { 957 int pos; 958 verify_type t; 959 } args[6]; 960 } calls[] = { 961 "aescrypt_encryptBlock", 962 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad }, 963 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 964 "aescrypt_decryptBlock", 965 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad }, 966 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 967 "multiplyToLen", 968 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad }, { TypeFunc::Parms+4, ShenandoahStore }, 969 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 970 "squareToLen", 971 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad }, { -1, ShenandoahNone}, 972 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 973 "montgomery_multiply", 974 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad }, 975 { TypeFunc::Parms+6, ShenandoahStore }, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 976 "montgomery_square", 977 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+5, ShenandoahStore }, 978 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 979 "mulAdd", 980 { { TypeFunc::Parms, ShenandoahStore }, { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone}, 981 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 982 "vectorizedMismatch", 983 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone}, 984 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 985 "updateBytesCRC32", 986 { { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone}, 987 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 988 "updateBytesAdler32", 989 { { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone}, 990 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 991 "updateBytesCRC32C", 992 { { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahLoad}, { -1, ShenandoahNone}, 993 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 994 "counterMode_AESCrypt", 995 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad }, 996 { TypeFunc::Parms+3, ShenandoahStore }, { TypeFunc::Parms+5, ShenandoahStore }, { TypeFunc::Parms+6, ShenandoahStore } }, 997 "cipherBlockChaining_encryptAESCrypt", 998 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad }, 999 { TypeFunc::Parms+3, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1000 "cipherBlockChaining_decryptAESCrypt", 1001 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad }, 1002 { TypeFunc::Parms+3, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1003 "shenandoah_clone_barrier", 1004 { { TypeFunc::Parms, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone}, 1005 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1006 "ghash_processBlocks", 1007 { { TypeFunc::Parms, ShenandoahStore }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad }, 1008 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1009 "sha1_implCompress", 1010 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone }, 1011 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1012 "sha256_implCompress", 1013 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone }, 1014 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1015 "sha512_implCompress", 1016 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone }, 1017 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1018 "sha1_implCompressMB", 1019 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone }, 1020 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1021 "sha256_implCompressMB", 1022 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone }, 1023 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1024 "sha512_implCompressMB", 1025 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone }, 1026 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1027 "encodeBlock", 1028 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahStore }, { -1, ShenandoahNone }, 1029 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1030 }; 1031 1032 if (call->is_call_to_arraycopystub()) { 1033 Node* dest = NULL; 1034 const TypeTuple* args = n->as_Call()->_tf->domain(); 1035 for (uint i = TypeFunc::Parms, j = 0; i < args->cnt(); i++) { 1036 if (args->field_at(i)->isa_ptr()) { 1037 j++; 1038 if (j == 2) { 1039 dest = n->in(i); 1040 break; 1041 } 1042 } 1043 } 1044 if (!ShenandoahBarrierNode::verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahLoad, trace, barriers_used) || 1045 !ShenandoahBarrierNode::verify_helper(dest, phis, visited, ShenandoahStore, trace, barriers_used)) { 1046 report_verify_failure("Shenandoah verification: ArrayCopy should have barriers", n); 1047 } 1048 } else if (strlen(call->_name) > 5 && 1049 !strcmp(call->_name + strlen(call->_name) - 5, "_fill")) { 1050 if (!ShenandoahBarrierNode::verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahStore, trace, barriers_used)) { 1051 report_verify_failure("Shenandoah verification: _fill should have barriers", n); 1052 } 1053 } else if (!strcmp(call->_name, "shenandoah_wb_pre")) { 1054 // skip 1055 } else { 1056 const int calls_len = sizeof(calls) / sizeof(calls[0]); 1057 int i = 0; 1058 for (; i < calls_len; i++) { 1059 if (!strcmp(calls[i].name, call->_name)) { 1060 break; 1061 } 1062 } 1063 if (i != calls_len) { 1064 const uint args_len = sizeof(calls[0].args) / sizeof(calls[0].args[0]); 1065 for (uint j = 0; j < args_len; j++) { 1066 int pos = calls[i].args[j].pos; 1067 if (pos == -1) { 1068 break; 1069 } 1070 if (!ShenandoahBarrierNode::verify_helper(call->in(pos), phis, visited, calls[i].args[j].t, trace, barriers_used)) { 1071 report_verify_failure("Shenandoah verification: intrinsic calls should have barriers", n); 1072 } 1073 } 1074 for (uint j = TypeFunc::Parms; j < call->req(); j++) { 1075 if (call->in(j)->bottom_type()->make_ptr() && 1076 call->in(j)->bottom_type()->make_ptr()->isa_oopptr()) { 1077 uint k = 0; 1078 for (; k < args_len && calls[i].args[k].pos != (int)j; k++); 1079 if (k == args_len) { 1080 fatal("arg %d for call %s not covered", j, call->_name); 1081 } 1082 } 1083 } 1084 } else { 1085 for (uint j = TypeFunc::Parms; j < call->req(); j++) { 1086 if (call->in(j)->bottom_type()->make_ptr() && 1087 call->in(j)->bottom_type()->make_ptr()->isa_oopptr()) { 1088 fatal("%s not covered", call->_name); 1089 } 1090 } 1091 } 1092 } 1093 } else if (n->is_ShenandoahBarrier()) { 1094 assert(!barriers.contains(n), ""); 1095 assert(n->Opcode() != Op_ShenandoahWriteBarrier || n->find_out_with(Op_ShenandoahWBMemProj) != NULL, "bad shenandoah write barrier"); 1096 assert(n->Opcode() != Op_ShenandoahWriteBarrier || n->outcnt() > 1, "bad shenandoah write barrier"); 1097 barriers.push(n); 1098 } else if (n->Opcode() == Op_ShenandoahEnqueueBarrier) { 1099 // skip 1100 } else if (n->Opcode() == Op_ShenandoahWBMemProj) { 1101 assert(n->in(0) == NULL && n->in(ShenandoahWBMemProjNode::WriteBarrier)->Opcode() == Op_ShenandoahWriteBarrier, "strange ShenandoahWBMemProj"); 1102 } else if (n->is_AddP() 1103 || n->is_Phi() 1104 || n->is_ConstraintCast() 1105 || n->Opcode() == Op_Return 1106 || n->Opcode() == Op_CMoveP 1107 || n->Opcode() == Op_CMoveN 1108 || n->Opcode() == Op_Rethrow 1109 || n->is_MemBar() 1110 || n->Opcode() == Op_Conv2B 1111 || n->Opcode() == Op_SafePoint 1112 || n->is_CallJava() 1113 || n->Opcode() == Op_Unlock 1114 || n->Opcode() == Op_EncodeP 1115 || n->Opcode() == Op_DecodeN) { 1116 // nothing to do 1117 } else { 1118 static struct { 1119 int opcode; 1120 struct { 1121 int pos; 1122 verify_type t; 1123 } inputs[2]; 1124 } others[] = { 1125 Op_FastLock, 1126 { { 1, ShenandoahLoad }, { -1, ShenandoahNone} }, 1127 Op_Lock, 1128 { { TypeFunc::Parms, ShenandoahLoad }, { -1, ShenandoahNone} }, 1129 Op_ArrayCopy, 1130 { { ArrayCopyNode::Src, ShenandoahLoad }, { ArrayCopyNode::Dest, ShenandoahStore } }, 1131 Op_StrCompressedCopy, 1132 { { 2, ShenandoahLoad }, { 3, ShenandoahStore } }, 1133 Op_StrInflatedCopy, 1134 { { 2, ShenandoahLoad }, { 3, ShenandoahStore } }, 1135 Op_AryEq, 1136 { { 2, ShenandoahLoad }, { 3, ShenandoahLoad } }, 1137 Op_StrIndexOf, 1138 { { 2, ShenandoahLoad }, { 4, ShenandoahLoad } }, 1139 Op_StrComp, 1140 { { 2, ShenandoahLoad }, { 4, ShenandoahLoad } }, 1141 Op_StrEquals, 1142 { { 2, ShenandoahLoad }, { 3, ShenandoahLoad } }, 1143 Op_EncodeISOArray, 1144 { { 2, ShenandoahLoad }, { 3, ShenandoahStore } }, 1145 Op_HasNegatives, 1146 { { 2, ShenandoahLoad }, { -1, ShenandoahNone} }, 1147 Op_CastP2X, 1148 { { 1, ShenandoahLoad }, { -1, ShenandoahNone} }, 1149 Op_StrIndexOfChar, 1150 { { 2, ShenandoahLoad }, { -1, ShenandoahNone } }, 1151 }; 1152 1153 const int others_len = sizeof(others) / sizeof(others[0]); 1154 int i = 0; 1155 for (; i < others_len; i++) { 1156 if (others[i].opcode == n->Opcode()) { 1157 break; 1158 } 1159 } 1160 uint stop = n->is_Call() ? n->as_Call()->tf()->domain()->cnt() : n->req(); 1161 if (i != others_len) { 1162 const uint inputs_len = sizeof(others[0].inputs) / sizeof(others[0].inputs[0]); 1163 for (uint j = 0; j < inputs_len; j++) { 1164 int pos = others[i].inputs[j].pos; 1165 if (pos == -1) { 1166 break; 1167 } 1168 if (!ShenandoahBarrierNode::verify_helper(n->in(pos), phis, visited, others[i].inputs[j].t, trace, barriers_used)) { 1169 report_verify_failure("Shenandoah verification: intrinsic calls should have barriers", n); 1170 } 1171 } 1172 for (uint j = 1; j < stop; j++) { 1173 if (n->in(j) != NULL && n->in(j)->bottom_type()->make_ptr() && 1174 n->in(j)->bottom_type()->make_ptr()->make_oopptr()) { 1175 uint k = 0; 1176 for (; k < inputs_len && others[i].inputs[k].pos != (int)j; k++); 1177 if (k == inputs_len) { 1178 fatal("arg %d for node %s not covered", j, n->Name()); 1179 } 1180 } 1181 } 1182 } else { 1183 for (uint j = 1; j < stop; j++) { 1184 if (n->in(j) != NULL && n->in(j)->bottom_type()->make_ptr() && 1185 n->in(j)->bottom_type()->make_ptr()->make_oopptr()) { 1186 fatal("%s not covered", n->Name()); 1187 } 1188 } 1189 } 1190 } 1191 1192 if (n->is_SafePoint()) { 1193 SafePointNode* sfpt = n->as_SafePoint(); 1194 if (verify_no_useless_barrier && sfpt->jvms() != NULL) { 1195 for (uint i = sfpt->jvms()->scloff(); i < sfpt->jvms()->endoff(); i++) { 1196 if (!ShenandoahBarrierNode::verify_helper(sfpt->in(i), phis, visited, ShenandoahLoad, trace, barriers_used)) { 1197 phis.clear(); 1198 visited.Reset(); 1199 } 1200 } 1201 } 1202 } 1203 for( uint i = 0; i < n->len(); ++i ) { 1204 Node *m = n->in(i); 1205 if (m == NULL) continue; 1206 1207 // In most cases, inputs should be known to be non null. If it's 1208 // not the case, it could be a missing cast_not_null() in an 1209 // intrinsic or support might be needed in AddPNode::Ideal() to 1210 // avoid a NULL+offset input. 1211 if (!(n->is_Phi() || 1212 (n->is_SafePoint() && (!n->is_CallRuntime() || !strcmp(n->as_Call()->_name, "shenandoah_wb_pre") || !strcmp(n->as_Call()->_name, "unsafe_arraycopy"))) || 1213 n->Opcode() == Op_CmpP || 1214 n->Opcode() == Op_CmpN || 1215 (n->Opcode() == Op_StoreP && i == StoreNode::ValueIn) || 1216 (n->Opcode() == Op_StoreN && i == StoreNode::ValueIn) || 1217 n->is_ConstraintCast() || 1218 n->Opcode() == Op_Return || 1219 n->Opcode() == Op_Conv2B || 1220 n->is_AddP() || 1221 n->Opcode() == Op_CMoveP || 1222 n->Opcode() == Op_CMoveN || 1223 n->Opcode() == Op_Rethrow || 1224 n->is_MemBar() || 1225 n->is_Mem() || 1226 n->Opcode() == Op_AryEq || 1227 n->Opcode() == Op_SCMemProj || 1228 n->Opcode() == Op_EncodeP || 1229 n->Opcode() == Op_DecodeN || 1230 n->Opcode() == Op_ShenandoahWriteBarrier || 1231 n->Opcode() == Op_ShenandoahWBMemProj || 1232 n->Opcode() == Op_ShenandoahEnqueueBarrier)) { 1233 if (m->bottom_type()->make_oopptr() && m->bottom_type()->make_oopptr()->meet(TypePtr::NULL_PTR) == m->bottom_type()) { 1234 report_verify_failure("Shenandoah verification: null input", n, m); 1235 } 1236 } 1237 1238 wq.push(m); 1239 } 1240 } 1241 1242 if (verify_no_useless_barrier) { 1243 for (int i = 0; i < barriers.length(); i++) { 1244 Node* n = barriers.at(i); 1245 if (!barriers_used.member(n)) { 1246 tty->print("XXX useless barrier"); n->dump(-2); 1247 ShouldNotReachHere(); 1248 } 1249 } 1250 } 1251 } 1252 #endif 1253 1254 bool ShenandoahBarrierNode::is_dominator_same_ctrl(Node*c, Node* d, Node* n, PhaseIdealLoop* phase) { 1255 // That both nodes have the same control is not sufficient to prove 1256 // domination, verify that there's no path from d to n 1257 ResourceMark rm; 1258 Unique_Node_List wq; 1259 wq.push(d); 1260 for (uint next = 0; next < wq.size(); next++) { 1261 Node *m = wq.at(next); 1262 if (m == n) { 1263 return false; 1264 } 1265 if (m->is_Phi() && m->in(0)->is_Loop()) { 1266 assert(phase->ctrl_or_self(m->in(LoopNode::EntryControl)) != c, "following loop entry should lead to new control"); 1267 } else { 1268 for (uint i = 0; i < m->req(); i++) { 1269 if (m->in(i) != NULL && phase->ctrl_or_self(m->in(i)) == c) { 1270 wq.push(m->in(i)); 1271 } 1272 } 1273 } 1274 } 1275 return true; 1276 } 1277 1278 bool ShenandoahBarrierNode::is_dominator(Node *d_c, Node *n_c, Node* d, Node* n, PhaseIdealLoop* phase) { 1279 if (d_c != n_c) { 1280 return phase->is_dominator(d_c, n_c); 1281 } 1282 return is_dominator_same_ctrl(d_c, d, n, phase); 1283 } 1284 1285 Node* next_mem(Node* mem, int alias) { 1286 Node* res = NULL; 1287 if (mem->is_Proj()) { 1288 res = mem->in(0); 1289 } else if (mem->is_SafePoint() || mem->is_MemBar()) { 1290 res = mem->in(TypeFunc::Memory); 1291 } else if (mem->is_Phi()) { 1292 res = mem->in(1); 1293 } else if (mem->is_ShenandoahBarrier()) { 1294 res = mem->in(ShenandoahBarrierNode::Memory); 1295 } else if (mem->is_MergeMem()) { 1296 res = mem->as_MergeMem()->memory_at(alias); 1297 } else if (mem->is_Store() || mem->is_LoadStore() || mem->is_ClearArray()) { 1298 assert(alias = Compile::AliasIdxRaw, "following raw memory can't lead to a barrier"); 1299 res = mem->in(MemNode::Memory); 1300 } else if (mem->Opcode() == Op_ShenandoahWBMemProj) { 1301 res = mem->in(ShenandoahWBMemProjNode::WriteBarrier); 1302 } else { 1303 #ifdef ASSERT 1304 mem->dump(); 1305 #endif 1306 ShouldNotReachHere(); 1307 } 1308 return res; 1309 } 1310 1311 Node* ShenandoahBarrierNode::no_branches(Node* c, Node* dom, bool allow_one_proj, PhaseIdealLoop* phase) { 1312 Node* iffproj = NULL; 1313 while (c != dom) { 1314 Node* next = phase->idom(c); 1315 assert(next->unique_ctrl_out() == c || c->is_Proj() || c->is_Region(), "multiple control flow out but no proj or region?"); 1316 if (c->is_Region()) { 1317 ResourceMark rm; 1318 Unique_Node_List wq; 1319 wq.push(c); 1320 for (uint i = 0; i < wq.size(); i++) { 1321 Node *n = wq.at(i); 1322 if (n == next) { 1323 continue; 1324 } 1325 if (n->is_Region()) { 1326 for (uint j = 1; j < n->req(); j++) { 1327 wq.push(n->in(j)); 1328 } 1329 } else { 1330 wq.push(n->in(0)); 1331 } 1332 } 1333 for (uint i = 0; i < wq.size(); i++) { 1334 Node *n = wq.at(i); 1335 assert(n->is_CFG(), ""); 1336 if (n->is_Multi()) { 1337 for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) { 1338 Node* u = n->fast_out(j); 1339 if (u->is_CFG()) { 1340 if (!wq.member(u) && !u->as_Proj()->is_uncommon_trap_proj(Deoptimization::Reason_none)) { 1341 return NodeSentinel; 1342 } 1343 } 1344 } 1345 } 1346 } 1347 } else if (c->is_Proj()) { 1348 if (c->is_IfProj()) { 1349 if (c->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none) != NULL) { 1350 // continue; 1351 } else { 1352 if (!allow_one_proj) { 1353 return NodeSentinel; 1354 } 1355 if (iffproj == NULL) { 1356 iffproj = c; 1357 } else { 1358 return NodeSentinel; 1359 } 1360 } 1361 } else if (c->Opcode() == Op_JumpProj) { 1362 return NodeSentinel; // unsupported 1363 } else if (c->Opcode() == Op_CatchProj) { 1364 return NodeSentinel; // unsupported 1365 } else if (c->Opcode() == Op_CProj && next->Opcode() == Op_NeverBranch) { 1366 return NodeSentinel; // unsupported 1367 } else { 1368 assert(next->unique_ctrl_out() == c, "unsupported branch pattern"); 1369 } 1370 } 1371 c = next; 1372 } 1373 return iffproj; 1374 } 1375 1376 bool ShenandoahBarrierNode::build_loop_late_post(PhaseIdealLoop* phase, Node* n) { 1377 if (n->Opcode() == Op_ShenandoahReadBarrier || 1378 n->Opcode() == Op_ShenandoahWriteBarrier || 1379 n->Opcode() == Op_ShenandoahWBMemProj) { 1380 1381 phase->build_loop_late_post_work(n, false); 1382 1383 if (n->Opcode() == Op_ShenandoahWriteBarrier) { 1384 // The write barrier and its memory proj must have the same 1385 // control otherwise some loop opts could put nodes (Phis) between 1386 // them 1387 Node* proj = n->find_out_with(Op_ShenandoahWBMemProj); 1388 if (proj != NULL) { 1389 phase->set_ctrl_and_loop(proj, phase->get_ctrl(n)); 1390 } 1391 } 1392 return true; 1393 } 1394 return false; 1395 } 1396 1397 bool ShenandoahBarrierNode::sink_node(PhaseIdealLoop* phase, Node* ctrl, Node* n_ctrl) { 1398 ctrl = phase->find_non_split_ctrl(ctrl); 1399 assert(phase->dom_depth(n_ctrl) <= phase->dom_depth(ctrl), "n is later than its clone"); 1400 set_req(0, ctrl); 1401 phase->register_new_node(this, ctrl); 1402 return true; 1403 } 1404 1405 #ifdef ASSERT 1406 void ShenandoahWriteBarrierNode::memory_dominates_all_paths_helper(Node* c, Node* rep_ctrl, Unique_Node_List& controls, PhaseIdealLoop* phase) { 1407 const bool trace = false; 1408 if (trace) { tty->print("X control is"); c->dump(); } 1409 1410 uint start = controls.size(); 1411 controls.push(c); 1412 for (uint i = start; i < controls.size(); i++) { 1413 Node *n = controls.at(i); 1414 1415 if (trace) { tty->print("X from"); n->dump(); } 1416 1417 if (n == rep_ctrl) { 1418 continue; 1419 } 1420 1421 if (n->is_Proj()) { 1422 Node* n_dom = n->in(0); 1423 IdealLoopTree* n_dom_loop = phase->get_loop(n_dom); 1424 if (n->is_IfProj() && n_dom->outcnt() == 2) { 1425 n_dom_loop = phase->get_loop(n_dom->as_If()->proj_out(n->as_Proj()->_con == 0 ? 1 : 0)); 1426 } 1427 if (n_dom_loop != phase->ltree_root()) { 1428 Node* tail = n_dom_loop->tail(); 1429 if (tail->is_Region()) { 1430 for (uint j = 1; j < tail->req(); j++) { 1431 if (phase->is_dominator(n_dom, tail->in(j)) && !phase->is_dominator(n, tail->in(j))) { 1432 assert(phase->is_dominator(rep_ctrl, tail->in(j)), "why are we here?"); 1433 // entering loop from below, mark backedge 1434 if (trace) { tty->print("X pushing backedge"); tail->in(j)->dump(); } 1435 controls.push(tail->in(j)); 1436 //assert(n->in(0) == n_dom, "strange flow control"); 1437 } 1438 } 1439 } else if (phase->get_loop(n) != n_dom_loop && phase->is_dominator(n_dom, tail)) { 1440 // entering loop from below, mark backedge 1441 if (trace) { tty->print("X pushing backedge"); tail->dump(); } 1442 controls.push(tail); 1443 //assert(n->in(0) == n_dom, "strange flow control"); 1444 } 1445 } 1446 } 1447 1448 if (n->is_Loop()) { 1449 Node* c = n->in(LoopNode::EntryControl); 1450 if (trace) { tty->print("X pushing"); c->dump(); } 1451 controls.push(c); 1452 } else if (n->is_Region()) { 1453 for (uint i = 1; i < n->req(); i++) { 1454 Node* c = n->in(i); 1455 if (trace) { tty->print("X pushing"); c->dump(); } 1456 controls.push(c); 1457 } 1458 } else { 1459 Node* c = n->in(0); 1460 if (trace) { tty->print("X pushing"); c->dump(); } 1461 controls.push(c); 1462 } 1463 } 1464 } 1465 1466 bool ShenandoahWriteBarrierNode::memory_dominates_all_paths(Node* mem, Node* rep_ctrl, int alias, PhaseIdealLoop* phase) { 1467 const bool trace = false; 1468 if (trace) { 1469 tty->print("XXX mem is"); mem->dump(); 1470 tty->print("XXX rep ctrl is"); rep_ctrl->dump(); 1471 tty->print_cr("XXX alias is %d", alias); 1472 } 1473 ResourceMark rm; 1474 Unique_Node_List wq; 1475 Unique_Node_List controls; 1476 wq.push(mem); 1477 for (uint next = 0; next < wq.size(); next++) { 1478 Node *nn = wq.at(next); 1479 if (trace) { tty->print("XX from mem"); nn->dump(); } 1480 assert(nn->bottom_type() == Type::MEMORY, "memory only"); 1481 1482 if (nn->is_Phi()) { 1483 Node* r = nn->in(0); 1484 for (DUIterator_Fast jmax, j = r->fast_outs(jmax); j < jmax; j++) { 1485 Node* u = r->fast_out(j); 1486 if (u->is_Phi() && u->bottom_type() == Type::MEMORY && u != nn && 1487 (u->adr_type() == TypePtr::BOTTOM || phase->C->get_alias_index(u->adr_type()) == alias)) { 1488 if (trace) { tty->print("XX Next mem (other phi)"); u->dump(); } 1489 wq.push(u); 1490 } 1491 } 1492 } 1493 1494 for (DUIterator_Fast imax, i = nn->fast_outs(imax); i < imax; i++) { 1495 Node* use = nn->fast_out(i); 1496 1497 if (trace) { tty->print("XX use %p", use->adr_type()); use->dump(); } 1498 if (use->is_CFG() && use->in(TypeFunc::Memory) == nn) { 1499 Node* c = use->in(0); 1500 if (phase->is_dominator(rep_ctrl, c)) { 1501 memory_dominates_all_paths_helper(c, rep_ctrl, controls, phase); 1502 } else if (use->is_CallStaticJava() && use->as_CallStaticJava()->uncommon_trap_request() != 0 && c->is_Region()) { 1503 Node* region = c; 1504 if (trace) { tty->print("XX unc region"); region->dump(); } 1505 for (uint j = 1; j < region->req(); j++) { 1506 if (phase->is_dominator(rep_ctrl, region->in(j))) { 1507 if (trace) { tty->print("XX unc follows"); region->in(j)->dump(); } 1508 memory_dominates_all_paths_helper(region->in(j), rep_ctrl, controls, phase); 1509 } 1510 } 1511 } 1512 //continue; 1513 } else if (use->is_Phi()) { 1514 assert(use->bottom_type() == Type::MEMORY, "bad phi"); 1515 if ((use->adr_type() == TypePtr::BOTTOM) || 1516 phase->C->get_alias_index(use->adr_type()) == alias) { 1517 for (uint j = 1; j < use->req(); j++) { 1518 if (use->in(j) == nn) { 1519 Node* c = use->in(0)->in(j); 1520 if (phase->is_dominator(rep_ctrl, c)) { 1521 memory_dominates_all_paths_helper(c, rep_ctrl, controls, phase); 1522 } 1523 } 1524 } 1525 } 1526 // continue; 1527 } 1528 1529 if (use->is_MergeMem()) { 1530 if (use->as_MergeMem()->memory_at(alias) == nn) { 1531 if (trace) { tty->print("XX Next mem"); use->dump(); } 1532 // follow the memory edges 1533 wq.push(use); 1534 } 1535 } else if (use->is_Phi()) { 1536 assert(use->bottom_type() == Type::MEMORY, "bad phi"); 1537 if ((use->adr_type() == TypePtr::BOTTOM) || 1538 phase->C->get_alias_index(use->adr_type()) == alias) { 1539 if (trace) { tty->print("XX Next mem"); use->dump(); } 1540 // follow the memory edges 1541 wq.push(use); 1542 } 1543 } else if (use->bottom_type() == Type::MEMORY && 1544 (use->adr_type() == TypePtr::BOTTOM || phase->C->get_alias_index(use->adr_type()) == alias)) { 1545 if (trace) { tty->print("XX Next mem"); use->dump(); } 1546 // follow the memory edges 1547 wq.push(use); 1548 } else if ((use->is_SafePoint() || use->is_MemBar()) && 1549 (use->adr_type() == TypePtr::BOTTOM || phase->C->get_alias_index(use->adr_type()) == alias)) { 1550 for (DUIterator_Fast jmax, j = use->fast_outs(jmax); j < jmax; j++) { 1551 Node* u = use->fast_out(j); 1552 if (u->bottom_type() == Type::MEMORY) { 1553 if (trace) { tty->print("XX Next mem"); u->dump(); } 1554 // follow the memory edges 1555 wq.push(u); 1556 } 1557 } 1558 } else if (use->Opcode() == Op_ShenandoahWriteBarrier && phase->C->get_alias_index(use->adr_type()) == alias) { 1559 Node* m = use->find_out_with(Op_ShenandoahWBMemProj); 1560 if (m != NULL) { 1561 if (trace) { tty->print("XX Next mem"); m->dump(); } 1562 // follow the memory edges 1563 wq.push(m); 1564 } 1565 } 1566 } 1567 } 1568 1569 if (controls.size() == 0) { 1570 return false; 1571 } 1572 1573 for (uint i = 0; i < controls.size(); i++) { 1574 Node *n = controls.at(i); 1575 1576 if (trace) { tty->print("X checking"); n->dump(); } 1577 1578 if (n->unique_ctrl_out() != NULL) { 1579 continue; 1580 } 1581 1582 if (n->Opcode() == Op_NeverBranch) { 1583 Node* taken = n->as_Multi()->proj_out(0); 1584 if (!controls.member(taken)) { 1585 if (trace) { tty->print("X not seen"); taken->dump(); } 1586 return false; 1587 } 1588 continue; 1589 } 1590 1591 for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) { 1592 Node* u = n->fast_out(j); 1593 1594 if (u->is_CFG()) { 1595 if (!controls.member(u)) { 1596 if (u->is_Proj() && u->as_Proj()->is_uncommon_trap_proj(Deoptimization::Reason_none)) { 1597 if (trace) { tty->print("X not seen but unc"); u->dump(); } 1598 } else { 1599 Node* c = u; 1600 do { 1601 c = c->unique_ctrl_out(); 1602 } while (c != NULL && c->is_Region()); 1603 if (c != NULL && c->Opcode() == Op_Halt) { 1604 if (trace) { tty->print("X not seen but halt"); c->dump(); } 1605 } else { 1606 if (trace) { tty->print("X not seen"); u->dump(); } 1607 return false; 1608 } 1609 } 1610 } else { 1611 if (trace) { tty->print("X seen"); u->dump(); } 1612 } 1613 } 1614 } 1615 } 1616 return true; 1617 } 1618 #endif 1619 1620 Node* ShenandoahBarrierNode::dom_mem(Node* mem, Node*& mem_ctrl, Node* n, Node* rep_ctrl, int alias, PhaseIdealLoop* phase) { 1621 ResourceMark rm; 1622 VectorSet wq(Thread::current()->resource_area()); 1623 wq.set(mem->_idx); 1624 mem_ctrl = phase->get_ctrl(mem); 1625 while (!is_dominator(mem_ctrl, rep_ctrl, mem, n, phase)) { 1626 mem = next_mem(mem, alias); 1627 if (wq.test_set(mem->_idx)) { 1628 return NULL; // hit an unexpected loop 1629 } 1630 mem_ctrl = phase->ctrl_or_self(mem); 1631 } 1632 if (mem->is_MergeMem()) { 1633 mem = mem->as_MergeMem()->memory_at(alias); 1634 mem_ctrl = phase->ctrl_or_self(mem); 1635 } 1636 return mem; 1637 } 1638 1639 Node* ShenandoahBarrierNode::dom_mem(Node* mem, Node* ctrl, int alias, Node*& mem_ctrl, PhaseIdealLoop* phase) { 1640 ResourceMark rm; 1641 VectorSet wq(Thread::current()->resource_area()); 1642 wq.set(mem->_idx); 1643 mem_ctrl = phase->ctrl_or_self(mem); 1644 while (!phase->is_dominator(mem_ctrl, ctrl) || mem_ctrl == ctrl) { 1645 mem = next_mem(mem, alias); 1646 if (wq.test_set(mem->_idx)) { 1647 return NULL; 1648 } 1649 mem_ctrl = phase->ctrl_or_self(mem); 1650 } 1651 if (mem->is_MergeMem()) { 1652 mem = mem->as_MergeMem()->memory_at(alias); 1653 mem_ctrl = phase->ctrl_or_self(mem); 1654 } 1655 return mem; 1656 } 1657 1658 static void disconnect_barrier_mem(Node* wb, PhaseIterGVN& igvn) { 1659 Node* mem_in = wb->in(ShenandoahBarrierNode::Memory); 1660 Node* proj = wb->find_out_with(Op_ShenandoahWBMemProj); 1661 1662 for (DUIterator_Last imin, i = proj->last_outs(imin); i >= imin; ) { 1663 Node* u = proj->last_out(i); 1664 igvn.rehash_node_delayed(u); 1665 int nb = u->replace_edge(proj, mem_in); 1666 assert(nb > 0, "no replacement?"); 1667 i -= nb; 1668 } 1669 } 1670 1671 Node* ShenandoahWriteBarrierNode::move_above_predicates(LoopNode* cl, Node* val_ctrl, PhaseIdealLoop* phase) { 1672 Node* entry = cl->skip_strip_mined(-1)->in(LoopNode::EntryControl); 1673 Node* above_pred = phase->skip_all_loop_predicates(entry); 1674 Node* ctrl = entry; 1675 while (ctrl != above_pred) { 1676 Node* next = ctrl->in(0); 1677 if (!phase->is_dominator(val_ctrl, next)) { 1678 break; 1679 } 1680 ctrl = next; 1681 } 1682 return ctrl; 1683 } 1684 1685 static MemoryGraphFixer* find_fixer(GrowableArray<MemoryGraphFixer*>& memory_graph_fixers, int alias) { 1686 for (int i = 0; i < memory_graph_fixers.length(); i++) { 1687 if (memory_graph_fixers.at(i)->alias() == alias) { 1688 return memory_graph_fixers.at(i); 1689 } 1690 } 1691 return NULL; 1692 } 1693 1694 static MemoryGraphFixer* create_fixer(GrowableArray<MemoryGraphFixer*>& memory_graph_fixers, int alias, PhaseIdealLoop* phase, bool include_lsm) { 1695 assert(find_fixer(memory_graph_fixers, alias) == NULL, "none should exist yet"); 1696 MemoryGraphFixer* fixer = new MemoryGraphFixer(alias, include_lsm, phase); 1697 memory_graph_fixers.push(fixer); 1698 return fixer; 1699 } 1700 1701 void ShenandoahWriteBarrierNode::try_move_before_loop_helper(LoopNode* cl, Node* val_ctrl, GrowableArray<MemoryGraphFixer*>& memory_graph_fixers, PhaseIdealLoop* phase, bool include_lsm, Unique_Node_List& uses) { 1702 assert(cl->is_Loop(), "bad control"); 1703 Node* ctrl = move_above_predicates(cl, val_ctrl, phase); 1704 Node* mem_ctrl = NULL; 1705 int alias = phase->C->get_alias_index(adr_type()); 1706 1707 MemoryGraphFixer* fixer = find_fixer(memory_graph_fixers, alias); 1708 if (fixer == NULL) { 1709 fixer = create_fixer(memory_graph_fixers, alias, phase, include_lsm); 1710 } 1711 1712 Node* proj = find_out_with(Op_ShenandoahWBMemProj); 1713 1714 fixer->remove(proj); 1715 Node* mem = fixer->find_mem(ctrl, NULL); 1716 1717 assert(!ShenandoahVerifyOptoBarriers || memory_dominates_all_paths(mem, ctrl, alias, phase), "can't fix the memory graph"); 1718 1719 phase->set_ctrl_and_loop(this, ctrl); 1720 phase->igvn().replace_input_of(this, Control, ctrl); 1721 1722 disconnect_barrier_mem(this, phase->igvn()); 1723 1724 phase->igvn().replace_input_of(this, Memory, mem); 1725 phase->set_ctrl_and_loop(proj, ctrl); 1726 1727 fixer->fix_mem(ctrl, ctrl, mem, mem, proj, uses); 1728 assert(proj->outcnt() > 0, "disconnected write barrier"); 1729 } 1730 1731 LoopNode* ShenandoahWriteBarrierNode::try_move_before_pre_loop(Node* c, Node* val_ctrl, PhaseIdealLoop* phase) { 1732 // A write barrier between a pre and main loop can get in the way of 1733 // vectorization. Move it above the pre loop if possible 1734 CountedLoopNode* cl = NULL; 1735 if (c->is_IfFalse() && 1736 c->in(0)->is_CountedLoopEnd()) { 1737 cl = c->in(0)->as_CountedLoopEnd()->loopnode(); 1738 } else if (c->is_IfProj() && 1739 c->in(0)->is_If() && 1740 c->in(0)->in(0)->is_IfFalse() && 1741 c->in(0)->in(0)->in(0)->is_CountedLoopEnd()) { 1742 cl = c->in(0)->in(0)->in(0)->as_CountedLoopEnd()->loopnode(); 1743 } 1744 if (cl != NULL && 1745 cl->is_pre_loop() && 1746 val_ctrl != cl && 1747 phase->is_dominator(val_ctrl, cl)) { 1748 return cl; 1749 } 1750 return NULL; 1751 } 1752 1753 void ShenandoahWriteBarrierNode::try_move_before_loop(GrowableArray<MemoryGraphFixer*>& memory_graph_fixers, PhaseIdealLoop* phase, bool include_lsm, Unique_Node_List& uses) { 1754 Node *n_ctrl = phase->get_ctrl(this); 1755 IdealLoopTree *n_loop = phase->get_loop(n_ctrl); 1756 Node* val = in(ValueIn); 1757 Node* val_ctrl = phase->get_ctrl(val); 1758 if (n_loop != phase->ltree_root() && !n_loop->_irreducible) { 1759 IdealLoopTree *val_loop = phase->get_loop(val_ctrl); 1760 Node* mem = in(Memory); 1761 IdealLoopTree *mem_loop = phase->get_loop(phase->get_ctrl(mem)); 1762 if (!n_loop->is_member(val_loop) && 1763 n_loop->is_member(mem_loop)) { 1764 Node* n_loop_head = n_loop->_head; 1765 1766 if (n_loop_head->is_Loop()) { 1767 LoopNode* loop = n_loop_head->as_Loop(); 1768 if (n_loop_head->is_CountedLoop() && n_loop_head->as_CountedLoop()->is_main_loop()) { 1769 LoopNode* res = try_move_before_pre_loop(n_loop_head->in(LoopNode::EntryControl), val_ctrl, phase); 1770 if (res != NULL) { 1771 loop = res; 1772 } 1773 } 1774 1775 try_move_before_loop_helper(loop, val_ctrl, memory_graph_fixers, phase, include_lsm, uses); 1776 } 1777 } 1778 } 1779 LoopNode* ctrl = try_move_before_pre_loop(in(0), val_ctrl, phase); 1780 if (ctrl != NULL) { 1781 try_move_before_loop_helper(ctrl, val_ctrl, memory_graph_fixers, phase, include_lsm, uses); 1782 } 1783 } 1784 1785 Node* ShenandoahWriteBarrierNode::would_subsume(ShenandoahBarrierNode* other, PhaseIdealLoop* phase) { 1786 Node* val = in(ValueIn); 1787 Node* val_ctrl = phase->get_ctrl(val); 1788 Node* other_mem = other->in(Memory); 1789 Node* other_ctrl = phase->get_ctrl(other); 1790 Node* this_ctrl = phase->get_ctrl(this); 1791 IdealLoopTree* this_loop = phase->get_loop(this_ctrl); 1792 IdealLoopTree* other_loop = phase->get_loop(other_ctrl); 1793 1794 Node* ctrl = phase->dom_lca(other_ctrl, this_ctrl); 1795 1796 if (ctrl->is_Proj() && 1797 ctrl->in(0)->is_Call() && 1798 ctrl->unique_ctrl_out() != NULL && 1799 ctrl->unique_ctrl_out()->Opcode() == Op_Catch && 1800 !phase->is_dominator(val_ctrl, ctrl->in(0)->in(0))) { 1801 return NULL; 1802 } 1803 1804 IdealLoopTree* loop = phase->get_loop(ctrl); 1805 1806 // We don't want to move a write barrier in a loop 1807 // If the LCA is in a inner loop, try a control out of loop if possible 1808 while (!loop->is_member(this_loop) && (other->Opcode() != Op_ShenandoahWriteBarrier || !loop->is_member(other_loop))) { 1809 ctrl = phase->idom(ctrl); 1810 if (ctrl->is_MultiBranch()) { 1811 ctrl = ctrl->in(0); 1812 } 1813 if (ctrl != val_ctrl && phase->is_dominator(ctrl, val_ctrl)) { 1814 return NULL; 1815 } 1816 loop = phase->get_loop(ctrl); 1817 } 1818 1819 if (ShenandoahDontIncreaseWBFreq) { 1820 Node* this_iffproj = no_branches(this_ctrl, ctrl, true, phase); 1821 if (other->Opcode() == Op_ShenandoahWriteBarrier) { 1822 Node* other_iffproj = no_branches(other_ctrl, ctrl, true, phase); 1823 if (other_iffproj == NULL || this_iffproj == NULL) { 1824 return ctrl; 1825 } else if (other_iffproj != NodeSentinel && this_iffproj != NodeSentinel && 1826 other_iffproj->in(0) == this_iffproj->in(0)) { 1827 return ctrl; 1828 } 1829 } else if (this_iffproj == NULL) { 1830 return ctrl; 1831 } 1832 return NULL; 1833 } 1834 1835 return ctrl; 1836 } 1837 1838 void ShenandoahWriteBarrierNode::optimize_before_expansion(PhaseIdealLoop* phase, GrowableArray<MemoryGraphFixer*> memory_graph_fixers, bool include_lsm) { 1839 bool progress = false; 1840 Unique_Node_List uses; 1841 do { 1842 progress = false; 1843 for (int i = 0; i < ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count(); i++) { 1844 ShenandoahWriteBarrierNode* wb = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barrier(i); 1845 1846 wb->try_move_before_loop(memory_graph_fixers, phase, include_lsm, uses); 1847 1848 Node* val = wb->in(ValueIn); 1849 1850 for (DUIterator_Fast jmax, j = val->fast_outs(jmax); j < jmax; j++) { 1851 Node* u = val->fast_out(j); 1852 if (u != wb && u->is_ShenandoahBarrier()) { 1853 Node* rep_ctrl = wb->would_subsume(u->as_ShenandoahBarrier(), phase); 1854 1855 if (rep_ctrl != NULL) { 1856 Node* other = u; 1857 Node* val_ctrl = phase->get_ctrl(val); 1858 if (rep_ctrl->is_Proj() && 1859 rep_ctrl->in(0)->is_Call() && 1860 rep_ctrl->unique_ctrl_out() != NULL && 1861 rep_ctrl->unique_ctrl_out()->Opcode() == Op_Catch) { 1862 rep_ctrl = rep_ctrl->in(0)->in(0); 1863 1864 assert(phase->is_dominator(val_ctrl, rep_ctrl), "bad control"); 1865 } else { 1866 LoopNode* c = ShenandoahWriteBarrierNode::try_move_before_pre_loop(rep_ctrl, val_ctrl, phase); 1867 if (c != NULL) { 1868 rep_ctrl = ShenandoahWriteBarrierNode::move_above_predicates(c, val_ctrl, phase); 1869 } else { 1870 while (rep_ctrl->is_IfProj()) { 1871 CallStaticJavaNode* unc = rep_ctrl->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none); 1872 if (unc != NULL) { 1873 int req = unc->uncommon_trap_request(); 1874 Deoptimization::DeoptReason trap_reason = Deoptimization::trap_request_reason(req); 1875 if ((trap_reason == Deoptimization::Reason_loop_limit_check || 1876 trap_reason == Deoptimization::Reason_predicate || 1877 trap_reason == Deoptimization::Reason_profile_predicate) && 1878 phase->is_dominator(val_ctrl, rep_ctrl->in(0)->in(0))) { 1879 rep_ctrl = rep_ctrl->in(0)->in(0); 1880 continue; 1881 } 1882 } 1883 break; 1884 } 1885 } 1886 } 1887 1888 Node* wb_ctrl = phase->get_ctrl(wb); 1889 Node* other_ctrl = phase->get_ctrl(other); 1890 int alias = phase->C->get_alias_index(wb->adr_type()); 1891 MemoryGraphFixer* fixer = find_fixer(memory_graph_fixers, alias);; 1892 if (!is_dominator(wb_ctrl, other_ctrl, wb, other, phase)) { 1893 if (fixer == NULL) { 1894 fixer = create_fixer(memory_graph_fixers, alias, phase, include_lsm); 1895 } 1896 Node* mem = fixer->find_mem(rep_ctrl, phase->get_ctrl(other) == rep_ctrl ? other : NULL); 1897 1898 if (mem->has_out_with(Op_Lock) || mem->has_out_with(Op_Unlock)) { 1899 continue; 1900 } 1901 1902 Node* wb_proj = wb->find_out_with(Op_ShenandoahWBMemProj); 1903 fixer->remove(wb_proj); 1904 Node* mem_for_ctrl = fixer->find_mem(rep_ctrl, NULL); 1905 1906 if (wb->in(Memory) != mem) { 1907 disconnect_barrier_mem(wb, phase->igvn()); 1908 phase->igvn().replace_input_of(wb, Memory, mem); 1909 } 1910 if (rep_ctrl != wb_ctrl) { 1911 phase->set_ctrl_and_loop(wb, rep_ctrl); 1912 phase->igvn().replace_input_of(wb, Control, rep_ctrl); 1913 phase->set_ctrl_and_loop(wb_proj, rep_ctrl); 1914 progress = true; 1915 } 1916 1917 fixer->fix_mem(rep_ctrl, rep_ctrl, mem, mem_for_ctrl, wb_proj, uses); 1918 1919 assert(!ShenandoahVerifyOptoBarriers || ShenandoahWriteBarrierNode::memory_dominates_all_paths(mem, rep_ctrl, alias, phase), "can't fix the memory graph"); 1920 } 1921 1922 if (other->Opcode() == Op_ShenandoahWriteBarrier) { 1923 Node* other_proj = other->find_out_with(Op_ShenandoahWBMemProj); 1924 if (fixer != NULL) { 1925 fixer->remove(other_proj); 1926 } 1927 phase->igvn().replace_node(other_proj, other->in(Memory)); 1928 } 1929 phase->igvn().replace_node(other, wb); 1930 --j; --jmax; 1931 } 1932 } 1933 } 1934 } 1935 } while(progress); 1936 } 1937 1938 // Some code duplication with PhaseIdealLoop::split_if_with_blocks_pre() 1939 Node* ShenandoahWriteBarrierNode::try_split_thru_phi(PhaseIdealLoop* phase) { 1940 Node *ctrl = phase->get_ctrl(this); 1941 if (ctrl == NULL) { 1942 return this; 1943 } 1944 Node *blk = phase->has_local_phi_input(this); 1945 if (blk == NULL) { 1946 return this; 1947 } 1948 1949 if (in(0) != blk) { 1950 return this; 1951 } 1952 1953 int policy = blk->req() >> 2; 1954 1955 if (blk->is_CountedLoop()) { 1956 IdealLoopTree *lp = phase->get_loop(blk); 1957 if (lp && lp->_rce_candidate) { 1958 return this; 1959 } 1960 } 1961 1962 if (phase->C->live_nodes() > 35000) { 1963 return this; 1964 } 1965 1966 uint unique = phase->C->unique(); 1967 Node *phi = phase->split_thru_phi(this, blk, policy); 1968 if (phi == NULL) { 1969 return this; 1970 } 1971 1972 Node* mem_phi = new PhiNode(blk, Type::MEMORY, phase->C->alias_type(adr_type())->adr_type()); 1973 for (uint i = 1; i < blk->req(); i++) { 1974 Node* n = phi->in(i); 1975 if (n->Opcode() == Op_ShenandoahWriteBarrier && 1976 n->_idx >= unique) { 1977 Node* proj = new ShenandoahWBMemProjNode(n); 1978 phase->register_new_node(proj, phase->get_ctrl(n)); 1979 mem_phi->init_req(i, proj); 1980 } else { 1981 Node* mem = in(ShenandoahBarrierNode::Memory); 1982 if (mem->is_Phi() && mem->in(0) == blk) { 1983 mem = mem->in(i); 1984 } 1985 mem_phi->init_req(i, mem); 1986 } 1987 } 1988 phase->register_new_node(mem_phi, blk); 1989 1990 1991 Node* proj = find_out_with(Op_ShenandoahWBMemProj); 1992 phase->igvn().replace_node(proj, mem_phi); 1993 phase->igvn().replace_node(this, phi); 1994 1995 return phi; 1996 } 1997 1998 void ShenandoahReadBarrierNode::try_move(PhaseIdealLoop* phase) { 1999 Node *n_ctrl = phase->get_ctrl(this); 2000 if (n_ctrl == NULL) { 2001 return; 2002 } 2003 Node* mem = in(MemNode::Memory); 2004 int alias = phase->C->get_alias_index(adr_type()); 2005 const bool trace = false; 2006 2007 #ifdef ASSERT 2008 if (trace) { tty->print("Trying to move mem of"); dump(); } 2009 #endif 2010 2011 Node* new_mem = mem; 2012 2013 ResourceMark rm; 2014 VectorSet seen(Thread::current()->resource_area()); 2015 Node_List phis; 2016 2017 for (;;) { 2018 #ifdef ASSERT 2019 if (trace) { tty->print("Looking for dominator from"); mem->dump(); } 2020 #endif 2021 if (mem->is_Proj() && mem->in(0)->is_Start()) { 2022 if (new_mem != in(MemNode::Memory)) { 2023 #ifdef ASSERT 2024 if (trace) { tty->print("XXX Setting mem to"); new_mem->dump(); tty->print(" for "); dump(); } 2025 #endif 2026 phase->igvn().replace_input_of(this, MemNode::Memory, new_mem); 2027 } 2028 return; 2029 } 2030 2031 Node* candidate = mem; 2032 do { 2033 if (!is_independent(mem)) { 2034 if (trace) { tty->print_cr("Not independent"); } 2035 if (new_mem != in(MemNode::Memory)) { 2036 #ifdef ASSERT 2037 if (trace) { tty->print("XXX Setting mem to"); new_mem->dump(); tty->print(" for "); dump(); } 2038 #endif 2039 phase->igvn().replace_input_of(this, MemNode::Memory, new_mem); 2040 } 2041 return; 2042 } 2043 if (seen.test_set(mem->_idx)) { 2044 if (trace) { tty->print_cr("Already seen"); } 2045 ShouldNotReachHere(); 2046 // Strange graph 2047 if (new_mem != in(MemNode::Memory)) { 2048 #ifdef ASSERT 2049 if (trace) { tty->print("XXX Setting mem to"); new_mem->dump(); tty->print(" for "); dump(); } 2050 #endif 2051 phase->igvn().replace_input_of(this, MemNode::Memory, new_mem); 2052 } 2053 return; 2054 } 2055 if (mem->is_Phi()) { 2056 phis.push(mem); 2057 } 2058 mem = next_mem(mem, alias); 2059 if (mem->bottom_type() == Type::MEMORY) { 2060 candidate = mem; 2061 } 2062 assert(is_dominator(phase->ctrl_or_self(mem), n_ctrl, mem, this, phase) == phase->is_dominator(phase->ctrl_or_self(mem), n_ctrl), "strange dominator"); 2063 #ifdef ASSERT 2064 if (trace) { tty->print("Next mem is"); mem->dump(); } 2065 #endif 2066 } while (mem->bottom_type() != Type::MEMORY || !phase->is_dominator(phase->ctrl_or_self(mem), n_ctrl)); 2067 2068 assert(mem->bottom_type() == Type::MEMORY, "bad mem"); 2069 2070 bool not_dom = false; 2071 for (uint i = 0; i < phis.size() && !not_dom; i++) { 2072 Node* nn = phis.at(i); 2073 2074 #ifdef ASSERT 2075 if (trace) { tty->print("Looking from phi"); nn->dump(); } 2076 #endif 2077 assert(nn->is_Phi(), "phis only"); 2078 for (uint j = 2; j < nn->req() && !not_dom; j++) { 2079 Node* m = nn->in(j); 2080 #ifdef ASSERT 2081 if (trace) { tty->print("Input %d is", j); m->dump(); } 2082 #endif 2083 while (m != mem && !seen.test_set(m->_idx)) { 2084 if (is_dominator(phase->ctrl_or_self(m), phase->ctrl_or_self(mem), m, mem, phase)) { 2085 not_dom = true; 2086 // Scheduling anomaly 2087 #ifdef ASSERT 2088 if (trace) { tty->print("Giving up"); m->dump(); } 2089 #endif 2090 break; 2091 } 2092 if (!is_independent(m)) { 2093 if (trace) { tty->print_cr("Not independent"); } 2094 if (new_mem != in(MemNode::Memory)) { 2095 #ifdef ASSERT 2096 if (trace) { tty->print("XXX Setting mem to"); new_mem->dump(); tty->print(" for "); dump(); } 2097 #endif 2098 phase->igvn().replace_input_of(this, MemNode::Memory, new_mem); 2099 } 2100 return; 2101 } 2102 if (m->is_Phi()) { 2103 phis.push(m); 2104 } 2105 m = next_mem(m, alias); 2106 #ifdef ASSERT 2107 if (trace) { tty->print("Next mem is"); m->dump(); } 2108 #endif 2109 } 2110 } 2111 } 2112 if (!not_dom) { 2113 new_mem = mem; 2114 phis.clear(); 2115 } else { 2116 seen.Clear(); 2117 } 2118 } 2119 } 2120 2121 CallStaticJavaNode* ShenandoahWriteBarrierNode::pin_and_expand_null_check(PhaseIterGVN& igvn) { 2122 Node* val = in(ValueIn); 2123 2124 const Type* val_t = igvn.type(val); 2125 2126 if (val_t->meet(TypePtr::NULL_PTR) != val_t && 2127 val->Opcode() == Op_CastPP && 2128 val->in(0) != NULL && 2129 val->in(0)->Opcode() == Op_IfTrue && 2130 val->in(0)->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none) && 2131 val->in(0)->in(0)->is_If() && 2132 val->in(0)->in(0)->in(1)->Opcode() == Op_Bool && 2133 val->in(0)->in(0)->in(1)->as_Bool()->_test._test == BoolTest::ne && 2134 val->in(0)->in(0)->in(1)->in(1)->Opcode() == Op_CmpP && 2135 val->in(0)->in(0)->in(1)->in(1)->in(1) == val->in(1) && 2136 val->in(0)->in(0)->in(1)->in(1)->in(2)->bottom_type() == TypePtr::NULL_PTR) { 2137 assert(val->in(0)->in(0)->in(1)->in(1)->in(1) == val->in(1), ""); 2138 CallStaticJavaNode* unc = val->in(0)->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none); 2139 return unc; 2140 } 2141 return NULL; 2142 } 2143 2144 void ShenandoahWriteBarrierNode::pin_and_expand_move_barrier(PhaseIdealLoop* phase, GrowableArray<MemoryGraphFixer*>& memory_graph_fixers, Unique_Node_List& uses) { 2145 Node* unc = pin_and_expand_null_check(phase->igvn()); 2146 Node* val = in(ValueIn); 2147 2148 if (unc != NULL) { 2149 Node* ctrl = phase->get_ctrl(this); 2150 Node* unc_ctrl = val->in(0); 2151 2152 // Don't move write barrier in a loop 2153 IdealLoopTree* loop = phase->get_loop(ctrl); 2154 IdealLoopTree* unc_loop = phase->get_loop(unc_ctrl); 2155 2156 if (!unc_loop->is_member(loop)) { 2157 return; 2158 } 2159 2160 Node* branch = no_branches(ctrl, unc_ctrl, false, phase); 2161 assert(branch == NULL || branch == NodeSentinel, "was not looking for a branch"); 2162 if (branch == NodeSentinel) { 2163 return; 2164 } 2165 2166 RegionNode* r = new RegionNode(3); 2167 IfNode* iff = unc_ctrl->in(0)->as_If(); 2168 2169 Node* ctrl_use = unc_ctrl->unique_ctrl_out(); 2170 Node* unc_ctrl_clone = unc_ctrl->clone(); 2171 phase->register_control(unc_ctrl_clone, loop, iff); 2172 Node* c = unc_ctrl_clone; 2173 Node* new_cast = clone_null_check(c, val, unc_ctrl_clone, phase); 2174 r->init_req(1, new_cast->in(0)->in(0)->as_If()->proj_out(0)); 2175 2176 phase->igvn().replace_input_of(unc_ctrl, 0, c->in(0)); 2177 phase->set_idom(unc_ctrl, c->in(0), phase->dom_depth(unc_ctrl)); 2178 phase->lazy_replace(c, unc_ctrl); 2179 c = NULL;; 2180 phase->igvn().replace_input_of(val, 0, unc_ctrl_clone); 2181 phase->set_ctrl(val, unc_ctrl_clone); 2182 2183 IfNode* new_iff = new_cast->in(0)->in(0)->as_If(); 2184 fix_null_check(unc, unc_ctrl_clone, r, uses, phase); 2185 Node* iff_proj = iff->proj_out(0); 2186 r->init_req(2, iff_proj); 2187 phase->register_control(r, phase->ltree_root(), iff); 2188 2189 Node* new_bol = new_iff->in(1)->clone(); 2190 Node* new_cmp = new_bol->in(1)->clone(); 2191 assert(new_cmp->Opcode() == Op_CmpP, "broken"); 2192 assert(new_cmp->in(1) == val->in(1), "broken"); 2193 new_bol->set_req(1, new_cmp); 2194 new_cmp->set_req(1, this); 2195 phase->register_new_node(new_bol, new_iff->in(0)); 2196 phase->register_new_node(new_cmp, new_iff->in(0)); 2197 phase->igvn().replace_input_of(new_iff, 1, new_bol); 2198 phase->igvn().replace_input_of(new_cast, 1, this); 2199 2200 for (DUIterator_Fast imax, i = this->fast_outs(imax); i < imax; i++) { 2201 Node* u = this->fast_out(i); 2202 if (u == new_cast || u->Opcode() == Op_ShenandoahWBMemProj || u == new_cmp) { 2203 continue; 2204 } 2205 phase->igvn().rehash_node_delayed(u); 2206 int nb = u->replace_edge(this, new_cast); 2207 assert(nb > 0, "no update?"); 2208 --i; imax -= nb; 2209 } 2210 2211 for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) { 2212 Node* u = val->fast_out(i); 2213 if (u == this) { 2214 continue; 2215 } 2216 phase->igvn().rehash_node_delayed(u); 2217 int nb = u->replace_edge(val, new_cast); 2218 assert(nb > 0, "no update?"); 2219 --i; imax -= nb; 2220 } 2221 2222 Node* new_ctrl = unc_ctrl_clone; 2223 2224 int alias = phase->C->get_alias_index(adr_type()); 2225 MemoryGraphFixer* fixer = find_fixer(memory_graph_fixers, alias); 2226 if (fixer == NULL) { 2227 fixer = create_fixer(memory_graph_fixers, alias, phase, true); 2228 } 2229 2230 Node* proj = find_out_with(Op_ShenandoahWBMemProj); 2231 fixer->remove(proj); 2232 Node* mem = fixer->find_mem(new_ctrl, NULL); 2233 2234 if (in(Memory) != mem) { 2235 disconnect_barrier_mem(this, phase->igvn()); 2236 phase->igvn().replace_input_of(this, Memory, mem); 2237 } 2238 2239 phase->set_ctrl_and_loop(this, new_ctrl); 2240 phase->igvn().replace_input_of(this, Control, new_ctrl); 2241 phase->set_ctrl_and_loop(proj, new_ctrl); 2242 2243 fixer->fix_mem(new_ctrl, new_ctrl, mem, mem, proj, uses); 2244 } 2245 } 2246 2247 void ShenandoahWriteBarrierNode::pin_and_expand_helper(PhaseIdealLoop* phase) { 2248 Node* val = in(ValueIn); 2249 CallStaticJavaNode* unc = pin_and_expand_null_check(phase->igvn()); 2250 Node* rep = this; 2251 Node* ctrl = phase->get_ctrl(this); 2252 if (unc != NULL && val->in(0) == ctrl) { 2253 Node* unc_ctrl = val->in(0); 2254 IfNode* other_iff = unc_ctrl->unique_ctrl_out()->as_If(); 2255 ProjNode* other_unc_ctrl = other_iff->proj_out(1); 2256 Node* cast = NULL; 2257 for (DUIterator_Fast imax, i = other_unc_ctrl->fast_outs(imax); i < imax && cast == NULL; i++) { 2258 Node* u = other_unc_ctrl->fast_out(i); 2259 if (u->Opcode() == Op_CastPP && u->in(1) == this) { 2260 cast = u; 2261 } 2262 } 2263 assert(other_unc_ctrl->is_uncommon_trap_if_pattern(Deoptimization::Reason_none) == unc, "broken"); 2264 rep = cast; 2265 } 2266 2267 // Replace all uses of barrier's input that are dominated by ctrl 2268 // with the value returned by the barrier: no need to keep both 2269 // live. 2270 for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) { 2271 Node* u = val->fast_out(i); 2272 if (u != this) { 2273 if (u->is_Phi()) { 2274 int nb = 0; 2275 for (uint j = 1; j < u->req(); j++) { 2276 if (u->in(j) == val) { 2277 Node* c = u->in(0)->in(j); 2278 if (phase->is_dominator(ctrl, c)) { 2279 phase->igvn().replace_input_of(u, j, rep); 2280 nb++; 2281 } 2282 } 2283 } 2284 if (nb > 0) { 2285 imax -= nb; 2286 --i; 2287 } 2288 } else { 2289 Node* c = phase->ctrl_or_self(u); 2290 if (is_dominator(ctrl, c, this, u, phase)) { 2291 phase->igvn().rehash_node_delayed(u); 2292 int nb = u->replace_edge(val, rep); 2293 assert(nb > 0, "no update?"); 2294 --i, imax -= nb; 2295 } 2296 } 2297 } 2298 } 2299 } 2300 2301 Node* ShenandoahWriteBarrierNode::find_bottom_mem(Node* ctrl, PhaseIdealLoop* phase) { 2302 Node* mem = NULL; 2303 Node* c = ctrl; 2304 do { 2305 if (c->is_Region()) { 2306 Node* phi_bottom = NULL; 2307 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax && mem == NULL; i++) { 2308 Node* u = c->fast_out(i); 2309 if (u->is_Phi() && u->bottom_type() == Type::MEMORY) { 2310 if (u->adr_type() == TypePtr::BOTTOM) { 2311 mem = u; 2312 } 2313 } 2314 } 2315 } else { 2316 if (c->is_Call() && c->as_Call()->adr_type() != NULL) { 2317 CallProjections projs; 2318 c->as_Call()->extract_projections(&projs, true, false); 2319 if (projs.fallthrough_memproj != NULL) { 2320 if (projs.fallthrough_memproj->adr_type() == TypePtr::BOTTOM) { 2321 if (projs.catchall_memproj == NULL) { 2322 mem = projs.fallthrough_memproj; 2323 } else { 2324 if (phase->is_dominator(projs.fallthrough_catchproj, ctrl)) { 2325 mem = projs.fallthrough_memproj; 2326 } else { 2327 assert(phase->is_dominator(projs.catchall_catchproj, ctrl), "one proj must dominate barrier"); 2328 mem = projs.catchall_memproj; 2329 } 2330 } 2331 } 2332 } else { 2333 Node* proj = c->as_Call()->proj_out(TypeFunc::Memory); 2334 if (proj != NULL && 2335 proj->adr_type() == TypePtr::BOTTOM) { 2336 mem = proj; 2337 } 2338 } 2339 } else { 2340 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) { 2341 Node* u = c->fast_out(i); 2342 if (u->is_Proj() && 2343 u->bottom_type() == Type::MEMORY && 2344 u->adr_type() == TypePtr::BOTTOM) { 2345 assert(c->is_SafePoint() || c->is_MemBar() || c->is_Start(), ""); 2346 assert(mem == NULL, "only one proj"); 2347 mem = u; 2348 } 2349 } 2350 assert(!c->is_Call() || c->as_Call()->adr_type() != NULL || mem == NULL, "no mem projection expected"); 2351 } 2352 } 2353 c = phase->idom(c); 2354 } while (mem == NULL); 2355 return mem; 2356 } 2357 2358 void ShenandoahWriteBarrierNode::follow_barrier_uses(Node* n, Node* ctrl, Unique_Node_List& uses, PhaseIdealLoop* phase) { 2359 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { 2360 Node* u = n->fast_out(i); 2361 if (!u->is_CFG() && phase->get_ctrl(u) == ctrl && (!u->is_Phi() || !u->in(0)->is_Loop() || u->in(LoopNode::LoopBackControl) != n)) { 2362 uses.push(u); 2363 } 2364 } 2365 } 2366 2367 static void hide_strip_mined_loop(OuterStripMinedLoopNode* outer, CountedLoopNode* inner, PhaseIdealLoop* phase) { 2368 OuterStripMinedLoopEndNode* le = inner->outer_loop_end(); 2369 Node* new_outer = new LoopNode(outer->in(LoopNode::EntryControl), outer->in(LoopNode::LoopBackControl)); 2370 phase->register_control(new_outer, phase->get_loop(outer), outer->in(LoopNode::EntryControl)); 2371 Node* new_le = new IfNode(le->in(0), le->in(1), le->_prob, le->_fcnt); 2372 phase->register_control(new_le, phase->get_loop(le), le->in(0)); 2373 phase->lazy_replace(outer, new_outer); 2374 phase->lazy_replace(le, new_le); 2375 inner->clear_strip_mined(); 2376 } 2377 2378 void ShenandoahWriteBarrierNode::test_heap_stable(Node*& ctrl, Node* raw_mem, Node*& heap_stable_ctrl, 2379 PhaseIdealLoop* phase) { 2380 IdealLoopTree* loop = phase->get_loop(ctrl); 2381 Node* thread = new ThreadLocalNode(); 2382 phase->register_new_node(thread, ctrl); 2383 Node* offset = phase->igvn().MakeConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset())); 2384 phase->set_ctrl(offset, phase->C->root()); 2385 Node* gc_state_addr = new AddPNode(phase->C->top(), thread, offset); 2386 phase->register_new_node(gc_state_addr, ctrl); 2387 uint gc_state_idx = Compile::AliasIdxRaw; 2388 const TypePtr* gc_state_adr_type = NULL; // debug-mode-only argument 2389 debug_only(gc_state_adr_type = phase->C->get_adr_type(gc_state_idx)); 2390 2391 Node* gc_state = new LoadBNode(ctrl, raw_mem, gc_state_addr, gc_state_adr_type, TypeInt::BYTE, MemNode::unordered); 2392 phase->register_new_node(gc_state, ctrl); 2393 Node* heap_stable_and = new AndINode(gc_state, phase->igvn().intcon(ShenandoahHeap::HAS_FORWARDED)); 2394 phase->register_new_node(heap_stable_and, ctrl); 2395 Node* heap_stable_cmp = new CmpINode(heap_stable_and, phase->igvn().zerocon(T_INT)); 2396 phase->register_new_node(heap_stable_cmp, ctrl); 2397 Node* heap_stable_test = new BoolNode(heap_stable_cmp, BoolTest::ne); 2398 phase->register_new_node(heap_stable_test, ctrl); 2399 IfNode* heap_stable_iff = new IfNode(ctrl, heap_stable_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN); 2400 phase->register_control(heap_stable_iff, loop, ctrl); 2401 2402 heap_stable_ctrl = new IfFalseNode(heap_stable_iff); 2403 phase->register_control(heap_stable_ctrl, loop, heap_stable_iff); 2404 ctrl = new IfTrueNode(heap_stable_iff); 2405 phase->register_control(ctrl, loop, heap_stable_iff); 2406 2407 assert(is_heap_stable_test(heap_stable_iff), "Should match the shape"); 2408 } 2409 2410 void ShenandoahWriteBarrierNode::test_null(Node*& ctrl, Node* val, Node*& null_ctrl, PhaseIdealLoop* phase) { 2411 const Type* val_t = phase->igvn().type(val); 2412 if (val_t->meet(TypePtr::NULL_PTR) == val_t) { 2413 IdealLoopTree* loop = phase->get_loop(ctrl); 2414 Node* null_cmp = new CmpPNode(val, phase->igvn().zerocon(T_OBJECT)); 2415 phase->register_new_node(null_cmp, ctrl); 2416 Node* null_test = new BoolNode(null_cmp, BoolTest::ne); 2417 phase->register_new_node(null_test, ctrl); 2418 IfNode* null_iff = new IfNode(ctrl, null_test, PROB_LIKELY(0.999), COUNT_UNKNOWN); 2419 phase->register_control(null_iff, loop, ctrl); 2420 ctrl = new IfTrueNode(null_iff); 2421 phase->register_control(ctrl, loop, null_iff); 2422 null_ctrl = new IfFalseNode(null_iff); 2423 phase->register_control(null_ctrl, loop, null_iff); 2424 } 2425 } 2426 2427 Node* ShenandoahWriteBarrierNode::clone_null_check(Node*& c, Node* val, Node* unc_ctrl, PhaseIdealLoop* phase) { 2428 IdealLoopTree *loop = phase->get_loop(c); 2429 Node* iff = unc_ctrl->in(0); 2430 assert(iff->is_If(), "broken"); 2431 Node* new_iff = iff->clone(); 2432 new_iff->set_req(0, c); 2433 phase->register_control(new_iff, loop, c); 2434 Node* iffalse = new IfFalseNode(new_iff->as_If()); 2435 phase->register_control(iffalse, loop, new_iff); 2436 Node* iftrue = new IfTrueNode(new_iff->as_If()); 2437 phase->register_control(iftrue, loop, new_iff); 2438 c = iftrue; 2439 const Type *t = phase->igvn().type(val); 2440 assert(val->Opcode() == Op_CastPP, "expect cast to non null here"); 2441 Node* uncasted_val = val->in(1); 2442 val = new CastPPNode(uncasted_val, t); 2443 val->init_req(0, c); 2444 phase->register_new_node(val, c); 2445 return val; 2446 } 2447 2448 void ShenandoahWriteBarrierNode::fix_null_check(Node* unc, Node* unc_ctrl, Node* new_unc_ctrl, 2449 Unique_Node_List& uses, PhaseIdealLoop* phase) { 2450 IfNode* iff = unc_ctrl->in(0)->as_If(); 2451 Node* proj = iff->proj_out(0); 2452 assert(proj != unc_ctrl, "bad projection"); 2453 Node* use = proj->unique_ctrl_out(); 2454 2455 assert(use == unc || use->is_Region(), "what else?"); 2456 2457 uses.clear(); 2458 if (use == unc) { 2459 phase->set_idom(use, new_unc_ctrl, phase->dom_depth(use)); 2460 for (uint i = 1; i < unc->req(); i++) { 2461 Node* n = unc->in(i); 2462 if (phase->has_ctrl(n) && phase->get_ctrl(n) == proj) { 2463 uses.push(n); 2464 } 2465 } 2466 } else { 2467 assert(use->is_Region(), "what else?"); 2468 uint idx = 1; 2469 for (; use->in(idx) != proj; idx++); 2470 for (DUIterator_Fast imax, i = use->fast_outs(imax); i < imax; i++) { 2471 Node* u = use->fast_out(i); 2472 if (u->is_Phi() && phase->get_ctrl(u->in(idx)) == proj) { 2473 uses.push(u->in(idx)); 2474 } 2475 } 2476 } 2477 for(uint next = 0; next < uses.size(); next++ ) { 2478 Node *n = uses.at(next); 2479 assert(phase->get_ctrl(n) == proj, "bad control"); 2480 phase->set_ctrl_and_loop(n, new_unc_ctrl); 2481 if (n->in(0) == proj) { 2482 phase->igvn().replace_input_of(n, 0, new_unc_ctrl); 2483 } 2484 for (uint i = 0; i < n->req(); i++) { 2485 Node* m = n->in(i); 2486 if (m != NULL && phase->has_ctrl(m) && phase->get_ctrl(m) == proj) { 2487 uses.push(m); 2488 } 2489 } 2490 } 2491 2492 phase->igvn().rehash_node_delayed(use); 2493 int nb = use->replace_edge(proj, new_unc_ctrl); 2494 assert(nb == 1, "only use expected"); 2495 } 2496 2497 void ShenandoahWriteBarrierNode::in_cset_fast_test(Node*& ctrl, Node*& not_cset_ctrl, Node* val, Node* raw_mem, PhaseIdealLoop* phase) { 2498 IdealLoopTree *loop = phase->get_loop(ctrl); 2499 Node* raw_rbtrue = new CastP2XNode(ctrl, val); 2500 phase->register_new_node(raw_rbtrue, ctrl); 2501 Node* cset_offset = new URShiftXNode(raw_rbtrue, phase->igvn().intcon(ShenandoahHeapRegion::region_size_bytes_shift_jint())); 2502 phase->register_new_node(cset_offset, ctrl); 2503 Node* in_cset_fast_test_base_addr = phase->igvn().makecon(TypeRawPtr::make(ShenandoahHeap::in_cset_fast_test_addr())); 2504 phase->set_ctrl(in_cset_fast_test_base_addr, phase->C->root()); 2505 Node* in_cset_fast_test_adr = new AddPNode(phase->C->top(), in_cset_fast_test_base_addr, cset_offset); 2506 phase->register_new_node(in_cset_fast_test_adr, ctrl); 2507 uint in_cset_fast_test_idx = Compile::AliasIdxRaw; 2508 const TypePtr* in_cset_fast_test_adr_type = NULL; // debug-mode-only argument 2509 debug_only(in_cset_fast_test_adr_type = phase->C->get_adr_type(in_cset_fast_test_idx)); 2510 Node* in_cset_fast_test_load = new LoadBNode(ctrl, raw_mem, in_cset_fast_test_adr, in_cset_fast_test_adr_type, TypeInt::BYTE, MemNode::unordered); 2511 phase->register_new_node(in_cset_fast_test_load, ctrl); 2512 Node* in_cset_fast_test_cmp = new CmpINode(in_cset_fast_test_load, phase->igvn().zerocon(T_INT)); 2513 phase->register_new_node(in_cset_fast_test_cmp, ctrl); 2514 Node* in_cset_fast_test_test = new BoolNode(in_cset_fast_test_cmp, BoolTest::eq); 2515 phase->register_new_node(in_cset_fast_test_test, ctrl); 2516 IfNode* in_cset_fast_test_iff = new IfNode(ctrl, in_cset_fast_test_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN); 2517 phase->register_control(in_cset_fast_test_iff, loop, ctrl); 2518 2519 not_cset_ctrl = new IfTrueNode(in_cset_fast_test_iff); 2520 phase->register_control(not_cset_ctrl, loop, in_cset_fast_test_iff); 2521 2522 ctrl = new IfFalseNode(in_cset_fast_test_iff); 2523 phase->register_control(ctrl, loop, in_cset_fast_test_iff); 2524 } 2525 2526 void ShenandoahWriteBarrierNode::call_wb_stub(Node*& ctrl, Node*& val, Node*& result_mem, 2527 Node* raw_mem, Node* wb_mem, 2528 int alias, 2529 PhaseIdealLoop* phase) { 2530 IdealLoopTree*loop = phase->get_loop(ctrl); 2531 const TypePtr* obj_type = phase->igvn().type(val)->is_oopptr()->cast_to_nonconst(); 2532 2533 // The slow path stub consumes and produces raw memory in addition 2534 // to the existing memory edges 2535 Node* base = find_bottom_mem(ctrl, phase); 2536 2537 MergeMemNode* mm = MergeMemNode::make(base); 2538 mm->set_memory_at(alias, wb_mem); 2539 mm->set_memory_at(Compile::AliasIdxRaw, raw_mem); 2540 phase->register_new_node(mm, ctrl); 2541 2542 Node* call = new CallLeafNode(ShenandoahBarrierSetC2::shenandoah_write_barrier_Type(), CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_JRT), "shenandoah_write_barrier", TypeRawPtr::BOTTOM); 2543 call->init_req(TypeFunc::Control, ctrl); 2544 call->init_req(TypeFunc::I_O, phase->C->top()); 2545 call->init_req(TypeFunc::Memory, mm); 2546 call->init_req(TypeFunc::FramePtr, phase->C->top()); 2547 call->init_req(TypeFunc::ReturnAdr, phase->C->top()); 2548 call->init_req(TypeFunc::Parms, val); 2549 phase->register_control(call, loop, ctrl); 2550 ctrl = new ProjNode(call, TypeFunc::Control); 2551 phase->register_control(ctrl, loop, call); 2552 result_mem = new ProjNode(call, TypeFunc::Memory); 2553 phase->register_new_node(result_mem, call); 2554 val = new ProjNode(call, TypeFunc::Parms); 2555 phase->register_new_node(val, call); 2556 val = new CheckCastPPNode(ctrl, val, obj_type); 2557 phase->register_new_node(val, ctrl); 2558 } 2559 2560 void ShenandoahWriteBarrierNode::fix_ctrl(Node* barrier, Node* region, const MemoryGraphFixer& fixer, Unique_Node_List& uses, Unique_Node_List& uses_to_ignore, uint last, PhaseIdealLoop* phase) { 2561 Node* ctrl = phase->get_ctrl(barrier); 2562 Node* init_raw_mem = fixer.find_mem(ctrl, barrier); 2563 2564 // Update the control of all nodes that should be after the 2565 // barrier control flow 2566 uses.clear(); 2567 // Every node that is control dependent on the barrier's input 2568 // control will be after the expanded barrier. The raw memory (if 2569 // its memory is control dependent on the barrier's input control) 2570 // must stay above the barrier. 2571 uses_to_ignore.clear(); 2572 if (phase->has_ctrl(init_raw_mem) && phase->get_ctrl(init_raw_mem) == ctrl && !init_raw_mem->is_Phi()) { 2573 uses_to_ignore.push(init_raw_mem); 2574 } 2575 for (uint next = 0; next < uses_to_ignore.size(); next++) { 2576 Node *n = uses_to_ignore.at(next); 2577 for (uint i = 0; i < n->req(); i++) { 2578 Node* in = n->in(i); 2579 if (in != NULL && phase->has_ctrl(in) && phase->get_ctrl(in) == ctrl) { 2580 uses_to_ignore.push(in); 2581 } 2582 } 2583 } 2584 for (DUIterator_Fast imax, i = ctrl->fast_outs(imax); i < imax; i++) { 2585 Node* u = ctrl->fast_out(i); 2586 if (u->_idx < last && 2587 u != barrier && 2588 !uses_to_ignore.member(u) && 2589 (u->in(0) != ctrl || (!u->is_Region() && !u->is_Phi())) && 2590 (ctrl->Opcode() != Op_CatchProj || u->Opcode() != Op_CreateEx)) { 2591 Node* old_c = phase->ctrl_or_self(u); 2592 Node* c = old_c; 2593 if (c != ctrl || 2594 is_dominator_same_ctrl(old_c, barrier, u, phase) || 2595 ShenandoahBarrierSetC2::is_shenandoah_state_load(u)) { 2596 phase->igvn().rehash_node_delayed(u); 2597 int nb = u->replace_edge(ctrl, region); 2598 if (u->is_CFG()) { 2599 if (phase->idom(u) == ctrl) { 2600 phase->set_idom(u, region, phase->dom_depth(region)); 2601 } 2602 } else if (phase->get_ctrl(u) == ctrl) { 2603 assert(u != init_raw_mem, "should leave input raw mem above the barrier"); 2604 uses.push(u); 2605 } 2606 assert(nb == 1, "more than 1 ctrl input?"); 2607 --i, imax -= nb; 2608 } 2609 } 2610 } 2611 } 2612 2613 void ShenandoahWriteBarrierNode::pin_and_expand(PhaseIdealLoop* phase) { 2614 Node_List enqueue_barriers; 2615 if (ShenandoahStoreValEnqueueBarrier) { 2616 Unique_Node_List wq; 2617 wq.push(phase->C->root()); 2618 for (uint i = 0; i < wq.size(); i++) { 2619 Node* n = wq.at(i); 2620 if (n->Opcode() == Op_ShenandoahEnqueueBarrier) { 2621 enqueue_barriers.push(n); 2622 } 2623 for (uint i = 0; i < n->req(); i++) { 2624 Node* in = n->in(i); 2625 if (in != NULL) { 2626 wq.push(in); 2627 } 2628 } 2629 } 2630 } 2631 2632 const bool trace = false; 2633 2634 // Collect raw memory state at CFG points in the entire graph and 2635 // record it in memory_nodes. Optimize the raw memory graph in the 2636 // process. Optimizing the memory graph also makes the memory graph 2637 // simpler. 2638 GrowableArray<MemoryGraphFixer*> memory_graph_fixers; 2639 2640 // Let's try to common write barriers again 2641 optimize_before_expansion(phase, memory_graph_fixers, true); 2642 2643 Unique_Node_List uses; 2644 for (int i = 0; i < ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count(); i++) { 2645 ShenandoahWriteBarrierNode* wb = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barrier(i); 2646 Node* ctrl = phase->get_ctrl(wb); 2647 2648 Node* val = wb->in(ValueIn); 2649 if (ctrl->is_Proj() && ctrl->in(0)->is_CallJava()) { 2650 assert(is_dominator(phase->get_ctrl(val), ctrl->in(0)->in(0), val, ctrl->in(0), phase), "can't move"); 2651 phase->set_ctrl(wb, ctrl->in(0)->in(0)); 2652 } else if (ctrl->is_CallRuntime()) { 2653 assert(is_dominator(phase->get_ctrl(val), ctrl->in(0), val, ctrl, phase), "can't move"); 2654 phase->set_ctrl(wb, ctrl->in(0)); 2655 } 2656 2657 assert(wb->Opcode() == Op_ShenandoahWriteBarrier, "only for write barriers"); 2658 // Look for a null check that dominates this barrier and move the 2659 // barrier right after the null check to enable implicit null 2660 // checks 2661 wb->pin_and_expand_move_barrier(phase, memory_graph_fixers, uses); 2662 2663 wb->pin_and_expand_helper(phase); 2664 } 2665 2666 for (uint i = 0; i < enqueue_barriers.size(); i++) { 2667 Node* barrier = enqueue_barriers.at(i); 2668 Node* ctrl = phase->get_ctrl(barrier); 2669 IdealLoopTree* loop = phase->get_loop(ctrl); 2670 if (loop->_head->is_OuterStripMinedLoop()) { 2671 // Expanding a barrier here will break loop strip mining 2672 // verification. Transform the loop so the loop nest doesn't 2673 // appear as strip mined. 2674 OuterStripMinedLoopNode* outer = loop->_head->as_OuterStripMinedLoop(); 2675 hide_strip_mined_loop(outer, outer->unique_ctrl_out()->as_CountedLoop(), phase); 2676 } 2677 } 2678 2679 for (int i = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count(); i > 0; i--) { 2680 int cnt = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count(); 2681 ShenandoahWriteBarrierNode* wb = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barrier(i-1); 2682 Node* ctrl = phase->get_ctrl(wb); 2683 IdealLoopTree* loop = phase->get_loop(ctrl); 2684 if (loop->_head->is_OuterStripMinedLoop()) { 2685 // Expanding a barrier here will break loop strip mining 2686 // verification. Transform the loop so the loop nest doesn't 2687 // appear as strip mined. 2688 OuterStripMinedLoopNode* outer = loop->_head->as_OuterStripMinedLoop(); 2689 hide_strip_mined_loop(outer, outer->unique_ctrl_out()->as_CountedLoop(), phase); 2690 } 2691 } 2692 2693 MemoryGraphFixer fixer(Compile::AliasIdxRaw, true, phase); 2694 Unique_Node_List uses_to_ignore; 2695 for (uint i = 0; i < enqueue_barriers.size(); i++) { 2696 Node* barrier = enqueue_barriers.at(i); 2697 Node* pre_val = barrier->in(1); 2698 2699 if (phase->igvn().type(pre_val)->higher_equal(TypePtr::NULL_PTR)) { 2700 ShouldNotReachHere(); 2701 continue; 2702 } 2703 2704 Node* ctrl = phase->get_ctrl(barrier); 2705 2706 if (ctrl->is_Proj() && ctrl->in(0)->is_CallJava()) { 2707 assert(is_dominator(phase->get_ctrl(pre_val), ctrl->in(0)->in(0), pre_val, ctrl->in(0), phase), "can't move"); 2708 ctrl = ctrl->in(0)->in(0); 2709 phase->set_ctrl(barrier, ctrl); 2710 } else if (ctrl->is_CallRuntime()) { 2711 assert(is_dominator(phase->get_ctrl(pre_val), ctrl->in(0), pre_val, ctrl, phase), "can't move"); 2712 ctrl = ctrl->in(0); 2713 phase->set_ctrl(barrier, ctrl); 2714 } 2715 2716 Node* init_ctrl = ctrl; 2717 IdealLoopTree* loop = phase->get_loop(ctrl); 2718 Node* raw_mem = fixer.find_mem(ctrl, barrier); 2719 Node* init_raw_mem = raw_mem; 2720 Node* raw_mem_for_ctrl = fixer.find_mem(ctrl, NULL); 2721 Node* heap_stable_ctrl = NULL; 2722 Node* null_ctrl = NULL; 2723 uint last = phase->C->unique(); 2724 2725 enum { _heap_stable = 1, _heap_unstable, PATH_LIMIT }; 2726 Node* region = new RegionNode(PATH_LIMIT); 2727 Node* phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM); 2728 2729 enum { _fast_path = 1, _slow_path, _null_path, PATH_LIMIT2 }; 2730 Node* region2 = new RegionNode(PATH_LIMIT2); 2731 Node* phi2 = PhiNode::make(region2, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM); 2732 2733 // Stable path. 2734 test_heap_stable(ctrl, raw_mem, heap_stable_ctrl, phase); 2735 region->init_req(_heap_stable, heap_stable_ctrl); 2736 phi->init_req(_heap_stable, raw_mem); 2737 2738 // Null path 2739 Node* reg2_ctrl = NULL; 2740 test_null(ctrl, pre_val, null_ctrl, phase); 2741 if (null_ctrl != NULL) { 2742 reg2_ctrl = null_ctrl->in(0); 2743 region2->init_req(_null_path, null_ctrl); 2744 phi2->init_req(_null_path, raw_mem); 2745 } else { 2746 region2->del_req(_null_path); 2747 phi2->del_req(_null_path); 2748 } 2749 2750 const int index_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()); 2751 const int buffer_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()); 2752 Node* thread = new ThreadLocalNode(); 2753 phase->register_new_node(thread, ctrl); 2754 Node* buffer_adr = new AddPNode(phase->C->top(), thread, phase->igvn().MakeConX(buffer_offset)); 2755 phase->register_new_node(buffer_adr, ctrl); 2756 Node* index_adr = new AddPNode(phase->C->top(), thread, phase->igvn().MakeConX(index_offset)); 2757 phase->register_new_node(index_adr, ctrl); 2758 2759 BasicType index_bt = TypeX_X->basic_type(); 2760 assert(sizeof(size_t) == type2aelembytes(index_bt), "Loading G1 SATBMarkQueue::_index with wrong size."); 2761 const TypePtr* adr_type = TypeRawPtr::BOTTOM; 2762 Node* index = new LoadXNode(ctrl, raw_mem, index_adr, adr_type, TypeX_X, MemNode::unordered); 2763 phase->register_new_node(index, ctrl); 2764 Node* index_cmp = new CmpXNode(index, phase->igvn().MakeConX(0)); 2765 phase->register_new_node(index_cmp, ctrl); 2766 Node* index_test = new BoolNode(index_cmp, BoolTest::ne); 2767 phase->register_new_node(index_test, ctrl); 2768 IfNode* queue_full_iff = new IfNode(ctrl, index_test, PROB_LIKELY(0.999), COUNT_UNKNOWN); 2769 if (reg2_ctrl == NULL) reg2_ctrl = queue_full_iff; 2770 phase->register_control(queue_full_iff, loop, ctrl); 2771 Node* not_full = new IfTrueNode(queue_full_iff); 2772 phase->register_control(not_full, loop, queue_full_iff); 2773 Node* full = new IfFalseNode(queue_full_iff); 2774 phase->register_control(full, loop, queue_full_iff); 2775 2776 ctrl = not_full; 2777 2778 Node* next_index = new SubXNode(index, phase->igvn().MakeConX(sizeof(intptr_t))); 2779 phase->register_new_node(next_index, ctrl); 2780 2781 Node* buffer = new LoadPNode(ctrl, raw_mem, buffer_adr, adr_type, TypeRawPtr::NOTNULL, MemNode::unordered); 2782 phase->register_new_node(buffer, ctrl); 2783 Node *log_addr = new AddPNode(phase->C->top(), buffer, next_index); 2784 phase->register_new_node(log_addr, ctrl); 2785 Node* log_store = new StorePNode(ctrl, raw_mem, log_addr, adr_type, pre_val, MemNode::unordered); 2786 phase->register_new_node(log_store, ctrl); 2787 // update the index 2788 Node* index_update = new StoreXNode(ctrl, log_store, index_adr, adr_type, next_index, MemNode::unordered); 2789 phase->register_new_node(index_update, ctrl); 2790 2791 // Fast-path case 2792 region2->init_req(_fast_path, ctrl); 2793 phi2->init_req(_fast_path, index_update); 2794 2795 ctrl = full; 2796 2797 Node* base = find_bottom_mem(ctrl, phase); 2798 2799 MergeMemNode* mm = MergeMemNode::make(base); 2800 mm->set_memory_at(Compile::AliasIdxRaw, raw_mem); 2801 phase->register_new_node(mm, ctrl); 2802 2803 Node* call = new CallLeafNode(ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type(), CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), "shenandoah_wb_pre", TypeRawPtr::BOTTOM); 2804 call->init_req(TypeFunc::Control, ctrl); 2805 call->init_req(TypeFunc::I_O, phase->C->top()); 2806 call->init_req(TypeFunc::Memory, mm); 2807 call->init_req(TypeFunc::FramePtr, phase->C->top()); 2808 call->init_req(TypeFunc::ReturnAdr, phase->C->top()); 2809 call->init_req(TypeFunc::Parms, pre_val); 2810 call->init_req(TypeFunc::Parms+1, thread); 2811 phase->register_control(call, loop, ctrl); 2812 2813 Node* ctrl_proj = new ProjNode(call, TypeFunc::Control); 2814 phase->register_control(ctrl_proj, loop, call); 2815 Node* mem_proj = new ProjNode(call, TypeFunc::Memory); 2816 phase->register_new_node(mem_proj, call); 2817 2818 // Slow-path case 2819 region2->init_req(_slow_path, ctrl_proj); 2820 phi2->init_req(_slow_path, mem_proj); 2821 2822 phase->register_control(region2, loop, reg2_ctrl); 2823 phase->register_new_node(phi2, region2); 2824 2825 region->init_req(_heap_unstable, region2); 2826 phi->init_req(_heap_unstable, phi2); 2827 2828 phase->register_control(region, loop, heap_stable_ctrl->in(0)); 2829 phase->register_new_node(phi, region); 2830 2831 fix_ctrl(barrier, region, fixer, uses, uses_to_ignore, last, phase); 2832 for(uint next = 0; next < uses.size(); next++ ) { 2833 Node *n = uses.at(next); 2834 assert(phase->get_ctrl(n) == init_ctrl, "bad control"); 2835 assert(n != init_raw_mem, "should leave input raw mem above the barrier"); 2836 phase->set_ctrl(n, region); 2837 follow_barrier_uses(n, init_ctrl, uses, phase); 2838 } 2839 fixer.fix_mem(init_ctrl, region, init_raw_mem, raw_mem_for_ctrl, phi, uses); 2840 2841 phase->igvn().replace_node(barrier, pre_val); 2842 } 2843 2844 for (int i = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count(); i > 0; i--) { 2845 int cnt = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count(); 2846 ShenandoahWriteBarrierNode* wb = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barrier(i-1); 2847 2848 uint last = phase->C->unique(); 2849 Node* ctrl = phase->get_ctrl(wb); 2850 Node* orig_ctrl = ctrl; 2851 2852 Node* raw_mem = fixer.find_mem(ctrl, wb); 2853 Node* init_raw_mem = raw_mem; 2854 Node* raw_mem_for_ctrl = fixer.find_mem(ctrl, NULL); 2855 int alias = phase->C->get_alias_index(wb->adr_type()); 2856 Node* wb_mem = wb->in(Memory); 2857 Node* init_wb_mem = wb_mem; 2858 2859 Node* val = wb->in(ValueIn); 2860 Node* wbproj = wb->find_out_with(Op_ShenandoahWBMemProj); 2861 IdealLoopTree *loop = phase->get_loop(ctrl); 2862 2863 assert(val->Opcode() != Op_ShenandoahWriteBarrier, "No chain of write barriers"); 2864 2865 CallStaticJavaNode* unc = wb->pin_and_expand_null_check(phase->igvn()); 2866 Node* unc_ctrl = NULL; 2867 if (unc != NULL) { 2868 if (val->in(0) != ctrl) { 2869 unc = NULL; 2870 } else { 2871 unc_ctrl = val->in(0); 2872 } 2873 } 2874 2875 Node* uncasted_val = val; 2876 if (unc != NULL) { 2877 uncasted_val = val->in(1); 2878 } 2879 2880 Node* heap_stable_ctrl = NULL; 2881 Node* null_ctrl = NULL; 2882 2883 assert(val->bottom_type()->make_oopptr(), "need oop"); 2884 assert(val->bottom_type()->make_oopptr()->const_oop() == NULL, "expect non-constant"); 2885 2886 enum { _heap_stable = 1, _heap_unstable, PATH_LIMIT }; 2887 Node* region = new RegionNode(PATH_LIMIT); 2888 Node* val_phi = new PhiNode(region, uncasted_val->bottom_type()->is_oopptr()); 2889 Node* mem_phi = PhiNode::make(region, wb_mem, Type::MEMORY, phase->C->alias_type(wb->adr_type())->adr_type()); 2890 Node* raw_mem_phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM); 2891 2892 enum { _not_cset = 1, _not_equal, _evac_path, _null_path, PATH_LIMIT2 }; 2893 Node* region2 = new RegionNode(PATH_LIMIT2); 2894 Node* val_phi2 = new PhiNode(region2, uncasted_val->bottom_type()->is_oopptr()); 2895 Node* mem_phi2 = PhiNode::make(region2, wb_mem, Type::MEMORY, phase->C->alias_type(wb->adr_type())->adr_type()); 2896 Node* raw_mem_phi2 = PhiNode::make(region2, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM); 2897 2898 // Stable path. 2899 test_heap_stable(ctrl, raw_mem, heap_stable_ctrl, phase); 2900 IfNode* heap_stable_iff = heap_stable_ctrl->in(0)->as_If(); 2901 2902 // Heap stable case 2903 region->init_req(_heap_stable, heap_stable_ctrl); 2904 val_phi->init_req(_heap_stable, uncasted_val); 2905 mem_phi->init_req(_heap_stable, wb_mem); 2906 raw_mem_phi->init_req(_heap_stable, raw_mem); 2907 2908 Node* reg2_ctrl = NULL; 2909 // Null case 2910 test_null(ctrl, val, null_ctrl, phase); 2911 if (null_ctrl != NULL) { 2912 reg2_ctrl = null_ctrl->in(0); 2913 region2->init_req(_null_path, null_ctrl); 2914 val_phi2->init_req(_null_path, uncasted_val); 2915 mem_phi2->init_req(_null_path, wb_mem); 2916 raw_mem_phi2->init_req(_null_path, raw_mem); 2917 } else { 2918 region2->del_req(_null_path); 2919 val_phi2->del_req(_null_path); 2920 mem_phi2->del_req(_null_path); 2921 raw_mem_phi2->del_req(_null_path); 2922 } 2923 2924 // Test for in-cset. 2925 // Wires !in_cset(obj) to slot 2 of region and phis 2926 Node* not_cset_ctrl = NULL; 2927 in_cset_fast_test(ctrl, not_cset_ctrl, uncasted_val, raw_mem, phase); 2928 if (not_cset_ctrl != NULL) { 2929 if (reg2_ctrl == NULL) reg2_ctrl = not_cset_ctrl->in(0); 2930 region2->init_req(_not_cset, not_cset_ctrl); 2931 val_phi2->init_req(_not_cset, uncasted_val); 2932 mem_phi2->init_req(_not_cset, wb_mem); 2933 raw_mem_phi2->init_req(_not_cset, raw_mem); 2934 } 2935 2936 // Resolve object when orig-value is in cset. 2937 // Make the unconditional resolve for fwdptr, not the read barrier. 2938 Node* new_val = uncasted_val; 2939 if (unc_ctrl != NULL) { 2940 // Clone the null check in this branch to allow implicit null check 2941 new_val = clone_null_check(ctrl, val, unc_ctrl, phase); 2942 fix_null_check(unc, unc_ctrl, ctrl->in(0)->as_If()->proj_out(0), uses, phase); 2943 2944 IfNode* iff = unc_ctrl->in(0)->as_If(); 2945 phase->igvn().replace_input_of(iff, 1, phase->igvn().intcon(1)); 2946 } 2947 Node* addr = new AddPNode(new_val, uncasted_val, phase->igvn().MakeConX(ShenandoahBrooksPointer::byte_offset())); 2948 phase->register_new_node(addr, ctrl); 2949 assert(val->bottom_type()->isa_oopptr(), "what else?"); 2950 const TypePtr* obj_type = val->bottom_type()->is_oopptr(); 2951 const TypePtr* adr_type = ShenandoahBarrierNode::brooks_pointer_type(obj_type); 2952 Node* fwd = new LoadPNode(ctrl, wb_mem, addr, adr_type, obj_type, MemNode::unordered); 2953 phase->register_new_node(fwd, ctrl); 2954 2955 // Only branch to WB stub if object is not forwarded; otherwise reply with fwd ptr 2956 Node* cmp = new CmpPNode(fwd, new_val); 2957 phase->register_new_node(cmp, ctrl); 2958 Node* bol = new BoolNode(cmp, BoolTest::eq); 2959 phase->register_new_node(bol, ctrl); 2960 2961 IfNode* iff = new IfNode(ctrl, bol, PROB_UNLIKELY(0.999), COUNT_UNKNOWN); 2962 if (reg2_ctrl == NULL) reg2_ctrl = iff; 2963 phase->register_control(iff, loop, ctrl); 2964 Node* if_not_eq = new IfFalseNode(iff); 2965 phase->register_control(if_not_eq, loop, iff); 2966 Node* if_eq = new IfTrueNode(iff); 2967 phase->register_control(if_eq, loop, iff); 2968 2969 // Wire up not-equal-path in slots 3. 2970 region2->init_req(_not_equal, if_not_eq); 2971 val_phi2->init_req(_not_equal, fwd); 2972 mem_phi2->init_req(_not_equal, wb_mem); 2973 raw_mem_phi2->init_req(_not_equal, raw_mem); 2974 2975 // Call wb-stub and wire up that path in slots 4 2976 Node* result_mem = NULL; 2977 ctrl = if_eq; 2978 call_wb_stub(ctrl, new_val, result_mem, 2979 raw_mem, wb_mem, 2980 alias, phase); 2981 region2->init_req(_evac_path, ctrl); 2982 val_phi2->init_req(_evac_path, new_val); 2983 mem_phi2->init_req(_evac_path, result_mem); 2984 raw_mem_phi2->init_req(_evac_path, result_mem); 2985 2986 phase->register_control(region2, loop, reg2_ctrl); 2987 phase->register_new_node(val_phi2, region2); 2988 phase->register_new_node(mem_phi2, region2); 2989 phase->register_new_node(raw_mem_phi2, region2); 2990 2991 region->init_req(_heap_unstable, region2); 2992 val_phi->init_req(_heap_unstable, val_phi2); 2993 mem_phi->init_req(_heap_unstable, mem_phi2); 2994 raw_mem_phi->init_req(_heap_unstable, raw_mem_phi2); 2995 2996 phase->register_control(region, loop, heap_stable_iff); 2997 Node* out_val = val_phi; 2998 phase->register_new_node(val_phi, region); 2999 phase->register_new_node(mem_phi, region); 3000 phase->register_new_node(raw_mem_phi, region); 3001 3002 fix_ctrl(wb, region, fixer, uses, uses_to_ignore, last, phase); 3003 3004 ctrl = orig_ctrl; 3005 3006 phase->igvn().replace_input_of(wbproj, ShenandoahWBMemProjNode::WriteBarrier, phase->C->top()); 3007 phase->igvn().replace_node(wbproj, mem_phi); 3008 if (unc != NULL) { 3009 for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) { 3010 Node* u = val->fast_out(i); 3011 Node* c = phase->ctrl_or_self(u); 3012 if (u != wb && (c != ctrl || is_dominator_same_ctrl(c, wb, u, phase))) { 3013 phase->igvn().rehash_node_delayed(u); 3014 int nb = u->replace_edge(val, out_val); 3015 --i, imax -= nb; 3016 } 3017 } 3018 if (val->outcnt() == 0) { 3019 phase->igvn()._worklist.push(val); 3020 } 3021 } 3022 phase->igvn().replace_node(wb, out_val); 3023 3024 follow_barrier_uses(mem_phi, ctrl, uses, phase); 3025 follow_barrier_uses(out_val, ctrl, uses, phase); 3026 3027 for(uint next = 0; next < uses.size(); next++ ) { 3028 Node *n = uses.at(next); 3029 assert(phase->get_ctrl(n) == ctrl, "bad control"); 3030 assert(n != init_raw_mem, "should leave input raw mem above the barrier"); 3031 phase->set_ctrl(n, region); 3032 follow_barrier_uses(n, ctrl, uses, phase); 3033 } 3034 3035 // The slow path call produces memory: hook the raw memory phi 3036 // from the expanded write barrier with the rest of the graph 3037 // which may require adding memory phis at every post dominated 3038 // region and at enclosing loop heads. Use the memory state 3039 // collected in memory_nodes to fix the memory graph. Update that 3040 // memory state as we go. 3041 fixer.fix_mem(ctrl, region, init_raw_mem, raw_mem_for_ctrl, raw_mem_phi, uses); 3042 assert(ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count() == cnt - 1, "not replaced"); 3043 } 3044 3045 assert(ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count() == 0, "all write barrier nodes should have been replaced"); 3046 } 3047 3048 void ShenandoahWriteBarrierNode::move_heap_stable_test_out_of_loop(IfNode* iff, PhaseIdealLoop* phase) { 3049 IdealLoopTree *loop = phase->get_loop(iff); 3050 Node* loop_head = loop->_head; 3051 Node* entry_c = loop_head->in(LoopNode::EntryControl); 3052 3053 Node* bol = iff->in(1); 3054 Node* cmp = bol->in(1); 3055 Node* andi = cmp->in(1); 3056 Node* load = andi->in(1); 3057 3058 assert(is_gc_state_load(load), "broken"); 3059 if (!phase->is_dominator(load->in(0), entry_c)) { 3060 Node* mem_ctrl = NULL; 3061 Node* mem = dom_mem(load->in(MemNode::Memory), loop_head, Compile::AliasIdxRaw, mem_ctrl, phase); 3062 load = load->clone(); 3063 load->set_req(MemNode::Memory, mem); 3064 load->set_req(0, entry_c); 3065 phase->register_new_node(load, entry_c); 3066 andi = andi->clone(); 3067 andi->set_req(1, load); 3068 phase->register_new_node(andi, entry_c); 3069 cmp = cmp->clone(); 3070 cmp->set_req(1, andi); 3071 phase->register_new_node(cmp, entry_c); 3072 bol = bol->clone(); 3073 bol->set_req(1, cmp); 3074 phase->register_new_node(bol, entry_c); 3075 3076 Node* old_bol =iff->in(1); 3077 phase->igvn().replace_input_of(iff, 1, bol); 3078 } 3079 } 3080 3081 bool ShenandoahWriteBarrierNode::identical_backtoback_ifs(Node *n, PhaseIdealLoop* phase) { 3082 if (!n->is_If() || n->is_CountedLoopEnd()) { 3083 return false; 3084 } 3085 Node* region = n->in(0); 3086 3087 if (!region->is_Region()) { 3088 return false; 3089 } 3090 Node* dom = phase->idom(region); 3091 if (!dom->is_If()) { 3092 return false; 3093 } 3094 3095 if (!is_heap_stable_test(n) || !is_heap_stable_test(dom)) { 3096 return false; 3097 } 3098 3099 IfNode* dom_if = dom->as_If(); 3100 Node* proj_true = dom_if->proj_out(1); 3101 Node* proj_false = dom_if->proj_out(0); 3102 3103 for (uint i = 1; i < region->req(); i++) { 3104 if (phase->is_dominator(proj_true, region->in(i))) { 3105 continue; 3106 } 3107 if (phase->is_dominator(proj_false, region->in(i))) { 3108 continue; 3109 } 3110 return false; 3111 } 3112 3113 return true; 3114 } 3115 3116 void ShenandoahWriteBarrierNode::merge_back_to_back_tests(Node* n, PhaseIdealLoop* phase) { 3117 assert(is_heap_stable_test(n), "no other tests"); 3118 if (identical_backtoback_ifs(n, phase)) { 3119 Node* n_ctrl = n->in(0); 3120 if (phase->can_split_if(n_ctrl)) { 3121 IfNode* dom_if = phase->idom(n_ctrl)->as_If(); 3122 if (is_heap_stable_test(n)) { 3123 Node* gc_state_load = n->in(1)->in(1)->in(1)->in(1); 3124 assert(is_gc_state_load(gc_state_load), "broken"); 3125 Node* dom_gc_state_load = dom_if->in(1)->in(1)->in(1)->in(1); 3126 assert(is_gc_state_load(dom_gc_state_load), "broken"); 3127 if (gc_state_load != dom_gc_state_load) { 3128 phase->igvn().replace_node(gc_state_load, dom_gc_state_load); 3129 } 3130 } 3131 PhiNode* bolphi = PhiNode::make_blank(n_ctrl, n->in(1)); 3132 Node* proj_true = dom_if->proj_out(1); 3133 Node* proj_false = dom_if->proj_out(0); 3134 Node* con_true = phase->igvn().makecon(TypeInt::ONE); 3135 Node* con_false = phase->igvn().makecon(TypeInt::ZERO); 3136 3137 for (uint i = 1; i < n_ctrl->req(); i++) { 3138 if (phase->is_dominator(proj_true, n_ctrl->in(i))) { 3139 bolphi->init_req(i, con_true); 3140 } else { 3141 assert(phase->is_dominator(proj_false, n_ctrl->in(i)), "bad if"); 3142 bolphi->init_req(i, con_false); 3143 } 3144 } 3145 phase->register_new_node(bolphi, n_ctrl); 3146 phase->igvn().replace_input_of(n, 1, bolphi); 3147 phase->do_split_if(n); 3148 } 3149 } 3150 } 3151 3152 IfNode* ShenandoahWriteBarrierNode::find_unswitching_candidate(const IdealLoopTree *loop, PhaseIdealLoop* phase) { 3153 // Find first invariant test that doesn't exit the loop 3154 LoopNode *head = loop->_head->as_Loop(); 3155 IfNode* unswitch_iff = NULL; 3156 Node* n = head->in(LoopNode::LoopBackControl); 3157 int loop_has_sfpts = -1; 3158 while (n != head) { 3159 Node* n_dom = phase->idom(n); 3160 if (n->is_Region()) { 3161 if (n_dom->is_If()) { 3162 IfNode* iff = n_dom->as_If(); 3163 if (iff->in(1)->is_Bool()) { 3164 BoolNode* bol = iff->in(1)->as_Bool(); 3165 if (bol->in(1)->is_Cmp()) { 3166 // If condition is invariant and not a loop exit, 3167 // then found reason to unswitch. 3168 if (is_heap_stable_test(iff) && 3169 (loop_has_sfpts == -1 || loop_has_sfpts == 0)) { 3170 assert(!loop->is_loop_exit(iff), "both branches should be in the loop"); 3171 if (loop_has_sfpts == -1) { 3172 for(uint i = 0; i < loop->_body.size(); i++) { 3173 Node *m = loop->_body[i]; 3174 if (m->is_SafePoint() && !m->is_CallLeaf()) { 3175 loop_has_sfpts = 1; 3176 break; 3177 } 3178 } 3179 if (loop_has_sfpts == -1) { 3180 loop_has_sfpts = 0; 3181 } 3182 } 3183 if (!loop_has_sfpts) { 3184 unswitch_iff = iff; 3185 } 3186 } 3187 } 3188 } 3189 } 3190 } 3191 n = n_dom; 3192 } 3193 return unswitch_iff; 3194 } 3195 3196 3197 void ShenandoahWriteBarrierNode::optimize_after_expansion(VectorSet &visited, Node_Stack &stack, Node_List &old_new, PhaseIdealLoop* phase) { 3198 Node_List heap_stable_tests; 3199 Node_List gc_state_loads; 3200 3201 stack.push(phase->C->start(), 0); 3202 do { 3203 Node* n = stack.node(); 3204 uint i = stack.index(); 3205 3206 if (i < n->outcnt()) { 3207 Node* u = n->raw_out(i); 3208 stack.set_index(i+1); 3209 if (!visited.test_set(u->_idx)) { 3210 stack.push(u, 0); 3211 } 3212 } else { 3213 stack.pop(); 3214 if (ShenandoahCommonGCStateLoads && is_gc_state_load(n)) { 3215 gc_state_loads.push(n); 3216 } 3217 if (n->is_If() && is_heap_stable_test(n)) { 3218 heap_stable_tests.push(n); 3219 } 3220 } 3221 } while (stack.size() > 0); 3222 3223 bool progress; 3224 do { 3225 progress = false; 3226 for (uint i = 0; i < gc_state_loads.size(); i++) { 3227 Node* n = gc_state_loads.at(i); 3228 if (n->outcnt() != 0) { 3229 progress |= try_common_gc_state_load(n, phase); 3230 } 3231 } 3232 } while (progress); 3233 3234 for (uint i = 0; i < heap_stable_tests.size(); i++) { 3235 Node* n = heap_stable_tests.at(i); 3236 assert(is_heap_stable_test(n), "only evacuation test"); 3237 merge_back_to_back_tests(n, phase); 3238 } 3239 3240 if (!phase->C->major_progress()) { 3241 VectorSet seen(Thread::current()->resource_area()); 3242 for (uint i = 0; i < heap_stable_tests.size(); i++) { 3243 Node* n = heap_stable_tests.at(i); 3244 IdealLoopTree* loop = phase->get_loop(n); 3245 if (loop != phase->ltree_root() && 3246 loop->_child == NULL && 3247 !loop->_irreducible) { 3248 LoopNode* head = loop->_head->as_Loop(); 3249 if ((!head->is_CountedLoop() || head->as_CountedLoop()->is_main_loop() || head->as_CountedLoop()->is_normal_loop()) && 3250 !seen.test_set(head->_idx)) { 3251 IfNode* iff = find_unswitching_candidate(loop, phase); 3252 if (iff != NULL) { 3253 Node* bol = iff->in(1); 3254 if (head->is_strip_mined()) { 3255 head->verify_strip_mined(0); 3256 } 3257 move_heap_stable_test_out_of_loop(iff, phase); 3258 if (loop->policy_unswitching(phase)) { 3259 if (head->is_strip_mined()) { 3260 OuterStripMinedLoopNode* outer = head->as_CountedLoop()->outer_loop(); 3261 hide_strip_mined_loop(outer, head->as_CountedLoop(), phase); 3262 } 3263 phase->do_unswitching(loop, old_new); 3264 } else { 3265 // Not proceeding with unswitching. Move load back in 3266 // the loop. 3267 phase->igvn().replace_input_of(iff, 1, bol); 3268 } 3269 } 3270 } 3271 } 3272 } 3273 } 3274 } 3275 3276 #ifdef ASSERT 3277 void ShenandoahBarrierNode::verify_raw_mem(RootNode* root) { 3278 const bool trace = false; 3279 ResourceMark rm; 3280 Unique_Node_List nodes; 3281 Unique_Node_List controls; 3282 Unique_Node_List memories; 3283 3284 nodes.push(root); 3285 for (uint next = 0; next < nodes.size(); next++) { 3286 Node *n = nodes.at(next); 3287 if (ShenandoahBarrierSetC2::is_shenandoah_wb_call(n)) { 3288 controls.push(n); 3289 if (trace) { tty->print("XXXXXX verifying"); n->dump(); } 3290 for (uint next2 = 0; next2 < controls.size(); next2++) { 3291 Node *m = controls.at(next2); 3292 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) { 3293 Node* u = m->fast_out(i); 3294 if (u->is_CFG() && !u->is_Root() && 3295 !(u->Opcode() == Op_CProj && u->in(0)->Opcode() == Op_NeverBranch && u->as_Proj()->_con == 1) && 3296 !(u->is_Region() && u->unique_ctrl_out()->Opcode() == Op_Halt)) { 3297 if (trace) { tty->print("XXXXXX pushing control"); u->dump(); } 3298 controls.push(u); 3299 } 3300 } 3301 } 3302 memories.push(n->as_Call()->proj_out(TypeFunc::Memory)); 3303 for (uint next2 = 0; next2 < memories.size(); next2++) { 3304 Node *m = memories.at(next2); 3305 assert(m->bottom_type() == Type::MEMORY, ""); 3306 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) { 3307 Node* u = m->fast_out(i); 3308 if (u->bottom_type() == Type::MEMORY && (u->is_Mem() || u->is_ClearArray())) { 3309 if (trace) { tty->print("XXXXXX pushing memory"); u->dump(); } 3310 memories.push(u); 3311 } else if (u->is_LoadStore()) { 3312 if (trace) { tty->print("XXXXXX pushing memory"); u->find_out_with(Op_SCMemProj)->dump(); } 3313 memories.push(u->find_out_with(Op_SCMemProj)); 3314 } else if (u->is_MergeMem() && u->as_MergeMem()->memory_at(Compile::AliasIdxRaw) == m) { 3315 if (trace) { tty->print("XXXXXX pushing memory"); u->dump(); } 3316 memories.push(u); 3317 } else if (u->is_Phi()) { 3318 assert(u->bottom_type() == Type::MEMORY, ""); 3319 if (u->adr_type() == TypeRawPtr::BOTTOM || u->adr_type() == TypePtr::BOTTOM) { 3320 assert(controls.member(u->in(0)), ""); 3321 if (trace) { tty->print("XXXXXX pushing memory"); u->dump(); } 3322 memories.push(u); 3323 } 3324 } else if (u->is_SafePoint() || u->is_MemBar()) { 3325 for (DUIterator_Fast jmax, j = u->fast_outs(jmax); j < jmax; j++) { 3326 Node* uu = u->fast_out(j); 3327 if (uu->bottom_type() == Type::MEMORY) { 3328 if (trace) { tty->print("XXXXXX pushing memory"); uu->dump(); } 3329 memories.push(uu); 3330 } 3331 } 3332 } 3333 } 3334 } 3335 for (uint next2 = 0; next2 < controls.size(); next2++) { 3336 Node *m = controls.at(next2); 3337 if (m->is_Region()) { 3338 bool all_in = true; 3339 for (uint i = 1; i < m->req(); i++) { 3340 if (!controls.member(m->in(i))) { 3341 all_in = false; 3342 break; 3343 } 3344 } 3345 if (trace) { tty->print("XXX verifying %s", all_in ? "all in" : ""); m->dump(); } 3346 bool found_phi = false; 3347 for (DUIterator_Fast jmax, j = m->fast_outs(jmax); j < jmax && !found_phi; j++) { 3348 Node* u = m->fast_out(j); 3349 if (u->is_Phi() && memories.member(u)) { 3350 found_phi = true; 3351 for (uint i = 1; i < u->req() && found_phi; i++) { 3352 Node* k = u->in(i); 3353 if (memories.member(k) != controls.member(m->in(i))) { 3354 found_phi = false; 3355 } 3356 } 3357 } 3358 } 3359 assert(found_phi || all_in, ""); 3360 } 3361 } 3362 controls.clear(); 3363 memories.clear(); 3364 } 3365 for( uint i = 0; i < n->len(); ++i ) { 3366 Node *m = n->in(i); 3367 if (m != NULL) { 3368 nodes.push(m); 3369 } 3370 } 3371 } 3372 } 3373 #endif 3374 3375 const Type* ShenandoahEnqueueBarrierNode::bottom_type() const { 3376 if (in(1) == NULL || in(1)->is_top()) { 3377 return Type::TOP; 3378 } 3379 const Type* t = in(1)->bottom_type(); 3380 if (t == TypePtr::NULL_PTR) { 3381 return t; 3382 } 3383 return t->is_oopptr()->cast_to_nonconst(); 3384 } 3385 3386 const Type* ShenandoahEnqueueBarrierNode::Value(PhaseGVN* phase) const { 3387 if (in(1) == NULL) { 3388 return Type::TOP; 3389 } 3390 const Type* t = phase->type(in(1)); 3391 if (t == Type::TOP) { 3392 return Type::TOP; 3393 } 3394 if (t == TypePtr::NULL_PTR) { 3395 return t; 3396 } 3397 return t->is_oopptr()->cast_to_nonconst(); 3398 } 3399 3400 int ShenandoahEnqueueBarrierNode::needed(Node* n) { 3401 if (n == NULL || 3402 n->is_Allocate() || 3403 n->bottom_type() == TypePtr::NULL_PTR || 3404 (n->bottom_type()->make_oopptr() != NULL && n->bottom_type()->make_oopptr()->const_oop() != NULL)) { 3405 return NotNeeded; 3406 } 3407 if (n->is_Phi() || 3408 n->is_CMove()) { 3409 return MaybeNeeded; 3410 } 3411 return Needed; 3412 } 3413 3414 Node* ShenandoahEnqueueBarrierNode::next(Node* n) { 3415 for (;;) { 3416 if (n == NULL) { 3417 return n; 3418 } else if (n->bottom_type() == TypePtr::NULL_PTR) { 3419 return n; 3420 } else if (n->bottom_type()->make_oopptr() != NULL && n->bottom_type()->make_oopptr()->const_oop() != NULL) { 3421 return n; 3422 } else if (n->is_ConstraintCast() || 3423 n->Opcode() == Op_DecodeN || 3424 n->Opcode() == Op_EncodeP) { 3425 n = n->in(1); 3426 } else if (n->is_Proj()) { 3427 n = n->in(0); 3428 } else { 3429 return n; 3430 } 3431 } 3432 ShouldNotReachHere(); 3433 return NULL; 3434 } 3435 3436 Node* ShenandoahEnqueueBarrierNode::Identity(PhaseGVN* phase) { 3437 PhaseIterGVN* igvn = phase->is_IterGVN(); 3438 3439 Node* n = next(in(1)); 3440 3441 int cont = needed(n); 3442 3443 if (cont == NotNeeded) { 3444 return in(1); 3445 } else if (cont == MaybeNeeded) { 3446 if (igvn == NULL) { 3447 phase->record_for_igvn(this); 3448 return this; 3449 } else { 3450 ResourceMark rm; 3451 Unique_Node_List wq; 3452 uint wq_i = 0; 3453 3454 for (;;) { 3455 if (n->is_Phi()) { 3456 for (uint i = 1; i < n->req(); i++) { 3457 Node* m = n->in(i); 3458 if (m != NULL) { 3459 wq.push(m); 3460 } 3461 } 3462 } else { 3463 assert(n->is_CMove(), "nothing else here"); 3464 Node* m = n->in(CMoveNode::IfFalse); 3465 wq.push(m); 3466 m = n->in(CMoveNode::IfTrue); 3467 wq.push(m); 3468 } 3469 Node* orig_n = NULL; 3470 do { 3471 if (wq_i >= wq.size()) { 3472 return in(1); 3473 } 3474 n = wq.at(wq_i); 3475 wq_i++; 3476 orig_n = n; 3477 n = next(n); 3478 cont = needed(n); 3479 if (cont == Needed) { 3480 return this; 3481 } 3482 } while (cont != MaybeNeeded || (orig_n != n && wq.member(n))); 3483 } 3484 } 3485 } 3486 3487 return this; 3488 } 3489 3490 #ifdef ASSERT 3491 static bool has_never_branch(Node* root) { 3492 for (uint i = 1; i < root->req(); i++) { 3493 Node* in = root->in(i); 3494 if (in != NULL && in->Opcode() == Op_Halt && in->in(0)->is_Proj() && in->in(0)->in(0)->Opcode() == Op_NeverBranch) { 3495 return true; 3496 } 3497 } 3498 return false; 3499 } 3500 #endif 3501 3502 void MemoryGraphFixer::collect_memory_nodes() { 3503 Node_Stack stack(0); 3504 VectorSet visited(Thread::current()->resource_area()); 3505 Node_List regions; 3506 3507 // Walk the raw memory graph and create a mapping from CFG node to 3508 // memory node. Exclude phis for now. 3509 stack.push(_phase->C->root(), 1); 3510 do { 3511 Node* n = stack.node(); 3512 int opc = n->Opcode(); 3513 uint i = stack.index(); 3514 if (i < n->req()) { 3515 Node* mem = NULL; 3516 if (opc == Op_Root) { 3517 Node* in = n->in(i); 3518 int in_opc = in->Opcode(); 3519 if (in_opc == Op_Return || in_opc == Op_Rethrow) { 3520 mem = in->in(TypeFunc::Memory); 3521 } else if (in_opc == Op_Halt) { 3522 if (!in->in(0)->is_Region()) { 3523 Node* proj = in->in(0); 3524 assert(proj->is_Proj(), ""); 3525 Node* in = proj->in(0); 3526 assert(in->is_CallStaticJava() || in->Opcode() == Op_NeverBranch || in->Opcode() == Op_Catch || proj->is_IfProj(), ""); 3527 if (in->is_CallStaticJava()) { 3528 mem = in->in(TypeFunc::Memory); 3529 } else if (in->Opcode() == Op_Catch) { 3530 Node* call = in->in(0)->in(0); 3531 assert(call->is_Call(), ""); 3532 mem = call->in(TypeFunc::Memory); 3533 } 3534 } 3535 } else { 3536 #ifdef ASSERT 3537 n->dump(); 3538 in->dump(); 3539 #endif 3540 ShouldNotReachHere(); 3541 } 3542 } else { 3543 assert(n->is_Phi() && n->bottom_type() == Type::MEMORY, ""); 3544 assert(n->adr_type() == TypePtr::BOTTOM || _phase->C->get_alias_index(n->adr_type()) == _alias, ""); 3545 mem = n->in(i); 3546 } 3547 i++; 3548 stack.set_index(i); 3549 if (mem == NULL) { 3550 continue; 3551 } 3552 for (;;) { 3553 if (visited.test_set(mem->_idx) || mem->is_Start()) { 3554 break; 3555 } 3556 if (mem->is_Phi()) { 3557 stack.push(mem, 2); 3558 mem = mem->in(1); 3559 } else if (mem->is_Proj()) { 3560 stack.push(mem, mem->req()); 3561 mem = mem->in(0); 3562 } else if (mem->is_SafePoint() || mem->is_MemBar()) { 3563 mem = mem->in(TypeFunc::Memory); 3564 } else if (mem->is_MergeMem()) { 3565 MergeMemNode* mm = mem->as_MergeMem(); 3566 mem = mm->memory_at(_alias); 3567 } else if (mem->is_Store() || mem->is_LoadStore() || mem->is_ClearArray()) { 3568 assert(_alias == Compile::AliasIdxRaw, ""); 3569 stack.push(mem, mem->req()); 3570 mem = mem->in(MemNode::Memory); 3571 } else if (mem->Opcode() == Op_ShenandoahWriteBarrier) { 3572 assert(_alias != Compile::AliasIdxRaw, ""); 3573 mem = mem->in(ShenandoahBarrierNode::Memory); 3574 } else if (mem->Opcode() == Op_ShenandoahWBMemProj) { 3575 stack.push(mem, mem->req()); 3576 mem = mem->in(ShenandoahWBMemProjNode::WriteBarrier); 3577 } else { 3578 #ifdef ASSERT 3579 mem->dump(); 3580 #endif 3581 ShouldNotReachHere(); 3582 } 3583 } 3584 } else { 3585 if (n->is_Phi()) { 3586 // Nothing 3587 } else if (!n->is_Root()) { 3588 Node* c = get_ctrl(n); 3589 _memory_nodes.map(c->_idx, n); 3590 } 3591 stack.pop(); 3592 } 3593 } while(stack.is_nonempty()); 3594 3595 // Iterate over CFG nodes in rpo and propagate memory state to 3596 // compute memory state at regions, creating new phis if needed. 3597 Node_List rpo_list; 3598 visited.Clear(); 3599 _phase->rpo(_phase->C->root(), stack, visited, rpo_list); 3600 Node* root = rpo_list.pop(); 3601 assert(root == _phase->C->root(), ""); 3602 3603 const bool trace = false; 3604 #ifdef ASSERT 3605 if (trace) { 3606 for (int i = rpo_list.size() - 1; i >= 0; i--) { 3607 Node* c = rpo_list.at(i); 3608 if (_memory_nodes[c->_idx] != NULL) { 3609 tty->print("X %d", c->_idx); _memory_nodes[c->_idx]->dump(); 3610 } 3611 } 3612 } 3613 #endif 3614 uint last = _phase->C->unique(); 3615 3616 #ifdef ASSERT 3617 uint8_t max_depth = 0; 3618 for (LoopTreeIterator iter(_phase->ltree_root()); !iter.done(); iter.next()) { 3619 IdealLoopTree* lpt = iter.current(); 3620 max_depth = MAX2(max_depth, lpt->_nest); 3621 } 3622 #endif 3623 3624 bool progress = true; 3625 int iteration = 0; 3626 Node_List dead_phis; 3627 while (progress) { 3628 progress = false; 3629 iteration++; 3630 assert(iteration <= 2+max_depth || _phase->C->has_irreducible_loop(), ""); 3631 if (trace) { tty->print_cr("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"); } 3632 IdealLoopTree* last_updated_ilt = NULL; 3633 for (int i = rpo_list.size() - 1; i >= 0; i--) { 3634 Node* c = rpo_list.at(i); 3635 3636 Node* prev_mem = _memory_nodes[c->_idx]; 3637 if (c->is_Region() && (_include_lsm || !c->is_OuterStripMinedLoop())) { 3638 Node* prev_region = regions[c->_idx]; 3639 Node* unique = NULL; 3640 for (uint j = 1; j < c->req() && unique != NodeSentinel; j++) { 3641 Node* m = _memory_nodes[c->in(j)->_idx]; 3642 assert(m != NULL || (c->is_Loop() && j == LoopNode::LoopBackControl && iteration == 1) || _phase->C->has_irreducible_loop() || has_never_branch(_phase->C->root()), "expect memory state"); 3643 if (m != NULL) { 3644 if (m == prev_region && ((c->is_Loop() && j == LoopNode::LoopBackControl) || (prev_region->is_Phi() && prev_region->in(0) == c))) { 3645 assert(c->is_Loop() && j == LoopNode::LoopBackControl || _phase->C->has_irreducible_loop(), ""); 3646 // continue 3647 } else if (unique == NULL) { 3648 unique = m; 3649 } else if (m == unique) { 3650 // continue 3651 } else { 3652 unique = NodeSentinel; 3653 } 3654 } 3655 } 3656 assert(unique != NULL, "empty phi???"); 3657 if (unique != NodeSentinel) { 3658 if (prev_region != NULL && prev_region->is_Phi() && prev_region->in(0) == c) { 3659 dead_phis.push(prev_region); 3660 } 3661 regions.map(c->_idx, unique); 3662 } else { 3663 Node* phi = NULL; 3664 if (prev_region != NULL && prev_region->is_Phi() && prev_region->in(0) == c && prev_region->_idx >= last) { 3665 phi = prev_region; 3666 for (uint k = 1; k < c->req(); k++) { 3667 Node* m = _memory_nodes[c->in(k)->_idx]; 3668 assert(m != NULL, "expect memory state"); 3669 phi->set_req(k, m); 3670 } 3671 } else { 3672 for (DUIterator_Fast jmax, j = c->fast_outs(jmax); j < jmax && phi == NULL; j++) { 3673 Node* u = c->fast_out(j); 3674 if (u->is_Phi() && u->bottom_type() == Type::MEMORY && 3675 (u->adr_type() == TypePtr::BOTTOM || _phase->C->get_alias_index(u->adr_type()) == _alias)) { 3676 phi = u; 3677 for (uint k = 1; k < c->req() && phi != NULL; k++) { 3678 Node* m = _memory_nodes[c->in(k)->_idx]; 3679 assert(m != NULL, "expect memory state"); 3680 if (u->in(k) != m) { 3681 phi = NULL; 3682 } 3683 } 3684 } 3685 } 3686 if (phi == NULL) { 3687 phi = new PhiNode(c, Type::MEMORY, _phase->C->get_adr_type(_alias)); 3688 for (uint k = 1; k < c->req(); k++) { 3689 Node* m = _memory_nodes[c->in(k)->_idx]; 3690 assert(m != NULL, "expect memory state"); 3691 phi->init_req(k, m); 3692 } 3693 } 3694 } 3695 assert(phi != NULL, ""); 3696 regions.map(c->_idx, phi); 3697 } 3698 Node* current_region = regions[c->_idx]; 3699 if (current_region != prev_region) { 3700 progress = true; 3701 if (prev_region == prev_mem) { 3702 _memory_nodes.map(c->_idx, current_region); 3703 } 3704 } 3705 } else if (prev_mem == NULL || prev_mem->is_Phi() || ctrl_or_self(prev_mem) != c) { 3706 Node* m = _memory_nodes[_phase->idom(c)->_idx]; 3707 assert(m != NULL, "expect memory state"); 3708 if (m != prev_mem) { 3709 _memory_nodes.map(c->_idx, m); 3710 progress = true; 3711 } 3712 } 3713 #ifdef ASSERT 3714 if (trace) { tty->print("X %d", c->_idx); _memory_nodes[c->_idx]->dump(); } 3715 #endif 3716 } 3717 } 3718 3719 // Replace existing phi with computed memory state for that region 3720 // if different (could be a new phi or a dominating memory node if 3721 // that phi was found to be useless). 3722 while (dead_phis.size() > 0) { 3723 Node* n = dead_phis.pop(); 3724 n->replace_by(_phase->C->top()); 3725 n->destruct(); 3726 } 3727 for (int i = rpo_list.size() - 1; i >= 0; i--) { 3728 Node* c = rpo_list.at(i); 3729 if (c->is_Region() && (_include_lsm || !c->is_OuterStripMinedLoop())) { 3730 Node* n = regions[c->_idx]; 3731 if (n->is_Phi() && n->_idx >= last && n->in(0) == c) { 3732 _phase->register_new_node(n, c); 3733 } 3734 } 3735 } 3736 for (int i = rpo_list.size() - 1; i >= 0; i--) { 3737 Node* c = rpo_list.at(i); 3738 if (c->is_Region() && (_include_lsm || !c->is_OuterStripMinedLoop())) { 3739 Node* n = regions[c->_idx]; 3740 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) { 3741 Node* u = c->fast_out(i); 3742 if (u->is_Phi() && u->bottom_type() == Type::MEMORY && 3743 u != n) { 3744 if (u->adr_type() == TypePtr::BOTTOM) { 3745 fix_memory_uses(u, n, n, c); 3746 } else if (_phase->C->get_alias_index(u->adr_type()) == _alias) { 3747 _phase->lazy_replace(u, n); 3748 --i; --imax; 3749 } 3750 } 3751 } 3752 } 3753 } 3754 } 3755 3756 Node* MemoryGraphFixer::get_ctrl(Node* n) const { 3757 Node* c = _phase->get_ctrl(n); 3758 if (n->is_Proj() && n->in(0) != NULL && n->in(0)->is_Call()) { 3759 assert(c == n->in(0), ""); 3760 CallNode* call = c->as_Call(); 3761 CallProjections projs; 3762 call->extract_projections(&projs, true, false); 3763 if (projs.catchall_memproj != NULL) { 3764 if (projs.fallthrough_memproj == n) { 3765 c = projs.fallthrough_catchproj; 3766 } else { 3767 assert(projs.catchall_memproj == n, ""); 3768 c = projs.catchall_catchproj; 3769 } 3770 } 3771 } 3772 return c; 3773 } 3774 3775 Node* MemoryGraphFixer::ctrl_or_self(Node* n) const { 3776 if (_phase->has_ctrl(n)) 3777 return get_ctrl(n); 3778 else { 3779 assert (n->is_CFG(), "must be a CFG node"); 3780 return n; 3781 } 3782 } 3783 3784 bool MemoryGraphFixer::mem_is_valid(Node* m, Node* c) const { 3785 return m != NULL && get_ctrl(m) == c; 3786 } 3787 3788 Node* MemoryGraphFixer::find_mem(Node* ctrl, Node* n) const { 3789 assert(n == NULL || _phase->ctrl_or_self(n) == ctrl, ""); 3790 Node* mem = _memory_nodes[ctrl->_idx]; 3791 Node* c = ctrl; 3792 while (!mem_is_valid(mem, c) && 3793 (!c->is_CatchProj() || mem == NULL || c->in(0)->in(0)->in(0) != get_ctrl(mem))) { 3794 c = _phase->idom(c); 3795 mem = _memory_nodes[c->_idx]; 3796 } 3797 if (n != NULL && mem_is_valid(mem, c)) { 3798 while (!ShenandoahWriteBarrierNode::is_dominator_same_ctrl(c, mem, n, _phase) && _phase->ctrl_or_self(mem) == ctrl) { 3799 mem = next_mem(mem, _alias); 3800 } 3801 if (mem->is_MergeMem()) { 3802 mem = mem->as_MergeMem()->memory_at(_alias); 3803 } 3804 if (!mem_is_valid(mem, c)) { 3805 do { 3806 c = _phase->idom(c); 3807 mem = _memory_nodes[c->_idx]; 3808 } while (!mem_is_valid(mem, c) && 3809 (!c->is_CatchProj() || mem == NULL || c->in(0)->in(0)->in(0) != get_ctrl(mem))); 3810 } 3811 } 3812 assert(mem->bottom_type() == Type::MEMORY, ""); 3813 return mem; 3814 } 3815 3816 bool MemoryGraphFixer::has_mem_phi(Node* region) const { 3817 for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) { 3818 Node* use = region->fast_out(i); 3819 if (use->is_Phi() && use->bottom_type() == Type::MEMORY && 3820 (_phase->C->get_alias_index(use->adr_type()) == _alias)) { 3821 return true; 3822 } 3823 } 3824 return false; 3825 } 3826 3827 void MemoryGraphFixer::fix_mem(Node* ctrl, Node* new_ctrl, Node* mem, Node* mem_for_ctrl, Node* new_mem, Unique_Node_List& uses) { 3828 assert(_phase->ctrl_or_self(new_mem) == new_ctrl, ""); 3829 const bool trace = false; 3830 DEBUG_ONLY(if (trace) { tty->print("ZZZ control is"); ctrl->dump(); }); 3831 DEBUG_ONLY(if (trace) { tty->print("ZZZ mem is"); mem->dump(); }); 3832 GrowableArray<Node*> phis; 3833 if (mem_for_ctrl != mem) { 3834 Node* old = mem_for_ctrl; 3835 Node* prev = NULL; 3836 while (old != mem) { 3837 prev = old; 3838 if (old->is_Store() || old->is_ClearArray() || old->is_LoadStore()) { 3839 assert(_alias == Compile::AliasIdxRaw, ""); 3840 old = old->in(MemNode::Memory); 3841 } else if (old->Opcode() == Op_SCMemProj) { 3842 assert(_alias == Compile::AliasIdxRaw, ""); 3843 old = old->in(0); 3844 } else if (old->Opcode() == Op_ShenandoahWBMemProj) { 3845 assert(_alias != Compile::AliasIdxRaw, ""); 3846 old = old->in(ShenandoahWBMemProjNode::WriteBarrier); 3847 } else if (old->Opcode() == Op_ShenandoahWriteBarrier) { 3848 assert(_alias != Compile::AliasIdxRaw, ""); 3849 old = old->in(ShenandoahBarrierNode::Memory); 3850 } else { 3851 ShouldNotReachHere(); 3852 } 3853 } 3854 assert(prev != NULL, ""); 3855 if (new_ctrl != ctrl) { 3856 _memory_nodes.map(ctrl->_idx, mem); 3857 _memory_nodes.map(new_ctrl->_idx, mem_for_ctrl); 3858 } 3859 uint input = prev->Opcode() == Op_ShenandoahWriteBarrier ? (uint)ShenandoahBarrierNode::Memory : (uint)MemNode::Memory; 3860 _phase->igvn().replace_input_of(prev, input, new_mem); 3861 } else { 3862 uses.clear(); 3863 _memory_nodes.map(new_ctrl->_idx, new_mem); 3864 uses.push(new_ctrl); 3865 for(uint next = 0; next < uses.size(); next++ ) { 3866 Node *n = uses.at(next); 3867 assert(n->is_CFG(), ""); 3868 DEBUG_ONLY(if (trace) { tty->print("ZZZ ctrl"); n->dump(); }); 3869 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { 3870 Node* u = n->fast_out(i); 3871 if (!u->is_Root() && u->is_CFG() && u != n) { 3872 Node* m = _memory_nodes[u->_idx]; 3873 if (u->is_Region() && (!u->is_OuterStripMinedLoop() || _include_lsm) && 3874 !has_mem_phi(u) && 3875 u->unique_ctrl_out()->Opcode() != Op_Halt) { 3876 DEBUG_ONLY(if (trace) { tty->print("ZZZ region"); u->dump(); }); 3877 DEBUG_ONLY(if (trace && m != NULL) { tty->print("ZZZ mem"); m->dump(); }); 3878 3879 if (!mem_is_valid(m, u) || !m->is_Phi()) { 3880 bool push = true; 3881 bool create_phi = true; 3882 if (_phase->is_dominator(new_ctrl, u)) { 3883 create_phi = false; 3884 } else if (!_phase->C->has_irreducible_loop()) { 3885 IdealLoopTree* loop = _phase->get_loop(ctrl); 3886 bool do_check = true; 3887 IdealLoopTree* l = loop; 3888 create_phi = false; 3889 while (l != _phase->ltree_root()) { 3890 if (_phase->is_dominator(l->_head, u) && _phase->is_dominator(_phase->idom(u), l->_head)) { 3891 create_phi = true; 3892 do_check = false; 3893 break; 3894 } 3895 l = l->_parent; 3896 } 3897 3898 if (do_check) { 3899 assert(!create_phi, ""); 3900 IdealLoopTree* u_loop = _phase->get_loop(u); 3901 if (u_loop != _phase->ltree_root() && u_loop->is_member(loop)) { 3902 Node* c = ctrl; 3903 while (!_phase->is_dominator(c, u_loop->tail())) { 3904 c = _phase->idom(c); 3905 } 3906 if (!_phase->is_dominator(c, u)) { 3907 do_check = false; 3908 } 3909 } 3910 } 3911 3912 if (do_check && _phase->is_dominator(_phase->idom(u), new_ctrl)) { 3913 create_phi = true; 3914 } 3915 } 3916 if (create_phi) { 3917 Node* phi = new PhiNode(u, Type::MEMORY, _phase->C->get_adr_type(_alias)); 3918 _phase->register_new_node(phi, u); 3919 phis.push(phi); 3920 DEBUG_ONLY(if (trace) { tty->print("ZZZ new phi"); phi->dump(); }); 3921 if (!mem_is_valid(m, u)) { 3922 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting mem"); phi->dump(); }); 3923 _memory_nodes.map(u->_idx, phi); 3924 } else { 3925 DEBUG_ONLY(if (trace) { tty->print("ZZZ NOT setting mem"); m->dump(); }); 3926 for (;;) { 3927 assert(m->is_Mem() || m->is_LoadStore() || m->is_Proj() || m->Opcode() == Op_ShenandoahWriteBarrier || m->Opcode() == Op_ShenandoahWBMemProj, ""); 3928 Node* next = NULL; 3929 if (m->is_Proj()) { 3930 next = m->in(0); 3931 } else if (m->Opcode() == Op_ShenandoahWBMemProj) { 3932 next = m->in(ShenandoahWBMemProjNode::WriteBarrier); 3933 } else if (m->is_Mem() || m->is_LoadStore()) { 3934 assert(_alias == Compile::AliasIdxRaw, ""); 3935 next = m->in(MemNode::Memory); 3936 } else { 3937 assert(_alias != Compile::AliasIdxRaw, ""); 3938 assert (m->Opcode() == Op_ShenandoahWriteBarrier, ""); 3939 next = m->in(ShenandoahBarrierNode::Memory); 3940 } 3941 if (_phase->get_ctrl(next) != u) { 3942 break; 3943 } 3944 if (next->is_MergeMem()) { 3945 assert(_phase->get_ctrl(next->as_MergeMem()->memory_at(_alias)) != u, ""); 3946 break; 3947 } 3948 if (next->is_Phi()) { 3949 assert(next->adr_type() == TypePtr::BOTTOM && next->in(0) == u, ""); 3950 break; 3951 } 3952 m = next; 3953 } 3954 3955 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting to phi"); m->dump(); }); 3956 assert(m->is_Mem() || m->is_LoadStore() || m->Opcode() == Op_ShenandoahWriteBarrier, ""); 3957 uint input = (m->is_Mem() || m->is_LoadStore()) ? (uint)MemNode::Memory : (uint)ShenandoahBarrierNode::Memory; 3958 _phase->igvn().replace_input_of(m, input, phi); 3959 push = false; 3960 } 3961 } else { 3962 DEBUG_ONLY(if (trace) { tty->print("ZZZ skipping region"); u->dump(); }); 3963 } 3964 if (push) { 3965 uses.push(u); 3966 } 3967 } 3968 } else if (!mem_is_valid(m, u) && 3969 !(u->Opcode() == Op_CProj && u->in(0)->Opcode() == Op_NeverBranch && u->as_Proj()->_con == 1)) { 3970 uses.push(u); 3971 } 3972 } 3973 } 3974 } 3975 for (int i = 0; i < phis.length(); i++) { 3976 Node* n = phis.at(i); 3977 Node* r = n->in(0); 3978 DEBUG_ONLY(if (trace) { tty->print("ZZZ fixing new phi"); n->dump(); }); 3979 for (uint j = 1; j < n->req(); j++) { 3980 Node* m = find_mem(r->in(j), NULL); 3981 _phase->igvn().replace_input_of(n, j, m); 3982 DEBUG_ONLY(if (trace) { tty->print("ZZZ fixing new phi: %d", j); m->dump(); }); 3983 } 3984 } 3985 } 3986 uint last = _phase->C->unique(); 3987 MergeMemNode* mm = NULL; 3988 int alias = _alias; 3989 DEBUG_ONLY(if (trace) { tty->print("ZZZ raw mem is"); mem->dump(); }); 3990 for (DUIterator i = mem->outs(); mem->has_out(i); i++) { 3991 Node* u = mem->out(i); 3992 if (u->_idx < last) { 3993 if (u->is_Mem()) { 3994 if (_phase->C->get_alias_index(u->adr_type()) == alias) { 3995 Node* m = find_mem(_phase->get_ctrl(u), u); 3996 if (m != mem) { 3997 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting memory of use"); u->dump(); }); 3998 _phase->igvn().replace_input_of(u, MemNode::Memory, m); 3999 --i; 4000 } 4001 } 4002 } else if (u->is_MergeMem()) { 4003 MergeMemNode* u_mm = u->as_MergeMem(); 4004 if (u_mm->memory_at(alias) == mem) { 4005 MergeMemNode* newmm = NULL; 4006 for (DUIterator_Fast jmax, j = u->fast_outs(jmax); j < jmax; j++) { 4007 Node* uu = u->fast_out(j); 4008 assert(!uu->is_MergeMem(), "chain of MergeMems?"); 4009 if (uu->is_Phi()) { 4010 assert(uu->adr_type() == TypePtr::BOTTOM, ""); 4011 Node* region = uu->in(0); 4012 int nb = 0; 4013 for (uint k = 1; k < uu->req(); k++) { 4014 if (uu->in(k) == u) { 4015 Node* m = find_mem(region->in(k), NULL); 4016 if (m != mem) { 4017 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting memory of phi %d", k); uu->dump(); }); 4018 newmm = clone_merge_mem(u, mem, m, _phase->ctrl_or_self(m), i); 4019 if (newmm != u) { 4020 _phase->igvn().replace_input_of(uu, k, newmm); 4021 nb++; 4022 --jmax; 4023 } 4024 } 4025 } 4026 } 4027 if (nb > 0) { 4028 --j; 4029 } 4030 } else { 4031 Node* m = find_mem(_phase->ctrl_or_self(uu), uu); 4032 if (m != mem) { 4033 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting memory of use"); uu->dump(); }); 4034 newmm = clone_merge_mem(u, mem, m, _phase->ctrl_or_self(m), i); 4035 if (newmm != u) { 4036 _phase->igvn().replace_input_of(uu, uu->find_edge(u), newmm); 4037 --j, --jmax; 4038 } 4039 } 4040 } 4041 } 4042 } 4043 } else if (u->is_Phi()) { 4044 assert(u->bottom_type() == Type::MEMORY, "what else?"); 4045 if (_phase->C->get_alias_index(u->adr_type()) == alias || u->adr_type() == TypePtr::BOTTOM) { 4046 Node* region = u->in(0); 4047 bool replaced = false; 4048 for (uint j = 1; j < u->req(); j++) { 4049 if (u->in(j) == mem) { 4050 Node* m = find_mem(region->in(j), NULL); 4051 Node* nnew = m; 4052 if (m != mem) { 4053 if (u->adr_type() == TypePtr::BOTTOM) { 4054 mm = allocate_merge_mem(mem, m, _phase->ctrl_or_self(m)); 4055 nnew = mm; 4056 } 4057 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting memory of phi %d", j); u->dump(); }); 4058 _phase->igvn().replace_input_of(u, j, nnew); 4059 replaced = true; 4060 } 4061 } 4062 } 4063 if (replaced) { 4064 --i; 4065 } 4066 } 4067 } else if ((u->adr_type() == TypePtr::BOTTOM && u->Opcode() != Op_StrInflatedCopy) || 4068 u->adr_type() == NULL) { 4069 assert(u->adr_type() != NULL || 4070 u->Opcode() == Op_Rethrow || 4071 u->Opcode() == Op_Return || 4072 u->Opcode() == Op_SafePoint || 4073 (u->is_CallStaticJava() && u->as_CallStaticJava()->uncommon_trap_request() != 0) || 4074 (u->is_CallStaticJava() && u->as_CallStaticJava()->_entry_point == OptoRuntime::rethrow_stub()) || 4075 u->Opcode() == Op_CallLeaf, ""); 4076 Node* m = find_mem(_phase->ctrl_or_self(u), u); 4077 if (m != mem) { 4078 mm = allocate_merge_mem(mem, m, _phase->get_ctrl(m)); 4079 _phase->igvn().replace_input_of(u, u->find_edge(mem), mm); 4080 --i; 4081 } 4082 } else if (_phase->C->get_alias_index(u->adr_type()) == alias) { 4083 Node* m = find_mem(_phase->ctrl_or_self(u), u); 4084 if (m != mem) { 4085 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting memory of use"); u->dump(); }); 4086 _phase->igvn().replace_input_of(u, u->find_edge(mem), m); 4087 --i; 4088 } 4089 } else if (u->adr_type() != TypePtr::BOTTOM && 4090 _memory_nodes[_phase->ctrl_or_self(u)->_idx] == u) { 4091 Node* m = find_mem(_phase->ctrl_or_self(u), u); 4092 assert(m != mem, ""); 4093 // u is on the wrong slice... 4094 assert(u->is_ClearArray(), ""); 4095 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting memory of use"); u->dump(); }); 4096 _phase->igvn().replace_input_of(u, u->find_edge(mem), m); 4097 --i; 4098 } 4099 } 4100 } 4101 #ifdef ASSERT 4102 assert(new_mem->outcnt() > 0, ""); 4103 for (int i = 0; i < phis.length(); i++) { 4104 Node* n = phis.at(i); 4105 assert(n->outcnt() > 0, "new phi must have uses now"); 4106 } 4107 #endif 4108 } 4109 4110 MergeMemNode* MemoryGraphFixer::allocate_merge_mem(Node* mem, Node* rep_proj, Node* rep_ctrl) const { 4111 MergeMemNode* mm = MergeMemNode::make(mem); 4112 mm->set_memory_at(_alias, rep_proj); 4113 _phase->register_new_node(mm, rep_ctrl); 4114 return mm; 4115 } 4116 4117 MergeMemNode* MemoryGraphFixer::clone_merge_mem(Node* u, Node* mem, Node* rep_proj, Node* rep_ctrl, DUIterator& i) const { 4118 MergeMemNode* newmm = NULL; 4119 MergeMemNode* u_mm = u->as_MergeMem(); 4120 Node* c = _phase->get_ctrl(u); 4121 if (_phase->is_dominator(c, rep_ctrl)) { 4122 c = rep_ctrl; 4123 } else { 4124 assert(_phase->is_dominator(rep_ctrl, c), "one must dominate the other"); 4125 } 4126 if (u->outcnt() == 1) { 4127 if (u->req() > (uint)_alias && u->in(_alias) == mem) { 4128 _phase->igvn().replace_input_of(u, _alias, rep_proj); 4129 --i; 4130 } else { 4131 _phase->igvn().rehash_node_delayed(u); 4132 u_mm->set_memory_at(_alias, rep_proj); 4133 } 4134 newmm = u_mm; 4135 _phase->set_ctrl_and_loop(u, c); 4136 } else { 4137 // can't simply clone u and then change one of its input because 4138 // it adds and then removes an edge which messes with the 4139 // DUIterator 4140 newmm = MergeMemNode::make(u_mm->base_memory()); 4141 for (uint j = 0; j < u->req(); j++) { 4142 if (j < newmm->req()) { 4143 if (j == (uint)_alias) { 4144 newmm->set_req(j, rep_proj); 4145 } else if (newmm->in(j) != u->in(j)) { 4146 newmm->set_req(j, u->in(j)); 4147 } 4148 } else if (j == (uint)_alias) { 4149 newmm->add_req(rep_proj); 4150 } else { 4151 newmm->add_req(u->in(j)); 4152 } 4153 } 4154 if ((uint)_alias >= u->req()) { 4155 newmm->set_memory_at(_alias, rep_proj); 4156 } 4157 _phase->register_new_node(newmm, c); 4158 } 4159 return newmm; 4160 } 4161 4162 bool MemoryGraphFixer::should_process_phi(Node* phi) const { 4163 if (phi->adr_type() == TypePtr::BOTTOM) { 4164 Node* region = phi->in(0); 4165 for (DUIterator_Fast jmax, j = region->fast_outs(jmax); j < jmax; j++) { 4166 Node* uu = region->fast_out(j); 4167 if (uu->is_Phi() && uu != phi && uu->bottom_type() == Type::MEMORY && _phase->C->get_alias_index(uu->adr_type()) == _alias) { 4168 return false; 4169 } 4170 } 4171 return true; 4172 } 4173 return _phase->C->get_alias_index(phi->adr_type()) == _alias; 4174 } 4175 4176 void MemoryGraphFixer::fix_memory_uses(Node* mem, Node* replacement, Node* rep_proj, Node* rep_ctrl) const { 4177 uint last = _phase-> C->unique(); 4178 MergeMemNode* mm = NULL; 4179 assert(mem->bottom_type() == Type::MEMORY, ""); 4180 for (DUIterator i = mem->outs(); mem->has_out(i); i++) { 4181 Node* u = mem->out(i); 4182 if (u != replacement && u->_idx < last) { 4183 if (u->is_ShenandoahBarrier() && _alias != Compile::AliasIdxRaw) { 4184 if (_phase->C->get_alias_index(u->adr_type()) == _alias && ShenandoahWriteBarrierNode::is_dominator(rep_ctrl, _phase->ctrl_or_self(u), replacement, u, _phase)) { 4185 _phase->igvn().replace_input_of(u, u->find_edge(mem), rep_proj); 4186 assert(u->find_edge(mem) == -1, "only one edge"); 4187 --i; 4188 } 4189 } else if (u->is_Mem()) { 4190 if (_phase->C->get_alias_index(u->adr_type()) == _alias && ShenandoahWriteBarrierNode::is_dominator(rep_ctrl, _phase->ctrl_or_self(u), replacement, u, _phase)) { 4191 assert(_alias == Compile::AliasIdxRaw , "only raw memory can lead to a memory operation"); 4192 _phase->igvn().replace_input_of(u, u->find_edge(mem), rep_proj); 4193 assert(u->find_edge(mem) == -1, "only one edge"); 4194 --i; 4195 } 4196 } else if (u->is_MergeMem()) { 4197 MergeMemNode* u_mm = u->as_MergeMem(); 4198 if (u_mm->memory_at(_alias) == mem) { 4199 MergeMemNode* newmm = NULL; 4200 for (DUIterator_Fast jmax, j = u->fast_outs(jmax); j < jmax; j++) { 4201 Node* uu = u->fast_out(j); 4202 assert(!uu->is_MergeMem(), "chain of MergeMems?"); 4203 if (uu->is_Phi()) { 4204 if (should_process_phi(uu)) { 4205 Node* region = uu->in(0); 4206 int nb = 0; 4207 for (uint k = 1; k < uu->req(); k++) { 4208 if (uu->in(k) == u && _phase->is_dominator(rep_ctrl, region->in(k))) { 4209 if (newmm == NULL) { 4210 newmm = clone_merge_mem(u, mem, rep_proj, rep_ctrl, i); 4211 } 4212 if (newmm != u) { 4213 _phase->igvn().replace_input_of(uu, k, newmm); 4214 nb++; 4215 --jmax; 4216 } 4217 } 4218 } 4219 if (nb > 0) { 4220 --j; 4221 } 4222 } 4223 } else { 4224 if (rep_ctrl != uu && ShenandoahWriteBarrierNode::is_dominator(rep_ctrl, _phase->ctrl_or_self(uu), replacement, uu, _phase)) { 4225 if (newmm == NULL) { 4226 newmm = clone_merge_mem(u, mem, rep_proj, rep_ctrl, i); 4227 } 4228 if (newmm != u) { 4229 _phase->igvn().replace_input_of(uu, uu->find_edge(u), newmm); 4230 --j, --jmax; 4231 } 4232 } 4233 } 4234 } 4235 } 4236 } else if (u->is_Phi()) { 4237 assert(u->bottom_type() == Type::MEMORY, "what else?"); 4238 Node* region = u->in(0); 4239 if (should_process_phi(u)) { 4240 bool replaced = false; 4241 for (uint j = 1; j < u->req(); j++) { 4242 if (u->in(j) == mem && _phase->is_dominator(rep_ctrl, region->in(j))) { 4243 Node* nnew = rep_proj; 4244 if (u->adr_type() == TypePtr::BOTTOM) { 4245 if (mm == NULL) { 4246 mm = allocate_merge_mem(mem, rep_proj, rep_ctrl); 4247 } 4248 nnew = mm; 4249 } 4250 _phase->igvn().replace_input_of(u, j, nnew); 4251 replaced = true; 4252 } 4253 } 4254 if (replaced) { 4255 --i; 4256 } 4257 4258 } 4259 } else if ((u->adr_type() == TypePtr::BOTTOM && u->Opcode() != Op_StrInflatedCopy) || 4260 u->adr_type() == NULL) { 4261 assert(u->adr_type() != NULL || 4262 u->Opcode() == Op_Rethrow || 4263 u->Opcode() == Op_Return || 4264 u->Opcode() == Op_SafePoint || 4265 (u->is_CallStaticJava() && u->as_CallStaticJava()->uncommon_trap_request() != 0) || 4266 (u->is_CallStaticJava() && u->as_CallStaticJava()->_entry_point == OptoRuntime::rethrow_stub()) || 4267 u->Opcode() == Op_CallLeaf, ""); 4268 if (ShenandoahWriteBarrierNode::is_dominator(rep_ctrl, _phase->ctrl_or_self(u), replacement, u, _phase)) { 4269 if (mm == NULL) { 4270 mm = allocate_merge_mem(mem, rep_proj, rep_ctrl); 4271 } 4272 _phase->igvn().replace_input_of(u, u->find_edge(mem), mm); 4273 --i; 4274 } 4275 } else if (_phase->C->get_alias_index(u->adr_type()) == _alias) { 4276 if (ShenandoahWriteBarrierNode::is_dominator(rep_ctrl, _phase->ctrl_or_self(u), replacement, u, _phase)) { 4277 _phase->igvn().replace_input_of(u, u->find_edge(mem), rep_proj); 4278 --i; 4279 } 4280 } 4281 } 4282 } 4283 } 4284 4285 void MemoryGraphFixer::remove(Node* n) { 4286 assert(n->Opcode() == Op_ShenandoahWBMemProj, ""); 4287 Node* c = _phase->get_ctrl(n); 4288 Node* mem = find_mem(c, NULL); 4289 if (mem == n) { 4290 _memory_nodes.map(c->_idx, mem->in(ShenandoahWBMemProjNode::WriteBarrier)->in(ShenandoahBarrierNode::Memory)); 4291 } 4292 }