1 /* 2 * Copyright (c) 2015, Red Hat, Inc. and/or its affiliates. 3 * 4 * This code is free software; you can redistribute it and/or modify it 5 * under the terms of the GNU General Public License version 2 only, as 6 * published by the Free Software Foundation. 7 * 8 * This code is distributed in the hope that it will be useful, but WITHOUT 9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 11 * version 2 for more details (a copy is included in the LICENSE file that 12 * accompanied this code). 13 * 14 * You should have received a copy of the GNU General Public License version 15 * 2 along with this work; if not, write to the Free Software Foundation, 16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 17 * 18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 19 * or visit www.oracle.com if you need additional information or have any 20 * questions. 21 * 22 */ 23 24 #include "gc/shenandoah/brooksPointer.hpp" 25 #include "gc/shenandoah/shenandoahHeap.hpp" 26 #include "gc/shenandoah/shenandoahHeapRegion.hpp" 27 #include "opto/arraycopynode.hpp" 28 #include "opto/block.hpp" 29 #include "opto/callnode.hpp" 30 #include "opto/castnode.hpp" 31 #include "opto/movenode.hpp" 32 #include "opto/phaseX.hpp" 33 #include "opto/rootnode.hpp" 34 #include "opto/runtime.hpp" 35 #include "opto/shenandoahSupport.hpp" 36 #include "opto/subnode.hpp" 37 38 Node* ShenandoahBarrierNode::skip_through_barrier(Node* n) { 39 if (n == NULL) { 40 return NULL; 41 } else if (n->is_ShenandoahBarrier()) { 42 return n->in(ValueIn); 43 } else if (n->is_Phi() && 44 n->req() == 3 && 45 n->in(1) != NULL && 46 n->in(1)->is_ShenandoahBarrier() && 47 n->in(2) != NULL && 48 n->in(2)->bottom_type() == TypePtr::NULL_PTR && 49 n->in(0) != NULL && 50 n->in(0)->in(1) != NULL && 51 n->in(0)->in(1)->is_IfProj() && 52 n->in(0)->in(2) != NULL && 53 n->in(0)->in(2)->is_IfProj() && 54 n->in(0)->in(1)->in(0) != NULL && 55 n->in(0)->in(1)->in(0) == n->in(0)->in(2)->in(0) && 56 n->in(1)->in(ValueIn)->Opcode() == Op_CastPP) { 57 Node* iff = n->in(0)->in(1)->in(0); 58 Node* res = n->in(1)->in(ValueIn)->in(1); 59 if (iff->is_If() && 60 iff->in(1) != NULL && 61 iff->in(1)->is_Bool() && 62 iff->in(1)->as_Bool()->_test._test == BoolTest::ne && 63 iff->in(1)->in(1) != NULL && 64 iff->in(1)->in(1)->Opcode() == Op_CmpP && 65 iff->in(1)->in(1)->in(1) != NULL && 66 iff->in(1)->in(1)->in(1) == res && 67 iff->in(1)->in(1)->in(2) != NULL && 68 iff->in(1)->in(1)->in(2)->bottom_type() == TypePtr::NULL_PTR) { 69 return res; 70 } 71 } 72 return n; 73 } 74 75 bool ShenandoahBarrierNode::needs_barrier(PhaseGVN* phase, ShenandoahBarrierNode* orig, Node* n, Node* rb_mem, bool allow_fromspace) { 76 Unique_Node_List visited; 77 return needs_barrier_impl(phase, orig, n, rb_mem, allow_fromspace, visited); 78 } 79 80 bool ShenandoahBarrierNode::needs_barrier_impl(PhaseGVN* phase, ShenandoahBarrierNode* orig, Node* n, Node* rb_mem, bool allow_fromspace, Unique_Node_List &visited) { 81 82 if (visited.member(n)) { 83 return false; // Been there. 84 } 85 visited.push(n); 86 87 if (n->is_Allocate()) { 88 // tty->print_cr("killed barrier for newly allocated object"); 89 return false; 90 } 91 92 if (n->is_CallJava() || n->Opcode() == Op_CallLeafNoFP) { 93 return true; 94 } 95 96 const Type* type = phase->type(n); 97 if (type == Type::TOP) { 98 return false; 99 } 100 if (type->make_ptr()->higher_equal(TypePtr::NULL_PTR)) { 101 // tty->print_cr("killed barrier for NULL object"); 102 return false; 103 } 104 if (type->make_oopptr() && type->make_oopptr()->const_oop() != NULL) { 105 // tty->print_cr("killed barrier for constant object"); 106 return !ShenandoahNoBarriersForConst; 107 } 108 109 if (ShenandoahOptimizeFinals) { 110 const TypeAryPtr* ary = type->isa_aryptr(); 111 if (ary && ary->is_stable() && allow_fromspace) { 112 return false; 113 } 114 } 115 116 if (n->is_CheckCastPP() || n->is_ConstraintCast()) { 117 return needs_barrier_impl(phase, orig, n->in(1), rb_mem, allow_fromspace, visited); 118 } 119 if (n->is_Parm()) { 120 return true; 121 } 122 if (n->is_Proj()) { 123 return needs_barrier_impl(phase, orig, n->in(0), rb_mem, allow_fromspace, visited); 124 } 125 if (n->is_Phi()) { 126 bool need_barrier = false; 127 for (uint i = 1; i < n->req() && ! need_barrier; i++) { 128 Node* input = n->in(i); 129 if (input == NULL) { 130 need_barrier = true; // Phi not complete yet? 131 } else if (needs_barrier_impl(phase, orig, input, rb_mem, allow_fromspace, visited)) { 132 need_barrier = true; 133 } 134 } 135 return need_barrier; 136 } 137 if (n->is_CMove()) { 138 return needs_barrier_impl(phase, orig, n->in(CMoveNode::IfFalse), rb_mem, allow_fromspace, visited) || 139 needs_barrier_impl(phase, orig, n->in(CMoveNode::IfTrue ), rb_mem, allow_fromspace, visited); 140 } 141 if (n->Opcode() == Op_CreateEx) { 142 return true; 143 } 144 if (n->Opcode() == Op_ShenandoahWriteBarrier) { 145 // tty->print_cr("skipped barrier for chained write barrier object"); 146 return false; 147 } 148 if (n->Opcode() == Op_ShenandoahReadBarrier) { 149 if (rb_mem == n->in(Memory)) { 150 // tty->print_cr("Eliminated chained read barrier"); 151 return false; 152 } else { 153 return true; 154 } 155 } 156 157 if (n->Opcode() == Op_LoadP || 158 n->Opcode() == Op_LoadN || 159 n->Opcode() == Op_GetAndSetP || 160 n->Opcode() == Op_CompareAndExchangeP || 161 n->Opcode() == Op_GetAndSetN || 162 n->Opcode() == Op_CompareAndExchangeN) { 163 return true; 164 } 165 if (n->Opcode() == Op_DecodeN || 166 n->Opcode() == Op_EncodeP) { 167 return needs_barrier_impl(phase, orig, n->in(1), rb_mem, allow_fromspace, visited); 168 } 169 170 #ifdef ASSERT 171 tty->print("need barrier on?: "); n->dump(); 172 ShouldNotReachHere(); 173 #endif 174 return true; 175 } 176 177 bool ShenandoahReadBarrierNode::dominates_memory_rb_impl(PhaseGVN* phase, 178 Node* b1, 179 Node* b2, 180 Node* current, 181 bool linear) { 182 ResourceMark rm; 183 VectorSet visited(Thread::current()->resource_area()); 184 Node_Stack phis(0); 185 186 for(int i = 0; i < 10; i++) { 187 if (current == NULL) { 188 return false; 189 } else if (visited.test_set(current->_idx) || current->is_top() || current == b1) { 190 current = NULL; 191 while (phis.is_nonempty() && current == NULL) { 192 uint idx = phis.index(); 193 Node* phi = phis.node(); 194 if (idx >= phi->req()) { 195 phis.pop(); 196 } else { 197 current = phi->in(idx); 198 phis.set_index(idx+1); 199 } 200 } 201 if (current == NULL) { 202 return true; 203 } 204 } else if (current == phase->C->immutable_memory()) { 205 return false; 206 } else if (current->isa_Phi()) { 207 if (!linear) { 208 return false; 209 } 210 phis.push(current, 2); 211 current = current->in(1); 212 } else if (current->Opcode() == Op_ShenandoahWriteBarrier) { 213 const Type* in_type = current->bottom_type(); 214 const Type* this_type = b2->bottom_type(); 215 if (is_independent(in_type, this_type)) { 216 current = current->in(Memory); 217 } else { 218 return false; 219 } 220 } else if (current->Opcode() == Op_ShenandoahWBMemProj) { 221 current = current->in(0); 222 } else if (current->is_Proj()) { 223 current = current->in(0); 224 } else if (current->is_Call()) { 225 return false; // TODO: Maybe improve by looking at the call's memory effects? 226 } else if (current->is_MemBar()) { 227 return false; // TODO: Do we need to stop at *any* membar? 228 } else if (current->is_MergeMem()) { 229 // if (true) return false; 230 // tty->print_cr("current == mergemem: "); current->dump(); 231 const TypePtr* adr_type = brooks_pointer_type(phase->type(b2)); 232 uint alias_idx = phase->C->get_alias_index(adr_type); 233 current = current->as_MergeMem()->memory_at(alias_idx); 234 } else { 235 // tty->print_cr("what else can we see here:"); 236 #ifdef ASSERT 237 current->dump(); 238 #endif 239 ShouldNotReachHere(); 240 return false; 241 } 242 } 243 return false; 244 } 245 246 bool ShenandoahReadBarrierNode::is_independent(Node* mem) { 247 if (mem->is_Phi() || mem->is_Proj() || mem->is_MergeMem()) { 248 return true; 249 } else if (mem->Opcode() == Op_ShenandoahWriteBarrier) { 250 const Type* mem_type = mem->bottom_type(); 251 const Type* this_type = bottom_type(); 252 if (is_independent(mem_type, this_type)) { 253 return true; 254 } else { 255 return false; 256 } 257 } else if (mem->is_Call() || mem->is_MemBar()) { 258 return false; 259 } 260 #ifdef ASSERT 261 mem->dump(); 262 #endif 263 ShouldNotReachHere(); 264 return true; 265 } 266 267 268 bool ShenandoahReadBarrierNode::dominates_memory_rb(PhaseGVN* phase, Node* b1, Node* b2, bool linear) { 269 return dominates_memory_rb_impl(phase, b1->in(Memory), b2, b2->in(Memory), linear); 270 } 271 272 bool ShenandoahReadBarrierNode::is_independent(const Type* in_type, const Type* this_type) { 273 assert(in_type->isa_oopptr(), "expect oop ptr"); 274 assert(this_type->isa_oopptr(), "expect oop ptr"); 275 /* 276 if ((! in_type->isa_oopptr()) || (! this_type->isa_oopptr())) { 277 #ifdef ASSERT 278 tty->print_cr("not oopptr"); 279 tty->print("in: "); in_type->dump(); tty->print_cr(" "); 280 tty->print("this: "); this_type->dump(); tty->print_cr(" "); 281 #endif 282 return false; 283 } 284 */ 285 286 ciKlass* in_kls = in_type->is_oopptr()->klass(); 287 ciKlass* this_kls = this_type->is_oopptr()->klass(); 288 if (in_kls != NULL && this_kls != NULL && 289 in_kls->is_loaded() && this_kls->is_loaded() && 290 (!in_kls->is_subclass_of(this_kls)) && 291 (!this_kls->is_subclass_of(in_kls))) { 292 #ifdef ASSERT 293 // tty->print_cr("independent: "); 294 // tty->print("in: "); in_kls->print(); tty->print_cr(" "); 295 // tty->print("this: "); this_kls->print(); tty->print_cr(" "); 296 #endif 297 return true; 298 } 299 #ifdef ASSERT 300 // tty->print_cr("possibly dependend?"); 301 // tty->print("in: "); in_type->dump(); tty->print_cr(" "); 302 // tty->print("this: "); this_type->dump(); tty->print_cr(" "); 303 #endif 304 return false; 305 } 306 307 Node* ShenandoahReadBarrierNode::Ideal(PhaseGVN *phase, bool can_reshape) { 308 309 if (! can_reshape) { 310 return NULL; 311 } 312 313 if (in(Memory) == phase->C->immutable_memory()) return NULL; 314 315 // If memory input is a MergeMem, take the appropriate slice out of it. 316 Node* mem_in = in(Memory); 317 if (mem_in->isa_MergeMem()) { 318 const TypePtr* adr_type = brooks_pointer_type(bottom_type()); 319 uint alias_idx = phase->C->get_alias_index(adr_type); 320 mem_in = mem_in->as_MergeMem()->memory_at(alias_idx); 321 set_req(Memory, mem_in); 322 return this; 323 } 324 325 Node* input = in(Memory); 326 if (input->Opcode() == Op_ShenandoahWBMemProj) { 327 Node* wb = input->in(0); 328 if (wb->is_top()) return NULL; // Dead path. 329 assert(wb->Opcode() == Op_ShenandoahWriteBarrier, "expect write barrier"); 330 const Type* in_type = phase->type(wb); 331 if (is_independent(in_type, _type)) { 332 phase->igvn_rehash_node_delayed(wb); 333 set_req(Memory, wb->in(Memory)); 334 if (can_reshape && input->outcnt() == 0) { 335 phase->is_IterGVN()->_worklist.push(input); 336 } 337 return this; 338 } 339 } 340 return NULL; 341 } 342 343 Node* ShenandoahWriteBarrierNode::Identity(PhaseGVN* phase) { 344 assert(in(0) != NULL, "should have control"); 345 PhaseIterGVN* igvn = phase->is_IterGVN(); 346 Node* mem_in = in(Memory); 347 Node* mem_proj = NULL; 348 349 if (igvn != NULL) { 350 mem_proj = find_out_with(Op_ShenandoahWBMemProj); 351 if (mem_proj == NULL || mem_in == mem_proj) { 352 return this; 353 } 354 } 355 356 Node* replacement = Identity_impl(phase); 357 if (igvn != NULL) { 358 if (replacement != NULL && replacement != this) { 359 igvn->replace_node(mem_proj, mem_in); 360 } 361 } 362 return replacement; 363 } 364 365 366 Node* ShenandoahWriteBarrierNode::Ideal(PhaseGVN *phase, bool can_reshape) { 367 assert(in(0) != NULL, "should have control"); 368 if (!can_reshape) { 369 return NULL; 370 } 371 372 PhaseIterGVN* igvn = phase->is_IterGVN(); 373 Node* mem_proj = find_out_with(Op_ShenandoahWBMemProj); 374 Node* mem_in = in(Memory); 375 376 if (mem_in == phase->C->immutable_memory()) return NULL; 377 378 if (mem_in->isa_MergeMem()) { 379 const TypePtr* adr_type = brooks_pointer_type(bottom_type()); 380 uint alias_idx = phase->C->get_alias_index(adr_type); 381 mem_in = mem_in->as_MergeMem()->memory_at(alias_idx); 382 set_req(Memory, mem_in); 383 return this; 384 } 385 386 return NULL; 387 } 388 389 bool ShenandoahWriteBarrierNode::expand(Compile* C, PhaseIterGVN& igvn, int& loop_opts_cnt) { 390 if (ShenandoahWriteBarrierToIR) { 391 if (C->shenandoah_barriers_count() > 0) { 392 bool attempt_more_loopopts = ShenandoahLoopOptsAfterExpansion && (C->shenandoah_barriers_count() > 1 || C->has_loops()); 393 C->clear_major_progress(); 394 PhaseIdealLoop ideal_loop(igvn, LoopOptsShenandoahExpand); 395 if (C->failing()) return false; 396 PhaseIdealLoop::verify(igvn); 397 #ifdef ASSERT 398 if (UseShenandoahGC) { 399 ShenandoahBarrierNode::verify_raw_mem(C->root()); 400 } 401 #endif 402 if (attempt_more_loopopts) { 403 C->set_major_progress(); 404 if (!C->optimize_loops(loop_opts_cnt, igvn, LoopOptsShenandoahPostExpand)) { 405 return false; 406 } 407 C->clear_major_progress(); 408 } 409 } 410 } 411 return true; 412 } 413 414 bool ShenandoahWriteBarrierNode::is_evacuation_in_progress_test(Node* iff) { 415 assert(iff->is_If(), "bad input"); 416 if (!iff->Opcode() == Op_If) { 417 return false; 418 } 419 Node* bol = iff->in(1); 420 if (!bol->is_Bool() || bol->as_Bool()->_test._test != BoolTest::ne) { 421 return false; 422 } 423 Node* cmp = bol->in(1); 424 if (cmp->Opcode() != Op_CmpI) { 425 return false; 426 } 427 Node* in1 = cmp->in(1); 428 Node* in2 = cmp->in(2); 429 if (in2->find_int_con(-1) != 0) { 430 return false; 431 } 432 if (in1->Opcode() != Op_LoadUB) { 433 return false; 434 } 435 Node* addp = in1->in(MemNode::Address); 436 if (!addp->is_AddP()) { 437 return false; 438 } 439 Node* base = addp->in(AddPNode::Address); 440 Node* off = addp->in(AddPNode::Offset); 441 if (base->Opcode() != Op_ThreadLocal) { 442 return false; 443 } 444 if (off->find_intptr_t_con(-1) != in_bytes(JavaThread::evacuation_in_progress_offset())) { 445 return false; 446 } 447 return true; 448 } 449 450 Node* ShenandoahWriteBarrierNode::evacuation_in_progress_test_ctrl(Node* iff) { 451 assert(is_evacuation_in_progress_test(iff), "bad input"); 452 Node* c = iff; 453 do { 454 assert(c->in(0)->is_Proj() && c->in(0)->in(0)->is_MemBar(), "where's the mem bar?"); 455 c = c->in(0)->in(0); 456 } while (c->adr_type() != TypeRawPtr::BOTTOM); 457 return c->in(0); 458 } 459 460 bool ShenandoahBarrierNode::dominates_memory_impl(PhaseGVN* phase, 461 Node* b1, 462 Node* b2, 463 Node* current, 464 bool linear) { 465 ResourceMark rm; 466 VectorSet visited(Thread::current()->resource_area()); 467 Node_Stack phis(0); 468 469 470 for(int i = 0; i < 10; i++) { 471 if (current == NULL) { 472 return false; 473 } else if (visited.test_set(current->_idx) || current->is_top() || current == b1) { 474 current = NULL; 475 while (phis.is_nonempty() && current == NULL) { 476 uint idx = phis.index(); 477 Node* phi = phis.node(); 478 if (idx >= phi->req()) { 479 phis.pop(); 480 } else { 481 current = phi->in(idx); 482 phis.set_index(idx+1); 483 } 484 } 485 if (current == NULL) { 486 return true; 487 } 488 } else if (current == b2) { 489 return false; 490 } else if (current == phase->C->immutable_memory()) { 491 return false; 492 } else if (current->isa_Phi()) { 493 if (!linear) { 494 return false; 495 } 496 phis.push(current, 2); 497 current = current->in(1); 498 } else if (current->Opcode() == Op_ShenandoahWriteBarrier) { 499 current = current->in(Memory); 500 } else if (current->Opcode() == Op_ShenandoahWBMemProj) { 501 current = current->in(0); 502 } else if (current->is_Proj()) { 503 current = current->in(0); 504 } else if (current->is_Call()) { 505 current = current->in(TypeFunc::Memory); 506 } else if (current->is_MemBar()) { 507 current = current->in(TypeFunc::Memory); 508 } else if (current->is_MergeMem()) { 509 const TypePtr* adr_type = brooks_pointer_type(phase->type(b2)); 510 uint alias_idx = phase->C->get_alias_index(adr_type); 511 current = current->as_MergeMem()->memory_at(alias_idx); 512 } else { 513 #ifdef ASSERT 514 current->dump(); 515 #endif 516 ShouldNotReachHere(); 517 return false; 518 } 519 } 520 return false; 521 } 522 523 /** 524 * Determines if b1 dominates b2 through memory inputs. It returns true if: 525 * - b1 can be reached by following each branch in b2's memory input (through phis, etc) 526 * - or we get back to b2 (i.e. through a loop) without seeing b1 527 * In all other cases, (in particular, if we reach immutable_memory without having seen b1) 528 * we return false. 529 */ 530 bool ShenandoahBarrierNode::dominates_memory(PhaseGVN* phase, Node* b1, Node* b2, bool linear) { 531 return dominates_memory_impl(phase, b1, b2, b2->in(Memory), linear); 532 } 533 534 Node* ShenandoahBarrierNode::Identity_impl(PhaseGVN* phase) { 535 Node* n = in(ValueIn); 536 537 Node* rb_mem = Opcode() == Op_ShenandoahReadBarrier ? in(Memory) : NULL; 538 if (! needs_barrier(phase, this, n, rb_mem, _allow_fromspace)) { 539 return n; 540 } 541 542 // tty->print_cr("find sibling for: "); dump(2); 543 // Try to find a write barrier sibling with identical inputs that we can fold into. 544 for (DUIterator i = n->outs(); n->has_out(i); i++) { 545 Node* sibling = n->out(i); 546 if (sibling == this) { 547 continue; 548 } 549 /* 550 assert(sibling->Opcode() != Op_ShenandoahWriteBarrier || 551 Opcode() != Op_ShenandoahWriteBarrier || hash() == sibling->hash(), 552 "if this is a write barrier, then sibling can't be write barrier too"); 553 */ 554 if (sibling->Opcode() != Op_ShenandoahWriteBarrier) { 555 continue; 556 } 557 /* 558 if (sibling->outcnt() == 0) { 559 // Some dead node. 560 continue; 561 } 562 */ 563 assert(sibling->in(ValueIn) == in(ValueIn), "sanity"); 564 assert(sibling->Opcode() == Op_ShenandoahWriteBarrier, "sanity"); 565 // tty->print_cr("candidate: "); sibling->dump(); 566 567 if (dominates_memory(phase, sibling, this, phase->is_IterGVN() == NULL)) { 568 569 /* 570 tty->print_cr("matched barrier:"); 571 sibling->dump(); 572 tty->print_cr("for: "); 573 dump(); 574 */ 575 return sibling; 576 } 577 578 /* 579 tty->print_cr("couldn't match candidate:"); 580 sibling->dump(2); 581 */ 582 } 583 /* 584 tty->print_cr("couldn't match barrier to any:"); 585 dump(); 586 */ 587 return this; 588 } 589 590 #ifndef PRODUCT 591 void ShenandoahBarrierNode::dump_spec(outputStream *st) const { 592 const TypePtr* adr = adr_type(); 593 if (adr == NULL) { 594 return; 595 } 596 st->print(" @"); 597 adr->dump_on(st); 598 st->print(" ("); 599 Compile::current()->alias_type(adr)->adr_type()->dump_on(st); 600 st->print(") "); 601 } 602 #endif 603 604 Node* ShenandoahReadBarrierNode::Identity(PhaseGVN* phase) { 605 606 // if (true) return this; 607 608 // tty->print("optimizing rb: "); dump(); 609 Node* id = Identity_impl(phase); 610 611 if (id == this && phase->is_IterGVN()) { 612 Node* n = in(ValueIn); 613 // No success in super call. Try to combine identical read barriers. 614 for (DUIterator i = n->outs(); n->has_out(i); i++) { 615 Node* sibling = n->out(i); 616 if (sibling == this || sibling->Opcode() != Op_ShenandoahReadBarrier) { 617 continue; 618 } 619 assert(sibling->in(ValueIn) == in(ValueIn), "sanity"); 620 if (phase->is_IterGVN()->hash_find(sibling) && 621 sibling->bottom_type() == bottom_type() && 622 sibling->in(Control) == in(Control) && 623 dominates_memory_rb(phase, sibling, this, phase->is_IterGVN() == NULL)) { 624 /* 625 if (in(Memory) != sibling->in(Memory)) { 626 tty->print_cr("interesting rb-fold"); 627 dump(); 628 sibling->dump(); 629 } 630 */ 631 return sibling; 632 } 633 } 634 } 635 return id; 636 } 637 638 const Type* ShenandoahBarrierNode::Value(PhaseGVN* phase) const { 639 // Either input is TOP ==> the result is TOP 640 const Type *t1 = phase->type(in(Memory)); 641 if (t1 == Type::TOP) return Type::TOP; 642 const Type *t2 = phase->type(in(ValueIn)); 643 if( t2 == Type::TOP ) return Type::TOP; 644 645 Node* input = in(ValueIn); 646 const Type* type = phase->type(input)->is_oopptr()->cast_to_nonconst(); 647 return type; 648 } 649 650 uint ShenandoahBarrierNode::hash() const { 651 return TypeNode::hash() + _allow_fromspace; 652 } 653 654 uint ShenandoahBarrierNode::cmp(const Node& n) const { 655 return _allow_fromspace == ((ShenandoahBarrierNode&) n)._allow_fromspace 656 && TypeNode::cmp(n); 657 } 658 659 uint ShenandoahBarrierNode::size_of() const { 660 return sizeof(*this); 661 } 662 663 Node* ShenandoahWBMemProjNode::Identity(PhaseGVN* phase) { 664 665 Node* wb = in(0); 666 if (wb->is_top()) return phase->C->top(); // Dead path. 667 668 assert(wb->Opcode() == Op_ShenandoahWriteBarrier, "expect write barrier"); 669 PhaseIterGVN* igvn = phase->is_IterGVN(); 670 // We can't do the below unless the graph is fully constructed. 671 if (igvn == NULL) { 672 return this; 673 } 674 675 // If the mem projection has no barrier users, it's not needed anymore. 676 Unique_Node_List visited; 677 if (wb->outcnt() == 1) { 678 return wb->in(ShenandoahBarrierNode::Memory); 679 } 680 681 return this; 682 } 683 684 #ifdef ASSERT 685 bool ShenandoahBarrierNode::verify_helper(Node* in, Node_Stack& phis, VectorSet& visited, verify_type t, bool trace, Unique_Node_List& barriers_used) { 686 assert(phis.size() == 0, ""); 687 688 while (true) { 689 if (!in->bottom_type()->make_ptr()->make_oopptr()) { 690 if (trace) {tty->print_cr("Non oop");} 691 } else if (t == ShenandoahLoad && ShenandoahOptimizeFinals && 692 in->bottom_type()->make_ptr()->isa_aryptr() && 693 in->bottom_type()->make_ptr()->is_aryptr()->is_stable()) { 694 if (trace) {tty->print_cr("Stable array load");} 695 } else { 696 in = in->uncast(); 697 if (in->is_AddP()) { 698 assert(!in->in(AddPNode::Address)->is_top(), "no raw memory access"); 699 in = in->in(AddPNode::Address); 700 continue; 701 } else if (in->is_Con()) { 702 if (trace) {tty->print("Found constant"); in->dump();} 703 } else if (in->is_ShenandoahBarrier()) { 704 if (t == ShenandoahStore && in->Opcode() != Op_ShenandoahWriteBarrier) { 705 return false; 706 } 707 barriers_used.push(in); 708 if (trace) {tty->print("Found barrier"); in->dump();} 709 } else if (in->is_Proj() && in->in(0)->is_Allocate()) { 710 if (trace) {tty->print("Found alloc"); in->in(0)->dump();} 711 } else if (in->is_Phi()) { 712 if (!visited.test_set(in->_idx)) { 713 if (trace) {tty->print("Pushed phi:"); in->dump();} 714 phis.push(in, 2); 715 in = in->in(1); 716 continue; 717 } 718 if (trace) {tty->print("Already seen phi:"); in->dump();} 719 } else if (in->Opcode() == Op_CMoveP) { 720 if (!visited.test_set(in->_idx)) { 721 if (trace) {tty->print("Pushed cmovep:"); in->dump();} 722 phis.push(in, CMoveNode::IfTrue); 723 in = in->in(CMoveNode::IfFalse); 724 continue; 725 } 726 if (trace) {tty->print("Already seen cmovep:"); in->dump();} 727 } else if (in->Opcode() == Op_EncodeP || in->Opcode() == Op_DecodeN) { 728 in = in->in(1); 729 continue; 730 } else { 731 return false; 732 } 733 } 734 bool cont = false; 735 while (phis.is_nonempty()) { 736 uint idx = phis.index(); 737 Node* phi = phis.node(); 738 if (idx >= phi->req()) { 739 if (trace) {tty->print("Popped phi:"); phi->dump();} 740 phis.pop(); 741 continue; 742 } 743 if (trace) {tty->print("Next entry(%d) for phi:", idx); phi->dump();} 744 in = phi->in(idx); 745 phis.set_index(idx+1); 746 cont = true; 747 break; 748 } 749 if (!cont) { 750 break; 751 } 752 } 753 return true; 754 } 755 #endif 756 757 void ShenandoahBarrierNode::verify(RootNode* root) { 758 #ifdef ASSERT 759 ResourceMark rm; 760 Unique_Node_List wq; 761 GrowableArray<Node*> barriers; 762 Unique_Node_List barriers_used; 763 Node_Stack phis(0); 764 VectorSet visited(Thread::current()->resource_area()); 765 const bool trace = false; 766 const bool verify_no_useless_barrier = false; 767 768 wq.push(root); 769 for (uint next = 0; next < wq.size(); next++) { 770 Node *n = wq.at(next); 771 if (n->is_Load()) { 772 const bool trace = false; 773 if (trace) {tty->print("Verifying"); n->dump();} 774 if (n->Opcode() == Op_LoadRange || n->Opcode() == Op_LoadKlass || n->Opcode() == Op_LoadNKlass) { 775 if (trace) {tty->print_cr("Load range/klass");} 776 } else { 777 const TypePtr* adr_type = n->as_Load()->adr_type(); 778 779 if (adr_type->isa_oopptr() && adr_type->is_oopptr()->offset() == oopDesc::mark_offset_in_bytes()) { 780 if (trace) {tty->print_cr("Mark load");} 781 } else if (adr_type->isa_instptr() && 782 adr_type->is_instptr()->klass()->is_subtype_of(Compile::current()->env()->Reference_klass()) && 783 adr_type->is_instptr()->offset() == java_lang_ref_Reference::referent_offset) { 784 if (trace) {tty->print_cr("Reference.get()");} 785 } else { 786 bool verify = true; 787 if (adr_type->isa_instptr() && ShenandoahOptimizeFinals) { 788 ciKlass* k = adr_type->is_instptr()->klass(); 789 assert(k->is_instance_klass(), ""); 790 ciInstanceKlass* ik = (ciInstanceKlass*)k; 791 int offset = adr_type->offset(); 792 793 if (ik->debug_final_or_stable_field_at(offset)) { 794 if (trace) {tty->print_cr("Final/stable");} 795 verify = false; 796 } 797 } 798 799 if (verify && !ShenandoahBarrierNode::verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahLoad, trace, barriers_used)) { 800 n->dump(10); 801 ShouldNotReachHere(); 802 } 803 } 804 } 805 } else if (n->is_Store()) { 806 const bool trace = false; 807 808 if (trace) {tty->print("Verifying"); n->dump();} 809 if (n->in(MemNode::ValueIn)->bottom_type()->make_oopptr()) { 810 Node* adr = n->in(MemNode::Address); 811 bool verify = true; 812 813 if (adr->is_AddP() && adr->in(AddPNode::Base)->is_top()) { 814 adr = adr->in(AddPNode::Address); 815 if (adr->is_AddP()) { 816 assert(adr->in(AddPNode::Base)->is_top(), ""); 817 adr = adr->in(AddPNode::Address); 818 if (adr->Opcode() == Op_LoadP && 819 adr->in(MemNode::Address)->in(AddPNode::Base)->is_top() && 820 adr->in(MemNode::Address)->in(AddPNode::Address)->Opcode() == Op_ThreadLocal && 821 adr->in(MemNode::Address)->in(AddPNode::Offset)->find_intptr_t_con(-1) == in_bytes(JavaThread::satb_mark_queue_offset() + 822 SATBMarkQueue::byte_offset_of_buf())) { 823 if (trace) {tty->print_cr("G1 prebarrier");} 824 verify = false; 825 } 826 } 827 } 828 829 if (verify && !ShenandoahBarrierNode::verify_helper(n->in(MemNode::ValueIn), phis, visited, ShenandoahValue, trace, barriers_used)) { 830 n->dump(10); 831 ShouldNotReachHere(); 832 } 833 } 834 if (!ShenandoahBarrierNode::verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahStore, trace, barriers_used)) { 835 n->dump(10); 836 ShouldNotReachHere(); 837 } 838 } else if (n->Opcode() == Op_CmpP) { 839 const bool trace = false; 840 841 Node* in1 = n->in(1); 842 Node* in2 = n->in(2); 843 if (in1->bottom_type()->isa_oopptr()) { 844 if (trace) {tty->print("Verifying"); n->dump();} 845 846 bool mark_inputs = false; 847 if (in1->is_Con() || in2->is_Con()) { 848 if (trace) {tty->print_cr("Comparison against a constant");} 849 mark_inputs = true; 850 } else if ((in1->is_CheckCastPP() && in1->in(1)->is_Proj() && in1->in(1)->in(0)->is_Allocate()) || 851 (in2->is_CheckCastPP() && in2->in(1)->is_Proj() && in2->in(1)->in(0)->is_Allocate())) { 852 if (trace) {tty->print_cr("Comparison with newly alloc'ed object");} 853 mark_inputs = true; 854 } else { 855 assert(in2->bottom_type()->isa_oopptr(), ""); 856 857 if (!ShenandoahBarrierNode::verify_helper(in1, phis, visited, ShenandoahStore, trace, barriers_used) || 858 !ShenandoahBarrierNode::verify_helper(in2, phis, visited, ShenandoahStore, trace, barriers_used)) { 859 n->dump(10); 860 ShouldNotReachHere(); 861 } 862 } 863 if (verify_no_useless_barrier && 864 mark_inputs && 865 (!ShenandoahBarrierNode::verify_helper(in1, phis, visited, ShenandoahValue, trace, barriers_used) || 866 !ShenandoahBarrierNode::verify_helper(in2, phis, visited, ShenandoahValue, trace, barriers_used))) { 867 phis.clear(); 868 visited.Reset(); 869 } 870 } 871 } else if (n->is_LoadStore()) { 872 if (n->in(MemNode::ValueIn)->bottom_type()->isa_ptr() && 873 !ShenandoahBarrierNode::verify_helper(n->in(MemNode::ValueIn), phis, visited, ShenandoahLoad, trace, barriers_used)) { 874 n->dump(10); 875 ShouldNotReachHere(); 876 } 877 878 if (n->in(MemNode::Address)->bottom_type()->isa_oopptr() && !ShenandoahBarrierNode::verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahStore, trace, barriers_used)) { 879 n->dump(10); 880 ShouldNotReachHere(); 881 } 882 } else if (n->Opcode() == Op_CallLeafNoFP || n->Opcode() == Op_CallLeaf) { 883 CallNode* call = n->as_Call(); 884 885 static struct { 886 const char* name; 887 struct { 888 int pos; 889 verify_type t; 890 } args[6]; 891 } calls[] = { 892 "aescrypt_encryptBlock", 893 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad }, 894 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 895 "aescrypt_decryptBlock", 896 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad }, 897 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 898 "multiplyToLen", 899 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad }, { TypeFunc::Parms+4, ShenandoahStore }, 900 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 901 "squareToLen", 902 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad }, { -1, ShenandoahNone}, 903 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 904 "montgomery_multiply", 905 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad }, 906 { TypeFunc::Parms+6, ShenandoahStore }, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 907 "montgomery_square", 908 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+5, ShenandoahStore }, 909 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 910 "mulAdd", 911 { { TypeFunc::Parms, ShenandoahStore }, { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone}, 912 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 913 "vectorizedMismatch", 914 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone}, 915 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 916 "updateBytesCRC32", 917 { { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone}, 918 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 919 "updateBytesAdler32", 920 { { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone}, 921 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 922 "updateBytesCRC32C", 923 { { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahLoad}, { -1, ShenandoahNone}, 924 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 925 "counterMode_AESCrypt", 926 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad }, 927 { TypeFunc::Parms+3, ShenandoahStore }, { TypeFunc::Parms+5, ShenandoahStore }, { TypeFunc::Parms+6, ShenandoahStore } }, 928 "cipherBlockChaining_encryptAESCrypt", 929 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad }, 930 { TypeFunc::Parms+3, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 931 "cipherBlockChaining_decryptAESCrypt", 932 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad }, 933 { TypeFunc::Parms+3, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 934 "shenandoah_clone_barrier", 935 { { TypeFunc::Parms, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone}, 936 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 937 "ghash_processBlocks", 938 { { TypeFunc::Parms, ShenandoahStore }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad }, 939 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 940 "sha1_implCompress", 941 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone }, 942 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 943 "sha256_implCompress", 944 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone }, 945 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 946 "sha512_implCompress", 947 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone }, 948 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 949 "sha1_implCompressMB", 950 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone }, 951 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 952 "sha256_implCompressMB", 953 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone }, 954 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 955 "sha512_implCompressMB", 956 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone }, 957 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 958 }; 959 960 if (call->is_call_to_arraycopystub()) { 961 Node* dest = NULL; 962 const TypeTuple* args = n->as_Call()->_tf->domain(); 963 for (uint i = TypeFunc::Parms, j = 0; i < args->cnt(); i++) { 964 if (args->field_at(i)->isa_ptr()) { 965 j++; 966 if (j == 2) { 967 dest = n->in(i); 968 break; 969 } 970 } 971 } 972 if (!ShenandoahBarrierNode::verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahLoad, trace, barriers_used) || 973 !ShenandoahBarrierNode::verify_helper(dest, phis, visited, ShenandoahStore, trace, barriers_used)) { 974 n->dump(10); 975 ShouldNotReachHere(); 976 } 977 } else if (strlen(call->_name) > 5 && 978 !strcmp(call->_name + strlen(call->_name) - 5, "_fill")) { 979 if (!ShenandoahBarrierNode::verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahStore, trace, barriers_used)) { 980 n->dump(10); 981 ShouldNotReachHere(); 982 } 983 } else if (!strcmp(call->_name, "g1_wb_pre")) { 984 // skip 985 } else { 986 const int calls_len = sizeof(calls) / sizeof(calls[0]); 987 int i = 0; 988 for (; i < calls_len; i++) { 989 if (!strcmp(calls[i].name, call->_name)) { 990 break; 991 } 992 } 993 if (i != calls_len) { 994 const uint args_len = sizeof(calls[0].args) / sizeof(calls[0].args[0]); 995 for (uint j = 0; j < args_len; j++) { 996 int pos = calls[i].args[j].pos; 997 if (pos == -1) { 998 break; 999 } 1000 if (!ShenandoahBarrierNode::verify_helper(call->in(pos), phis, visited, calls[i].args[j].t, trace, barriers_used)) { 1001 n->dump(10); 1002 ShouldNotReachHere(); 1003 } 1004 } 1005 for (uint j = TypeFunc::Parms; j < call->req(); j++) { 1006 if (call->in(j)->bottom_type()->make_ptr() && 1007 call->in(j)->bottom_type()->make_ptr()->isa_oopptr()) { 1008 uint k = 0; 1009 for (; k < args_len && calls[i].args[k].pos != (int)j; k++); 1010 if (k == args_len) { 1011 fatal("arg %d for call %s not covered", j, call->_name); 1012 } 1013 } 1014 } 1015 } else { 1016 for (uint j = TypeFunc::Parms; j < call->req(); j++) { 1017 if (call->in(j)->bottom_type()->make_ptr() && 1018 call->in(j)->bottom_type()->make_ptr()->isa_oopptr()) { 1019 fatal("%s not covered", call->_name); 1020 } 1021 } 1022 } 1023 } 1024 } else if (n->is_ShenandoahBarrier()) { 1025 assert(!barriers.contains(n), ""); 1026 assert(n->Opcode() != Op_ShenandoahWriteBarrier || n->find_out_with(Op_ShenandoahWBMemProj) != NULL, "bad shenandoah write barrier"); 1027 assert(n->Opcode() != Op_ShenandoahWriteBarrier || n->outcnt() > 1, "bad shenandoah write barrier"); 1028 barriers.push(n); 1029 } else if (n->is_AddP() 1030 || n->is_Phi() 1031 || n->is_ConstraintCast() 1032 || n->Opcode() == Op_Return 1033 || n->Opcode() == Op_CMoveP 1034 || n->Opcode() == Op_CMoveN 1035 || n->Opcode() == Op_Rethrow 1036 || n->is_MemBar() 1037 || n->Opcode() == Op_Conv2B 1038 || n->Opcode() == Op_SafePoint 1039 || n->is_CallJava() 1040 || n->Opcode() == Op_Unlock 1041 || n->Opcode() == Op_EncodeP 1042 || n->Opcode() == Op_DecodeN) { 1043 // nothing to do 1044 } else { 1045 static struct { 1046 int opcode; 1047 struct { 1048 int pos; 1049 verify_type t; 1050 } inputs[2]; 1051 } others[] = { 1052 Op_FastLock, 1053 { { 1, ShenandoahLoad }, { -1, ShenandoahNone} }, 1054 Op_Lock, 1055 { { TypeFunc::Parms, ShenandoahLoad }, { -1, ShenandoahNone} }, 1056 Op_ArrayCopy, 1057 { { ArrayCopyNode::Src, ShenandoahLoad }, { ArrayCopyNode::Dest, ShenandoahStore } }, 1058 Op_StrCompressedCopy, 1059 { { 2, ShenandoahLoad }, { 3, ShenandoahStore } }, 1060 Op_StrInflatedCopy, 1061 { { 2, ShenandoahLoad }, { 3, ShenandoahStore } }, 1062 Op_AryEq, 1063 { { 2, ShenandoahLoad }, { 3, ShenandoahLoad } }, 1064 Op_StrIndexOf, 1065 { { 2, ShenandoahLoad }, { 4, ShenandoahLoad } }, 1066 Op_StrComp, 1067 { { 2, ShenandoahLoad }, { 4, ShenandoahLoad } }, 1068 Op_StrEquals, 1069 { { 2, ShenandoahLoad }, { 3, ShenandoahLoad } }, 1070 Op_EncodeISOArray, 1071 { { 2, ShenandoahLoad }, { 3, ShenandoahStore } }, 1072 Op_HasNegatives, 1073 { { 2, ShenandoahLoad }, { -1, ShenandoahNone} }, 1074 Op_CastP2X, 1075 { { 1, ShenandoahLoad }, { -1, ShenandoahNone} }, 1076 Op_StrIndexOfChar, 1077 { { 2, ShenandoahLoad }, { -1, ShenandoahNone } }, 1078 }; 1079 1080 const int others_len = sizeof(others) / sizeof(others[0]); 1081 int i = 0; 1082 for (; i < others_len; i++) { 1083 if (others[i].opcode == n->Opcode()) { 1084 break; 1085 } 1086 } 1087 uint stop = n->is_Call() ? n->as_Call()->tf()->domain()->cnt() : n->req(); 1088 if (i != others_len) { 1089 const uint inputs_len = sizeof(others[0].inputs) / sizeof(others[0].inputs[0]); 1090 for (uint j = 0; j < inputs_len; j++) { 1091 int pos = others[i].inputs[j].pos; 1092 if (pos == -1) { 1093 break; 1094 } 1095 if (!ShenandoahBarrierNode::verify_helper(n->in(pos), phis, visited, others[i].inputs[j].t, trace, barriers_used)) { 1096 n->dump(10); 1097 ShouldNotReachHere(); 1098 } 1099 } 1100 for (uint j = 1; j < stop; j++) { 1101 if (n->in(j) != NULL && n->in(j)->bottom_type()->make_ptr() && 1102 n->in(j)->bottom_type()->make_ptr()->make_oopptr()) { 1103 uint k = 0; 1104 for (; k < inputs_len && others[i].inputs[k].pos != (int)j; k++); 1105 if (k == inputs_len) { 1106 fatal("arg %d for node %s not covered", j, n->Name()); 1107 } 1108 } 1109 } 1110 } else { 1111 for (uint j = 1; j < stop; j++) { 1112 if (n->in(j) != NULL && n->in(j)->bottom_type()->make_ptr() && 1113 n->in(j)->bottom_type()->make_ptr()->make_oopptr()) { 1114 fatal("%s not covered", n->Name()); 1115 } 1116 } 1117 } 1118 } 1119 1120 if (n->is_SafePoint()) { 1121 SafePointNode* sfpt = n->as_SafePoint(); 1122 if (verify_no_useless_barrier && sfpt->jvms() != NULL) { 1123 for (uint i = sfpt->jvms()->scloff(); i < sfpt->jvms()->endoff(); i++) { 1124 if (!ShenandoahBarrierNode::verify_helper(sfpt->in(i), phis, visited, ShenandoahLoad, trace, barriers_used)) { 1125 phis.clear(); 1126 visited.Reset(); 1127 } 1128 } 1129 } 1130 } 1131 for( uint i = 0; i < n->len(); ++i ) { 1132 Node *m = n->in(i); 1133 if (m == NULL) continue; 1134 1135 // In most cases, inputs should be known to be non null. If it's 1136 // not the case, it could be a missing cast_not_null() in an 1137 // intrinsic or support might be needed in AddPNode::Ideal() to 1138 // avoid a NULL+offset input. 1139 if (!(n->is_Phi() || 1140 (n->is_SafePoint() && (!n->is_CallRuntime() || !strcmp(n->as_Call()->_name, "g1_wb_pre") || !strcmp(n->as_Call()->_name, "unsafe_arraycopy"))) || 1141 n->Opcode() == Op_CmpP || 1142 n->Opcode() == Op_CmpN || 1143 (n->Opcode() == Op_StoreP && i == StoreNode::ValueIn) || 1144 (n->Opcode() == Op_StoreN && i == StoreNode::ValueIn) || 1145 n->is_ConstraintCast() || 1146 n->Opcode() == Op_Return || 1147 n->Opcode() == Op_Conv2B || 1148 n->is_AddP() || 1149 n->Opcode() == Op_CMoveP || 1150 n->Opcode() == Op_CMoveN || 1151 n->Opcode() == Op_Rethrow || 1152 n->is_MemBar() || 1153 n->is_Mem() || 1154 n->Opcode() == Op_AryEq || 1155 n->Opcode() == Op_SCMemProj || 1156 n->Opcode() == Op_EncodeP || 1157 n->Opcode() == Op_DecodeN)) { 1158 if (m->bottom_type()->make_oopptr() && m->bottom_type()->make_oopptr()->meet(TypePtr::NULL_PTR) == m->bottom_type()) { 1159 n->dump(); 1160 m->dump(); 1161 ShouldNotReachHere(); 1162 } 1163 } 1164 1165 wq.push(m); 1166 } 1167 } 1168 1169 if (verify_no_useless_barrier) { 1170 for (int i = 0; i < barriers.length(); i++) { 1171 Node* n = barriers.at(i); 1172 if (!barriers_used.member(n)) { 1173 tty->print("XXX useless barrier"); n->dump(-2); 1174 ShouldNotReachHere(); 1175 } 1176 } 1177 } 1178 #endif 1179 } 1180 1181 #include "opto/loopnode.hpp" 1182 1183 MergeMemNode* ShenandoahWriteBarrierNode::allocate_merge_mem(Node* mem, int alias, Node* rep_proj, Node* rep_ctrl, PhaseIdealLoop* phase) { 1184 MergeMemNode* mm = MergeMemNode::make(mem); 1185 mm->set_memory_at(alias, rep_proj); 1186 phase->register_new_node(mm, rep_ctrl); 1187 return mm; 1188 } 1189 1190 MergeMemNode* ShenandoahWriteBarrierNode::clone_merge_mem(Node* u, Node* mem, int alias, Node* rep_proj, Node* rep_ctrl, DUIterator& i, PhaseIdealLoop* phase) { 1191 MergeMemNode* newmm = NULL; 1192 MergeMemNode* u_mm = u->as_MergeMem(); 1193 Node* c = phase->get_ctrl(u); 1194 if (phase->is_dominator(c, rep_ctrl)) { 1195 c = rep_ctrl; 1196 } else { 1197 assert(phase->is_dominator(rep_ctrl, c), "one must dominate the other"); 1198 } 1199 if (u->outcnt() == 1) { 1200 if (u->req() > (uint)alias && u->in(alias) == mem) { 1201 phase->igvn().replace_input_of(u, alias, rep_proj); 1202 --i; 1203 } else { 1204 phase->igvn().rehash_node_delayed(u); 1205 u_mm->set_memory_at(alias, rep_proj); 1206 } 1207 newmm = u_mm; 1208 phase->set_ctrl_and_loop(u, c); 1209 } else { 1210 // can't simply clone u and then change one of its input because 1211 // it adds and then removes an edge which messes with the 1212 // DUIterator 1213 newmm = MergeMemNode::make(u_mm->base_memory()); 1214 for (uint j = 0; j < u->req(); j++) { 1215 if (j < newmm->req()) { 1216 if (j == (uint)alias) { 1217 newmm->set_req(j, rep_proj); 1218 } else if (newmm->in(j) != u->in(j)) { 1219 newmm->set_req(j, u->in(j)); 1220 } 1221 } else if (j == (uint)alias) { 1222 newmm->add_req(rep_proj); 1223 } else { 1224 newmm->add_req(u->in(j)); 1225 } 1226 } 1227 if ((uint)alias >= u->req()) { 1228 newmm->set_memory_at(alias, rep_proj); 1229 } 1230 phase->register_new_node(newmm, c); 1231 } 1232 return newmm; 1233 } 1234 1235 bool ShenandoahWriteBarrierNode::should_process_phi(Node* phi, int alias, Compile* C) { 1236 if (phi->adr_type() == TypePtr::BOTTOM) { 1237 Node* region = phi->in(0); 1238 for (DUIterator_Fast jmax, j = region->fast_outs(jmax); j < jmax; j++) { 1239 Node* uu = region->fast_out(j); 1240 if (uu->is_Phi() && uu != phi && uu->bottom_type() == Type::MEMORY && C->get_alias_index(uu->adr_type()) == alias) { 1241 return false; 1242 } 1243 } 1244 return true; 1245 } 1246 return C->get_alias_index(phi->adr_type()) == alias; 1247 } 1248 1249 bool ShenandoahBarrierNode::is_dominator_same_ctrl(Node*c, Node* d, Node* n, PhaseIdealLoop* phase) { 1250 // That both nodes have the same control is not sufficient to prove 1251 // domination, verify that there's no path from d to n 1252 ResourceMark rm; 1253 Unique_Node_List wq; 1254 wq.push(d); 1255 for (uint next = 0; next < wq.size(); next++) { 1256 Node *m = wq.at(next); 1257 if (m == n) { 1258 return false; 1259 } 1260 if (m->is_Phi() && m->in(0)->is_Loop()) { 1261 assert(phase->ctrl_or_self(m->in(LoopNode::EntryControl)) != c, "following loop entry should lead to new control"); 1262 } else { 1263 for (uint i = 0; i < m->req(); i++) { 1264 if (m->in(i) != NULL && phase->ctrl_or_self(m->in(i)) == c) { 1265 wq.push(m->in(i)); 1266 } 1267 } 1268 } 1269 } 1270 return true; 1271 } 1272 1273 bool ShenandoahBarrierNode::is_dominator(Node *d_c, Node *n_c, Node* d, Node* n, PhaseIdealLoop* phase) { 1274 if (d_c != n_c) { 1275 return phase->is_dominator(d_c, n_c); 1276 } 1277 return is_dominator_same_ctrl(d_c, d, n, phase); 1278 } 1279 1280 Node* next_mem(Node* mem, int alias) { 1281 Node* res = NULL; 1282 if (mem->is_Proj()) { 1283 res = mem->in(0); 1284 } else if (mem->is_SafePoint() || mem->is_MemBar()) { 1285 res = mem->in(TypeFunc::Memory); 1286 } else if (mem->is_Phi()) { 1287 res = mem->in(1); 1288 } else if (mem->is_ShenandoahBarrier()) { 1289 res = mem->in(ShenandoahBarrierNode::Memory); 1290 } else if (mem->is_MergeMem()) { 1291 res = mem->as_MergeMem()->memory_at(alias); 1292 } else if (mem->is_Store() || mem->is_LoadStore() || mem->is_ClearArray()) { 1293 assert(alias = Compile::AliasIdxRaw, "following raw memory can't lead to a barrier"); 1294 res = mem->in(MemNode::Memory); 1295 } else { 1296 #ifdef ASSERT 1297 mem->dump(); 1298 #endif 1299 ShouldNotReachHere(); 1300 } 1301 return res; 1302 } 1303 1304 bool suitable_mem(Node* mem, Node* old_mem, Node* rep_proj) { 1305 for (DUIterator_Fast imax, i = mem->fast_outs(imax); i < imax; i++) { 1306 Node* u = mem->fast_out(i); 1307 if (u->is_MergeMem()) { 1308 if (u->has_out_with(Op_MergeMem)) { 1309 // too complicated for now 1310 return false; 1311 } 1312 if (old_mem == u && rep_proj->has_out_with(Op_MergeMem)) { 1313 return false; 1314 } 1315 } 1316 if (u->Opcode() == Op_Unlock && mem->is_Proj() && mem->in(0)->Opcode() == Op_MemBarReleaseLock) { 1317 // would require a merge mem between unlock and the 1318 // preceding membar. Would confuse logic that eliminates 1319 // lock/unlock nodes. 1320 return false; 1321 } 1322 } 1323 return true; 1324 } 1325 1326 void ShenandoahWriteBarrierNode::fix_memory_uses(Node* mem, Node* replacement, Node* rep_proj, Node* rep_ctrl, int alias, PhaseIdealLoop* phase) { 1327 uint last =phase-> C->unique(); 1328 MergeMemNode* mm = NULL; 1329 assert(mem->bottom_type() == Type::MEMORY, ""); 1330 for (DUIterator i = mem->outs(); mem->has_out(i); i++) { 1331 Node* u = mem->out(i); 1332 if (u != replacement && u->_idx < last) { 1333 if (u->is_ShenandoahBarrier() && alias != Compile::AliasIdxRaw) { 1334 if (phase->C->get_alias_index(u->adr_type()) == alias && is_dominator(rep_ctrl, phase->ctrl_or_self(u), replacement, u, phase)) { 1335 phase->igvn().replace_input_of(u, u->find_edge(mem), rep_proj); 1336 assert(u->find_edge(mem) == -1, "only one edge"); 1337 --i; 1338 } 1339 } else if (u->is_Mem()) { 1340 if (phase->C->get_alias_index(u->adr_type()) == alias && is_dominator(rep_ctrl, phase->ctrl_or_self(u), replacement, u, phase)) { 1341 assert(alias == Compile::AliasIdxRaw , "only raw memory can lead to a memory operation"); 1342 phase->igvn().replace_input_of(u, u->find_edge(mem), rep_proj); 1343 assert(u->find_edge(mem) == -1, "only one edge"); 1344 --i; 1345 } 1346 } else if (u->is_MergeMem()) { 1347 MergeMemNode* u_mm = u->as_MergeMem(); 1348 if (u_mm->memory_at(alias) == mem) { 1349 MergeMemNode* newmm = NULL; 1350 for (DUIterator_Fast jmax, j = u->fast_outs(jmax); j < jmax; j++) { 1351 Node* uu = u->fast_out(j); 1352 assert(!uu->is_MergeMem(), "chain of MergeMems?"); 1353 if (uu->is_Phi()) { 1354 if (should_process_phi(uu, alias, phase->C)) { 1355 Node* region = uu->in(0); 1356 int nb = 0; 1357 for (uint k = 1; k < uu->req(); k++) { 1358 if (uu->in(k) == u && phase->is_dominator(rep_ctrl, region->in(k))) { 1359 if (newmm == NULL) { 1360 newmm = clone_merge_mem(u, mem, alias, rep_proj, rep_ctrl, i, phase); 1361 } 1362 if (newmm != u) { 1363 phase->igvn().replace_input_of(uu, k, newmm); 1364 nb++; 1365 --jmax; 1366 } 1367 } 1368 } 1369 if (nb > 0) { 1370 --j; 1371 } 1372 } 1373 } else { 1374 if (rep_ctrl != uu && is_dominator(rep_ctrl, phase->ctrl_or_self(uu), replacement, uu, phase)) { 1375 if (newmm == NULL) { 1376 newmm = clone_merge_mem(u, mem, alias, rep_proj, rep_ctrl, i, phase); 1377 } 1378 if (newmm != u) { 1379 phase->igvn().replace_input_of(uu, uu->find_edge(u), newmm); 1380 --j, --jmax; 1381 } 1382 } 1383 } 1384 } 1385 } 1386 } else if (u->is_Phi()) { 1387 assert(u->bottom_type() == Type::MEMORY, "what else?"); 1388 Node* region = u->in(0); 1389 if (should_process_phi(u, alias, phase->C)) { 1390 bool replaced = false; 1391 for (uint j = 1; j < u->req(); j++) { 1392 if (u->in(j) == mem && phase->is_dominator(rep_ctrl, region->in(j))) { 1393 Node* nnew = rep_proj; 1394 if (u->adr_type() == TypePtr::BOTTOM) { 1395 if (mm == NULL) { 1396 mm = allocate_merge_mem(mem, alias, rep_proj, rep_ctrl, phase); 1397 } 1398 nnew = mm; 1399 } 1400 phase->igvn().replace_input_of(u, j, nnew); 1401 replaced = true; 1402 } 1403 } 1404 if (replaced) { 1405 --i; 1406 } 1407 1408 } 1409 } else if ((u->adr_type() == TypePtr::BOTTOM && u->Opcode() != Op_StrInflatedCopy) || 1410 u->adr_type() == NULL) { 1411 assert(u->adr_type() != NULL || 1412 u->Opcode() == Op_Rethrow || 1413 u->Opcode() == Op_Return || 1414 u->Opcode() == Op_SafePoint || 1415 (u->is_CallStaticJava() && u->as_CallStaticJava()->uncommon_trap_request() != 0) || 1416 (u->is_CallStaticJava() && u->as_CallStaticJava()->_entry_point == OptoRuntime::rethrow_stub()) || 1417 u->Opcode() == Op_CallLeaf, ""); 1418 if (is_dominator(rep_ctrl, phase->ctrl_or_self(u), replacement, u, phase)) { 1419 if (mm == NULL) { 1420 mm = allocate_merge_mem(mem, alias, rep_proj, rep_ctrl, phase); 1421 } 1422 phase->igvn().replace_input_of(u, u->find_edge(mem), mm); 1423 --i; 1424 } 1425 } else if (phase->C->get_alias_index(u->adr_type()) == alias) { 1426 if (is_dominator(rep_ctrl, phase->ctrl_or_self(u), replacement, u, phase)) { 1427 phase->igvn().replace_input_of(u, u->find_edge(mem), rep_proj); 1428 --i; 1429 } 1430 } 1431 } 1432 } 1433 } 1434 1435 Node* ShenandoahBarrierNode::no_branches(Node* c, Node* dom, bool allow_one_proj, PhaseIdealLoop* phase) { 1436 Node* iffproj = NULL; 1437 while (c != dom) { 1438 Node* next = phase->idom(c); 1439 assert(next->unique_ctrl_out() == c || c->is_Proj() || c->is_Region(), "multiple control flow out but no proj or region?"); 1440 if (c->is_Region()) { 1441 ResourceMark rm; 1442 Unique_Node_List wq; 1443 wq.push(c); 1444 for (uint i = 0; i < wq.size(); i++) { 1445 Node *n = wq.at(i); 1446 if (n->is_Region()) { 1447 for (uint j = 1; j < n->req(); j++) { 1448 if (n->in(j) != next) { 1449 wq.push(n->in(j)); 1450 } 1451 } 1452 } else { 1453 if (n->in(0) != next) { 1454 wq.push(n->in(0)); 1455 } 1456 } 1457 } 1458 for (DUIterator_Fast imax, i = next->fast_outs(imax); i < imax; i++) { 1459 Node* u = next->fast_out(i); 1460 if (u->is_CFG()) { 1461 if (!wq.member(u)) { 1462 return NodeSentinel; 1463 } 1464 } 1465 } 1466 1467 } else if (c->is_Proj()) { 1468 if (c->is_IfProj()) { 1469 if (c->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none) != NULL) { 1470 // continue; 1471 } else { 1472 if (!allow_one_proj) { 1473 return NodeSentinel; 1474 } 1475 if (iffproj == NULL) { 1476 iffproj = c; 1477 } else { 1478 return NodeSentinel; 1479 } 1480 } 1481 } else if (c->Opcode() == Op_JumpProj) { 1482 return NodeSentinel; // unsupported 1483 } else if (c->Opcode() == Op_CatchProj) { 1484 return NodeSentinel; // unsupported 1485 } else if (c->Opcode() == Op_CProj && next->Opcode() == Op_NeverBranch) { 1486 return NodeSentinel; // unsupported 1487 } else { 1488 assert(next->unique_ctrl_out() == c, "unsupported branch pattern"); 1489 } 1490 } 1491 c = next; 1492 } 1493 return iffproj; 1494 } 1495 1496 #ifdef ASSERT 1497 void ShenandoahWriteBarrierNode::memory_dominates_all_paths_helper(Node* c, Node* rep_ctrl, Unique_Node_List& controls, PhaseIdealLoop* phase) { 1498 const bool trace = false; 1499 if (trace) { tty->print("X control is"); c->dump(); } 1500 1501 uint start = controls.size(); 1502 controls.push(c); 1503 for (uint i = start; i < controls.size(); i++) { 1504 Node *n = controls.at(i); 1505 1506 if (trace) { tty->print("X from"); n->dump(); } 1507 1508 if (n == rep_ctrl) { 1509 continue; 1510 } 1511 1512 if (n->is_Proj()) { 1513 Node* n_dom = n->in(0); 1514 IdealLoopTree* n_dom_loop = phase->get_loop(n_dom); 1515 if (n->is_IfProj() && n_dom->outcnt() == 2) { 1516 n_dom_loop = phase->get_loop(n_dom->as_If()->proj_out(n->as_Proj()->_con == 0 ? 1 : 0)); 1517 } 1518 if (n_dom_loop != phase->ltree_root()) { 1519 Node* tail = n_dom_loop->tail(); 1520 if (tail->is_Region()) { 1521 for (uint j = 1; j < tail->req(); j++) { 1522 if (phase->is_dominator(n_dom, tail->in(j)) && !phase->is_dominator(n, tail->in(j))) { 1523 assert(phase->is_dominator(rep_ctrl, tail->in(j)), "why are we here?"); 1524 // entering loop from below, mark backedge 1525 if (trace) { tty->print("X pushing backedge"); tail->in(j)->dump(); } 1526 controls.push(tail->in(j)); 1527 //assert(n->in(0) == n_dom, "strange flow control"); 1528 } 1529 } 1530 } else if (phase->get_loop(n) != n_dom_loop && phase->is_dominator(n_dom, tail)) { 1531 // entering loop from below, mark backedge 1532 if (trace) { tty->print("X pushing backedge"); tail->dump(); } 1533 controls.push(tail); 1534 //assert(n->in(0) == n_dom, "strange flow control"); 1535 } 1536 } 1537 } 1538 1539 if (n->is_Loop()) { 1540 Node* c = n->in(LoopNode::EntryControl); 1541 if (trace) { tty->print("X pushing"); c->dump(); } 1542 controls.push(c); 1543 } else if (n->is_Region()) { 1544 for (uint i = 1; i < n->req(); i++) { 1545 Node* c = n->in(i); 1546 if (trace) { tty->print("X pushing"); c->dump(); } 1547 controls.push(c); 1548 } 1549 } else { 1550 Node* c = n->in(0); 1551 if (trace) { tty->print("X pushing"); c->dump(); } 1552 controls.push(c); 1553 } 1554 } 1555 1556 } 1557 1558 bool ShenandoahWriteBarrierNode::memory_dominates_all_paths(Node* mem, Node* rep_ctrl, int alias, PhaseIdealLoop* phase) { 1559 const bool trace = false; 1560 if (trace) { 1561 tty->print("XXX mem is"); mem->dump(); 1562 tty->print("XXX rep ctrl is"); rep_ctrl->dump(); 1563 tty->print_cr("XXX alias is %d", alias); 1564 } 1565 ResourceMark rm; 1566 Unique_Node_List wq; 1567 Unique_Node_List controls; 1568 wq.push(mem); 1569 for (uint next = 0; next < wq.size(); next++) { 1570 Node *nn = wq.at(next); 1571 if (trace) { tty->print("XX from mem"); nn->dump(); } 1572 assert(nn->bottom_type() == Type::MEMORY, "memory only"); 1573 1574 if (nn->is_Phi()) { 1575 Node* r = nn->in(0); 1576 for (DUIterator_Fast jmax, j = r->fast_outs(jmax); j < jmax; j++) { 1577 Node* u = r->fast_out(j); 1578 if (u->is_Phi() && u->bottom_type() == Type::MEMORY && u != nn && 1579 (u->adr_type() == TypePtr::BOTTOM || phase->C->get_alias_index(u->adr_type()) == alias)) { 1580 if (trace) { tty->print("XX Next mem (other phi)"); u->dump(); } 1581 wq.push(u); 1582 } 1583 } 1584 } 1585 1586 for (DUIterator_Fast imax, i = nn->fast_outs(imax); i < imax; i++) { 1587 Node* use = nn->fast_out(i); 1588 1589 if (trace) { tty->print("XX use %p", use->adr_type()); use->dump(); } 1590 if (use->is_CFG()) { 1591 assert(use->in(TypeFunc::Memory) == nn, "bad cfg node"); 1592 Node* c = use->in(0); 1593 if (phase->is_dominator(rep_ctrl, c)) { 1594 memory_dominates_all_paths_helper(c, rep_ctrl, controls, phase); 1595 } else if (use->is_CallStaticJava() && use->as_CallStaticJava()->uncommon_trap_request() != 0 && c->is_Region()) { 1596 Node* region = c; 1597 if (trace) { tty->print("XX unc region"); region->dump(); } 1598 for (uint j = 1; j < region->req(); j++) { 1599 if (phase->is_dominator(rep_ctrl, region->in(j))) { 1600 if (trace) { tty->print("XX unc follows"); region->in(j)->dump(); } 1601 memory_dominates_all_paths_helper(region->in(j), rep_ctrl, controls, phase); 1602 } 1603 } 1604 } 1605 //continue; 1606 } else if (use->is_Phi()) { 1607 assert(use->bottom_type() == Type::MEMORY, "bad phi"); 1608 if ((use->adr_type() == TypePtr::BOTTOM /*&& !shenandoah_has_alias_phi(C, use, alias)*/) || 1609 phase->C->get_alias_index(use->adr_type()) == alias) { 1610 for (uint j = 1; j < use->req(); j++) { 1611 if (use->in(j) == nn) { 1612 Node* c = use->in(0)->in(j); 1613 if (phase->is_dominator(rep_ctrl, c)) { 1614 memory_dominates_all_paths_helper(c, rep_ctrl, controls, phase); 1615 } 1616 } 1617 } 1618 } 1619 // continue; 1620 } 1621 1622 if (use->is_MergeMem()) { 1623 if (use->as_MergeMem()->memory_at(alias) == nn) { 1624 if (trace) { tty->print("XX Next mem"); use->dump(); } 1625 // follow the memory edges 1626 wq.push(use); 1627 } 1628 } else if (use->is_Phi()) { 1629 assert(use->bottom_type() == Type::MEMORY, "bad phi"); 1630 if ((use->adr_type() == TypePtr::BOTTOM /*&& !shenandoah_has_alias_phi(C, use, alias)*/) || 1631 phase->C->get_alias_index(use->adr_type()) == alias) { 1632 if (trace) { tty->print("XX Next mem"); use->dump(); } 1633 // follow the memory edges 1634 wq.push(use); 1635 } 1636 } else if (use->bottom_type() == Type::MEMORY && 1637 (use->adr_type() == TypePtr::BOTTOM || phase->C->get_alias_index(use->adr_type()) == alias)) { 1638 if (trace) { tty->print("XX Next mem"); use->dump(); } 1639 // follow the memory edges 1640 wq.push(use); 1641 } else if ((use->is_SafePoint() || use->is_MemBar()) && 1642 (use->adr_type() == TypePtr::BOTTOM || phase->C->get_alias_index(use->adr_type()) == alias)) { 1643 for (DUIterator_Fast jmax, j = use->fast_outs(jmax); j < jmax; j++) { 1644 Node* u = use->fast_out(j); 1645 if (u->bottom_type() == Type::MEMORY) { 1646 if (trace) { tty->print("XX Next mem"); u->dump(); } 1647 // follow the memory edges 1648 wq.push(u); 1649 } 1650 } 1651 } else if (use->Opcode() == Op_ShenandoahWriteBarrier && phase->C->get_alias_index(use->adr_type()) == alias) { 1652 Node* m = use->find_out_with(Op_ShenandoahWBMemProj); 1653 if (m != NULL) { 1654 if (trace) { tty->print("XX Next mem"); m->dump(); } 1655 // follow the memory edges 1656 wq.push(m); 1657 } 1658 } 1659 } 1660 } 1661 1662 if (controls.size() == 0) { 1663 return false; 1664 } 1665 1666 for (uint i = 0; i < controls.size(); i++) { 1667 Node *n = controls.at(i); 1668 1669 if (trace) { tty->print("X checking"); n->dump(); } 1670 1671 if (n->unique_ctrl_out() != NULL) { 1672 continue; 1673 } 1674 1675 if (n->Opcode() == Op_NeverBranch) { 1676 Node* taken = n->as_Multi()->proj_out(0); 1677 if (!controls.member(taken)) { 1678 if (trace) { tty->print("X not seen"); taken->dump(); } 1679 return false; 1680 } 1681 continue; 1682 } 1683 1684 for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) { 1685 Node* u = n->fast_out(j); 1686 1687 if (u->is_CFG()) { 1688 if (!controls.member(u)) { 1689 if (u->is_Proj() && u->as_Proj()->is_uncommon_trap_proj(Deoptimization::Reason_none)) { 1690 if (trace) { tty->print("X not seen but unc"); u->dump(); } 1691 } else { 1692 if (trace) { tty->print("X not seen"); u->dump(); } 1693 return false; 1694 } 1695 } else { 1696 if (trace) { tty->print("X seen"); u->dump(); } 1697 } 1698 } 1699 } 1700 } 1701 return true; 1702 } 1703 #endif 1704 1705 static bool has_mem_phi(Compile* C, Node* region, int alias) { 1706 for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) { 1707 Node* use = region->fast_out(i); 1708 if (use->is_Phi() && use->bottom_type() == Type::MEMORY && 1709 (C->get_alias_index(use->adr_type()) == alias)) { 1710 return true; 1711 } 1712 } 1713 return false; 1714 } 1715 1716 bool ShenandoahWriteBarrierNode::fix_mem_phis_helper(Node* c, Node* mem, Node* mem_ctrl, Node* rep_ctrl, int alias, VectorSet& controls, GrowableArray<Node*>& regions, PhaseIdealLoop* phase) { 1717 const bool trace = false; 1718 Node_List wq; 1719 wq.push(c); 1720 1721 #ifdef ASSERT 1722 if (trace) { tty->print("YYY from"); c->dump(); } 1723 if (trace) { tty->print("YYY with mem"); mem->dump(); } 1724 #endif 1725 1726 while(wq.size() > 0) { 1727 c = wq.pop(); 1728 1729 while (!c->is_Region() || c->is_Loop()) { 1730 #ifdef ASSERT 1731 if (trace) { tty->print("YYY"); c->dump(); } 1732 #endif 1733 assert(c->is_CFG(), "node should be control node"); 1734 if (c == mem_ctrl || phase->is_dominator(c, rep_ctrl)) { 1735 c = NULL; 1736 break; 1737 } else if (c->is_Loop()) { 1738 c = c->in(LoopNode::EntryControl); 1739 } else { 1740 c = c->in(0); 1741 } 1742 } 1743 if (c == NULL) { 1744 continue; 1745 } 1746 1747 #ifdef ASSERT 1748 if (trace) { tty->print("YYY new region"); c->dump(); } 1749 #endif 1750 1751 bool has_phi = has_mem_phi(phase->C, c, alias); 1752 if (!has_phi) { 1753 1754 Node* m_ctrl = NULL; 1755 Node* m = dom_mem(mem, c, alias, m_ctrl, phase); 1756 if (m == NULL) { 1757 return false; 1758 } 1759 1760 #ifdef ASSERT 1761 if (trace) { tty->print("YYY mem "); m->dump(); } 1762 #endif 1763 1764 if (controls.test(c->_idx)) { 1765 int i = 0; 1766 for (; i < regions.length() && regions.at(i) != c; i+=2); 1767 assert(i < regions.length(), "missing region"); 1768 Node* prev_m = regions.at(i+1); 1769 if (prev_m == m) { 1770 continue; 1771 } 1772 #ifdef ASSERT 1773 if (trace) { tty->print("YYY prev mem "); prev_m->dump(); } 1774 #endif 1775 Node* prev_m_ctrl = phase->ctrl_or_self(prev_m); 1776 assert(is_dominator(m_ctrl, prev_m_ctrl, m, prev_m, phase) || 1777 is_dominator(prev_m_ctrl, m_ctrl, prev_m, m, phase), "one should dominate the other"); 1778 if (is_dominator(m_ctrl, prev_m_ctrl, m, prev_m, phase)) { 1779 continue; 1780 } 1781 #ifdef ASSERT 1782 if (trace) { tty->print("YYY Fixing "); c->dump(); } 1783 #endif 1784 regions.at_put(i+1, m); 1785 } else { 1786 #ifdef ASSERT 1787 if (trace) { tty->print("YYY Pushing "); c->dump(); } 1788 #endif 1789 regions.push(c); 1790 regions.push(m); 1791 } 1792 } else { 1793 continue; 1794 } 1795 1796 controls.set(c->_idx); 1797 1798 for (uint i = 1; i < c->req(); i++) { 1799 wq.push(c->in(i)); 1800 } 1801 } 1802 return true; 1803 } 1804 1805 1806 bool ShenandoahWriteBarrierNode::fix_mem_phis(Node* mem, Node* mem_ctrl, Node* rep_ctrl, int alias, PhaseIdealLoop* phase) { 1807 //ResourceMark rm; // register_new_node makes an internal grow 1808 GrowableArray<Node*> regions; 1809 VectorSet controls(Thread::current()->resource_area()); 1810 const bool trace = false; 1811 1812 #ifdef ASSERT 1813 if (trace) { tty->print("YYY mem is "); mem->dump(); } 1814 if (trace) { tty->print("YYY mem ctrl is "); mem_ctrl->dump(); } 1815 if (trace) { tty->print("YYY rep ctrl is "); rep_ctrl->dump(); } 1816 if (trace) { tty->print_cr("YYY alias is %d", alias); } 1817 #endif 1818 1819 // Walk memory edges from mem until we hit a memory point where 1820 // control is known then follow the control up looking for regions 1821 // with no memory Phi for alias 1822 Unique_Node_List wq; 1823 wq.push(mem); 1824 1825 for (uint next = 0; next < wq.size(); next++) { 1826 Node *n = wq.at(next); 1827 #ifdef ASSERT 1828 if (trace) { tty->print("YYY from (2) "); n->dump(); } 1829 #endif 1830 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { 1831 Node* u = n->fast_out(i); 1832 #ifdef ASSERT 1833 if (trace) { tty->print("YYY processing "); u->dump(); } 1834 #endif 1835 if (u->is_Phi()) { 1836 assert(u->bottom_type() == Type::MEMORY, "strange memory graph"); 1837 if (should_process_phi(u, alias, phase->C)) { 1838 for (uint j = 1; j < u->req(); j++) { 1839 if (u->in(j) == n) { 1840 Node *c = u->in(0)->in(j); 1841 if (!fix_mem_phis_helper(c, n, mem_ctrl, rep_ctrl, alias, controls, regions, phase)) { 1842 return false; 1843 } 1844 } 1845 } 1846 } 1847 #ifdef ASSERT 1848 } else if (u->is_CallStaticJava() && u->as_CallStaticJava()->uncommon_trap_request() != 0) { 1849 if (!fix_mem_phis_helper(u->in(0), n, mem_ctrl, rep_ctrl, alias, controls, regions, phase)) { 1850 return false; 1851 } 1852 #endif 1853 } else if ((u->is_CFG() && u->adr_type() == TypePtr::BOTTOM) || u->Opcode() == Op_Rethrow || u->Opcode() == Op_Return) { 1854 if (!fix_mem_phis_helper(u->in(0), n, mem_ctrl, rep_ctrl, alias, controls, regions, phase)) { 1855 return false; 1856 } 1857 } else if (u->is_MergeMem() && u->as_MergeMem()->memory_at(alias) == n) { 1858 wq.push(u); 1859 } else if (u->Opcode() == Op_ShenandoahWriteBarrier && phase->C->get_alias_index(u->adr_type()) == alias) { 1860 Node* m = u->find_out_with(Op_ShenandoahWBMemProj); 1861 if (m != NULL) { 1862 wq.push(m); 1863 } 1864 } 1865 } 1866 } 1867 #ifdef ASSERT 1868 if (trace) { 1869 tty->print_cr("XXXXXXXXXXXXXXXXXXXX"); 1870 for (int i = 0; i < regions.length(); i++) { 1871 Node* r = regions.at(i); 1872 tty->print("%d", i); r->dump(); 1873 } 1874 tty->print_cr("XXXXXXXXXXXXXXXXXXXX"); 1875 } 1876 #endif 1877 1878 if (regions.length() == 0) { 1879 return true; 1880 } 1881 1882 { 1883 int i = 0; 1884 for (; i < regions.length(); i+=2) { 1885 Node* region = regions.at(i); 1886 bool has_phi = false; 1887 for (DUIterator_Fast jmax, j = region->fast_outs(jmax); j < jmax && !has_phi; j++) { 1888 Node* u = region->fast_out(j); 1889 if (u->is_Phi() && u->bottom_type() == Type::MEMORY && 1890 (u->adr_type() == TypePtr::BOTTOM || phase->C->get_alias_index(u->adr_type()) == alias)) { 1891 has_phi = true; 1892 } 1893 } 1894 if (!has_phi) { 1895 break; 1896 } 1897 } 1898 if (i == regions.length()) { 1899 return true; 1900 } 1901 } 1902 1903 // Try to restrict the update to path that post dominates rep_ctrl 1904 int k = 0; 1905 int start = 0; 1906 int end = 0; 1907 do { 1908 start = end; 1909 end = k; 1910 for (int i = end; i < regions.length(); i+=2) { 1911 Node* r = regions.at(i); 1912 int prev = k; 1913 for (uint j = 1; j < r->req() && prev == k; j++) { 1914 if (end == 0) { 1915 if (phase->is_dominator(rep_ctrl, r->in(j))) { 1916 Node* mem = regions.at(i+1); 1917 regions.at_put(i, regions.at(k)); 1918 regions.at_put(i+1, regions.at(k+1)); 1919 regions.at_put(k, r); 1920 regions.at_put(k+1, mem); 1921 k+=2; 1922 } 1923 } else { 1924 for (int l = start; l < end && prev == k; l+=2) { 1925 Node* r2 = regions.at(l); 1926 if (phase->is_dominator(r2, r->in(j))) { 1927 Node* mem = regions.at(i+1); 1928 regions.at_put(i, regions.at(k)); 1929 regions.at_put(i+1, regions.at(k+1)); 1930 regions.at_put(k, r); 1931 regions.at_put(k+1, mem); 1932 k+=2; 1933 } 1934 } 1935 } 1936 } 1937 } 1938 #ifdef ASSERT 1939 if (trace) { tty->print_cr("k = %d start = %d end = %d", k, start, end); } 1940 #endif 1941 } while(k != end); 1942 1943 #ifdef ASSERT 1944 if (end != regions.length()) { 1945 if (trace) { tty->print_cr("Compacting %d -> %d", regions.length(), end); } 1946 } 1947 #endif 1948 regions.trunc_to(end); 1949 1950 #ifdef ASSERT 1951 if (trace) { 1952 tty->print_cr("XXXXXXXXXXXXXXXXXXXX"); 1953 for (int i = 0; i < regions.length(); i++) { 1954 Node* r = regions.at(i); 1955 tty->print("%d", i); r->dump(); 1956 } 1957 tty->print_cr("XXXXXXXXXXXXXXXXXXXX"); 1958 } 1959 #endif 1960 1961 // Creating new phis must be done in post order 1962 while (regions.length() > 0) { 1963 int i = 0; 1964 for (; i < regions.length(); i+=2) { 1965 Node* r1 = regions.at(i); 1966 bool is_dom = false; 1967 for (int j = 0; j < regions.length() && !is_dom; j+=2) { 1968 if (i != j) { 1969 Node* r2 = regions.at(j); 1970 for (uint k = 1; k < r2->req() && !is_dom; k++) { 1971 if (phase->is_dominator(r1, r2->in(k))) { 1972 is_dom = true; 1973 } 1974 } 1975 } 1976 } 1977 if (!is_dom) { 1978 break; 1979 } 1980 } 1981 assert(i < regions.length(), "need one"); 1982 Node* r = regions.at(i); 1983 Node* m = regions.at(i+1); 1984 regions.delete_at(i+1); 1985 regions.delete_at(i); 1986 1987 if (!suitable_mem(m, NULL, NULL)) { 1988 return false; 1989 } 1990 Node* phi = PhiNode::make(r, m, Type::MEMORY, phase->C->get_adr_type(alias)); 1991 #ifdef ASSERT 1992 if (trace) { tty->print("YYY Adding new mem phi "); phi->dump(); } 1993 #endif 1994 phase->register_new_node(phi, r); 1995 1996 fix_memory_uses(m, phi, phi, r, phase->C->get_alias_index(phi->adr_type()), phase); 1997 assert(phi->outcnt() != 0, "new proj should have uses"); 1998 if (phi->outcnt() == 0) { 1999 phase->igvn().remove_dead_node(phi); 2000 } 2001 } 2002 2003 return true; 2004 } 2005 2006 Node* ShenandoahBarrierNode::dom_mem(Node* mem, Node*& mem_ctrl, Node* n, Node* rep_ctrl, int alias, PhaseIdealLoop* phase) { 2007 ResourceMark rm; 2008 VectorSet wq(Thread::current()->resource_area()); 2009 wq.set(mem->_idx); 2010 mem_ctrl = phase->get_ctrl(mem); 2011 while (!is_dominator(mem_ctrl, rep_ctrl, mem, n, phase)) { 2012 mem = next_mem(mem, alias); 2013 if (wq.test_set(mem->_idx)) { 2014 return NULL; // hit an unexpected loop 2015 } 2016 mem_ctrl = phase->ctrl_or_self(mem); 2017 } 2018 if (mem->is_MergeMem()) { 2019 mem = mem->as_MergeMem()->memory_at(alias); 2020 mem_ctrl = phase->ctrl_or_self(mem); 2021 } 2022 return mem; 2023 } 2024 2025 Node* ShenandoahBarrierNode::dom_mem(Node* mem, Node* ctrl, int alias, Node*& mem_ctrl, PhaseIdealLoop* phase) { 2026 ResourceMark rm; 2027 VectorSet wq(Thread::current()->resource_area()); 2028 wq.set(mem->_idx); 2029 mem_ctrl = phase->ctrl_or_self(mem); 2030 while (!phase->is_dominator(mem_ctrl, ctrl) || mem_ctrl == ctrl) { 2031 mem = next_mem(mem, alias); 2032 if (wq.test_set(mem->_idx)) { 2033 return NULL; 2034 } 2035 mem_ctrl = phase->ctrl_or_self(mem); 2036 } 2037 if (mem->is_MergeMem()) { 2038 mem = mem->as_MergeMem()->memory_at(alias); 2039 mem_ctrl = phase->ctrl_or_self(mem); 2040 } 2041 return mem; 2042 } 2043 2044 Node* ShenandoahBarrierNode::try_common(Node *n_ctrl, PhaseIdealLoop* phase) { 2045 if (!phase->C->has_irreducible_loop()) { 2046 // We look for a write barrier whose memory edge dominates n 2047 // Either the replacement write barrier dominates n or we have, 2048 // for instance: 2049 // if ( ) { 2050 // read barrier n 2051 // } else { 2052 // write barrier 2053 // } 2054 // in which case replacing n by the write barrier causes the write 2055 // barrier to move above the if() and the memory Phi that merges 2056 // the memory state for both branches must be updated so both 2057 // inputs become the write barrier's memory projection (and the 2058 // Phi is optimized out) otherwise we risk loosing a memory 2059 // dependency. 2060 // Once we find a replacement write barrier, the code below fixes 2061 // the memory graph in cases like the one above. 2062 Node* val = in(ValueIn); 2063 Node* val_ctrl = phase->get_ctrl(val); 2064 Node* n_proj = find_out_with(Op_ShenandoahWBMemProj); 2065 Node* replacement = NULL; 2066 int alias = phase->C->get_alias_index(adr_type()); 2067 for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax && replacement == NULL; i++) { 2068 Node* u = val->fast_out(i); 2069 if (u != this && u->Opcode() == Op_ShenandoahWriteBarrier) { 2070 Node* u_mem = u->in(Memory); 2071 Node* u_proj = u->find_out_with(Op_ShenandoahWBMemProj); 2072 Node* u_ctrl = phase->get_ctrl(u); 2073 Node* u_mem_ctrl = phase->get_ctrl(u_mem); 2074 IdealLoopTree* n_loop = phase->get_loop(n_ctrl); 2075 IdealLoopTree* u_loop = phase->get_loop(u_ctrl); 2076 2077 Node* ctrl = phase->dom_lca(u_ctrl, n_ctrl); 2078 2079 if (ctrl->is_Proj() && 2080 ctrl->in(0)->is_Call() && 2081 ctrl->unique_ctrl_out() != NULL && 2082 ctrl->unique_ctrl_out()->Opcode() == Op_Catch & 2083 !phase->is_dominator(val_ctrl, ctrl->in(0)->in(0))) { 2084 continue; 2085 } 2086 2087 if (Opcode() == Op_ShenandoahWriteBarrier && u_proj == NULL && n_proj != NULL) { 2088 continue; 2089 } 2090 2091 IdealLoopTree* loop = phase->get_loop(ctrl); 2092 2093 // we don't want to move a write barrier in a loop 2094 if (loop->is_member(u_loop) || (Opcode() == Op_ShenandoahWriteBarrier && loop->is_member(n_loop))) { 2095 if (ShenandoahDontIncreaseWBFreq) { 2096 Node* u_iffproj = no_branches(u_ctrl, ctrl, true, phase); 2097 if (Opcode() == Op_ShenandoahWriteBarrier) { 2098 Node* n_iffproj = no_branches(n_ctrl, ctrl, true, phase); 2099 if (u_iffproj == NULL || n_iffproj == NULL) { 2100 replacement = u; 2101 } else if (u_iffproj != NodeSentinel && n_iffproj != NodeSentinel && u_iffproj->in(0) == n_iffproj->in(0)) { 2102 replacement = u; 2103 } 2104 } else if (u_iffproj == NULL) { 2105 replacement = u; 2106 } 2107 } else { 2108 replacement = u; 2109 } 2110 } 2111 } 2112 } 2113 if (replacement != NULL) { 2114 Node* old_ctrl = phase->get_ctrl(replacement); 2115 Node* rep_ctrl = phase->dom_lca(n_ctrl, old_ctrl); 2116 if (rep_ctrl->is_Proj() && 2117 rep_ctrl->in(0)->is_Call() && 2118 rep_ctrl->unique_ctrl_out() != NULL && 2119 rep_ctrl->unique_ctrl_out()->Opcode() == Op_Catch) { 2120 rep_ctrl = rep_ctrl->in(0)->in(0); 2121 assert(phase->is_dominator(val_ctrl, rep_ctrl), "bad control"); 2122 } else { 2123 LoopNode* c = ShenandoahWriteBarrierNode::try_move_before_pre_loop(rep_ctrl, val_ctrl, phase); 2124 if (c != NULL) { 2125 rep_ctrl = ShenandoahWriteBarrierNode::move_above_predicates(c, val_ctrl, phase); 2126 } else { 2127 while (rep_ctrl->is_IfProj()) { 2128 CallStaticJavaNode* unc = rep_ctrl->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none); 2129 if (unc != NULL) { 2130 int req = unc->uncommon_trap_request(); 2131 Deoptimization::DeoptReason trap_reason = Deoptimization::trap_request_reason(req); 2132 if ((trap_reason == Deoptimization::Reason_loop_limit_check || 2133 trap_reason == Deoptimization::Reason_predicate) && phase->is_dominator(val_ctrl, rep_ctrl->in(0)->in(0))) { 2134 rep_ctrl = rep_ctrl->in(0)->in(0); 2135 continue; 2136 } 2137 } 2138 break; 2139 } 2140 } 2141 } 2142 2143 Node* mem = replacement->in(Memory); 2144 Node* old_mem = mem; 2145 Node* rep_proj = replacement->find_out_with(Op_ShenandoahWBMemProj); 2146 { 2147 Node* mem_ctrl = NULL; 2148 2149 mem = dom_mem(mem, mem_ctrl, this, rep_ctrl, alias, phase); 2150 if (mem == NULL) { 2151 return NULL; 2152 } 2153 2154 // Add a memory Phi for the slice of the write barrier to any 2155 // region that post dominates rep_ctrl and doesn't have one 2156 // already. 2157 if (rep_proj != NULL && !ShenandoahWriteBarrierNode::fix_mem_phis(mem, mem_ctrl, rep_ctrl, alias, phase)) { 2158 return NULL; 2159 } 2160 2161 assert(!ShenandoahVerifyOptoBarriers || ShenandoahWriteBarrierNode::memory_dominates_all_paths(mem, rep_ctrl, alias, phase), "can't fix the memory graph"); 2162 } 2163 assert(phase->igvn().type(mem) == Type::MEMORY, "not memory"); 2164 2165 if (rep_proj != NULL) { 2166 Node* old_mem = replacement->in(Memory); 2167 if (!suitable_mem(mem, old_mem, rep_proj)) { 2168 return NULL; 2169 } 2170 2171 if (replacement->in(Memory) != mem) { 2172 // tty->print("XXX setting memory of"); replacement->dump(); 2173 // tty->print("XXX to"); mem->dump(); 2174 for (DUIterator_Last imin, i = rep_proj->last_outs(imin); i >= imin; ) { 2175 Node* u = rep_proj->last_out(i); 2176 phase->igvn().rehash_node_delayed(u); 2177 int uses_found = u->replace_edge(rep_proj, old_mem); 2178 i -= uses_found; 2179 } 2180 phase->igvn().replace_input_of(replacement, Memory, mem); 2181 } 2182 phase->set_ctrl_and_loop(replacement, rep_ctrl); 2183 phase->igvn().replace_input_of(replacement, Control, rep_ctrl); 2184 2185 ShenandoahWriteBarrierNode::fix_memory_uses(mem, replacement, rep_proj, rep_ctrl, phase->C->get_alias_index(replacement->adr_type()), phase); 2186 assert(rep_proj->outcnt() != 0, "new proj should have uses"); 2187 } else { 2188 if (replacement->in(Memory) != mem) { 2189 phase->igvn()._worklist.push(replacement->in(Memory)); 2190 phase->igvn().replace_input_of(replacement, Memory, mem); 2191 } 2192 phase->set_ctrl_and_loop(replacement, rep_ctrl); 2193 phase->igvn().replace_input_of(replacement, Control, rep_ctrl); 2194 } 2195 if (Opcode() == Op_ShenandoahWriteBarrier) { 2196 if (n_proj != NULL) { 2197 phase->lazy_replace(n_proj, in(Memory)); 2198 } 2199 } 2200 phase->lazy_replace(this, replacement); 2201 if (rep_proj != NULL) { 2202 phase->set_ctrl_and_loop(rep_proj, rep_ctrl); 2203 } 2204 return replacement; 2205 } 2206 } 2207 2208 return NULL; 2209 } 2210 2211 static Node* find_mem_phi(Node* n_loop_head, int alias, Compile* C) { 2212 Node* phi_in = NULL; 2213 Node* phi_bottom = NULL; 2214 2215 for (DUIterator_Fast imax, i = n_loop_head->fast_outs(imax); i < imax; i++) { 2216 Node* u = n_loop_head->fast_out(i); 2217 if (u->is_Phi() && 2218 u->bottom_type() == Type::MEMORY) { 2219 if (C->get_alias_index(u->adr_type()) == alias) { 2220 if (phi_in != NULL && phi_in != u) { 2221 return NULL; 2222 } 2223 phi_in = u; 2224 } else if (u->adr_type() == TypePtr::BOTTOM) { 2225 assert(phi_bottom == NULL, "only one phi"); 2226 phi_bottom = u; 2227 } 2228 } 2229 } 2230 if (phi_in != NULL) { 2231 return phi_in; 2232 } 2233 return phi_bottom; 2234 } 2235 2236 static void disconnect_barrier_mem(Node* wb, PhaseIterGVN& igvn) { 2237 Node* mem_in = wb->in(ShenandoahBarrierNode::Memory); 2238 Node* proj = wb->find_out_with(Op_ShenandoahWBMemProj); 2239 2240 for (DUIterator_Last imin, i = proj->last_outs(imin); i >= imin; ) { 2241 Node* u = proj->last_out(i); 2242 igvn.rehash_node_delayed(u); 2243 int nb = u->replace_edge(proj, mem_in); 2244 assert(nb > 0, "no replacement?"); 2245 i -= nb; 2246 } 2247 } 2248 2249 Node* ShenandoahWriteBarrierNode::move_above_predicates(LoopNode* cl, Node* val_ctrl, PhaseIdealLoop* phase) { 2250 Node* entry = cl->skip_strip_mined()->in(LoopNode::EntryControl); 2251 Node* above_pred = phase->skip_loop_predicates(entry); 2252 Node* ctrl = entry; 2253 while (ctrl != above_pred) { 2254 Node* next = ctrl->in(0); 2255 if (!phase->is_dominator(val_ctrl, next)) { 2256 break; 2257 } 2258 ctrl = next; 2259 } 2260 return ctrl; 2261 } 2262 2263 Node* ShenandoahWriteBarrierNode::try_move_before_loop_helper(LoopNode* cl, Node* val_ctrl, Node* mem, PhaseIdealLoop* phase) { 2264 assert(cl->is_Loop(), "bad control"); 2265 Node* ctrl = move_above_predicates(cl, val_ctrl, phase); 2266 Node* mem_ctrl = NULL; 2267 int alias = phase->C->get_alias_index(adr_type()); 2268 mem = dom_mem(mem, mem_ctrl, this, ctrl, alias, phase); 2269 if (mem == NULL) { 2270 return NULL; 2271 } 2272 2273 Node* old_mem = in(Memory); 2274 Node* proj = find_out_with(Op_ShenandoahWBMemProj); 2275 if (old_mem != mem && !suitable_mem(mem, old_mem, proj)) { 2276 return NULL; 2277 } 2278 2279 assert(!ShenandoahVerifyOptoBarriers || memory_dominates_all_paths(mem, ctrl, alias, phase), "can't fix the memory graph"); 2280 phase->set_ctrl_and_loop(this, ctrl); 2281 phase->igvn().replace_input_of(this, Control, ctrl); 2282 if (old_mem != mem) { 2283 if (proj != NULL) { 2284 disconnect_barrier_mem(this, phase->igvn()); 2285 fix_memory_uses(mem, this, proj, ctrl, phase->C->get_alias_index(adr_type()), phase); 2286 assert(proj->outcnt() > 0, "disconnected write barrier"); 2287 } 2288 phase->igvn().replace_input_of(this, Memory, mem); 2289 } 2290 if (proj != NULL) { 2291 phase->set_ctrl_and_loop(proj, ctrl); 2292 } 2293 return this; 2294 } 2295 2296 LoopNode* ShenandoahWriteBarrierNode::try_move_before_pre_loop(Node* c, Node* val_ctrl, PhaseIdealLoop* phase) { 2297 // A write barrier between a pre and main loop can get in the way of 2298 // vectorization. Move it above the pre loop if possible 2299 CountedLoopNode* cl = NULL; 2300 if (c->is_IfFalse() && 2301 c->in(0)->is_CountedLoopEnd()) { 2302 cl = c->in(0)->as_CountedLoopEnd()->loopnode(); 2303 } else if (c->is_IfProj() && 2304 c->in(0)->is_If() && 2305 c->in(0)->in(0)->is_IfFalse() && 2306 c->in(0)->in(0)->in(0)->is_CountedLoopEnd()) { 2307 cl = c->in(0)->in(0)->in(0)->as_CountedLoopEnd()->loopnode(); 2308 } 2309 if (cl != NULL && 2310 cl->is_pre_loop() && 2311 val_ctrl != cl && 2312 phase->is_dominator(val_ctrl, cl)) { 2313 return cl; 2314 } 2315 return NULL; 2316 } 2317 2318 Node* ShenandoahWriteBarrierNode::try_move_before_loop(Node *n_ctrl, PhaseIdealLoop* phase) { 2319 IdealLoopTree *n_loop = phase->get_loop(n_ctrl); 2320 Node* val = in(ValueIn); 2321 Node* val_ctrl = phase->get_ctrl(val); 2322 if (n_loop != phase->ltree_root() && !n_loop->_irreducible) { 2323 IdealLoopTree *val_loop = phase->get_loop(val_ctrl); 2324 Node* mem = in(Memory); 2325 IdealLoopTree *mem_loop = phase->get_loop(phase->get_ctrl(mem)); 2326 if (!n_loop->is_member(val_loop) && 2327 n_loop->is_member(mem_loop)) { 2328 Node* n_loop_head = n_loop->_head; 2329 2330 if (n_loop_head->is_Loop()) { 2331 int alias = phase->C->get_alias_index(adr_type()); 2332 Node* mem = find_mem_phi(n_loop_head, alias, phase->C); 2333 if (mem == NULL) { 2334 mem = in(Memory); 2335 } 2336 2337 LoopNode* loop = n_loop_head->as_Loop(); 2338 if (n_loop_head->is_CountedLoop() && n_loop_head->as_CountedLoop()->is_main_loop()) { 2339 LoopNode* res = try_move_before_pre_loop(n_loop_head->in(LoopNode::EntryControl), val_ctrl, phase); 2340 if (res != NULL) { 2341 loop = res; 2342 } 2343 } 2344 2345 return try_move_before_loop_helper(loop, val_ctrl, mem, phase); 2346 } 2347 } 2348 } 2349 LoopNode* ctrl = try_move_before_pre_loop(in(0), val_ctrl, phase); 2350 if (ctrl != NULL) { 2351 return try_move_before_loop_helper(ctrl, val_ctrl, in(Memory), phase); 2352 } 2353 return NULL; 2354 } 2355 2356 void ShenandoahReadBarrierNode::try_move(Node *n_ctrl, PhaseIdealLoop* phase) { 2357 Node* mem = in(MemNode::Memory); 2358 int alias = phase->C->get_alias_index(adr_type()); 2359 const bool trace = false; 2360 2361 #ifdef ASSERT 2362 if (trace) { tty->print("Trying to move mem of"); dump(); } 2363 #endif 2364 2365 Node* new_mem = mem; 2366 2367 ResourceMark rm; 2368 VectorSet seen(Thread::current()->resource_area()); 2369 Node_List phis; 2370 2371 for (;;) { 2372 #ifdef ASSERT 2373 if (trace) { tty->print("Looking for dominator from"); mem->dump(); } 2374 #endif 2375 if (mem->is_Proj() && mem->in(0)->is_Start()) { 2376 if (new_mem != in(MemNode::Memory)) { 2377 #ifdef ASSERT 2378 if (trace) { tty->print("XXX Setting mem to"); new_mem->dump(); tty->print(" for "); dump(); } 2379 #endif 2380 phase->igvn().replace_input_of(this, MemNode::Memory, new_mem); 2381 } 2382 return; 2383 } 2384 2385 Node* candidate = mem; 2386 do { 2387 if (!is_independent(mem)) { 2388 if (trace) { tty->print_cr("Not independent"); } 2389 if (new_mem != in(MemNode::Memory)) { 2390 #ifdef ASSERT 2391 if (trace) { tty->print("XXX Setting mem to"); new_mem->dump(); tty->print(" for "); dump(); } 2392 #endif 2393 phase->igvn().replace_input_of(this, MemNode::Memory, new_mem); 2394 } 2395 return; 2396 } 2397 if (seen.test_set(mem->_idx)) { 2398 if (trace) { tty->print_cr("Already seen"); } 2399 ShouldNotReachHere(); 2400 // Strange graph 2401 if (new_mem != in(MemNode::Memory)) { 2402 #ifdef ASSERT 2403 if (trace) { tty->print("XXX Setting mem to"); new_mem->dump(); tty->print(" for "); dump(); } 2404 #endif 2405 phase->igvn().replace_input_of(this, MemNode::Memory, new_mem); 2406 } 2407 return; 2408 } 2409 if (mem->is_Phi()) { 2410 phis.push(mem); 2411 } 2412 mem = next_mem(mem, alias); 2413 if (mem->bottom_type() == Type::MEMORY) { 2414 candidate = mem; 2415 } 2416 assert(is_dominator(phase->ctrl_or_self(mem), n_ctrl, mem, this, phase) == phase->is_dominator(phase->ctrl_or_self(mem), n_ctrl), "strange dominator"); 2417 #ifdef ASSERT 2418 if (trace) { tty->print("Next mem is"); mem->dump(); } 2419 #endif 2420 } while (mem->bottom_type() != Type::MEMORY || !phase->is_dominator(phase->ctrl_or_self(mem), n_ctrl)); 2421 2422 assert(mem->bottom_type() == Type::MEMORY, "bad mem"); 2423 2424 bool not_dom = false; 2425 for (uint i = 0; i < phis.size() && !not_dom; i++) { 2426 Node* nn = phis.at(i); 2427 2428 #ifdef ASSERT 2429 if (trace) { tty->print("Looking from phi"); nn->dump(); } 2430 #endif 2431 assert(nn->is_Phi(), "phis only"); 2432 for (uint j = 2; j < nn->req() && !not_dom; j++) { 2433 Node* m = nn->in(j); 2434 #ifdef ASSERT 2435 if (trace) { tty->print("Input %d is", j); m->dump(); } 2436 #endif 2437 while (m != mem && !seen.test_set(m->_idx)) { 2438 if (is_dominator(phase->ctrl_or_self(m), phase->ctrl_or_self(mem), m, mem, phase)) { 2439 not_dom = true; 2440 // Scheduling anomaly 2441 #ifdef ASSERT 2442 if (trace) { tty->print("Giving up"); m->dump(); } 2443 #endif 2444 break; 2445 } 2446 if (!is_independent(m)) { 2447 if (trace) { tty->print_cr("Not independent"); } 2448 if (new_mem != in(MemNode::Memory)) { 2449 #ifdef ASSERT 2450 if (trace) { tty->print("XXX Setting mem to"); new_mem->dump(); tty->print(" for "); dump(); } 2451 #endif 2452 phase->igvn().replace_input_of(this, MemNode::Memory, new_mem); 2453 } 2454 return; 2455 } 2456 if (m->is_Phi()) { 2457 phis.push(m); 2458 } 2459 m = next_mem(m, alias); 2460 #ifdef ASSERT 2461 if (trace) { tty->print("Next mem is"); m->dump(); } 2462 #endif 2463 } 2464 } 2465 } 2466 if (!not_dom) { 2467 new_mem = mem; 2468 phis.clear(); 2469 } else { 2470 seen.Clear(); 2471 } 2472 } 2473 } 2474 2475 CallStaticJavaNode* ShenandoahWriteBarrierNode::pin_and_expand_null_check(PhaseIterGVN& igvn) { 2476 Node* val = in(ValueIn); 2477 2478 #ifdef ASSERT 2479 const Type* val_t = igvn.type(val); 2480 assert(val_t->meet(TypePtr::NULL_PTR) != val_t, "should be not null"); 2481 #endif 2482 2483 if (val->Opcode() == Op_CastPP && 2484 val->in(0)->Opcode() == Op_IfTrue && 2485 val->in(0)->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none) && 2486 val->in(0)->in(0)->is_If() && 2487 val->in(0)->in(0)->in(1)->Opcode() == Op_Bool && 2488 val->in(0)->in(0)->in(1)->as_Bool()->_test._test == BoolTest::ne && 2489 val->in(0)->in(0)->in(1)->in(1)->Opcode() == Op_CmpP && 2490 val->in(0)->in(0)->in(1)->in(1)->in(1) == val->in(1) && 2491 val->in(0)->in(0)->in(1)->in(1)->in(2)->bottom_type() == TypePtr::NULL_PTR) { 2492 assert(val->in(0)->in(0)->in(1)->in(1)->in(1) == val->in(1), ""); 2493 CallStaticJavaNode* unc = val->in(0)->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none); 2494 return unc; 2495 } 2496 return NULL; 2497 } 2498 2499 void ShenandoahWriteBarrierNode::pin_and_expand_move_barrier(PhaseIdealLoop* phase) { 2500 Node* unc = pin_and_expand_null_check(phase->igvn()); 2501 Node* val = in(ValueIn); 2502 2503 if (unc != NULL) { 2504 Node* ctrl = phase->get_ctrl(this); 2505 Node* unc_ctrl = val->in(0); 2506 2507 // Don't move write barrier in a loop 2508 IdealLoopTree* loop = phase->get_loop(ctrl); 2509 IdealLoopTree* unc_loop = phase->get_loop(unc_ctrl); 2510 2511 if (!unc_loop->is_member(loop)) { 2512 return; 2513 } 2514 2515 Node* branch = no_branches(ctrl, unc_ctrl, false, phase); 2516 assert(branch == NULL || branch == NodeSentinel, "was not looking for a branch"); 2517 if (branch == NodeSentinel) { 2518 return; 2519 } 2520 2521 Node* mem = in(Memory); 2522 Node* old_mem = mem; 2523 2524 Node* mem_ctrl = NULL; 2525 int alias = phase->C->get_alias_index(adr_type()); 2526 mem = dom_mem(mem, mem_ctrl, this, unc_ctrl, alias, phase); 2527 if (mem == NULL) { 2528 return; 2529 } 2530 2531 Node* proj = find_out_with(Op_ShenandoahWBMemProj); 2532 if (proj != NULL && mem != old_mem && !fix_mem_phis(mem, mem_ctrl, unc_ctrl, alias, phase)) { 2533 return; 2534 } 2535 2536 assert(proj == NULL || mem == old_mem || memory_dominates_all_paths(mem, unc_ctrl, alias, phase), "can't fix the memory graph"); 2537 phase->set_ctrl_and_loop(this, unc_ctrl); 2538 if (in(Control) != NULL) { 2539 phase->igvn().replace_input_of(this, Control, unc_ctrl); 2540 } 2541 if (old_mem != mem) { 2542 if (proj != NULL) { 2543 disconnect_barrier_mem(this, phase->igvn()); 2544 fix_memory_uses(mem, this, proj, unc_ctrl, phase->C->get_alias_index(adr_type()), phase); 2545 assert(proj->outcnt() > 0, "disconnected write barrier"); 2546 } 2547 phase->igvn().replace_input_of(this, Memory, mem); 2548 } 2549 if (proj != NULL) { 2550 phase->set_ctrl_and_loop(proj, unc_ctrl); 2551 } 2552 } 2553 } 2554 2555 void ShenandoahWriteBarrierNode::pin_and_expand_helper(PhaseIdealLoop* phase) { 2556 Node* val = in(ValueIn); 2557 Node* ctrl = phase->get_ctrl(this); 2558 // Replace all uses of barrier's input that are dominated by ctrl 2559 // with the value returned by the barrier: no need to keep both 2560 // live. 2561 for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) { 2562 Node* u = val->fast_out(i); 2563 if (u != this) { 2564 if (u->is_Phi()) { 2565 int nb = 0; 2566 for (uint j = 1; j < u->req(); j++) { 2567 if (u->in(j) == val) { 2568 Node* c = u->in(0)->in(j); 2569 if (phase->is_dominator(ctrl, c)) { 2570 phase->igvn().replace_input_of(u, j, this); 2571 nb++; 2572 } 2573 } 2574 } 2575 if (nb > 0) { 2576 imax -= nb; 2577 --i; 2578 } 2579 } else { 2580 Node* c = phase->ctrl_or_self(u); 2581 if (is_dominator(ctrl, c, this, u, phase)) { 2582 phase->igvn().rehash_node_delayed(u); 2583 int nb = u->replace_edge(val, this); 2584 assert(nb > 0, "no update?"); 2585 --i, imax -= nb; 2586 } 2587 } 2588 } 2589 } 2590 } 2591 2592 Node* ShenandoahWriteBarrierNode::pick_phi(Node* phi1, Node* phi2, Node_Stack& phis, VectorSet& visited, PhaseIdealLoop* phase) { 2593 assert(phis.size() == 0, "stack needs to be empty"); 2594 uint i = 1; 2595 int phi_dominates = -1; 2596 for (;;) { 2597 assert(phi1->req() == phi2->req(), "strange pair of phis"); 2598 assert(phis.size() % 2 == 0, ""); 2599 Node* in1 = phi1->in(i); 2600 Node* in2 = phi2->in(i); 2601 2602 if (in1->is_MergeMem()) { 2603 in1 = in1->as_MergeMem()->base_memory(); 2604 } 2605 if (in2->is_MergeMem()) { 2606 in2 = in2->as_MergeMem()->base_memory(); 2607 } 2608 2609 if (in1 == in2) { 2610 //continue 2611 } else if (in1->is_Phi() && in2->is_Phi() && in1->in(0) == in2->in(0)) { 2612 assert(!visited.test_set(in1->_idx), "no loop"); 2613 assert(!visited.test_set(in2->_idx), "no loop"); 2614 phis.push(phi1, i+1); 2615 phis.push(phi2, i+1); 2616 phi1 = in1; 2617 phi2 = in2; 2618 i = 1; 2619 } else { 2620 Node* in1_c = phase->get_ctrl(in1); 2621 Node* in2_c = phase->get_ctrl(in2); 2622 if (is_dominator(in1_c, in2_c, in1, in2, phase)) { 2623 assert(!is_dominator(in2_c, in1_c, in2, in1, phase), "one has to dominate the other"); 2624 assert(phi_dominates == -1 || phi_dominates == 1, "all inputs must dominate"); 2625 phi_dominates = 1; 2626 } else { 2627 assert(is_dominator(in2_c, in1_c, in2, in1, phase), "one must dominate the other"); 2628 assert(!is_dominator(in1_c, in2_c, in1, in2, phase), "one has to dominate the other"); 2629 assert(phi_dominates == -1 || phi_dominates == 2, "all inputs must dominate"); 2630 phi_dominates = 2; 2631 } 2632 } 2633 i++; 2634 2635 while (i >= phi1->req() && phis.size() > 0) { 2636 i = phis.index(); 2637 phi2 = phis.node(); 2638 phis.pop(); 2639 phi1 = phis.node(); 2640 phis.pop(); 2641 } 2642 2643 if (i >= phi1->req() && phis.size() == 0) { 2644 Node* phi = NULL; 2645 if (phi_dominates == 1) { 2646 return phi2; 2647 } else if (phi_dominates == 2) { 2648 return phi1; 2649 } else { 2650 return phi1; 2651 } 2652 } 2653 } 2654 return NULL; 2655 } 2656 2657 static Node* memory_for(Node* mem, const Node_List& phis) { 2658 Node *m = mem; 2659 while (m != NULL) { 2660 mem = m; 2661 m = phis[m->_idx]; 2662 } 2663 return mem; 2664 } 2665 2666 Node* ShenandoahWriteBarrierNode::find_raw_mem(Node* ctrl, Node* n, const Node_List& memory_nodes, const Node_List& phis, bool strict, PhaseIdealLoop* phase) { 2667 assert(n == NULL || phase->ctrl_or_self(n) == ctrl, ""); 2668 Node* raw_mem = memory_for(memory_nodes[ctrl->_idx], phis); 2669 Node* c = ctrl; 2670 while (raw_mem == NULL || (strict && get_ctrl(raw_mem, phase) != c && (!c->is_CatchProj() || c->in(0)->in(0)->in(0) != get_ctrl(raw_mem, phase)))) { 2671 c = phase->idom(c); 2672 raw_mem = memory_for(memory_nodes[c->_idx], phis); 2673 } 2674 if (n != NULL && get_ctrl(raw_mem, phase) == ctrl) { 2675 while (!is_dominator_same_ctrl(c, raw_mem, n, phase) && phase->ctrl_or_self(raw_mem) == ctrl) { 2676 raw_mem = next_mem(raw_mem, Compile::AliasIdxRaw); 2677 } 2678 if (raw_mem->is_MergeMem()) { 2679 raw_mem = raw_mem->as_MergeMem()->memory_at(Compile::AliasIdxRaw); 2680 } 2681 if (get_ctrl(raw_mem, phase) != ctrl) { 2682 do { 2683 c = phase->idom(c); 2684 raw_mem = memory_for(memory_nodes[c->_idx], phis); 2685 } while (raw_mem == NULL || (strict && get_ctrl(raw_mem, phase) != c && (!c->is_CatchProj() || c->in(0)->in(0)->in(0) != get_ctrl(raw_mem, phase)))); 2686 } 2687 } 2688 assert(raw_mem->bottom_type() == Type::MEMORY, ""); 2689 return raw_mem; 2690 } 2691 2692 Node* ShenandoahWriteBarrierNode::find_bottom_mem(Node* ctrl, PhaseIdealLoop* phase) { 2693 Node* mem = NULL; 2694 Node* c = ctrl; 2695 do { 2696 if (c->is_Region()) { 2697 Node* phi_bottom = NULL; 2698 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) { 2699 Node* u = c->fast_out(i); 2700 if (u->is_Phi() && u->bottom_type() == Type::MEMORY) { 2701 if (u->adr_type() == TypePtr::BOTTOM) { 2702 if (phi_bottom != NULL) { 2703 phi_bottom = NodeSentinel; 2704 } else { 2705 phi_bottom = u; 2706 } 2707 } 2708 } 2709 } 2710 if (phi_bottom != NULL) { 2711 if (phi_bottom != NodeSentinel) { 2712 mem = phi_bottom; 2713 } else { 2714 Node* phi = NULL; 2715 ResourceMark rm; 2716 Node_Stack phis(0); 2717 VectorSet visited(Thread::current()->resource_area()); 2718 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) { 2719 Node* u = c->fast_out(i); 2720 if (u->is_Phi() && u->bottom_type() == Type::MEMORY && u->adr_type() == TypePtr::BOTTOM) { 2721 if (phi == NULL) { 2722 phi = u; 2723 } else { 2724 phi = pick_phi(phi, u, phis, visited, phase); 2725 } 2726 } 2727 } 2728 mem = phi; 2729 } 2730 } 2731 } else { 2732 if (c->is_Call() && c->as_Call()->_entry_point != OptoRuntime::rethrow_stub()) { 2733 CallProjections projs; 2734 c->as_Call()->extract_projections(&projs, true, false); 2735 if (projs.fallthrough_memproj != NULL) { 2736 if (projs.fallthrough_memproj->adr_type() == TypePtr::BOTTOM) { 2737 if (projs.catchall_memproj == NULL) { 2738 mem = projs.fallthrough_memproj; 2739 } else { 2740 if (phase->is_dominator(projs.fallthrough_catchproj, ctrl)) { 2741 mem = projs.fallthrough_memproj; 2742 } else { 2743 assert(phase->is_dominator(projs.catchall_catchproj, ctrl), "one proj must dominate barrier"); 2744 mem = projs.catchall_memproj; 2745 } 2746 } 2747 } 2748 } else { 2749 Node* proj = c->as_Call()->proj_out(TypeFunc::Memory); 2750 if (proj != NULL && 2751 proj->adr_type() == TypePtr::BOTTOM) { 2752 mem = proj; 2753 } 2754 } 2755 } else { 2756 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) { 2757 Node* u = c->fast_out(i); 2758 if (u->is_Proj() && 2759 u->bottom_type() == Type::MEMORY && 2760 u->adr_type() == TypePtr::BOTTOM) { 2761 assert(c->is_SafePoint() || c->is_MemBar() || c->is_Start(), ""); 2762 assert(mem == NULL, "only one proj"); 2763 mem = u; 2764 } 2765 } 2766 } 2767 } 2768 c = phase->idom(c); 2769 } while (mem == NULL); 2770 return mem; 2771 } 2772 2773 void ShenandoahWriteBarrierNode::follow_barrier_uses(Node* n, Node* ctrl, Unique_Node_List& uses, PhaseIdealLoop* phase) { 2774 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { 2775 Node* u = n->fast_out(i); 2776 if (!u->is_CFG() && phase->get_ctrl(u) == ctrl) { 2777 uses.push(u); 2778 } 2779 } 2780 } 2781 2782 static void push_uses(Compile* C, Node* n, Unique_Node_List& wq, int alias) { 2783 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { 2784 Node* u = n->fast_out(i); 2785 if ((u->is_Phi() && (u->adr_type() == TypePtr::BOTTOM || C->get_alias_index(n->adr_type()) == alias)) || 2786 (u->is_MergeMem() && u->as_MergeMem()->memory_at(alias) == n)) { 2787 assert(!u->is_Phi() || u->in(0), ""); 2788 wq.push(u); 2789 } 2790 } 2791 } 2792 2793 void ShenandoahWriteBarrierNode::collect_memory_nodes_helper(Node* n, int alias, GrowableArray<Node*>& inputs, int adj, Node_List& memory_nodes, Node_List& phis, Node*& cur_mem, Unique_Node_List& wq, PhaseIdealLoop* phase) { 2794 const bool trace = false; 2795 DEBUG_ONLY(if (trace) { tty->print("YYY phi post"); n->dump(); }) 2796 Node* r = n->in(0); 2797 Node* self = memory_for(n, phis); 2798 if (self != n) { 2799 if (self->is_Phi()) { 2800 wq.push(self); 2801 } 2802 return; 2803 } 2804 bool differs = false; 2805 for (uint i = 1; i < n->req(); i++) { 2806 Node* mem = memory_for(inputs.at(i + adj), phis); 2807 DEBUG_ONLY(if (trace) { tty->print("YYY phi post input %d %s", i, mem != n->in(i) ? "differs" : "matches"); mem->dump(); }) 2808 if (mem != n->in(i)) { 2809 differs = true; 2810 } 2811 } 2812 Node* mem = NULL; 2813 if (differs) { 2814 Node* unique = NULL; 2815 for (uint i = 1; i < n->req() && unique != NodeSentinel; i++) { 2816 Node* in = memory_for(inputs.at(i + adj), phis); 2817 if (unique == NULL) { 2818 if (in != self) { 2819 unique = in; 2820 } 2821 } else if (in != self && in != unique) { 2822 unique = NodeSentinel; 2823 } 2824 } 2825 assert(unique != NULL, "at least one non backedge entry"); 2826 if (unique != NodeSentinel) { 2827 DEBUG_ONLY(if (trace) { tty->print("YYY phi post: unique input"); unique->dump(); }) 2828 if (phis[n->_idx] != unique) { 2829 phis.map(n->_idx, unique); 2830 push_uses(phase->C, n, wq, alias); 2831 } 2832 mem = unique; 2833 if (phase->C->get_alias_index(n->adr_type()) == alias) { 2834 phase->lazy_replace(n, unique); 2835 } else { 2836 fix_memory_uses(n, unique, unique, r, alias, phase); 2837 } 2838 if (n->outcnt() == 0) { 2839 wq.remove(n); 2840 } 2841 } else { 2842 Node* better_phi = NULL; 2843 for (DUIterator_Fast imax, i = r->fast_outs(imax); i < imax && better_phi == NULL; i++) { 2844 Node* u = r->fast_out(i); 2845 if (u != n && u->is_Phi() && u->bottom_type() == Type::MEMORY && 2846 (u->adr_type() == TypePtr::BOTTOM || phase->C->get_alias_index(u->adr_type()) == alias)) { 2847 uint i = 1; 2848 for (; i < u->req() && u->in(i) == memory_for(inputs.at(i + adj), phis); i++); 2849 if (i == u->req()) { 2850 better_phi = u; 2851 } 2852 } 2853 } 2854 if (better_phi != NULL) { 2855 DEBUG_ONLY(if (trace) { tty->print("YYY phi post: better phi"); better_phi->dump(); }) 2856 } else { 2857 better_phi = new PhiNode(r, Type::MEMORY, phase->C->get_adr_type(alias)); 2858 for (uint i = 1; i < better_phi->req(); i++) { 2859 better_phi->init_req(i, memory_for(inputs.at(i+adj), phis)); 2860 } 2861 DEBUG_ONLY(if (trace) { tty->print("YYY phi post: creating better phi"); better_phi->dump(); }) 2862 phase->register_new_node(better_phi, r); 2863 } 2864 mem = better_phi; 2865 if (phis[n->_idx] != better_phi) { 2866 phis.map(n->_idx, better_phi); 2867 push_uses(phase->C, n, wq, alias); 2868 } 2869 if (phase->C->get_alias_index(n->adr_type()) == alias) { 2870 phase->lazy_replace(n, better_phi); 2871 } else { 2872 fix_memory_uses(n, better_phi, better_phi, r, alias, phase); 2873 } 2874 if (n->outcnt() == 0) { 2875 wq.remove(n); 2876 } 2877 } 2878 } else { 2879 mem = n; 2880 } 2881 Node* other = memory_for(phis[r->_idx], phis); 2882 Node* other2 = memory_for(memory_nodes[r->_idx], phis); 2883 if (other != NULL) { 2884 DEBUG_ONLY(if (trace) { tty->print("YYY phi post: other"); other->dump(); }) 2885 if (other != mem) { 2886 if (other->is_Phi() && other->in(0) == r && mem->is_Phi() && mem->in(0) == r) { 2887 bool identical = true; 2888 for (uint i = 1; i < mem->req(); i++) { 2889 if (mem->in(i) != other->in(i)) { 2890 identical = false; 2891 } 2892 assert(mem->in(i) == other->in(i) || 2893 (phase->C->get_alias_index(mem->adr_type()) == alias && mem->in(i) == memory_for(phase->get_ctrl(other->in(i)), phis)), ""); 2894 } 2895 if (mem->adr_type() == TypePtr::BOTTOM || phase->C->get_alias_index(other->adr_type()) == alias || !identical) { 2896 assert(mem->adr_type() != TypePtr::BOTTOM || mem == n, ""); 2897 DEBUG_ONLY(if (trace) { tty->print("YYY phi post: replacing other with"); mem->dump(); }) 2898 if (phis[other->_idx] != mem) { 2899 phis.map(other->_idx, mem); 2900 push_uses(phase->C, other, wq, alias); 2901 } 2902 if (mem->adr_type() == TypePtr::BOTTOM || phase->C->get_alias_index(other->adr_type()) == alias) { 2903 phase->lazy_replace(other, mem); 2904 } 2905 wq.remove(other); 2906 phis.map(r->_idx, mem); 2907 cur_mem = mem; 2908 DEBUG_ONLY(if (trace) { tty->print("YYY setting cur_mem %d", __LINE__); cur_mem->dump(); }) 2909 } else { 2910 DEBUG_ONLY(if (trace) { tty->print("YYY phi post: replacing with other"); mem->dump(); }) 2911 assert(other->adr_type() == TypePtr::BOTTOM, ""); 2912 if (phis[mem->_idx] != other) { 2913 phis.map(mem->_idx, other); 2914 push_uses(phase->C, mem, wq, alias); 2915 } 2916 phase->lazy_replace(mem, other); 2917 wq.remove(mem); 2918 cur_mem = other; 2919 DEBUG_ONLY(if (trace) { tty->print("YYY setting cur_mem %d", __LINE__); cur_mem->dump(); }) 2920 } 2921 } else if (mem->is_Phi() && mem->in(0) == r) { 2922 assert (phase->is_dominator(phase->ctrl_or_self(other), phase->ctrl_or_self(mem)), ""); 2923 cur_mem = other; 2924 DEBUG_ONLY(if (trace) { tty->print("YYY setting cur_mem %d", __LINE__); cur_mem->dump(); }) 2925 } else if (other->is_Phi() && other->in(0) == r) { 2926 assert (phase->is_dominator(phase->ctrl_or_self(mem), phase->ctrl_or_self(other)), ""); 2927 cur_mem = mem; 2928 } else { 2929 ShouldNotReachHere(); 2930 } 2931 } else { 2932 cur_mem = mem; 2933 DEBUG_ONLY(if (trace) { tty->print("YYY setting cur_mem %d", __LINE__); cur_mem->dump(); }) 2934 } 2935 } else { 2936 phis.map(r->_idx, mem); 2937 cur_mem = mem; 2938 DEBUG_ONLY(if (trace) { tty->print("YYY setting cur_mem %d", __LINE__); cur_mem->dump(); }) 2939 } 2940 if (other == other2) { 2941 memory_nodes.map(r->_idx, memory_for(phis[r->_idx], phis)); 2942 } else if (other == NULL || phase->get_ctrl(other) != phase->get_ctrl(other2)) { 2943 if (other == NULL) { 2944 other = mem; 2945 } 2946 assert(is_dominator(phase->get_ctrl(other), phase->get_ctrl(other2), other, other2, phase) && !is_dominator(phase->get_ctrl(other2), phase->get_ctrl(other), other2, other, phase), ""); 2947 memory_nodes.map(r->_idx, other2); 2948 } else { 2949 assert(phase->ctrl_or_self(other2) == r && is_dominator_same_ctrl(r, other, other2, phase) && !is_dominator_same_ctrl(r, other2, other, phase), ""); 2950 } 2951 } 2952 2953 Node* ShenandoahWriteBarrierNode::get_ctrl(Node* n, PhaseIdealLoop* phase) { 2954 Node* c = phase->get_ctrl(n); 2955 if (n->is_Proj() && n->in(0)->is_Call()) { 2956 assert(c == n->in(0), ""); 2957 CallNode* call = c->as_Call(); 2958 CallProjections projs; 2959 call->extract_projections(&projs, true, false); 2960 if (projs.catchall_memproj != NULL) { 2961 if (projs.fallthrough_memproj == n) { 2962 c = projs.fallthrough_catchproj; 2963 } else { 2964 assert(projs.catchall_memproj == n, ""); 2965 c = projs.catchall_catchproj; 2966 } 2967 } 2968 } 2969 return c; 2970 } 2971 2972 void ShenandoahWriteBarrierNode::collect_memory_nodes(int alias, Node_List& memory_nodes, Node_List& phis, PhaseIdealLoop* phase) { 2973 const bool trace = false; 2974 2975 Node_Stack stack(0); 2976 VectorSet visited(Thread::current()->resource_area()); 2977 GrowableArray<Node*> inputs; 2978 Node* cur_mem = NULL; 2979 Unique_Node_List wq; 2980 2981 stack.push(phase->C->root(), 1); 2982 do { 2983 Node* n = stack.node(); 2984 int opc = n->Opcode(); 2985 uint i = stack.index(); 2986 if (i < n->req()) { 2987 Node* mem = NULL; 2988 if (opc == Op_Root) { 2989 Node* in = n->in(i); 2990 int in_opc = in->Opcode(); 2991 if (in_opc == Op_Return || in_opc == Op_Rethrow) { 2992 mem = in->in(TypeFunc::Memory); 2993 } else if (in_opc == Op_Halt) { 2994 if (in->in(0)->is_Region()) { 2995 #ifdef ASSERT 2996 Node* r = in->in(0); 2997 for (uint j = 1; j < r->req(); j++) { 2998 assert(r->in(j)->is_Proj() && r->in(j)->in(0)->Opcode() == Op_NeverBranch, ""); 2999 } 3000 #endif 3001 } else { 3002 Node* proj = in->in(0); 3003 assert(proj->is_Proj(), ""); 3004 Node* in = proj->in(0); 3005 assert(in->is_CallStaticJava() || in->Opcode() == Op_NeverBranch || in->Opcode() == Op_Catch, ""); 3006 if (in->is_CallStaticJava()) { 3007 mem = in->in(TypeFunc::Memory); 3008 } else if (in->Opcode() == Op_Catch) { 3009 Node* call = in->in(0)->in(0); 3010 assert(call->is_Call(), ""); 3011 mem = call->in(TypeFunc::Memory); 3012 } 3013 } 3014 } else { 3015 #ifdef ASSERT 3016 n->dump(); 3017 in->dump(); 3018 #endif 3019 ShouldNotReachHere(); 3020 } 3021 } else { 3022 assert(n->is_Phi() && n->bottom_type() == Type::MEMORY, ""); 3023 assert(n->adr_type() == TypePtr::BOTTOM || phase->C->get_alias_index(n->adr_type()) == alias, ""); 3024 assert(cur_mem != NULL, ""); 3025 DEBUG_ONLY(if (trace) { tty->print("YYY inputs: pushing %d", __LINE__); cur_mem->dump(); }) 3026 inputs.push(cur_mem); 3027 cur_mem = NULL; 3028 mem = n->in(i); 3029 } 3030 i++; 3031 stack.set_index(i); 3032 if (mem == NULL) { 3033 continue; 3034 } 3035 for (;;) { 3036 DEBUG_ONLY(if (trace) { tty->print("Y"); mem->dump(); }) 3037 if (mem->is_Start()) { 3038 break; 3039 } 3040 if (visited.test(mem->_idx)) { 3041 cur_mem = mem; 3042 DEBUG_ONLY(if (trace) { tty->print("YYY setting cur_mem %d", __LINE__); cur_mem->dump(); }) 3043 break; 3044 } 3045 if (mem->is_Phi()) { 3046 visited.set(mem->_idx); 3047 stack.push(mem, 2); 3048 mem = mem->in(1); 3049 } else if (mem->is_Proj()) { 3050 visited.set(mem->_idx); 3051 stack.push(mem, mem->req()); 3052 mem = mem->in(0); 3053 } else if (mem->is_SafePoint() || mem->is_MemBar()) { 3054 mem = mem->in(TypeFunc::Memory); 3055 } else if (mem->is_MergeMem()) { 3056 mem = mem->as_MergeMem()->memory_at(alias); 3057 } else if (mem->is_Store() || mem->is_LoadStore() || mem->is_ClearArray()) { 3058 visited.set(mem->_idx); 3059 stack.push(mem, mem->req()); 3060 mem = mem->in(MemNode::Memory); 3061 } else { 3062 #ifdef ASSERT 3063 mem->dump(); 3064 #endif 3065 ShouldNotReachHere(); 3066 } 3067 } 3068 } else { 3069 if (n->is_Phi()) { 3070 assert(cur_mem != NULL, ""); 3071 DEBUG_ONLY(if (trace) { tty->print("YYY inputs: pushing %d", __LINE__); cur_mem->dump(); }) 3072 inputs.push(cur_mem); 3073 cur_mem = NULL; 3074 int adj = inputs.length() - n->req(); 3075 collect_memory_nodes_helper(n, alias, inputs, adj, memory_nodes, phis, cur_mem, wq, phase); 3076 inputs.trunc_to(inputs.length() - n->req() + 1); 3077 } else if (!n->is_Root()) { 3078 cur_mem = n; 3079 DEBUG_ONLY(if (trace) { tty->print("YYY setting cur_mem %d", __LINE__); cur_mem->dump(); }) 3080 Node* c = get_ctrl(n, phase); 3081 Node* mem = memory_for(memory_nodes[c->_idx], phis); 3082 DEBUG_ONLY(if (trace) { tty->print("YYY post"); n->dump(); }) 3083 assert(mem == NULL || mem == n || is_dominator(phase->get_ctrl(mem), phase->get_ctrl(n), mem, n, phase) && !is_dominator(phase->get_ctrl(n), phase->get_ctrl(mem), n, mem, phase), ""); 3084 memory_nodes.map(c->_idx, n); 3085 } 3086 stack.pop(); 3087 } 3088 } while(stack.size() > 0); 3089 assert(inputs.length() == 0, ""); 3090 3091 #ifdef ASSERT 3092 if (trace) { 3093 tty->print_cr("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"); 3094 wq.dump(); 3095 } 3096 #endif 3097 3098 while (wq.size() > 0) { 3099 Node* n = wq.pop(); 3100 if (n->is_Phi()) { 3101 Node* r = n->in(0); 3102 assert(r != NULL, ""); 3103 for (uint i = 1; i < r->req(); i++) { 3104 inputs.push(find_raw_mem(r->in(i), NULL, memory_nodes, phis, false, phase)); 3105 } 3106 Node* dummy = NULL; 3107 collect_memory_nodes_helper(n, alias, inputs, -1, memory_nodes, phis, dummy, wq, phase); 3108 inputs.clear(); 3109 } else if (n->is_MergeMem()) { 3110 push_uses(phase->C, n, wq, alias); 3111 } 3112 } 3113 3114 #ifdef ASSERT 3115 if (trace) { 3116 for (uint i = 0; i < memory_nodes.Size(); i++) { 3117 if (memory_nodes[i] != NULL) { 3118 tty->print("XXX %d -> ", i); memory_nodes[i]->dump(); 3119 } 3120 } 3121 } 3122 #endif 3123 } 3124 3125 void ShenandoahWriteBarrierNode::fix_raw_mem(Node* ctrl, Node* region, Node* raw_mem, Node* raw_mem_for_ctrl, Node* raw_mem_phi, 3126 Node_List& memory_nodes, Node_List& memory_phis, Unique_Node_List& uses, PhaseIdealLoop* phase) { 3127 const bool trace = false; 3128 DEBUG_ONLY(if (trace) { tty->print("ZZZ control is"); ctrl->dump(); }); 3129 DEBUG_ONLY(if (trace) { tty->print("ZZZ mem is"); raw_mem->dump(); }); 3130 GrowableArray<Node*> phis; 3131 if (raw_mem_for_ctrl != raw_mem) { 3132 Node* old = raw_mem_for_ctrl; 3133 Node* prev = NULL; 3134 while (old != raw_mem) { 3135 assert(old->is_Store() || old->is_LoadStore() || old->is_ClearArray(), ""); 3136 prev = old; 3137 old = old->in(MemNode::Memory); 3138 } 3139 assert(prev != NULL, ""); 3140 memory_nodes.map(ctrl->_idx, raw_mem); 3141 memory_nodes.map(region->_idx, raw_mem_for_ctrl); 3142 phase->igvn().replace_input_of(prev, MemNode::Memory, raw_mem_phi); 3143 } else { 3144 memory_nodes.map(region->_idx, raw_mem_phi); 3145 uses.clear(); 3146 uses.push(region); 3147 for(uint next = 0; next < uses.size(); next++ ) { 3148 Node *n = uses.at(next); 3149 assert(n->is_CFG(), ""); 3150 DEBUG_ONLY(if (trace) { tty->print("ZZZ ctrl"); n->dump(); }) 3151 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { 3152 Node* u = n->fast_out(i); 3153 if (!u->is_Root() && u->is_CFG() && u != n) { 3154 Node* m = memory_nodes[u->_idx]; 3155 if (u->is_Region() && !has_mem_phi(phase->C, u, Compile::AliasIdxRaw)) { 3156 DEBUG_ONLY(if (trace) { tty->print("ZZZ region"); u->dump(); }); 3157 DEBUG_ONLY(if (trace && m != NULL) { tty->print("ZZZ mem"); m->dump(); }); 3158 3159 if (m == NULL || !m->is_Phi() || m->in(0) != u) { 3160 bool push = true; 3161 bool create_phi = true; 3162 if (phase->is_dominator(region, u)) { 3163 create_phi = false; 3164 } else { 3165 IdealLoopTree* loop = phase->get_loop(ctrl); 3166 bool do_check = true; 3167 IdealLoopTree* l = loop; 3168 create_phi = false; 3169 while (l != phase->ltree_root()) { 3170 if (phase->is_dominator(l->_head, u) && phase->is_dominator(phase->idom(u), l->_head)) { 3171 create_phi = true; 3172 do_check = false; 3173 break; 3174 } 3175 l = l->_parent; 3176 } 3177 3178 if (do_check) { 3179 assert(!create_phi, ""); 3180 IdealLoopTree* u_loop = phase->get_loop(u); 3181 if (u_loop != phase->ltree_root() && u_loop->is_member(loop)) { 3182 Node* c = ctrl; 3183 while (!phase->is_dominator(c, u_loop->tail())) { 3184 c = phase->idom(c); 3185 } 3186 if (do_check && !phase->is_dominator(c, u)) { 3187 do_check = false; 3188 } 3189 } 3190 } 3191 3192 if (do_check && phase->is_dominator(phase->idom(u), region)) { 3193 create_phi = true; 3194 } 3195 } 3196 if (create_phi) { 3197 Node* phi = new PhiNode(u, Type::MEMORY, TypeRawPtr::BOTTOM); 3198 phase->register_new_node(phi, u); 3199 phis.push(phi); 3200 DEBUG_ONLY(if (trace) { tty->print("ZZZ new phi"); phi->dump(); }) 3201 if (!(m != NULL && !m->is_Phi() && phase->get_ctrl(m) == u)) { 3202 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting mem"); phi->dump(); }) 3203 memory_nodes.map(u->_idx, phi); 3204 } else { 3205 DEBUG_ONLY(if (trace) { tty->print("ZZZ NOT setting mem"); m->dump(); }) 3206 for (;;) { 3207 assert(m->is_Mem() || m->is_LoadStore() || m->is_Proj() /*|| m->is_MergeMem()*/, ""); 3208 Node* next = NULL; 3209 if (m->is_Proj()) { 3210 next = m->in(0); 3211 } else { 3212 next = m->in(MemNode::Memory); 3213 } 3214 if (phase->get_ctrl(next) != u) { 3215 break; 3216 } 3217 if (next->is_MergeMem()) { 3218 assert(phase->get_ctrl(next->as_MergeMem()->memory_at(Compile::AliasIdxRaw)) != u, ""); 3219 break; 3220 } 3221 if (next->is_Phi()) { 3222 assert(next->adr_type() == TypePtr::BOTTOM && next->in(0) == u, ""); 3223 break; 3224 } 3225 m = next; 3226 } 3227 3228 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting to phi"); m->dump(); }) 3229 assert(m->is_Mem() || m->is_LoadStore(), ""); 3230 phase->igvn().replace_input_of(m, MemNode::Memory, phi); 3231 push = false; 3232 } 3233 } else { 3234 DEBUG_ONLY(if (trace) { tty->print("ZZZ skipping region"); u->dump(); }); 3235 } 3236 if (push) { 3237 uses.push(u); 3238 } 3239 } 3240 } else if (m == NULL) { 3241 uses.push(u); 3242 } 3243 } 3244 } 3245 } 3246 for (int i = 0; i < phis.length(); i++) { 3247 Node* n = phis.at(i); 3248 Node* r = n->in(0); 3249 DEBUG_ONLY(if (trace) { tty->print("ZZZ fixing new phi"); n->dump(); }) 3250 for (uint j = 1; j < n->req(); j++) { 3251 Node* m = find_raw_mem(r->in(j), NULL, memory_nodes, memory_phis, true, phase); 3252 phase->igvn().replace_input_of(n, j, m); 3253 DEBUG_ONLY(if (trace) { tty->print("ZZZ fixing new phi: %d", j); m->dump(); }) 3254 } 3255 } 3256 } 3257 uint last = phase->C->unique(); 3258 MergeMemNode* mm = NULL; 3259 int alias = Compile::AliasIdxRaw; 3260 DEBUG_ONLY(if (trace) { tty->print("ZZZ raw mem is"); raw_mem->dump(); }) 3261 for (DUIterator i = raw_mem->outs(); raw_mem->has_out(i); i++) { 3262 Node* u = raw_mem->out(i); 3263 if (u->_idx < last) { 3264 if (u->is_Mem()) { 3265 if (phase->C->get_alias_index(u->adr_type()) == alias) { 3266 Node* m = find_raw_mem(phase->get_ctrl(u), u, memory_nodes, memory_phis, true, phase); 3267 if (m != raw_mem) { 3268 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting memory of use"); u->dump(); }) 3269 phase->igvn().replace_input_of(u, MemNode::Memory, m); 3270 --i; 3271 } 3272 } 3273 } else if (u->is_MergeMem()) { 3274 MergeMemNode* u_mm = u->as_MergeMem(); 3275 if (u_mm->memory_at(alias) == raw_mem) { 3276 MergeMemNode* newmm = NULL; 3277 for (DUIterator_Fast jmax, j = u->fast_outs(jmax); j < jmax; j++) { 3278 Node* uu = u->fast_out(j); 3279 assert(!uu->is_MergeMem(), "chain of MergeMems?"); 3280 if (uu->is_Phi()) { 3281 assert(uu->adr_type() == TypePtr::BOTTOM, ""); 3282 Node* region = uu->in(0); 3283 int nb = 0; 3284 for (uint k = 1; k < uu->req(); k++) { 3285 if (uu->in(k) == u) { 3286 Node* m = find_raw_mem(region->in(k), NULL, memory_nodes, memory_phis, true, phase); 3287 if (m != raw_mem) { 3288 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting memory of phi %d", k); uu->dump(); }) 3289 if (newmm == NULL || 1) { 3290 newmm = clone_merge_mem(u, raw_mem, alias, m, phase->ctrl_or_self(m), i, phase); 3291 } 3292 if (newmm != u) { 3293 phase->igvn().replace_input_of(uu, k, newmm); 3294 nb++; 3295 --jmax; 3296 } 3297 } 3298 } 3299 } 3300 if (nb > 0) { 3301 --j; 3302 } 3303 } else { 3304 Node* m = find_raw_mem(phase->ctrl_or_self(uu), uu, memory_nodes, memory_phis, true, phase); 3305 if (m != raw_mem) { 3306 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting memory of use"); uu->dump(); }) 3307 if (newmm == NULL || 1) { 3308 newmm = clone_merge_mem(u, raw_mem, alias, m, phase->ctrl_or_self(m), i, phase); 3309 } 3310 if (newmm != u) { 3311 phase->igvn().replace_input_of(uu, uu->find_edge(u), newmm); 3312 --j, --jmax; 3313 } 3314 } 3315 } 3316 } 3317 } 3318 } else if (u->is_Phi()) { 3319 assert(u->bottom_type() == Type::MEMORY, "what else?"); 3320 if (u->adr_type() == TypeRawPtr::BOTTOM || u->adr_type() == TypePtr::BOTTOM) { 3321 Node* region = u->in(0); 3322 bool replaced = false; 3323 for (uint j = 1; j < u->req(); j++) { 3324 if (u->in(j) == raw_mem) { 3325 Node* m = find_raw_mem(region->in(j), NULL, memory_nodes, memory_phis, true, phase); 3326 Node* nnew = m; 3327 if (m != raw_mem) { 3328 if (u->adr_type() == TypePtr::BOTTOM) { 3329 if (mm == NULL || 1) { 3330 mm = allocate_merge_mem(raw_mem, alias, m, phase->ctrl_or_self(m), phase); 3331 } 3332 nnew = mm; 3333 } 3334 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting memory of phi %d", j); u->dump(); }) 3335 phase->igvn().replace_input_of(u, j, nnew); 3336 replaced = true; 3337 } 3338 } 3339 } 3340 if (replaced) { 3341 --i; 3342 } 3343 } 3344 } else if ((u->adr_type() == TypePtr::BOTTOM && u->Opcode() != Op_StrInflatedCopy) || 3345 u->adr_type() == NULL) { 3346 assert(u->adr_type() != NULL || 3347 u->Opcode() == Op_Rethrow || 3348 u->Opcode() == Op_Return || 3349 u->Opcode() == Op_SafePoint || 3350 (u->is_CallStaticJava() && u->as_CallStaticJava()->uncommon_trap_request() != 0) || 3351 (u->is_CallStaticJava() && u->as_CallStaticJava()->_entry_point == OptoRuntime::rethrow_stub()) || 3352 u->Opcode() == Op_CallLeaf, ""); 3353 Node* m = find_raw_mem(phase->ctrl_or_self(u), u, memory_nodes, memory_phis, true, phase); 3354 if (m != raw_mem) { 3355 if (mm == NULL || 1) { 3356 mm = allocate_merge_mem(raw_mem, alias, m, phase->get_ctrl(m), phase); 3357 } 3358 phase->igvn().replace_input_of(u, u->find_edge(raw_mem), mm); 3359 --i; 3360 } 3361 } else if (phase->C->get_alias_index(u->adr_type()) == alias) { 3362 Node* m = find_raw_mem(phase->ctrl_or_self(u), u, memory_nodes, memory_phis, true, phase); 3363 if (m != raw_mem) { 3364 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting memory of use"); u->dump(); }) 3365 phase->igvn().replace_input_of(u, u->find_edge(raw_mem), m); 3366 --i; 3367 } 3368 } 3369 } 3370 } 3371 #ifdef ASSERT 3372 assert(raw_mem_phi->outcnt() > 0, ""); 3373 for (int i = 0; i < phis.length(); i++) { 3374 Node* n = phis.at(i); 3375 assert(n->outcnt() > 0, "new phi must have uses now"); 3376 } 3377 #endif 3378 } 3379 3380 void ShenandoahWriteBarrierNode::test_evacuation_in_progress(Node* ctrl, int alias, Node*& raw_mem, Node*& wb_mem, 3381 IfNode*& evacuation_iff, Node*& evac_in_progress, 3382 Node*& evac_not_in_progress, PhaseIdealLoop* phase) { 3383 IdealLoopTree *loop = phase->get_loop(ctrl); 3384 Node* thread = new ThreadLocalNode(); 3385 phase->register_new_node(thread, ctrl); 3386 Node* offset = phase->igvn().MakeConX(in_bytes(JavaThread::evacuation_in_progress_offset())); 3387 phase->set_ctrl(offset, phase->C->root()); 3388 Node* evacuation_in_progress_adr = new AddPNode(phase->C->top(), thread, offset); 3389 phase->register_new_node(evacuation_in_progress_adr, ctrl); 3390 uint evacuation_in_progress_idx = Compile::AliasIdxRaw; 3391 const TypePtr* evacuation_in_progress_adr_type = NULL; // debug-mode-only argument 3392 debug_only(evacuation_in_progress_adr_type = phase->C->get_adr_type(evacuation_in_progress_idx)); 3393 3394 Node* evacuation_in_progress = new LoadUBNode(ctrl, raw_mem, evacuation_in_progress_adr, 3395 evacuation_in_progress_adr_type, TypeInt::BOOL, MemNode::unordered); 3396 phase->register_new_node(evacuation_in_progress, ctrl); 3397 3398 Node* mb = MemBarNode::make(phase->C, Op_MemBarAcquire, Compile::AliasIdxRaw); 3399 mb->init_req(TypeFunc::Control, ctrl); 3400 mb->init_req(TypeFunc::Memory, raw_mem); 3401 phase->register_control(mb, loop, ctrl); 3402 Node* ctrl_proj = new ProjNode(mb,TypeFunc::Control); 3403 phase->register_control(ctrl_proj, loop, mb); 3404 raw_mem = new ProjNode(mb, TypeFunc::Memory); 3405 phase->register_new_node(raw_mem, mb); 3406 3407 mb = MemBarNode::make(phase->C, Op_MemBarAcquire, alias); 3408 mb->init_req(TypeFunc::Control, ctrl_proj); 3409 mb->init_req(TypeFunc::Memory, wb_mem); 3410 phase->register_control(mb, loop, ctrl_proj); 3411 ctrl_proj = new ProjNode(mb,TypeFunc::Control); 3412 phase->register_control(ctrl_proj, loop, mb); 3413 wb_mem = new ProjNode(mb,TypeFunc::Memory); 3414 phase->register_new_node(wb_mem, mb); 3415 3416 Node* evacuation_in_progress_cmp = new CmpINode(evacuation_in_progress, phase->igvn().zerocon(T_INT)); 3417 phase->register_new_node(evacuation_in_progress_cmp, ctrl_proj); 3418 Node* evacuation_in_progress_test = new BoolNode(evacuation_in_progress_cmp, BoolTest::ne); 3419 phase->register_new_node(evacuation_in_progress_test, ctrl_proj); 3420 evacuation_iff = new IfNode(ctrl_proj, evacuation_in_progress_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN); 3421 phase->register_control(evacuation_iff, loop, ctrl_proj); 3422 3423 assert(is_evacuation_in_progress_test(evacuation_iff), "inconsistent"); 3424 3425 evac_not_in_progress = new IfFalseNode(evacuation_iff); 3426 phase->register_control(evac_not_in_progress, loop, evacuation_iff); 3427 evac_in_progress = new IfTrueNode(evacuation_iff); 3428 phase->register_control(evac_in_progress, loop, evacuation_iff); 3429 } 3430 3431 void ShenandoahWriteBarrierNode::evacuation_not_in_progress_null_check(Node*& c, Node*& val, Node* unc_ctrl, Node*& unc_region, PhaseIdealLoop* phase) { 3432 if (unc_ctrl != NULL) { 3433 // Clone the null check in this branch to allow implicit null check 3434 IdealLoopTree *loop = phase->get_loop(c); 3435 Node* iff = unc_ctrl->in(0); 3436 assert(iff->is_If(), "broken"); 3437 Node* new_iff = iff->clone(); 3438 new_iff->set_req(0, c); 3439 phase->register_control(new_iff, loop, c); 3440 Node* iffalse = new IfFalseNode(new_iff->as_If()); 3441 phase->register_control(iffalse, loop, new_iff); 3442 Node* iftrue = new IfTrueNode(new_iff->as_If()); 3443 phase->register_control(iftrue, loop, new_iff); 3444 c = iftrue; 3445 unc_region = new RegionNode(3); 3446 unc_region->init_req(1, iffalse); 3447 const Type *t = phase->igvn().type(val); 3448 assert(val->Opcode() == Op_CastPP, "expect cast to non null here"); 3449 Node* uncasted_val = val->in(1); 3450 val = new CastPPNode(uncasted_val, t); 3451 val->init_req(0, c); 3452 phase->register_new_node(val, c); 3453 } 3454 } 3455 3456 void ShenandoahWriteBarrierNode::evacuation_not_in_progress(Node* c, Node* val, Node* unc_ctrl, Node* raw_mem, Node* wb_mem, Node* region, 3457 Node* val_phi, Node* mem_phi, Node* raw_mem_phi, Node*& unc_region, PhaseIdealLoop* phase) { 3458 evacuation_not_in_progress_null_check(c, val, unc_ctrl, unc_region, phase); 3459 region->init_req(1, c); 3460 Node* rbfalse = new ShenandoahReadBarrierNode(c, wb_mem, val); 3461 phase->register_new_node(rbfalse, c); 3462 val_phi->init_req(1, rbfalse); 3463 mem_phi->init_req(1, wb_mem); 3464 raw_mem_phi->init_req(1, raw_mem); 3465 } 3466 3467 void ShenandoahWriteBarrierNode::evacuation_in_progress_null_check(Node*& c, Node*& val, Node* evacuation_iff, Node* unc, Node* unc_ctrl, 3468 Node* unc_region, Unique_Node_List& uses, PhaseIdealLoop* phase) { 3469 if (unc != NULL) { 3470 // Clone the null check in this branch to allow implicit null check 3471 IdealLoopTree *loop = phase->get_loop(c); 3472 Node* iff = unc_ctrl->in(0); 3473 assert(iff->is_If(), "broken"); 3474 Node* new_iff = iff->clone(); 3475 new_iff->set_req(0, c); 3476 phase->register_control(new_iff, loop, c); 3477 Node* iffalse = new IfFalseNode(new_iff->as_If()); 3478 phase->register_control(iffalse, loop, new_iff); 3479 Node* iftrue = new IfTrueNode(new_iff->as_If()); 3480 phase->register_control(iftrue, loop, new_iff); 3481 c = iftrue; 3482 unc_region->init_req(2, iffalse); 3483 3484 Node* proj = iff->as_If()->proj_out(0); 3485 assert(proj != unc_ctrl, "bad projection"); 3486 Node* use = proj->unique_ctrl_out(); 3487 3488 assert(use == unc || use->is_Region(), "what else?"); 3489 3490 uses.clear(); 3491 if (use == unc) { 3492 phase->set_idom(use, unc_region, phase->dom_depth(unc_region)+1); 3493 for (uint i = 1; i < unc->req(); i++) { 3494 Node* n = unc->in(i); 3495 if (phase->has_ctrl(n) && phase->get_ctrl(n) == proj) { 3496 uses.push(n); 3497 } 3498 } 3499 } else { 3500 assert(use->is_Region(), "what else?"); 3501 uint idx = 1; 3502 for (; use->in(idx) != proj; idx++); 3503 for (DUIterator_Fast imax, i = use->fast_outs(imax); i < imax; i++) { 3504 Node* u = use->fast_out(i); 3505 if (u->is_Phi() && phase->get_ctrl(u->in(idx)) == proj) { 3506 uses.push(u->in(idx)); 3507 } 3508 } 3509 } 3510 for(uint next = 0; next < uses.size(); next++ ) { 3511 Node *n = uses.at(next); 3512 assert(phase->get_ctrl(n) == proj, "bad control"); 3513 phase->set_ctrl_and_loop(n, unc_region); 3514 if (n->in(0) == proj) { 3515 phase->igvn().replace_input_of(n, 0, unc_region); 3516 } 3517 for (uint i = 0; i < n->req(); i++) { 3518 Node* m = n->in(i); 3519 if (m != NULL && phase->has_ctrl(m) && phase->get_ctrl(m) == proj) { 3520 uses.push(m); 3521 } 3522 } 3523 } 3524 3525 phase->igvn().rehash_node_delayed(use); 3526 int nb = use->replace_edge(proj, unc_region); 3527 assert(nb == 1, "only use expected"); 3528 phase->register_control(unc_region, phase->ltree_root(), evacuation_iff); 3529 3530 phase->igvn().replace_input_of(iff, 1, phase->igvn().intcon(1)); 3531 const Type *t = phase->igvn().type(val); 3532 assert(val->Opcode() == Op_CastPP, "expect cast to non null here"); 3533 Node* uncasted_val = val->in(1); 3534 val = new CastPPNode(uncasted_val, t); 3535 val->init_req(0, c); 3536 phase->register_new_node(val, c); 3537 } 3538 } 3539 3540 void ShenandoahWriteBarrierNode::in_cset_fast_test(Node*& c, Node* rbtrue, Node* raw_mem, Node* wb_mem, Node* region, Node* val_phi, Node* mem_phi, 3541 Node* raw_mem_phi, PhaseIdealLoop* phase) { 3542 if (ShenandoahWriteBarrierCsetTestInIR) { 3543 IdealLoopTree *loop = phase->get_loop(c); 3544 Node* raw_rbtrue = new CastP2XNode(c, rbtrue); 3545 phase->register_new_node(raw_rbtrue, c); 3546 Node* cset_offset = new URShiftXNode(raw_rbtrue, phase->igvn().intcon(ShenandoahHeapRegion::RegionSizeShift)); 3547 phase->register_new_node(cset_offset, c); 3548 Node* in_cset_fast_test_base_addr = phase->igvn().makecon(TypeRawPtr::make(ShenandoahHeap::in_cset_fast_test_addr())); 3549 phase->set_ctrl(in_cset_fast_test_base_addr, phase->C->root()); 3550 Node* in_cset_fast_test_adr = new AddPNode(phase->C->top(), in_cset_fast_test_base_addr, cset_offset); 3551 phase->register_new_node(in_cset_fast_test_adr, c); 3552 uint in_cset_fast_test_idx = Compile::AliasIdxRaw; 3553 const TypePtr* in_cset_fast_test_adr_type = NULL; // debug-mode-only argument 3554 debug_only(in_cset_fast_test_adr_type = phase->C->get_adr_type(in_cset_fast_test_idx)); 3555 Node* in_cset_fast_test_load = new LoadUBNode(c, raw_mem, in_cset_fast_test_adr, in_cset_fast_test_adr_type, TypeInt::BOOL, MemNode::unordered); 3556 phase->register_new_node(in_cset_fast_test_load, c); 3557 Node* in_cset_fast_test_cmp = new CmpINode(in_cset_fast_test_load, phase->igvn().zerocon(T_INT)); 3558 phase->register_new_node(in_cset_fast_test_cmp, c); 3559 Node* in_cset_fast_test_test = new BoolNode(in_cset_fast_test_cmp, BoolTest::ne); 3560 phase->register_new_node(in_cset_fast_test_test, c); 3561 IfNode* in_cset_fast_test_iff = new IfNode(c, in_cset_fast_test_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN); 3562 phase->register_control(in_cset_fast_test_iff, loop, c); 3563 3564 Node* in_cset_fast_test_success = new IfFalseNode(in_cset_fast_test_iff); 3565 phase->register_control(in_cset_fast_test_success, loop, in_cset_fast_test_iff); 3566 3567 region->init_req(3, in_cset_fast_test_success); 3568 val_phi->init_req(3, rbtrue); 3569 mem_phi->init_req(3, wb_mem); 3570 raw_mem_phi->init_req(3, raw_mem); 3571 3572 Node* in_cset_fast_test_failure = new IfTrueNode(in_cset_fast_test_iff); 3573 phase->register_control(in_cset_fast_test_failure, loop, in_cset_fast_test_iff); 3574 3575 c = in_cset_fast_test_failure; 3576 } 3577 } 3578 3579 void ShenandoahWriteBarrierNode::evacuation_in_progress(Node* c, Node* val, Node* evacuation_iff, Node* unc, Node* unc_ctrl, 3580 Node* raw_mem, Node* wb_mem, Node* region, Node* val_phi, Node* mem_phi, 3581 Node* raw_mem_phi, Node* unc_region, int alias, Unique_Node_List& uses, 3582 PhaseIdealLoop* phase) { 3583 evacuation_in_progress_null_check(c, val, evacuation_iff, unc, unc_ctrl, unc_region, uses, phase); 3584 3585 IdealLoopTree *loop = phase->get_loop(c); 3586 Node* rbtrue = new ShenandoahReadBarrierNode(c, wb_mem, val); 3587 phase->register_new_node(rbtrue, c); 3588 3589 Node* in_cset_fast_test_failure = NULL; 3590 in_cset_fast_test(c, rbtrue, raw_mem, wb_mem, region, val_phi, mem_phi, raw_mem_phi, phase); 3591 3592 // The slow path stub consumes and produces raw memory in addition 3593 // to the existing memory edges 3594 Node* base = find_bottom_mem(c, phase); 3595 3596 MergeMemNode* mm = MergeMemNode::make(base); 3597 mm->set_memory_at(alias, wb_mem); 3598 mm->set_memory_at(Compile::AliasIdxRaw, raw_mem); 3599 phase->register_new_node(mm, c); 3600 3601 Node* call = new CallLeafNoFPNode(OptoRuntime::shenandoah_write_barrier_Type(), StubRoutines::shenandoah_wb_C(), "shenandoah_write_barrier", TypeRawPtr::BOTTOM); 3602 call->init_req(TypeFunc::Control, c); 3603 call->init_req(TypeFunc::I_O, phase->C->top()); 3604 call->init_req(TypeFunc::Memory, mm); 3605 call->init_req(TypeFunc::FramePtr, phase->C->top()); 3606 call->init_req(TypeFunc::ReturnAdr, phase->C->top()); 3607 call->init_req(TypeFunc::Parms, rbtrue); 3608 phase->register_control(call, loop, c); 3609 Node* ctrl_proj = new ProjNode(call, TypeFunc::Control); 3610 phase->register_control(ctrl_proj, loop, call); 3611 Node* mem_proj = new ProjNode(call, TypeFunc::Memory); 3612 phase->register_new_node(mem_proj, call); 3613 Node* res_proj = new ProjNode(call, TypeFunc::Parms); 3614 phase->register_new_node(res_proj, call); 3615 Node* res = new CheckCastPPNode(ctrl_proj, res_proj, phase->igvn().type(val)); 3616 phase->register_new_node(res, ctrl_proj); 3617 region->init_req(2, ctrl_proj); 3618 val_phi->init_req(2, res); 3619 mem_phi->init_req(2, mem_proj); 3620 raw_mem_phi->init_req(2, mem_proj); 3621 phase->register_control(region, loop, evacuation_iff); 3622 3623 } 3624 3625 void ShenandoahWriteBarrierNode::pin_and_expand(PhaseIdealLoop* phase) { 3626 const bool trace = false; 3627 Node_List memory_nodes; 3628 Node_List memory_phis; 3629 3630 // Collect raw memory state at CFG points in the entire graph and 3631 // record it in memory_nodes. Optimize the raw memory graph in the 3632 // process and record all memory phis replaced by a more specific 3633 // memory state in memory_phis. Optimizing the memory graph also 3634 // makes the memory graph simpler. 3635 collect_memory_nodes(Compile::AliasIdxRaw, memory_nodes, memory_phis, phase); 3636 3637 // Let's try to common write barriers again 3638 for (;;) { 3639 bool progress = false; 3640 for (int i = phase->C->shenandoah_barriers_count(); i > 0; i--) { 3641 ShenandoahBarrierNode* wb = phase->C->shenandoah_barrier(i-1); 3642 Node* ctrl = phase->get_ctrl(wb); 3643 if (wb->try_common(ctrl, phase) != NULL) { 3644 progress = true; 3645 } 3646 } 3647 if (!progress) { 3648 break; 3649 } 3650 } 3651 3652 for (int i = 0; i < phase->C->shenandoah_barriers_count(); i++) { 3653 ShenandoahWriteBarrierNode* wb = phase->C->shenandoah_barrier(i); 3654 Node* ctrl = phase->get_ctrl(wb); 3655 3656 Node* val = wb->in(ValueIn); 3657 if (ctrl->is_Proj() && ctrl->in(0)->is_CallJava()) { 3658 assert(is_dominator(phase->get_ctrl(val), ctrl->in(0)->in(0), val, ctrl->in(0), phase), "can't move"); 3659 phase->set_ctrl(wb, ctrl->in(0)->in(0)); 3660 } else if (ctrl->is_CallRuntime()) { 3661 assert(is_dominator(phase->get_ctrl(val), ctrl->in(0), val, ctrl, phase), "can't move"); 3662 phase->set_ctrl(wb, ctrl->in(0)); 3663 } 3664 3665 assert(wb->Opcode() == Op_ShenandoahWriteBarrier, "only for write barriers"); 3666 // Look for a null check that dominates this barrier and move the 3667 // barrier right after the null check to enable implicit null 3668 // checks 3669 wb->pin_and_expand_move_barrier(phase); 3670 3671 ctrl = phase->get_ctrl(wb); 3672 wb->pin_and_expand_helper(phase); 3673 } 3674 3675 Unique_Node_List uses; 3676 Unique_Node_List uses_to_ignore; 3677 for (int i = phase->C->shenandoah_barriers_count(); i > 0; i--) { 3678 int cnt = phase->C->shenandoah_barriers_count(); 3679 ShenandoahWriteBarrierNode* wb = phase->C->shenandoah_barrier(i-1); 3680 3681 uint last = phase->C->unique(); 3682 Node* ctrl = phase->get_ctrl(wb); 3683 3684 Node* raw_mem = find_raw_mem(ctrl, wb, memory_nodes, memory_phis, true, phase); 3685 Node* init_raw_mem = raw_mem; 3686 Node* raw_mem_for_ctrl = find_raw_mem(ctrl, NULL, memory_nodes, memory_phis, true, phase); 3687 int alias = phase->C->get_alias_index(wb->adr_type()); 3688 Node* wb_mem = wb->in(Memory); 3689 3690 Node* val = wb->in(ValueIn); 3691 Node* wbproj = wb->find_out_with(Op_ShenandoahWBMemProj); 3692 IdealLoopTree *loop = phase->get_loop(ctrl); 3693 3694 assert(val->Opcode() != Op_ShenandoahWriteBarrier || phase->C->has_irreducible_loop(), "No chain of write barriers"); 3695 3696 CallStaticJavaNode* unc = wb->pin_and_expand_null_check(phase->igvn()); 3697 Node* unc_ctrl = NULL; 3698 if (unc != NULL) { 3699 if (val->in(0) != ctrl) { 3700 unc = NULL; 3701 } else { 3702 unc_ctrl = val->in(0); 3703 } 3704 } 3705 3706 Node* uncasted_val = val; 3707 if (unc != NULL) { 3708 uncasted_val = val->in(1); 3709 } 3710 3711 Node* evac_in_progress = NULL; 3712 Node* evac_not_in_progress = NULL; 3713 IfNode* evacuation_iff = NULL; 3714 test_evacuation_in_progress(ctrl, alias, raw_mem, wb_mem, evacuation_iff, evac_in_progress, evac_not_in_progress, phase); 3715 3716 Node* region = new RegionNode(4); 3717 Node* val_phi = PhiNode::make_blank(region, val); 3718 Node* mem_phi = PhiNode::make(region, wb_mem, Type::MEMORY, phase->C->alias_type(wb->adr_type())->adr_type()); 3719 Node* raw_mem_phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM); 3720 3721 Node* unc_region = NULL; 3722 evacuation_not_in_progress(evac_not_in_progress, val, unc_ctrl, raw_mem, wb_mem, 3723 region, val_phi, mem_phi, raw_mem_phi, unc_region, 3724 phase); 3725 3726 evacuation_in_progress(evac_in_progress, val, evacuation_iff, unc, unc_ctrl, 3727 raw_mem, wb_mem, region, val_phi, mem_phi, raw_mem_phi, 3728 unc_region, alias, uses, 3729 phase); 3730 Node* out_val = val_phi; 3731 phase->register_new_node(val_phi, region); 3732 phase->register_new_node(mem_phi, region); 3733 phase->register_new_node(raw_mem_phi, region); 3734 3735 // Update the control of all nodes that should be after the 3736 // barrier control flow 3737 uses.clear(); 3738 // Every node that is control dependent on the barrier's input 3739 // control will be after the expanded barrier. The raw memory (if 3740 // its memory is control dependent on the barrier's input control) 3741 // must stay above the barrier. 3742 uses_to_ignore.clear(); 3743 if (phase->has_ctrl(init_raw_mem) && phase->get_ctrl(init_raw_mem) == ctrl && !init_raw_mem->is_Phi()) { 3744 uses_to_ignore.push(init_raw_mem); 3745 } 3746 for (uint next = 0; next < uses_to_ignore.size(); next++) { 3747 Node *n = uses_to_ignore.at(next); 3748 for (uint i = 0; i < n->req(); i++) { 3749 Node* in = n->in(i); 3750 if (in != NULL && phase->has_ctrl(in) && phase->get_ctrl(in) == ctrl) { 3751 uses_to_ignore.push(in); 3752 } 3753 } 3754 } 3755 for (DUIterator_Fast imax, i = ctrl->fast_outs(imax); i < imax; i++) { 3756 Node* u = ctrl->fast_out(i); 3757 if (u->_idx < last && 3758 u != wb && 3759 !uses_to_ignore.member(u) && 3760 (u->in(0) != ctrl || (!u->is_Region() && !u->is_Phi())) && 3761 (ctrl->Opcode() != Op_CatchProj || u->Opcode() != Op_CreateEx)) { 3762 Node* old_c = phase->ctrl_or_self(u); 3763 Node* c = old_c; 3764 if (c != ctrl || 3765 is_dominator_same_ctrl(old_c, wb, u, phase) || 3766 u->is_g1_marking_load()) { 3767 phase->igvn().rehash_node_delayed(u); 3768 int nb = u->replace_edge(ctrl, region); 3769 if (u->is_CFG()) { 3770 if (phase->idom(u) == ctrl) { 3771 phase->set_idom(u, region, phase->dom_depth(region)); 3772 } 3773 } else if (phase->get_ctrl(u) == ctrl) { 3774 assert(u != init_raw_mem, "should leave input raw mem above the barrier"); 3775 uses.push(u); 3776 } 3777 assert(nb == 1, "more than 1 ctrl input?"); 3778 --i, imax -= nb; 3779 } 3780 } 3781 } 3782 3783 if (wbproj != NULL) { 3784 phase->igvn().replace_input_of(wbproj, 0, phase->C->top()); 3785 phase->lazy_replace(wbproj, mem_phi); 3786 } 3787 if (unc != NULL) { 3788 for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) { 3789 Node* u = val->fast_out(i); 3790 Node* c = phase->ctrl_or_self(u); 3791 if (u != wb && (c != ctrl || is_dominator_same_ctrl(c, wb, u, phase))) { 3792 phase->igvn().rehash_node_delayed(u); 3793 int nb = u->replace_edge(val, out_val); 3794 --i, imax -= nb; 3795 } 3796 } 3797 if (val->outcnt() == 0) { 3798 phase->lazy_update(val, out_val); 3799 phase->igvn()._worklist.push(val); 3800 } 3801 } 3802 phase->lazy_replace(wb, out_val); 3803 3804 follow_barrier_uses(mem_phi, ctrl, uses, phase); 3805 follow_barrier_uses(out_val, ctrl, uses, phase); 3806 3807 for(uint next = 0; next < uses.size(); next++ ) { 3808 Node *n = uses.at(next); 3809 assert(phase->get_ctrl(n) == ctrl, "bad control"); 3810 assert(n != init_raw_mem, "should leave input raw mem above the barrier"); 3811 phase->set_ctrl(n, region); 3812 follow_barrier_uses(n, ctrl, uses, phase); 3813 } 3814 3815 // The slow path call produces memory: hook the raw memory phi 3816 // from the expanded write barrier with the rest of the graph 3817 // which may require adding memory phis at every post dominated 3818 // region and at enclosing loop heads. Use the memory state 3819 // collected in memory_nodes to fix the memory graph. Update that 3820 // memory state as we go. 3821 fix_raw_mem(ctrl,region, init_raw_mem, raw_mem_for_ctrl, raw_mem_phi, memory_nodes, memory_phis, uses, phase); 3822 assert(phase->C->shenandoah_barriers_count() == cnt - 1, "not replaced"); 3823 } 3824 3825 assert(phase->C->shenandoah_barriers_count() == 0, "all write barrier nodes should have been replaced"); 3826 } 3827 3828 void ShenandoahWriteBarrierNode::move_evacuation_test_out_of_loop(IfNode* iff, PhaseIdealLoop* phase) { 3829 // move test and its mem barriers out of the loop 3830 assert(is_evacuation_in_progress_test(iff), "inconsistent"); 3831 3832 IdealLoopTree *loop = phase->get_loop(iff); 3833 Node* loop_head = loop->_head; 3834 Node* entry_c = loop_head->in(LoopNode::EntryControl); 3835 IdealLoopTree *entry_loop = phase->get_loop(entry_c); 3836 3837 GrowableArray<Node*> new_mbs; 3838 Node* c = iff->in(0); 3839 MemBarNode* mb = NULL; 3840 do { 3841 Node* proj_ctrl = c; 3842 assert(c->is_Proj(), "proj expected"); 3843 mb = proj_ctrl->in(0)->as_MemBar(); 3844 c = c->in(0)->in(0); 3845 3846 Node* proj_mem = mb->proj_out(TypeFunc::Memory); 3847 3848 MemBarNode* new_mb = mb->clone()->as_MemBar();; 3849 Node* new_proj_ctrl = new ProjNode(new_mb,TypeFunc::Control); 3850 Node* new_proj_mem = new ProjNode(new_mb,TypeFunc::Memory); 3851 3852 int alias = phase->C->get_alias_index(mb->adr_type()); 3853 Node* mem_ctrl = NULL; 3854 Node* mem = dom_mem(mb, loop_head, alias, mem_ctrl, phase); 3855 new_mb->set_req(TypeFunc::Memory, mem); 3856 phase->register_new_node(new_proj_mem, new_mb); 3857 fix_memory_uses(mem, new_mb, new_proj_mem, entry_c, alias, phase); 3858 assert(new_proj_mem->outcnt() >= 1, "memory projection is disconnected"); 3859 new_mbs.push(new_proj_ctrl); 3860 } while (mb->adr_type() != TypeRawPtr::BOTTOM); 3861 3862 c = entry_c; 3863 for (int i = new_mbs.length()-1; i >= 0; i--) { 3864 Node* proj_ctrl = new_mbs.at(i); 3865 Node* mb = proj_ctrl->in(0); 3866 mb->set_req(0, c); 3867 phase->set_idom(mb, mb->in(0), phase->dom_depth(mb->in(0))); 3868 phase->set_idom(proj_ctrl, mb, phase->dom_depth(mb)); 3869 c = proj_ctrl; 3870 phase->register_control(mb, entry_loop, mb->in(0)); 3871 phase->register_control(proj_ctrl, entry_loop, mb); 3872 } 3873 phase->igvn().replace_input_of(loop_head, LoopNode::EntryControl, c); 3874 phase->set_idom(loop_head, c, phase->dom_depth(c)); 3875 //phase->recompute_dom_depth(); 3876 3877 Node* load = iff->in(1)->in(1)->in(1); 3878 assert(load->Opcode() == Op_LoadUB, "inconsistent"); 3879 phase->igvn().replace_input_of(load, MemNode::Memory, new_mbs.at(new_mbs.length()-1)->in(0)->in(TypeFunc::Memory)); 3880 phase->igvn().replace_input_of(load, 0, entry_c); 3881 phase->set_ctrl_and_loop(load, entry_c); 3882 3883 c = iff->in(0); 3884 for (;;) { 3885 Node* next = c->in(0)->in(0); 3886 assert(c->is_Proj(), "proj expected"); 3887 Node* proj_ctrl = c; 3888 MemBarNode* mb = proj_ctrl->in(0)->as_MemBar(); 3889 Node* proj_mem = mb->proj_out(TypeFunc::Memory); 3890 Node* ctrl = mb->in(TypeFunc::Control); 3891 Node* mem = mb->in(TypeFunc::Memory); 3892 3893 phase->lazy_replace(proj_mem, mem); 3894 phase->lazy_replace(proj_ctrl, ctrl); 3895 phase->lazy_replace(mb, ctrl); 3896 loop->_body.yank(proj_ctrl); 3897 loop->_body.yank(proj_mem); 3898 loop->_body.yank(mb); 3899 if (mb->adr_type() == TypeRawPtr::BOTTOM) { 3900 break; 3901 } 3902 c = next; 3903 } 3904 3905 assert(phase->is_dominator(phase->get_ctrl(load->in(MemNode::Address)), entry_c), "address not out of loop?"); 3906 } 3907 3908 void ShenandoahWriteBarrierNode::backtoback_evacs(IfNode* iff, IfNode* dom_if, PhaseIdealLoop* phase) { 3909 // move all mem barriers from this evac test to the dominating one, 3910 // removing duplicates in the process 3911 IdealLoopTree *loop = phase->get_loop(dom_if); 3912 Node* c1 = iff->in(0); 3913 Node* mb1 = NULL; 3914 GrowableArray<Node*> new_mbs; 3915 for(;;) { 3916 mb1 = c1->in(0); 3917 c1 = c1->in(0)->in(0); 3918 assert(mb1->Opcode() == Op_MemBarAcquire, "mem bar expected"); 3919 if (mb1->adr_type() == TypeRawPtr::BOTTOM) { 3920 phase->lazy_replace(mb1->as_MemBar()->proj_out(TypeFunc::Memory), mb1->in(TypeFunc::Memory)); 3921 break; 3922 } 3923 Node* c2 = dom_if->in(0); 3924 Node* mb2 = NULL; 3925 do { 3926 mb2 = c2->in(0); 3927 c2 = c2->in(0)->in(0); 3928 assert(mb2->Opcode() == Op_MemBarAcquire, "mem bar expected"); 3929 if (mb2->adr_type() == TypeRawPtr::BOTTOM) { 3930 // Barrier on same slice doesn't exist at dominating if: 3931 // move barrier up 3932 MemBarNode* mb = mb1->clone()->as_MemBar(); 3933 Node* proj_ctrl = new ProjNode(mb,TypeFunc::Control); 3934 Node* proj_mem = new ProjNode(mb,TypeFunc::Memory); 3935 int alias = phase->C->get_alias_index(mb->adr_type()); 3936 Node* mem_ctrl = NULL; 3937 Node* mem = dom_mem(mb1, dom_if->in(0), alias, mem_ctrl, phase); 3938 mb->set_req(TypeFunc::Memory, mem); 3939 phase->register_new_node(proj_mem, mb); 3940 fix_memory_uses(mem, mb, proj_mem, dom_if->in(0), alias, phase); 3941 assert(proj_mem->outcnt() >= 1, "memory projection is disconnected"); 3942 new_mbs.push(proj_ctrl); 3943 break; 3944 } 3945 } while (mb2->adr_type() != mb1->adr_type()); 3946 phase->lazy_replace(mb1->as_MemBar()->proj_out(TypeFunc::Memory), mb1->in(TypeFunc::Memory)); 3947 } 3948 if (new_mbs.length() > 0) { 3949 Node* c = dom_if->in(0); 3950 for (int i = 0; i < new_mbs.length(); i++) { 3951 Node* proj_ctrl = new_mbs.at(i); 3952 Node* mb = proj_ctrl->in(0); 3953 mb->set_req(0, c); 3954 phase->set_idom(mb, mb->in(0), phase->dom_depth(mb->in(0))); 3955 phase->set_idom(proj_ctrl, mb, phase->dom_depth(mb)); 3956 c = proj_ctrl; 3957 phase->register_control(mb, loop, mb->in(0)); 3958 phase->register_control(proj_ctrl, loop, mb); 3959 } 3960 phase->igvn().replace_input_of(dom_if, 0, c); 3961 phase->set_idom(dom_if, dom_if->in(0), phase->dom_depth(dom_if->in(0))); 3962 //phase->recompute_dom_depth(); 3963 } 3964 Node* c = iff->in(0); 3965 for (;;) { 3966 Node* next = c->in(0)->in(0); 3967 assert(c->is_Proj(), "proj expected"); 3968 MemBarNode* mb = c->in(0)->as_MemBar(); 3969 Node* proj_ctrl = c; 3970 Node* ctrl = mb->in(TypeFunc::Control); 3971 3972 phase->lazy_replace(proj_ctrl, ctrl); 3973 phase->lazy_replace(mb, ctrl); 3974 if (mb->adr_type() == TypeRawPtr::BOTTOM) { 3975 break; 3976 } 3977 c = next; 3978 } 3979 } 3980 3981 void ShenandoahWriteBarrierNode::merge_back_to_back_evacuation_tests(Node* n, PhaseIdealLoop* phase) { 3982 if (phase->identical_backtoback_ifs(n)) { 3983 Node* n_ctrl = ShenandoahWriteBarrierNode::evacuation_in_progress_test_ctrl(n); 3984 if (phase->can_split_if(n_ctrl)) { 3985 IfNode* dom_if = phase->idom(n_ctrl)->as_If(); 3986 ShenandoahWriteBarrierNode::backtoback_evacs(n->as_If(), dom_if, phase); 3987 PhiNode* bolphi = PhiNode::make_blank(n_ctrl, n->in(1)); 3988 Node* proj_true = dom_if->proj_out(1); 3989 Node* proj_false = dom_if->proj_out(0); 3990 Node* con_true = phase->igvn().makecon(TypeInt::ONE); 3991 Node* con_false = phase->igvn().makecon(TypeInt::ZERO); 3992 3993 for (uint i = 1; i < n_ctrl->req(); i++) { 3994 if (phase->is_dominator(proj_true, n_ctrl->in(i))) { 3995 bolphi->init_req(i, con_true); 3996 } else { 3997 assert(phase->is_dominator(proj_false, n_ctrl->in(i)), "bad if"); 3998 bolphi->init_req(i, con_false); 3999 } 4000 } 4001 phase->register_new_node(bolphi, n_ctrl); 4002 phase->igvn().replace_input_of(n, 1, bolphi); 4003 phase->do_split_if(n); 4004 } 4005 } 4006 } 4007 4008 void ShenandoahWriteBarrierNode::optimize_after_expansion(const Node_List& evacuation_tests, Node_List &old_new, PhaseIdealLoop* phase) { 4009 for (uint i = 0; i < evacuation_tests.size(); i++) { 4010 Node* n = evacuation_tests.at(i); 4011 assert(is_evacuation_in_progress_test(n), "only evacuation test"); 4012 merge_back_to_back_evacuation_tests(n, phase); 4013 } 4014 if (!phase->C->major_progress()) { 4015 VectorSet seen(Thread::current()->resource_area()); 4016 for (uint i = 0; i < evacuation_tests.size(); i++) { 4017 Node* n = evacuation_tests.at(i); 4018 IdealLoopTree* loop = phase->get_loop(n); 4019 if (loop != phase->ltree_root() && 4020 loop->_child == NULL && 4021 !loop->_irreducible) { 4022 LoopNode* head = loop->_head->as_Loop(); 4023 if ((!head->is_CountedLoop() || head->as_CountedLoop()->is_main_loop() || head->as_CountedLoop()->is_normal_loop()) && 4024 !seen.test_set(head->_idx) && 4025 loop->policy_unswitching(phase)) { 4026 IfNode* iff = phase->find_unswitching_candidate(loop); 4027 if (iff != NULL && is_evacuation_in_progress_test(iff)) { 4028 if (head->is_strip_mined()) { 4029 head->verify_strip_mined(0); 4030 head->clear_strip_mined(); 4031 head->in(LoopNode::EntryControl)->as_Loop()->clear_strip_mined(); 4032 } 4033 phase->do_unswitching(loop, old_new); 4034 } 4035 } 4036 } 4037 } 4038 } 4039 } 4040 4041 #ifdef ASSERT 4042 void ShenandoahBarrierNode::verify_raw_mem(RootNode* root) { 4043 const bool trace = false; 4044 ResourceMark rm; 4045 Unique_Node_List nodes; 4046 Unique_Node_List controls; 4047 Unique_Node_List memories; 4048 4049 nodes.push(root); 4050 for (uint next = 0; next < nodes.size(); next++) { 4051 Node *n = nodes.at(next); 4052 if (n->Opcode() == Op_CallLeafNoFP && n->as_Call()->_entry_point == StubRoutines::shenandoah_wb_C()) { 4053 controls.push(n); 4054 if (trace) { tty->print("XXXXXX verifying"); n->dump(); } 4055 for (uint next2 = 0; next2 < controls.size(); next2++) { 4056 Node *m = controls.at(next2); 4057 if (!m->is_Loop() || controls.member(m->in(LoopNode::EntryControl)) || 1) { 4058 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) { 4059 Node* u = m->fast_out(i); 4060 if (u->is_CFG() && !u->is_Root()) { 4061 if (trace) { tty->print("XXXXXX pushing control"); u->dump(); } 4062 controls.push(u); 4063 } 4064 } 4065 } 4066 } 4067 memories.push(n->as_Call()->proj_out(TypeFunc::Memory)); 4068 for (uint next2 = 0; next2 < memories.size(); next2++) { 4069 Node *m = memories.at(next2); 4070 assert(m->bottom_type() == Type::MEMORY, ""); 4071 if (!m->is_Phi() || !m->in(0)->is_Loop() || controls.member(m->in(0)->in(LoopNode::EntryControl)) || 1) { 4072 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) { 4073 Node* u = m->fast_out(i); 4074 if (u->bottom_type() == Type::MEMORY && (u->is_Mem() || u->is_ClearArray())) { 4075 memories.push(u); 4076 } else if (u->is_LoadStore()) { 4077 memories.push(u->find_out_with(Op_SCMemProj)); 4078 } else if (u->is_MergeMem() && u->as_MergeMem()->memory_at(Compile::AliasIdxRaw) == m) { 4079 memories.push(u); 4080 } else if (u->is_Phi()) { 4081 assert(u->bottom_type() == Type::MEMORY, ""); 4082 if (u->adr_type() == TypeRawPtr::BOTTOM || u->adr_type() == TypePtr::BOTTOM) { 4083 assert(controls.member(u->in(0)), ""); 4084 memories.push(u); 4085 } 4086 } else if (u->is_SafePoint() || u->is_MemBar()) { 4087 for (DUIterator_Fast jmax, j = u->fast_outs(jmax); j < jmax; j++) { 4088 Node* uu = u->fast_out(j); 4089 if (uu->bottom_type() == Type::MEMORY) { 4090 memories.push(uu); 4091 } 4092 } 4093 } 4094 } 4095 } 4096 } 4097 for (uint next2 = 0; next2 < controls.size(); next2++) { 4098 Node *m = controls.at(next2); 4099 if (m->is_Region()) { 4100 bool all_in = true; 4101 for (uint i = 1; i < m->req(); i++) { 4102 if (!controls.member(m->in(i))) { 4103 all_in = false; 4104 break; 4105 } 4106 } 4107 if (trace) { tty->print("XXX verifying %s", all_in ? "all in" : ""); m->dump(); } 4108 bool found_phi = false; 4109 for (DUIterator_Fast jmax, j = m->fast_outs(jmax); j < jmax && !found_phi; j++) { 4110 Node* u = m->fast_out(j); 4111 if (u->is_Phi() && memories.member(u)) { 4112 found_phi = true; 4113 for (uint i = 1; i < u->req() && found_phi; i++) { 4114 Node* k = u->in(i); 4115 if (memories.member(k) != controls.member(m->in(i))) { 4116 found_phi = false; 4117 } 4118 } 4119 } 4120 } 4121 assert(found_phi || all_in, ""); 4122 } 4123 } 4124 controls.clear(); 4125 memories.clear(); 4126 } 4127 for( uint i = 0; i < n->len(); ++i ) { 4128 Node *m = n->in(i); 4129 if (m != NULL) { 4130 nodes.push(m); 4131 } 4132 } 4133 } 4134 } 4135 #endif 4136 4137 static bool is_on_null_check_path(Block* b, Block* null_check_block) { 4138 if (null_check_block == NULL) { 4139 return false; 4140 } 4141 do { 4142 assert(null_check_block->_num_succs == 1, "only one succ on the path to unc"); 4143 if (b == null_check_block) { 4144 return true; 4145 } 4146 null_check_block = null_check_block->_succs[0]; 4147 } while(!null_check_block->head()->is_Root()); 4148 4149 return false; 4150 } 4151 4152 int PhaseCFG::replace_uses_with_shenandoah_barrier_helper(Node* n, Node* use, Node* val, Block* block, Block* null_check_block) { 4153 int nb = 0; 4154 Block* buse = get_block_for_node(use); 4155 if (is_on_null_check_path(buse, null_check_block)) { 4156 return 0; 4157 } 4158 if (use->is_Phi()) { 4159 for (uint j = 1; j < use->req(); j++) { 4160 if (use->in(j) == val) { 4161 Block* b = get_block_for_node(use->in(0)->in(j)); 4162 if ((block != b && block->dom_lca(b) == block) || 4163 block == b) { 4164 use->set_req(j, n); 4165 nb++; 4166 } 4167 } 4168 } 4169 } else { 4170 if ((block != buse && block->dom_lca(buse) == block) || 4171 (block == buse && !use->is_scheduled())) { 4172 // Let precedence edges alone (can confuse anti-dependence verification code) 4173 for (uint i = 0; i < use->req(); i++) { 4174 if (use->in(i) == val) { 4175 use->set_req(i, n); 4176 nb++; 4177 } 4178 } 4179 assert(nb > 0 || use->find_prec_edge(val) != -1, "no replacement?"); 4180 } 4181 } 4182 4183 return nb; 4184 } 4185 4186 // acmp is compiled as: 4187 // if (a != b) { 4188 // a = rb(a); 4189 // b = rb(b); 4190 // } 4191 // if (a == b) { 4192 // 4193 // With compressed oops, the first comparison is a CmpN and the second 4194 // a CmpP. If we have a read barrier above: 4195 // 4196 // a = rb(a); 4197 // if (a != b) { 4198 // a = rb(a); 4199 // b = rb(b); 4200 // } 4201 // if (a == b) { 4202 // 4203 // replacing a by rb(a) where possible would lead to: 4204 // a = rb(a); 4205 // if (a != b) { 4206 // a = rb(rb(a)); 4207 // b = rb(b); 4208 // } 4209 // if (rb(a) == b) { 4210 // 4211 // The CmpN is unaffected but the CmpP is. If a == b, then the CmpN 4212 // succeeds but the CmpP can fail. So replacing the barrier's input 4213 // with the barrier's output at acmp is not safe. This 4214 // (conservatively) checks whether the use is part of a acmp. 4215 static bool replace_uses_skip_use(Node* use, Node* val) { 4216 if (use->is_Phi()) { 4217 bool found_cmpn = false; 4218 if (val->is_Mach() && val->as_Mach()->ideal_Opcode() == Op_DecodeN) { 4219 val = val->in(1); 4220 } 4221 for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) { 4222 Node* u = val->fast_out(i); 4223 if (u->is_Mach() && u->as_Mach()->ideal_Opcode() == Op_CmpN) { 4224 found_cmpn = true; 4225 break; 4226 } 4227 } 4228 if (!found_cmpn) { 4229 return false; 4230 } 4231 ResourceMark rm; 4232 Unique_Node_List wq; 4233 wq.push(use); 4234 for (uint next = 0; next < wq.size(); next++) { 4235 Node *n = wq.at(next); 4236 assert(n->is_Phi(), "only phi here"); 4237 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { 4238 Node* u = n->fast_out(i); 4239 if (u->is_Phi()) { 4240 wq.push(u); 4241 } else if (u->is_Mach() && u->as_Mach()->ideal_Opcode() == Op_CmpP) { 4242 return true; 4243 } 4244 } 4245 } 4246 } 4247 return false; 4248 } 4249 4250 void PhaseCFG::replace_uses_with_shenandoah_barrier(Node* n, Block* block, Node_List& worklist, GrowableArray<int>& ready_cnt, uint max_idx, uint& phi_cnt) { 4251 // Replace all uses of barrier's input that are dominated by the 4252 // barrier with the value returned by the barrier: no need to keep 4253 // both live. 4254 if (n->is_Mach() && n->as_Mach()->ideal_Opcode() == Op_ShenandoahReadBarrier) { 4255 MachNullCheckNode* null_check = NULL; 4256 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax && null_check == NULL; i++) { 4257 Node* use = n->fast_out(i); 4258 if (use->is_MachNullCheck()) { 4259 null_check = use->as_MachNullCheck(); 4260 } 4261 } 4262 Block* null_check_block = NULL; 4263 if (null_check != NULL) { 4264 Node* proj = null_check->find_out_with(Op_IfTrue); 4265 Node* head = proj->unique_out(); 4266 null_check_block = get_block_for_node(head); 4267 } 4268 4269 Node* val = n->in(ShenandoahBarrierNode::ValueIn); 4270 if (!val->bottom_type()->isa_narrowoop()) { 4271 for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) { 4272 Node* use = val->fast_out(i); 4273 if (use != n) { 4274 if (UseCompressedOops && replace_uses_skip_use(use, val)) { 4275 continue; 4276 } 4277 int nb = replace_uses_with_shenandoah_barrier_helper(n, use, val, block, null_check_block); 4278 if (nb > 0) { 4279 --i; imax -= nb; 4280 } 4281 } 4282 } 4283 } else { 4284 for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) { 4285 Node* u = val->fast_out(i); 4286 if (u->is_Mach() && u->as_Mach()->ideal_Opcode() == Op_DecodeN) { 4287 int projs = 0; 4288 for (DUIterator_Fast jmax, j = u->fast_outs(jmax); j < jmax; j++) { 4289 Node* uu = u->fast_out(j); 4290 assert(!uu->is_MachTemp(), ""); 4291 if (uu->is_MachProj() && uu->outcnt() == 0) { 4292 projs++; 4293 } else { 4294 if (replace_uses_skip_use(uu, val)) { 4295 continue; 4296 } 4297 int nb = replace_uses_with_shenandoah_barrier_helper(n, uu, u, block, null_check_block); 4298 if (nb > 0) { 4299 if (!u->is_scheduled()) { 4300 push_ready_nodes(n, uu, block, ready_cnt, worklist, max_idx, nb); 4301 } 4302 --j; jmax -= nb; 4303 } 4304 } 4305 } 4306 // The DecodeN may have gone dead 4307 if (u->outcnt() - projs == 0) { 4308 u->disconnect_inputs(NULL, C); 4309 Block* bu = get_block_for_node(u); 4310 unmap_node_from_block(u); 4311 if (bu == block) { 4312 if (u->is_scheduled()) { 4313 block->find_remove(u); 4314 phi_cnt--; 4315 } else { 4316 worklist.yank(u); 4317 block->remove_node(block->end_idx()-1); 4318 } 4319 } else { 4320 bu->find_remove(u); 4321 } 4322 for (DUIterator_Fast jmax, j = u->fast_outs(jmax); j < jmax; j++) { 4323 Node* uu = u->fast_out(j); 4324 assert(uu->is_MachProj() && uu->outcnt() == 0, ""); 4325 assert(bu == get_block_for_node(uu), ""); 4326 uu->disconnect_inputs(NULL, C); 4327 --j; --jmax; 4328 unmap_node_from_block(uu); 4329 if (bu == block) { 4330 if (u->is_scheduled()) { 4331 block->find_remove(uu); 4332 phi_cnt--; 4333 } else { 4334 worklist.yank(uu); 4335 block->remove_node(block->end_idx()-1); 4336 } 4337 } else { 4338 bu->find_remove(uu); 4339 } 4340 assert(uu->is_scheduled() == u->is_scheduled(), ""); 4341 } 4342 --i; --imax; 4343 } 4344 } 4345 } 4346 } 4347 } 4348 }