1 /* 2 * Copyright (c) 2015, Red Hat, Inc. and/or its affiliates. 3 * 4 * This code is free software; you can redistribute it and/or modify it 5 * under the terms of the GNU General Public License version 2 only, as 6 * published by the Free Software Foundation. 7 * 8 * This code is distributed in the hope that it will be useful, but WITHOUT 9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 11 * version 2 for more details (a copy is included in the LICENSE file that 12 * accompanied this code). 13 * 14 * You should have received a copy of the GNU General Public License version 15 * 2 along with this work; if not, write to the Free Software Foundation, 16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 17 * 18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 19 * or visit www.oracle.com if you need additional information or have any 20 * questions. 21 * 22 */ 23 24 #include "precompiled.hpp" 25 #include "gc/shenandoah/brooksPointer.hpp" 26 #include "gc/shenandoah/shenandoahHeap.hpp" 27 #include "gc/shenandoah/shenandoahHeapRegion.hpp" 28 #include "opto/arraycopynode.hpp" 29 #include "opto/block.hpp" 30 #include "opto/callnode.hpp" 31 #include "opto/castnode.hpp" 32 #include "opto/movenode.hpp" 33 #include "opto/phaseX.hpp" 34 #include "opto/rootnode.hpp" 35 #include "opto/runtime.hpp" 36 #include "opto/shenandoahSupport.hpp" 37 #include "opto/subnode.hpp" 38 39 Node* ShenandoahBarrierNode::skip_through_barrier(Node* n) { 40 if (n == NULL) { 41 return NULL; 42 } else if (n->is_ShenandoahBarrier()) { 43 return n->in(ValueIn); 44 } else if (n->is_Phi() && 45 n->req() == 3 && 46 n->in(1) != NULL && 47 n->in(1)->is_ShenandoahBarrier() && 48 n->in(2) != NULL && 49 n->in(2)->bottom_type() == TypePtr::NULL_PTR && 50 n->in(0) != NULL && 51 n->in(0)->in(1) != NULL && 52 n->in(0)->in(1)->is_IfProj() && 53 n->in(0)->in(2) != NULL && 54 n->in(0)->in(2)->is_IfProj() && 55 n->in(0)->in(1)->in(0) != NULL && 56 n->in(0)->in(1)->in(0) == n->in(0)->in(2)->in(0) && 57 n->in(1)->in(ValueIn)->Opcode() == Op_CastPP) { 58 Node* iff = n->in(0)->in(1)->in(0); 59 Node* res = n->in(1)->in(ValueIn)->in(1); 60 if (iff->is_If() && 61 iff->in(1) != NULL && 62 iff->in(1)->is_Bool() && 63 iff->in(1)->as_Bool()->_test._test == BoolTest::ne && 64 iff->in(1)->in(1) != NULL && 65 iff->in(1)->in(1)->Opcode() == Op_CmpP && 66 iff->in(1)->in(1)->in(1) != NULL && 67 iff->in(1)->in(1)->in(1) == res && 68 iff->in(1)->in(1)->in(2) != NULL && 69 iff->in(1)->in(1)->in(2)->bottom_type() == TypePtr::NULL_PTR) { 70 return res; 71 } 72 } 73 return n; 74 } 75 76 bool ShenandoahBarrierNode::needs_barrier(PhaseGVN* phase, ShenandoahBarrierNode* orig, Node* n, Node* rb_mem, bool allow_fromspace) { 77 Unique_Node_List visited; 78 return needs_barrier_impl(phase, orig, n, rb_mem, allow_fromspace, visited); 79 } 80 81 bool ShenandoahBarrierNode::needs_barrier_impl(PhaseGVN* phase, ShenandoahBarrierNode* orig, Node* n, Node* rb_mem, bool allow_fromspace, Unique_Node_List &visited) { 82 83 if (visited.member(n)) { 84 return false; // Been there. 85 } 86 visited.push(n); 87 88 if (n->is_Allocate()) { 89 // tty->print_cr("killed barrier for newly allocated object"); 90 return false; 91 } 92 93 if (n->is_CallJava() || n->Opcode() == Op_CallLeafNoFP) { 94 return true; 95 } 96 97 const Type* type = phase->type(n); 98 if (type == Type::TOP) { 99 return false; 100 } 101 if (type->make_ptr()->higher_equal(TypePtr::NULL_PTR)) { 102 // tty->print_cr("killed barrier for NULL object"); 103 return false; 104 } 105 if (type->make_oopptr() && type->make_oopptr()->const_oop() != NULL) { 106 // tty->print_cr("killed barrier for constant object"); 107 return ShenandoahBarriersForConst; 108 } 109 110 if (ShenandoahOptimizeStableFinals) { 111 const TypeAryPtr* ary = type->isa_aryptr(); 112 if (ary && ary->is_stable() && allow_fromspace) { 113 return false; 114 } 115 } 116 117 if (n->is_CheckCastPP() || n->is_ConstraintCast()) { 118 return needs_barrier_impl(phase, orig, n->in(1), rb_mem, allow_fromspace, visited); 119 } 120 if (n->is_Parm()) { 121 return true; 122 } 123 if (n->is_Proj()) { 124 return needs_barrier_impl(phase, orig, n->in(0), rb_mem, allow_fromspace, visited); 125 } 126 if (n->is_Phi()) { 127 bool need_barrier = false; 128 for (uint i = 1; i < n->req() && ! need_barrier; i++) { 129 Node* input = n->in(i); 130 if (input == NULL) { 131 need_barrier = true; // Phi not complete yet? 132 } else if (needs_barrier_impl(phase, orig, input, rb_mem, allow_fromspace, visited)) { 133 need_barrier = true; 134 } 135 } 136 return need_barrier; 137 } 138 if (n->is_CMove()) { 139 return needs_barrier_impl(phase, orig, n->in(CMoveNode::IfFalse), rb_mem, allow_fromspace, visited) || 140 needs_barrier_impl(phase, orig, n->in(CMoveNode::IfTrue ), rb_mem, allow_fromspace, visited); 141 } 142 if (n->Opcode() == Op_CreateEx) { 143 return true; 144 } 145 if (n->Opcode() == Op_ShenandoahWriteBarrier) { 146 // tty->print_cr("skipped barrier for chained write barrier object"); 147 return false; 148 } 149 if (n->Opcode() == Op_ShenandoahReadBarrier) { 150 if (rb_mem == n->in(Memory)) { 151 // tty->print_cr("Eliminated chained read barrier"); 152 return false; 153 } else { 154 return true; 155 } 156 } 157 158 if (n->Opcode() == Op_LoadP || 159 n->Opcode() == Op_LoadN || 160 n->Opcode() == Op_GetAndSetP || 161 n->Opcode() == Op_CompareAndExchangeP || 162 n->Opcode() == Op_GetAndSetN || 163 n->Opcode() == Op_CompareAndExchangeN) { 164 return true; 165 } 166 if (n->Opcode() == Op_DecodeN || 167 n->Opcode() == Op_EncodeP) { 168 return needs_barrier_impl(phase, orig, n->in(1), rb_mem, allow_fromspace, visited); 169 } 170 171 #ifdef ASSERT 172 tty->print("need barrier on?: "); n->dump(); 173 ShouldNotReachHere(); 174 #endif 175 return true; 176 } 177 178 /** 179 * In Shenandoah, we need barriers on acmp (and similar instructions that compare two 180 * oops) to avoid false negatives. If it compares a from-space and a to-space 181 * copy of an object, a regular acmp would return false, even though both are 182 * the same. The acmp barrier compares the two objects, and when they are 183 * *not equal* it does a read-barrier on both, and compares them again. When it 184 * failed because of different copies of the object, we know that the object 185 * must already have been evacuated (and therefore doesn't require a write-barrier). 186 */ 187 void ShenandoahBarrierNode::do_cmpp_if(GraphKit& kit, Node*& taken_branch, Node*& untaken_branch, Node*& taken_memory, Node*& untaken_memory) { 188 assert(taken_memory == NULL && untaken_memory == NULL, "unexpected memory inputs"); 189 if (!UseShenandoahGC || !ShenandoahAcmpBarrier || ShenandoahVerifyOptoBarriers) { 190 return; 191 } 192 if (taken_branch->is_top() || untaken_branch->is_top()) { 193 // one of the branches is known to be untaken 194 return; 195 } 196 assert(taken_branch->is_IfProj() && untaken_branch->is_IfProj(), "if projections only"); 197 assert(taken_branch->in(0) == untaken_branch->in(0), "should come from same if"); 198 IfNode* iff = taken_branch->in(0)->as_If(); 199 BoolNode* bol = iff->in(1)->as_Bool(); 200 Node* cmp = bol->in(1); 201 if (cmp->Opcode() != Op_CmpP) { 202 return; 203 } 204 Node* a = cmp->in(1); 205 Node* b = cmp->in(2); 206 const Type* a_type = kit.gvn().type(a); 207 const Type* b_type = kit.gvn().type(b); 208 if (a_type->higher_equal(TypePtr::NULL_PTR) || b_type->higher_equal(TypePtr::NULL_PTR)) { 209 // We know one arg is gonna be null. No need for barriers. 210 return; 211 } 212 213 const TypePtr* a_adr_type = ShenandoahBarrierNode::brooks_pointer_type(a_type); 214 const TypePtr* b_adr_type = ShenandoahBarrierNode::brooks_pointer_type(b_type); 215 if ((! ShenandoahBarrierNode::needs_barrier(&kit.gvn(), NULL, a, kit.memory(a_adr_type), false)) && 216 (! ShenandoahBarrierNode::needs_barrier(&kit.gvn(), NULL, b, kit.memory(b_adr_type), false))) { 217 // We know both args are in to-space already. No acmp barrier needed. 218 return; 219 } 220 221 Node* equal_path = iff->proj_out(true); 222 Node* not_equal_path = iff->proj_out(false); 223 224 if (bol->_test._test == BoolTest::ne) { 225 swap(equal_path, not_equal_path); 226 } 227 228 Node* init_equal_path = equal_path; 229 Node* init_not_equal_path = not_equal_path; 230 231 uint alias_a = kit.C->get_alias_index(a_adr_type); 232 uint alias_b = kit.C->get_alias_index(b_adr_type); 233 234 Node* equal_memory = NULL; 235 Node* not_equal_memory = NULL; 236 237 RegionNode* region = new RegionNode(3); 238 region->init_req(1, equal_path); 239 PhiNode* mem_phi = NULL; 240 if (alias_a == alias_b) { 241 mem_phi = PhiNode::make(region, kit.memory(alias_a), Type::MEMORY, kit.C->get_adr_type(alias_a)); 242 } else { 243 Node* mem = kit.reset_memory(); 244 mem_phi = PhiNode::make(region, mem, Type::MEMORY, TypePtr::BOTTOM); 245 kit.set_all_memory(mem); 246 } 247 248 kit.set_control(not_equal_path); 249 250 Node* mb = NULL; 251 if (alias_a == alias_b) { 252 Node* mem = kit.reset_memory(); 253 mb = MemBarNode::make(kit.C, Op_MemBarAcquire, alias_a); 254 mb->init_req(TypeFunc::Control, kit.control()); 255 mb->init_req(TypeFunc::Memory, mem); 256 Node* membar = kit.gvn().transform(mb); 257 kit.set_control(kit.gvn().transform(new ProjNode(membar, TypeFunc::Control))); 258 Node* newmem = kit.gvn().transform(new ProjNode(membar, TypeFunc::Memory)); 259 kit.set_all_memory(mem); 260 kit.set_memory(newmem, alias_a); 261 } else { 262 mb = kit.insert_mem_bar(Op_MemBarAcquire); 263 } 264 265 a = kit.shenandoah_read_barrier_acmp(a); 266 b = kit.shenandoah_read_barrier_acmp(b); 267 268 Node* cmp2 = kit.gvn().transform(new CmpPNode(a, b)); 269 Node* bol2 = bol->clone(); 270 bol2->set_req(1, cmp2); 271 bol2 = kit.gvn().transform(bol2); 272 Node* iff2 = iff->clone(); 273 iff2->set_req(0, kit.control()); 274 iff2->set_req(1, bol2); 275 kit.gvn().set_type(iff2, kit.gvn().type(iff)); 276 Node* equal_path2 = equal_path->clone(); 277 equal_path2->set_req(0, iff2); 278 equal_path2 = kit.gvn().transform(equal_path2); 279 Node* not_equal_path2 = not_equal_path->clone(); 280 not_equal_path2->set_req(0, iff2); 281 not_equal_path2 = kit.gvn().transform(not_equal_path2); 282 283 region->init_req(2, equal_path2); 284 not_equal_memory = kit.reset_memory(); 285 not_equal_path = not_equal_path2; 286 287 kit.set_all_memory(not_equal_memory); 288 289 if (alias_a == alias_b) { 290 mem_phi->init_req(2, kit.memory(alias_a)); 291 kit.set_memory(mem_phi, alias_a); 292 } else { 293 mem_phi->init_req(2, kit.reset_memory()); 294 } 295 296 kit.record_for_igvn(mem_phi); 297 kit.gvn().set_type(mem_phi, Type::MEMORY); 298 299 if (alias_a == alias_b) { 300 equal_memory = kit.reset_memory(); 301 } else { 302 equal_memory = mem_phi; 303 } 304 305 assert(kit.map()->memory() == NULL, "no live memory state"); 306 equal_path = kit.gvn().transform(region); 307 308 if (taken_branch == init_equal_path) { 309 assert(untaken_branch == init_not_equal_path, "inconsistent"); 310 taken_branch = equal_path; 311 untaken_branch = not_equal_path; 312 taken_memory = equal_memory; 313 untaken_memory = not_equal_memory; 314 } else { 315 assert(taken_branch == init_not_equal_path, "inconsistent"); 316 assert(untaken_branch == init_equal_path, "inconsistent"); 317 taken_branch = not_equal_path; 318 untaken_branch = equal_path; 319 taken_memory = not_equal_memory; 320 untaken_memory = equal_memory; 321 } 322 } 323 324 bool ShenandoahReadBarrierNode::dominates_memory_rb_impl(PhaseGVN* phase, 325 Node* b1, 326 Node* b2, 327 Node* current, 328 bool linear) { 329 ResourceMark rm; 330 VectorSet visited(Thread::current()->resource_area()); 331 Node_Stack phis(0); 332 333 for(int i = 0; i < 10; i++) { 334 if (current == NULL) { 335 return false; 336 } else if (visited.test_set(current->_idx) || current->is_top() || current == b1) { 337 current = NULL; 338 while (phis.is_nonempty() && current == NULL) { 339 uint idx = phis.index(); 340 Node* phi = phis.node(); 341 if (idx >= phi->req()) { 342 phis.pop(); 343 } else { 344 current = phi->in(idx); 345 phis.set_index(idx+1); 346 } 347 } 348 if (current == NULL) { 349 return true; 350 } 351 } else if (current == phase->C->immutable_memory()) { 352 return false; 353 } else if (current->isa_Phi()) { 354 if (!linear) { 355 return false; 356 } 357 phis.push(current, 2); 358 current = current->in(1); 359 } else if (current->Opcode() == Op_ShenandoahWriteBarrier) { 360 const Type* in_type = current->bottom_type(); 361 const Type* this_type = b2->bottom_type(); 362 if (is_independent(in_type, this_type)) { 363 current = current->in(Memory); 364 } else { 365 return false; 366 } 367 } else if (current->Opcode() == Op_ShenandoahWBMemProj) { 368 current = current->in(0); 369 } else if (current->is_Proj()) { 370 current = current->in(0); 371 } else if (current->is_Call()) { 372 return false; // TODO: Maybe improve by looking at the call's memory effects? 373 } else if (current->is_MemBar()) { 374 return false; // TODO: Do we need to stop at *any* membar? 375 } else if (current->is_MergeMem()) { 376 // if (true) return false; 377 // tty->print_cr("current == mergemem: "); current->dump(); 378 const TypePtr* adr_type = brooks_pointer_type(phase->type(b2)); 379 uint alias_idx = phase->C->get_alias_index(adr_type); 380 current = current->as_MergeMem()->memory_at(alias_idx); 381 } else { 382 // tty->print_cr("what else can we see here:"); 383 #ifdef ASSERT 384 current->dump(); 385 #endif 386 ShouldNotReachHere(); 387 return false; 388 } 389 } 390 return false; 391 } 392 393 bool ShenandoahReadBarrierNode::is_independent(Node* mem) { 394 if (mem->is_Phi() || mem->is_Proj() || mem->is_MergeMem()) { 395 return true; 396 } else if (mem->Opcode() == Op_ShenandoahWriteBarrier) { 397 const Type* mem_type = mem->bottom_type(); 398 const Type* this_type = bottom_type(); 399 if (is_independent(mem_type, this_type)) { 400 return true; 401 } else { 402 return false; 403 } 404 } else if (mem->is_Call() || mem->is_MemBar()) { 405 return false; 406 } 407 #ifdef ASSERT 408 mem->dump(); 409 #endif 410 ShouldNotReachHere(); 411 return true; 412 } 413 414 415 bool ShenandoahReadBarrierNode::dominates_memory_rb(PhaseGVN* phase, Node* b1, Node* b2, bool linear) { 416 return dominates_memory_rb_impl(phase, b1->in(Memory), b2, b2->in(Memory), linear); 417 } 418 419 bool ShenandoahReadBarrierNode::is_independent(const Type* in_type, const Type* this_type) { 420 assert(in_type->isa_oopptr(), "expect oop ptr"); 421 assert(this_type->isa_oopptr(), "expect oop ptr"); 422 /* 423 if ((! in_type->isa_oopptr()) || (! this_type->isa_oopptr())) { 424 #ifdef ASSERT 425 tty->print_cr("not oopptr"); 426 tty->print("in: "); in_type->dump(); tty->print_cr(" "); 427 tty->print("this: "); this_type->dump(); tty->print_cr(" "); 428 #endif 429 return false; 430 } 431 */ 432 433 ciKlass* in_kls = in_type->is_oopptr()->klass(); 434 ciKlass* this_kls = this_type->is_oopptr()->klass(); 435 if (in_kls != NULL && this_kls != NULL && 436 in_kls->is_loaded() && this_kls->is_loaded() && 437 (!in_kls->is_subclass_of(this_kls)) && 438 (!this_kls->is_subclass_of(in_kls))) { 439 #ifdef ASSERT 440 // tty->print_cr("independent: "); 441 // tty->print("in: "); in_kls->print(); tty->print_cr(" "); 442 // tty->print("this: "); this_kls->print(); tty->print_cr(" "); 443 #endif 444 return true; 445 } 446 #ifdef ASSERT 447 // tty->print_cr("possibly dependend?"); 448 // tty->print("in: "); in_type->dump(); tty->print_cr(" "); 449 // tty->print("this: "); this_type->dump(); tty->print_cr(" "); 450 #endif 451 return false; 452 } 453 454 Node* ShenandoahReadBarrierNode::Ideal(PhaseGVN *phase, bool can_reshape) { 455 456 if (! can_reshape) { 457 return NULL; 458 } 459 460 if (in(Memory) == phase->C->immutable_memory()) return NULL; 461 462 // If memory input is a MergeMem, take the appropriate slice out of it. 463 Node* mem_in = in(Memory); 464 if (mem_in->isa_MergeMem()) { 465 const TypePtr* adr_type = brooks_pointer_type(bottom_type()); 466 uint alias_idx = phase->C->get_alias_index(adr_type); 467 mem_in = mem_in->as_MergeMem()->memory_at(alias_idx); 468 set_req(Memory, mem_in); 469 return this; 470 } 471 472 Node* input = in(Memory); 473 if (input->Opcode() == Op_ShenandoahWBMemProj) { 474 Node* wb = input->in(0); 475 const Type* in_type = phase->type(wb); 476 // is_top() test not sufficient here: we can come here after CCP 477 // in a dead branch of the graph that has not yet been removed. 478 if (in_type == Type::TOP) return NULL; // Dead path. 479 assert(wb->Opcode() == Op_ShenandoahWriteBarrier, "expect write barrier"); 480 if (is_independent(in_type, _type)) { 481 phase->igvn_rehash_node_delayed(wb); 482 set_req(Memory, wb->in(Memory)); 483 if (can_reshape && input->outcnt() == 0) { 484 phase->is_IterGVN()->_worklist.push(input); 485 } 486 return this; 487 } 488 } 489 return NULL; 490 } 491 492 Node* ShenandoahWriteBarrierNode::Identity(PhaseGVN* phase) { 493 assert(in(0) != NULL, "should have control"); 494 PhaseIterGVN* igvn = phase->is_IterGVN(); 495 Node* mem_in = in(Memory); 496 Node* mem_proj = NULL; 497 498 if (igvn != NULL) { 499 mem_proj = find_out_with(Op_ShenandoahWBMemProj); 500 if (mem_proj == NULL || mem_in == mem_proj) { 501 return this; 502 } 503 } 504 505 Node* replacement = Identity_impl(phase); 506 if (igvn != NULL) { 507 if (replacement != NULL && replacement != this) { 508 igvn->replace_node(mem_proj, mem_in); 509 } 510 } 511 return replacement; 512 } 513 514 515 Node* ShenandoahWriteBarrierNode::Ideal(PhaseGVN *phase, bool can_reshape) { 516 assert(in(0) != NULL, "should have control"); 517 if (!can_reshape) { 518 return NULL; 519 } 520 521 PhaseIterGVN* igvn = phase->is_IterGVN(); 522 Node* mem_proj = find_out_with(Op_ShenandoahWBMemProj); 523 Node* mem_in = in(Memory); 524 525 if (mem_in == phase->C->immutable_memory()) return NULL; 526 527 if (mem_in->isa_MergeMem()) { 528 const TypePtr* adr_type = brooks_pointer_type(bottom_type()); 529 uint alias_idx = phase->C->get_alias_index(adr_type); 530 mem_in = mem_in->as_MergeMem()->memory_at(alias_idx); 531 set_req(Memory, mem_in); 532 return this; 533 } 534 535 return NULL; 536 } 537 538 bool ShenandoahWriteBarrierNode::expand(Compile* C, PhaseIterGVN& igvn, int& loop_opts_cnt) { 539 if (UseShenandoahGC && ShenandoahWriteBarrierToIR) { 540 if (C->shenandoah_barriers_count() > 0) { 541 bool attempt_more_loopopts = ShenandoahLoopOptsAfterExpansion && (C->shenandoah_barriers_count() > 1 || C->has_loops()); 542 C->clear_major_progress(); 543 PhaseIdealLoop ideal_loop(igvn, LoopOptsShenandoahExpand); 544 if (C->failing()) return false; 545 PhaseIdealLoop::verify(igvn); 546 DEBUG_ONLY(ShenandoahBarrierNode::verify_raw_mem(C->root());) 547 if (attempt_more_loopopts) { 548 C->set_major_progress(); 549 if (!C->optimize_loops(loop_opts_cnt, igvn, LoopOptsShenandoahPostExpand)) { 550 return false; 551 } 552 C->clear_major_progress(); 553 } 554 } 555 } 556 return true; 557 } 558 559 bool ShenandoahWriteBarrierNode::is_evacuation_in_progress_test(Node* iff) { 560 assert(iff->is_If(), "bad input"); 561 if (iff->Opcode() != Op_If) { 562 return false; 563 } 564 Node* bol = iff->in(1); 565 if (!bol->is_Bool() || bol->as_Bool()->_test._test != BoolTest::ne) { 566 return false; 567 } 568 Node* cmp = bol->in(1); 569 if (cmp->Opcode() != Op_CmpI) { 570 return false; 571 } 572 Node* in1 = cmp->in(1); 573 Node* in2 = cmp->in(2); 574 if (in2->find_int_con(-1) != 0) { 575 return false; 576 } 577 if (in1->Opcode() != Op_AndI) { 578 return false; 579 } 580 in2 = in1->in(2); 581 if (in2->find_int_con(-1) != ShenandoahHeap::EVACUATION) { 582 return false; 583 } 584 in1 = in1->in(1); 585 586 return is_gc_state_load(in1); 587 } 588 589 bool ShenandoahWriteBarrierNode::is_gc_state_load(Node *n) { 590 if (n->Opcode() != Op_LoadUB && n->Opcode() != Op_LoadB) { 591 return false; 592 } 593 Node* addp = n->in(MemNode::Address); 594 if (!addp->is_AddP()) { 595 return false; 596 } 597 Node* base = addp->in(AddPNode::Address); 598 Node* off = addp->in(AddPNode::Offset); 599 if (base->Opcode() != Op_ThreadLocal) { 600 return false; 601 } 602 if (off->find_intptr_t_con(-1) != in_bytes(JavaThread::gc_state_offset())) { 603 return false; 604 } 605 return true; 606 } 607 608 bool ShenandoahWriteBarrierNode::try_common_gc_state_load(Node *n, PhaseIdealLoop *phase) { 609 assert(is_gc_state_load(n), "inconsistent"); 610 Node* addp = n->in(MemNode::Address); 611 Node* dominator = NULL; 612 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) { 613 Node* u = addp->fast_out(i); 614 if (u != n && phase->is_dominator(u->in(0), n->in(0))) { 615 if (dominator == NULL) { 616 dominator = u; 617 } else { 618 if (phase->dom_depth(u->in(0)) < phase->dom_depth(dominator->in(0))) { 619 dominator = u; 620 } 621 } 622 } 623 } 624 if (dominator == NULL) { 625 return false; 626 } 627 ResourceMark rm; 628 Unique_Node_List wq; 629 wq.push(n->in(0)); 630 for (uint next = 0; next < wq.size(); next++) { 631 Node *m = wq.at(next); 632 if (m->is_SafePoint() && !m->is_CallLeaf()) { 633 return false; 634 } 635 if (m->is_Region()) { 636 for (uint i = 1; i < m->req(); i++) { 637 wq.push(m->in(i)); 638 } 639 } else { 640 wq.push(m->in(0)); 641 } 642 } 643 phase->igvn().replace_node(n, dominator); 644 645 return true; 646 } 647 648 Node* ShenandoahWriteBarrierNode::evacuation_in_progress_test_ctrl(Node* iff) { 649 assert(is_evacuation_in_progress_test(iff), "bad input"); 650 Node* c = iff; 651 if (ShenandoahWriteBarrierMemBar) { 652 do { 653 assert(c->in(0)->is_Proj() && c->in(0)->in(0)->is_MemBar(), "where's the mem bar?"); 654 c = c->in(0)->in(0); 655 } while (c->adr_type() != TypeRawPtr::BOTTOM); 656 } 657 return c->in(0); 658 } 659 660 bool ShenandoahBarrierNode::dominates_memory_impl(PhaseGVN* phase, 661 Node* b1, 662 Node* b2, 663 Node* current, 664 bool linear) { 665 ResourceMark rm; 666 VectorSet visited(Thread::current()->resource_area()); 667 Node_Stack phis(0); 668 669 670 for(int i = 0; i < 10; i++) { 671 if (current == NULL) { 672 return false; 673 } else if (visited.test_set(current->_idx) || current->is_top() || current == b1) { 674 current = NULL; 675 while (phis.is_nonempty() && current == NULL) { 676 uint idx = phis.index(); 677 Node* phi = phis.node(); 678 if (idx >= phi->req()) { 679 phis.pop(); 680 } else { 681 current = phi->in(idx); 682 phis.set_index(idx+1); 683 } 684 } 685 if (current == NULL) { 686 return true; 687 } 688 } else if (current == b2) { 689 return false; 690 } else if (current == phase->C->immutable_memory()) { 691 return false; 692 } else if (current->isa_Phi()) { 693 if (!linear) { 694 return false; 695 } 696 phis.push(current, 2); 697 current = current->in(1); 698 } else if (current->Opcode() == Op_ShenandoahWriteBarrier) { 699 current = current->in(Memory); 700 } else if (current->Opcode() == Op_ShenandoahWBMemProj) { 701 current = current->in(0); 702 } else if (current->is_Proj()) { 703 current = current->in(0); 704 } else if (current->is_Call()) { 705 current = current->in(TypeFunc::Memory); 706 } else if (current->is_MemBar()) { 707 current = current->in(TypeFunc::Memory); 708 } else if (current->is_MergeMem()) { 709 const TypePtr* adr_type = brooks_pointer_type(phase->type(b2)); 710 uint alias_idx = phase->C->get_alias_index(adr_type); 711 current = current->as_MergeMem()->memory_at(alias_idx); 712 } else { 713 #ifdef ASSERT 714 current->dump(); 715 #endif 716 ShouldNotReachHere(); 717 return false; 718 } 719 } 720 return false; 721 } 722 723 /** 724 * Determines if b1 dominates b2 through memory inputs. It returns true if: 725 * - b1 can be reached by following each branch in b2's memory input (through phis, etc) 726 * - or we get back to b2 (i.e. through a loop) without seeing b1 727 * In all other cases, (in particular, if we reach immutable_memory without having seen b1) 728 * we return false. 729 */ 730 bool ShenandoahBarrierNode::dominates_memory(PhaseGVN* phase, Node* b1, Node* b2, bool linear) { 731 return dominates_memory_impl(phase, b1, b2, b2->in(Memory), linear); 732 } 733 734 Node* ShenandoahBarrierNode::Identity_impl(PhaseGVN* phase) { 735 Node* n = in(ValueIn); 736 737 Node* rb_mem = Opcode() == Op_ShenandoahReadBarrier ? in(Memory) : NULL; 738 if (! needs_barrier(phase, this, n, rb_mem, _allow_fromspace)) { 739 return n; 740 } 741 742 // tty->print_cr("find sibling for: "); dump(2); 743 // Try to find a write barrier sibling with identical inputs that we can fold into. 744 for (DUIterator i = n->outs(); n->has_out(i); i++) { 745 Node* sibling = n->out(i); 746 if (sibling == this) { 747 continue; 748 } 749 /* 750 assert(sibling->Opcode() != Op_ShenandoahWriteBarrier || 751 Opcode() != Op_ShenandoahWriteBarrier || hash() == sibling->hash(), 752 "if this is a write barrier, then sibling can't be write barrier too"); 753 */ 754 if (sibling->Opcode() != Op_ShenandoahWriteBarrier) { 755 continue; 756 } 757 /* 758 if (sibling->outcnt() == 0) { 759 // Some dead node. 760 continue; 761 } 762 */ 763 assert(sibling->in(ValueIn) == in(ValueIn), "sanity"); 764 assert(sibling->Opcode() == Op_ShenandoahWriteBarrier, "sanity"); 765 // tty->print_cr("candidate: "); sibling->dump(); 766 767 if (dominates_memory(phase, sibling, this, phase->is_IterGVN() == NULL)) { 768 769 /* 770 tty->print_cr("matched barrier:"); 771 sibling->dump(); 772 tty->print_cr("for: "); 773 dump(); 774 */ 775 return sibling; 776 } 777 778 /* 779 tty->print_cr("couldn't match candidate:"); 780 sibling->dump(2); 781 */ 782 } 783 /* 784 tty->print_cr("couldn't match barrier to any:"); 785 dump(); 786 */ 787 return this; 788 } 789 790 #ifndef PRODUCT 791 void ShenandoahBarrierNode::dump_spec(outputStream *st) const { 792 const TypePtr* adr = adr_type(); 793 if (adr == NULL) { 794 return; 795 } 796 st->print(" @"); 797 adr->dump_on(st); 798 st->print(" ("); 799 Compile::current()->alias_type(adr)->adr_type()->dump_on(st); 800 st->print(") "); 801 } 802 #endif 803 804 Node* ShenandoahReadBarrierNode::Identity(PhaseGVN* phase) { 805 806 // if (true) return this; 807 808 // tty->print("optimizing rb: "); dump(); 809 Node* id = Identity_impl(phase); 810 811 if (id == this && phase->is_IterGVN()) { 812 Node* n = in(ValueIn); 813 // No success in super call. Try to combine identical read barriers. 814 for (DUIterator i = n->outs(); n->has_out(i); i++) { 815 Node* sibling = n->out(i); 816 if (sibling == this || sibling->Opcode() != Op_ShenandoahReadBarrier) { 817 continue; 818 } 819 assert(sibling->in(ValueIn) == in(ValueIn), "sanity"); 820 if (phase->is_IterGVN()->hash_find(sibling) && 821 sibling->bottom_type() == bottom_type() && 822 sibling->in(Control) == in(Control) && 823 dominates_memory_rb(phase, sibling, this, phase->is_IterGVN() == NULL)) { 824 /* 825 if (in(Memory) != sibling->in(Memory)) { 826 tty->print_cr("interesting rb-fold"); 827 dump(); 828 sibling->dump(); 829 } 830 */ 831 return sibling; 832 } 833 } 834 } 835 return id; 836 } 837 838 const Type* ShenandoahBarrierNode::Value(PhaseGVN* phase) const { 839 // Either input is TOP ==> the result is TOP 840 const Type *t1 = phase->type(in(Memory)); 841 if (t1 == Type::TOP) return Type::TOP; 842 const Type *t2 = phase->type(in(ValueIn)); 843 if( t2 == Type::TOP ) return Type::TOP; 844 845 Node* input = in(ValueIn); 846 const Type* type = phase->type(input)->is_oopptr()->cast_to_nonconst(); 847 return type; 848 } 849 850 uint ShenandoahBarrierNode::hash() const { 851 return TypeNode::hash() + _allow_fromspace; 852 } 853 854 uint ShenandoahBarrierNode::cmp(const Node& n) const { 855 return _allow_fromspace == ((ShenandoahBarrierNode&) n)._allow_fromspace 856 && TypeNode::cmp(n); 857 } 858 859 uint ShenandoahBarrierNode::size_of() const { 860 return sizeof(*this); 861 } 862 863 Node* ShenandoahWBMemProjNode::Identity(PhaseGVN* phase) { 864 865 Node* wb = in(0); 866 if (wb->is_top()) return phase->C->top(); // Dead path. 867 868 assert(wb->Opcode() == Op_ShenandoahWriteBarrier, "expect write barrier"); 869 PhaseIterGVN* igvn = phase->is_IterGVN(); 870 // We can't do the below unless the graph is fully constructed. 871 if (igvn == NULL) { 872 return this; 873 } 874 875 // If the mem projection has no barrier users, it's not needed anymore. 876 Unique_Node_List visited; 877 if (wb->outcnt() == 1) { 878 return wb->in(ShenandoahBarrierNode::Memory); 879 } 880 881 return this; 882 } 883 884 #ifdef ASSERT 885 bool ShenandoahBarrierNode::verify_helper(Node* in, Node_Stack& phis, VectorSet& visited, verify_type t, bool trace, Unique_Node_List& barriers_used) { 886 assert(phis.size() == 0, ""); 887 888 while (true) { 889 if (!in->bottom_type()->make_ptr()->make_oopptr()) { 890 if (trace) {tty->print_cr("Non oop");} 891 } else if (t == ShenandoahLoad && ShenandoahOptimizeStableFinals && 892 in->bottom_type()->make_ptr()->isa_aryptr() && 893 in->bottom_type()->make_ptr()->is_aryptr()->is_stable()) { 894 if (trace) {tty->print_cr("Stable array load");} 895 } else { 896 if (in->is_ConstraintCast()) { 897 in = in->in(1); 898 continue; 899 } else if (in->is_AddP()) { 900 assert(!in->in(AddPNode::Address)->is_top(), "no raw memory access"); 901 in = in->in(AddPNode::Address); 902 continue; 903 } else if (in->is_Con() && !ShenandoahBarriersForConst) { 904 if (trace) {tty->print("Found constant"); in->dump();} 905 } else if (in->is_ShenandoahBarrier()) { 906 if (t == ShenandoahStore && in->Opcode() != Op_ShenandoahWriteBarrier) { 907 return false; 908 } 909 barriers_used.push(in); 910 if (trace) {tty->print("Found barrier"); in->dump();} 911 } else if (in->is_Proj() && in->in(0)->is_Allocate()) { 912 if (trace) {tty->print("Found alloc"); in->in(0)->dump();} 913 } else if (in->is_Phi()) { 914 if (!visited.test_set(in->_idx)) { 915 if (trace) {tty->print("Pushed phi:"); in->dump();} 916 phis.push(in, 2); 917 in = in->in(1); 918 continue; 919 } 920 if (trace) {tty->print("Already seen phi:"); in->dump();} 921 } else if (in->Opcode() == Op_CMoveP || in->Opcode() == Op_CMoveN) { 922 if (!visited.test_set(in->_idx)) { 923 if (trace) {tty->print("Pushed cmovep:"); in->dump();} 924 phis.push(in, CMoveNode::IfTrue); 925 in = in->in(CMoveNode::IfFalse); 926 continue; 927 } 928 if (trace) {tty->print("Already seen cmovep:"); in->dump();} 929 } else if (in->Opcode() == Op_EncodeP || in->Opcode() == Op_DecodeN) { 930 in = in->in(1); 931 continue; 932 } else { 933 return false; 934 } 935 } 936 bool cont = false; 937 while (phis.is_nonempty()) { 938 uint idx = phis.index(); 939 Node* phi = phis.node(); 940 if (idx >= phi->req()) { 941 if (trace) {tty->print("Popped phi:"); phi->dump();} 942 phis.pop(); 943 continue; 944 } 945 if (trace) {tty->print("Next entry(%d) for phi:", idx); phi->dump();} 946 in = phi->in(idx); 947 phis.set_index(idx+1); 948 cont = true; 949 break; 950 } 951 if (!cont) { 952 break; 953 } 954 } 955 return true; 956 } 957 958 void ShenandoahBarrierNode::report_verify_failure(const char *msg, Node *n1, Node *n2) { 959 if (n1 != NULL) { 960 n1->dump(+10); 961 } 962 if (n2 != NULL) { 963 n2->dump(+10); 964 } 965 fatal("%s", msg); 966 } 967 968 void ShenandoahBarrierNode::verify(RootNode* root) { 969 ResourceMark rm; 970 Unique_Node_List wq; 971 GrowableArray<Node*> barriers; 972 Unique_Node_List barriers_used; 973 Node_Stack phis(0); 974 VectorSet visited(Thread::current()->resource_area()); 975 const bool trace = false; 976 const bool verify_no_useless_barrier = false; 977 978 wq.push(root); 979 for (uint next = 0; next < wq.size(); next++) { 980 Node *n = wq.at(next); 981 if (n->is_Load()) { 982 const bool trace = false; 983 if (trace) {tty->print("Verifying"); n->dump();} 984 if (n->Opcode() == Op_LoadRange || n->Opcode() == Op_LoadKlass || n->Opcode() == Op_LoadNKlass) { 985 if (trace) {tty->print_cr("Load range/klass");} 986 } else { 987 const TypePtr* adr_type = n->as_Load()->adr_type(); 988 989 if (adr_type->isa_oopptr() && adr_type->is_oopptr()->offset() == oopDesc::mark_offset_in_bytes()) { 990 if (trace) {tty->print_cr("Mark load");} 991 } else if (adr_type->isa_instptr() && 992 adr_type->is_instptr()->klass()->is_subtype_of(Compile::current()->env()->Reference_klass()) && 993 adr_type->is_instptr()->offset() == java_lang_ref_Reference::referent_offset) { 994 if (trace) {tty->print_cr("Reference.get()");} 995 } else { 996 bool verify = true; 997 if (adr_type->isa_instptr()) { 998 const TypeInstPtr* tinst = adr_type->is_instptr(); 999 ciKlass* k = tinst->klass(); 1000 assert(k->is_instance_klass(), ""); 1001 ciInstanceKlass* ik = (ciInstanceKlass*)k; 1002 int offset = adr_type->offset(); 1003 1004 if ((ik->debug_final_field_at(offset) && ShenandoahOptimizeInstanceFinals) || 1005 (ik->debug_stable_field_at(offset) && ShenandoahOptimizeStableFinals)) { 1006 if (trace) {tty->print_cr("Final/stable");} 1007 verify = false; 1008 } else if (k == ciEnv::current()->Class_klass() && 1009 tinst->const_oop() != NULL && 1010 tinst->offset() >= (ik->size_helper() * wordSize)) { 1011 ciInstanceKlass* k = tinst->const_oop()->as_instance()->java_lang_Class_klass()->as_instance_klass(); 1012 ciField* field = k->get_field_by_offset(tinst->offset(), true); 1013 if ((ShenandoahOptimizeStaticFinals && field->is_final()) || 1014 (ShenandoahOptimizeStableFinals && field->is_stable())) { 1015 verify = false; 1016 } 1017 } 1018 } 1019 1020 if (verify && !ShenandoahBarrierNode::verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahLoad, trace, barriers_used)) { 1021 report_verify_failure("Shenandoah verification: Load should have barriers", n); 1022 } 1023 } 1024 } 1025 } else if (n->is_Store()) { 1026 const bool trace = false; 1027 1028 if (trace) {tty->print("Verifying"); n->dump();} 1029 if (n->in(MemNode::ValueIn)->bottom_type()->make_oopptr()) { 1030 Node* adr = n->in(MemNode::Address); 1031 bool verify = true; 1032 1033 if (adr->is_AddP() && adr->in(AddPNode::Base)->is_top()) { 1034 adr = adr->in(AddPNode::Address); 1035 if (adr->is_AddP()) { 1036 assert(adr->in(AddPNode::Base)->is_top(), ""); 1037 adr = adr->in(AddPNode::Address); 1038 if (adr->Opcode() == Op_LoadP && 1039 adr->in(MemNode::Address)->in(AddPNode::Base)->is_top() && 1040 adr->in(MemNode::Address)->in(AddPNode::Address)->Opcode() == Op_ThreadLocal && 1041 adr->in(MemNode::Address)->in(AddPNode::Offset)->find_intptr_t_con(-1) == in_bytes(JavaThread::satb_mark_queue_offset() + 1042 SATBMarkQueue::byte_offset_of_buf())) { 1043 if (trace) {tty->print_cr("G1 prebarrier");} 1044 verify = false; 1045 } 1046 } 1047 } 1048 1049 if (verify && !ShenandoahBarrierNode::verify_helper(n->in(MemNode::ValueIn), phis, visited, ShenandoahValue, trace, barriers_used)) { 1050 report_verify_failure("Shenandoah verification: Store should have barriers", n); 1051 } 1052 } 1053 if (!ShenandoahBarrierNode::verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahStore, trace, barriers_used)) { 1054 report_verify_failure("Shenandoah verification: Store (address) should have barriers", n); 1055 } 1056 } else if (n->Opcode() == Op_CmpP) { 1057 const bool trace = false; 1058 1059 Node* in1 = n->in(1); 1060 Node* in2 = n->in(2); 1061 if (in1->bottom_type()->isa_oopptr()) { 1062 if (trace) {tty->print("Verifying"); n->dump();} 1063 1064 bool mark_inputs = false; 1065 if (in1->bottom_type() == TypePtr::NULL_PTR || in2->bottom_type() == TypePtr::NULL_PTR || 1066 ((in1->is_Con() || in2->is_Con()) && !ShenandoahBarriersForConst)) { 1067 if (trace) {tty->print_cr("Comparison against a constant");} 1068 mark_inputs = true; 1069 } else if ((in1->is_CheckCastPP() && in1->in(1)->is_Proj() && in1->in(1)->in(0)->is_Allocate()) || 1070 (in2->is_CheckCastPP() && in2->in(1)->is_Proj() && in2->in(1)->in(0)->is_Allocate())) { 1071 if (trace) {tty->print_cr("Comparison with newly alloc'ed object");} 1072 mark_inputs = true; 1073 } else { 1074 assert(in2->bottom_type()->isa_oopptr(), ""); 1075 1076 if (!ShenandoahBarrierNode::verify_helper(in1, phis, visited, ShenandoahStore, trace, barriers_used) || 1077 !ShenandoahBarrierNode::verify_helper(in2, phis, visited, ShenandoahStore, trace, barriers_used)) { 1078 report_verify_failure("Shenandoah verification: Cmp should have barriers", n); 1079 } 1080 } 1081 if (verify_no_useless_barrier && 1082 mark_inputs && 1083 (!ShenandoahBarrierNode::verify_helper(in1, phis, visited, ShenandoahValue, trace, barriers_used) || 1084 !ShenandoahBarrierNode::verify_helper(in2, phis, visited, ShenandoahValue, trace, barriers_used))) { 1085 phis.clear(); 1086 visited.Reset(); 1087 } 1088 } 1089 } else if (n->is_LoadStore()) { 1090 if (n->in(MemNode::ValueIn)->bottom_type()->isa_ptr() && 1091 !ShenandoahBarrierNode::verify_helper(n->in(MemNode::ValueIn), phis, visited, ShenandoahLoad, trace, barriers_used)) { 1092 report_verify_failure("Shenandoah verification: LoadStore (value) should have barriers", n); 1093 } 1094 1095 if (n->in(MemNode::Address)->bottom_type()->isa_oopptr() && !ShenandoahBarrierNode::verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahStore, trace, barriers_used)) { 1096 report_verify_failure("Shenandoah verification: LoadStore (address) should have barriers", n); 1097 } 1098 } else if (n->Opcode() == Op_CallLeafNoFP || n->Opcode() == Op_CallLeaf) { 1099 CallNode* call = n->as_Call(); 1100 1101 static struct { 1102 const char* name; 1103 struct { 1104 int pos; 1105 verify_type t; 1106 } args[6]; 1107 } calls[] = { 1108 "aescrypt_encryptBlock", 1109 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad }, 1110 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1111 "aescrypt_decryptBlock", 1112 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad }, 1113 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1114 "multiplyToLen", 1115 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad }, { TypeFunc::Parms+4, ShenandoahStore }, 1116 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1117 "squareToLen", 1118 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad }, { -1, ShenandoahNone}, 1119 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1120 "montgomery_multiply", 1121 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad }, 1122 { TypeFunc::Parms+6, ShenandoahStore }, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1123 "montgomery_square", 1124 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+5, ShenandoahStore }, 1125 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1126 "mulAdd", 1127 { { TypeFunc::Parms, ShenandoahStore }, { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone}, 1128 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1129 "vectorizedMismatch", 1130 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone}, 1131 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1132 "updateBytesCRC32", 1133 { { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone}, 1134 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1135 "updateBytesAdler32", 1136 { { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone}, 1137 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1138 "updateBytesCRC32C", 1139 { { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahLoad}, { -1, ShenandoahNone}, 1140 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1141 "counterMode_AESCrypt", 1142 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad }, 1143 { TypeFunc::Parms+3, ShenandoahStore }, { TypeFunc::Parms+5, ShenandoahStore }, { TypeFunc::Parms+6, ShenandoahStore } }, 1144 "cipherBlockChaining_encryptAESCrypt", 1145 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad }, 1146 { TypeFunc::Parms+3, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1147 "cipherBlockChaining_decryptAESCrypt", 1148 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad }, 1149 { TypeFunc::Parms+3, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1150 "shenandoah_clone_barrier", 1151 { { TypeFunc::Parms, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone}, 1152 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1153 "ghash_processBlocks", 1154 { { TypeFunc::Parms, ShenandoahStore }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad }, 1155 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1156 "sha1_implCompress", 1157 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone }, 1158 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1159 "sha256_implCompress", 1160 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone }, 1161 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1162 "sha512_implCompress", 1163 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone }, 1164 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1165 "sha1_implCompressMB", 1166 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone }, 1167 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1168 "sha256_implCompressMB", 1169 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone }, 1170 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1171 "sha512_implCompressMB", 1172 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone }, 1173 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1174 }; 1175 1176 if (call->is_call_to_arraycopystub()) { 1177 Node* dest = NULL; 1178 const TypeTuple* args = n->as_Call()->_tf->domain(); 1179 for (uint i = TypeFunc::Parms, j = 0; i < args->cnt(); i++) { 1180 if (args->field_at(i)->isa_ptr()) { 1181 j++; 1182 if (j == 2) { 1183 dest = n->in(i); 1184 break; 1185 } 1186 } 1187 } 1188 if (!ShenandoahBarrierNode::verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahLoad, trace, barriers_used) || 1189 !ShenandoahBarrierNode::verify_helper(dest, phis, visited, ShenandoahStore, trace, barriers_used)) { 1190 report_verify_failure("Shenandoah verification: ArrayCopy should have barriers", n); 1191 } 1192 } else if (strlen(call->_name) > 5 && 1193 !strcmp(call->_name + strlen(call->_name) - 5, "_fill")) { 1194 if (!ShenandoahBarrierNode::verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahStore, trace, barriers_used)) { 1195 report_verify_failure("Shenandoah verification: _fill should have barriers", n); 1196 } 1197 } else if (!strcmp(call->_name, "g1_wb_pre")) { 1198 // skip 1199 } else { 1200 const int calls_len = sizeof(calls) / sizeof(calls[0]); 1201 int i = 0; 1202 for (; i < calls_len; i++) { 1203 if (!strcmp(calls[i].name, call->_name)) { 1204 break; 1205 } 1206 } 1207 if (i != calls_len) { 1208 const uint args_len = sizeof(calls[0].args) / sizeof(calls[0].args[0]); 1209 for (uint j = 0; j < args_len; j++) { 1210 int pos = calls[i].args[j].pos; 1211 if (pos == -1) { 1212 break; 1213 } 1214 if (!ShenandoahBarrierNode::verify_helper(call->in(pos), phis, visited, calls[i].args[j].t, trace, barriers_used)) { 1215 report_verify_failure("Shenandoah verification: intrinsic calls should have barriers", n); 1216 } 1217 } 1218 for (uint j = TypeFunc::Parms; j < call->req(); j++) { 1219 if (call->in(j)->bottom_type()->make_ptr() && 1220 call->in(j)->bottom_type()->make_ptr()->isa_oopptr()) { 1221 uint k = 0; 1222 for (; k < args_len && calls[i].args[k].pos != (int)j; k++); 1223 if (k == args_len) { 1224 fatal("arg %d for call %s not covered", j, call->_name); 1225 } 1226 } 1227 } 1228 } else { 1229 for (uint j = TypeFunc::Parms; j < call->req(); j++) { 1230 if (call->in(j)->bottom_type()->make_ptr() && 1231 call->in(j)->bottom_type()->make_ptr()->isa_oopptr()) { 1232 fatal("%s not covered", call->_name); 1233 } 1234 } 1235 } 1236 } 1237 } else if (n->is_ShenandoahBarrier()) { 1238 assert(!barriers.contains(n), ""); 1239 assert(n->Opcode() != Op_ShenandoahWriteBarrier || n->find_out_with(Op_ShenandoahWBMemProj) != NULL, "bad shenandoah write barrier"); 1240 assert(n->Opcode() != Op_ShenandoahWriteBarrier || n->outcnt() > 1, "bad shenandoah write barrier"); 1241 barriers.push(n); 1242 } else if (n->is_AddP() 1243 || n->is_Phi() 1244 || n->is_ConstraintCast() 1245 || n->Opcode() == Op_Return 1246 || n->Opcode() == Op_CMoveP 1247 || n->Opcode() == Op_CMoveN 1248 || n->Opcode() == Op_Rethrow 1249 || n->is_MemBar() 1250 || n->Opcode() == Op_Conv2B 1251 || n->Opcode() == Op_SafePoint 1252 || n->is_CallJava() 1253 || n->Opcode() == Op_Unlock 1254 || n->Opcode() == Op_EncodeP 1255 || n->Opcode() == Op_DecodeN) { 1256 // nothing to do 1257 } else { 1258 static struct { 1259 int opcode; 1260 struct { 1261 int pos; 1262 verify_type t; 1263 } inputs[2]; 1264 } others[] = { 1265 Op_FastLock, 1266 { { 1, ShenandoahLoad }, { -1, ShenandoahNone} }, 1267 Op_Lock, 1268 { { TypeFunc::Parms, ShenandoahLoad }, { -1, ShenandoahNone} }, 1269 Op_ArrayCopy, 1270 { { ArrayCopyNode::Src, ShenandoahLoad }, { ArrayCopyNode::Dest, ShenandoahStore } }, 1271 Op_StrCompressedCopy, 1272 { { 2, ShenandoahLoad }, { 3, ShenandoahStore } }, 1273 Op_StrInflatedCopy, 1274 { { 2, ShenandoahLoad }, { 3, ShenandoahStore } }, 1275 Op_AryEq, 1276 { { 2, ShenandoahLoad }, { 3, ShenandoahLoad } }, 1277 Op_StrIndexOf, 1278 { { 2, ShenandoahLoad }, { 4, ShenandoahLoad } }, 1279 Op_StrComp, 1280 { { 2, ShenandoahLoad }, { 4, ShenandoahLoad } }, 1281 Op_StrEquals, 1282 { { 2, ShenandoahLoad }, { 3, ShenandoahLoad } }, 1283 Op_EncodeISOArray, 1284 { { 2, ShenandoahLoad }, { 3, ShenandoahStore } }, 1285 Op_HasNegatives, 1286 { { 2, ShenandoahLoad }, { -1, ShenandoahNone} }, 1287 Op_CastP2X, 1288 { { 1, ShenandoahLoad }, { -1, ShenandoahNone} }, 1289 Op_StrIndexOfChar, 1290 { { 2, ShenandoahLoad }, { -1, ShenandoahNone } }, 1291 }; 1292 1293 const int others_len = sizeof(others) / sizeof(others[0]); 1294 int i = 0; 1295 for (; i < others_len; i++) { 1296 if (others[i].opcode == n->Opcode()) { 1297 break; 1298 } 1299 } 1300 uint stop = n->is_Call() ? n->as_Call()->tf()->domain()->cnt() : n->req(); 1301 if (i != others_len) { 1302 const uint inputs_len = sizeof(others[0].inputs) / sizeof(others[0].inputs[0]); 1303 for (uint j = 0; j < inputs_len; j++) { 1304 int pos = others[i].inputs[j].pos; 1305 if (pos == -1) { 1306 break; 1307 } 1308 if (!ShenandoahBarrierNode::verify_helper(n->in(pos), phis, visited, others[i].inputs[j].t, trace, barriers_used)) { 1309 report_verify_failure("Shenandoah verification: intrinsic calls should have barriers", n); 1310 } 1311 } 1312 for (uint j = 1; j < stop; j++) { 1313 if (n->in(j) != NULL && n->in(j)->bottom_type()->make_ptr() && 1314 n->in(j)->bottom_type()->make_ptr()->make_oopptr()) { 1315 uint k = 0; 1316 for (; k < inputs_len && others[i].inputs[k].pos != (int)j; k++); 1317 if (k == inputs_len) { 1318 fatal("arg %d for node %s not covered", j, n->Name()); 1319 } 1320 } 1321 } 1322 } else { 1323 for (uint j = 1; j < stop; j++) { 1324 if (n->in(j) != NULL && n->in(j)->bottom_type()->make_ptr() && 1325 n->in(j)->bottom_type()->make_ptr()->make_oopptr()) { 1326 fatal("%s not covered", n->Name()); 1327 } 1328 } 1329 } 1330 } 1331 1332 if (n->is_SafePoint()) { 1333 SafePointNode* sfpt = n->as_SafePoint(); 1334 if (verify_no_useless_barrier && sfpt->jvms() != NULL) { 1335 for (uint i = sfpt->jvms()->scloff(); i < sfpt->jvms()->endoff(); i++) { 1336 if (!ShenandoahBarrierNode::verify_helper(sfpt->in(i), phis, visited, ShenandoahLoad, trace, barriers_used)) { 1337 phis.clear(); 1338 visited.Reset(); 1339 } 1340 } 1341 } 1342 } 1343 for( uint i = 0; i < n->len(); ++i ) { 1344 Node *m = n->in(i); 1345 if (m == NULL) continue; 1346 1347 // In most cases, inputs should be known to be non null. If it's 1348 // not the case, it could be a missing cast_not_null() in an 1349 // intrinsic or support might be needed in AddPNode::Ideal() to 1350 // avoid a NULL+offset input. 1351 if (!(n->is_Phi() || 1352 (n->is_SafePoint() && (!n->is_CallRuntime() || !strcmp(n->as_Call()->_name, "g1_wb_pre") || !strcmp(n->as_Call()->_name, "unsafe_arraycopy"))) || 1353 n->Opcode() == Op_CmpP || 1354 n->Opcode() == Op_CmpN || 1355 (n->Opcode() == Op_StoreP && i == StoreNode::ValueIn) || 1356 (n->Opcode() == Op_StoreN && i == StoreNode::ValueIn) || 1357 n->is_ConstraintCast() || 1358 n->Opcode() == Op_Return || 1359 n->Opcode() == Op_Conv2B || 1360 n->is_AddP() || 1361 n->Opcode() == Op_CMoveP || 1362 n->Opcode() == Op_CMoveN || 1363 n->Opcode() == Op_Rethrow || 1364 n->is_MemBar() || 1365 n->is_Mem() || 1366 n->Opcode() == Op_AryEq || 1367 n->Opcode() == Op_SCMemProj || 1368 n->Opcode() == Op_EncodeP || 1369 n->Opcode() == Op_DecodeN)) { 1370 if (m->bottom_type()->make_oopptr() && m->bottom_type()->make_oopptr()->meet(TypePtr::NULL_PTR) == m->bottom_type()) { 1371 report_verify_failure("Shenandoah verification: null input", n, m); 1372 } 1373 } 1374 1375 wq.push(m); 1376 } 1377 } 1378 1379 if (verify_no_useless_barrier) { 1380 for (int i = 0; i < barriers.length(); i++) { 1381 Node* n = barriers.at(i); 1382 if (!barriers_used.member(n)) { 1383 tty->print("XXX useless barrier"); n->dump(-2); 1384 ShouldNotReachHere(); 1385 } 1386 } 1387 } 1388 } 1389 #endif 1390 1391 MergeMemNode* ShenandoahWriteBarrierNode::allocate_merge_mem(Node* mem, int alias, Node* rep_proj, Node* rep_ctrl, PhaseIdealLoop* phase) { 1392 MergeMemNode* mm = MergeMemNode::make(mem); 1393 mm->set_memory_at(alias, rep_proj); 1394 phase->register_new_node(mm, rep_ctrl); 1395 return mm; 1396 } 1397 1398 MergeMemNode* ShenandoahWriteBarrierNode::clone_merge_mem(Node* u, Node* mem, int alias, Node* rep_proj, Node* rep_ctrl, DUIterator& i, PhaseIdealLoop* phase) { 1399 MergeMemNode* newmm = NULL; 1400 MergeMemNode* u_mm = u->as_MergeMem(); 1401 Node* c = phase->get_ctrl(u); 1402 if (phase->is_dominator(c, rep_ctrl)) { 1403 c = rep_ctrl; 1404 } else { 1405 assert(phase->is_dominator(rep_ctrl, c), "one must dominate the other"); 1406 } 1407 if (u->outcnt() == 1) { 1408 if (u->req() > (uint)alias && u->in(alias) == mem) { 1409 phase->igvn().replace_input_of(u, alias, rep_proj); 1410 --i; 1411 } else { 1412 phase->igvn().rehash_node_delayed(u); 1413 u_mm->set_memory_at(alias, rep_proj); 1414 } 1415 newmm = u_mm; 1416 phase->set_ctrl_and_loop(u, c); 1417 } else { 1418 // can't simply clone u and then change one of its input because 1419 // it adds and then removes an edge which messes with the 1420 // DUIterator 1421 newmm = MergeMemNode::make(u_mm->base_memory()); 1422 for (uint j = 0; j < u->req(); j++) { 1423 if (j < newmm->req()) { 1424 if (j == (uint)alias) { 1425 newmm->set_req(j, rep_proj); 1426 } else if (newmm->in(j) != u->in(j)) { 1427 newmm->set_req(j, u->in(j)); 1428 } 1429 } else if (j == (uint)alias) { 1430 newmm->add_req(rep_proj); 1431 } else { 1432 newmm->add_req(u->in(j)); 1433 } 1434 } 1435 if ((uint)alias >= u->req()) { 1436 newmm->set_memory_at(alias, rep_proj); 1437 } 1438 phase->register_new_node(newmm, c); 1439 } 1440 return newmm; 1441 } 1442 1443 bool ShenandoahWriteBarrierNode::should_process_phi(Node* phi, int alias, Compile* C) { 1444 if (phi->adr_type() == TypePtr::BOTTOM) { 1445 Node* region = phi->in(0); 1446 for (DUIterator_Fast jmax, j = region->fast_outs(jmax); j < jmax; j++) { 1447 Node* uu = region->fast_out(j); 1448 if (uu->is_Phi() && uu != phi && uu->bottom_type() == Type::MEMORY && C->get_alias_index(uu->adr_type()) == alias) { 1449 return false; 1450 } 1451 } 1452 return true; 1453 } 1454 return C->get_alias_index(phi->adr_type()) == alias; 1455 } 1456 1457 bool ShenandoahBarrierNode::is_dominator_same_ctrl(Node*c, Node* d, Node* n, PhaseIdealLoop* phase) { 1458 // That both nodes have the same control is not sufficient to prove 1459 // domination, verify that there's no path from d to n 1460 ResourceMark rm; 1461 Unique_Node_List wq; 1462 wq.push(d); 1463 for (uint next = 0; next < wq.size(); next++) { 1464 Node *m = wq.at(next); 1465 if (m == n) { 1466 return false; 1467 } 1468 if (m->is_Phi() && m->in(0)->is_Loop()) { 1469 assert(phase->ctrl_or_self(m->in(LoopNode::EntryControl)) != c, "following loop entry should lead to new control"); 1470 } else { 1471 for (uint i = 0; i < m->req(); i++) { 1472 if (m->in(i) != NULL && phase->ctrl_or_self(m->in(i)) == c) { 1473 wq.push(m->in(i)); 1474 } 1475 } 1476 } 1477 } 1478 return true; 1479 } 1480 1481 bool ShenandoahBarrierNode::is_dominator(Node *d_c, Node *n_c, Node* d, Node* n, PhaseIdealLoop* phase) { 1482 if (d_c != n_c) { 1483 return phase->is_dominator(d_c, n_c); 1484 } 1485 return is_dominator_same_ctrl(d_c, d, n, phase); 1486 } 1487 1488 Node* next_mem(Node* mem, int alias) { 1489 Node* res = NULL; 1490 if (mem->is_Proj()) { 1491 res = mem->in(0); 1492 } else if (mem->is_SafePoint() || mem->is_MemBar()) { 1493 res = mem->in(TypeFunc::Memory); 1494 } else if (mem->is_Phi()) { 1495 res = mem->in(1); 1496 } else if (mem->is_ShenandoahBarrier()) { 1497 res = mem->in(ShenandoahBarrierNode::Memory); 1498 } else if (mem->is_MergeMem()) { 1499 res = mem->as_MergeMem()->memory_at(alias); 1500 } else if (mem->is_Store() || mem->is_LoadStore() || mem->is_ClearArray()) { 1501 assert(alias = Compile::AliasIdxRaw, "following raw memory can't lead to a barrier"); 1502 res = mem->in(MemNode::Memory); 1503 } else { 1504 #ifdef ASSERT 1505 mem->dump(); 1506 #endif 1507 ShouldNotReachHere(); 1508 } 1509 return res; 1510 } 1511 1512 bool suitable_mem(Node* mem, Node* old_mem, Node* rep_proj) { 1513 for (DUIterator_Fast imax, i = mem->fast_outs(imax); i < imax; i++) { 1514 Node* u = mem->fast_out(i); 1515 if (u->is_MergeMem()) { 1516 if (u->has_out_with(Op_MergeMem)) { 1517 // too complicated for now 1518 return false; 1519 } 1520 if (old_mem == u && rep_proj->has_out_with(Op_MergeMem)) { 1521 return false; 1522 } 1523 } 1524 if (u->Opcode() == Op_Unlock && mem->is_Proj() && mem->in(0)->Opcode() == Op_MemBarReleaseLock) { 1525 // would require a merge mem between unlock and the 1526 // preceding membar. Would confuse logic that eliminates 1527 // lock/unlock nodes. 1528 return false; 1529 } 1530 } 1531 return true; 1532 } 1533 1534 void ShenandoahWriteBarrierNode::fix_memory_uses(Node* mem, Node* replacement, Node* rep_proj, Node* rep_ctrl, int alias, PhaseIdealLoop* phase) { 1535 uint last =phase-> C->unique(); 1536 MergeMemNode* mm = NULL; 1537 assert(mem->bottom_type() == Type::MEMORY, ""); 1538 for (DUIterator i = mem->outs(); mem->has_out(i); i++) { 1539 Node* u = mem->out(i); 1540 if (u != replacement && u->_idx < last) { 1541 if (u->is_ShenandoahBarrier() && alias != Compile::AliasIdxRaw) { 1542 if (phase->C->get_alias_index(u->adr_type()) == alias && is_dominator(rep_ctrl, phase->ctrl_or_self(u), replacement, u, phase)) { 1543 phase->igvn().replace_input_of(u, u->find_edge(mem), rep_proj); 1544 assert(u->find_edge(mem) == -1, "only one edge"); 1545 --i; 1546 } 1547 } else if (u->is_Mem()) { 1548 if (phase->C->get_alias_index(u->adr_type()) == alias && is_dominator(rep_ctrl, phase->ctrl_or_self(u), replacement, u, phase)) { 1549 assert(alias == Compile::AliasIdxRaw , "only raw memory can lead to a memory operation"); 1550 phase->igvn().replace_input_of(u, u->find_edge(mem), rep_proj); 1551 assert(u->find_edge(mem) == -1, "only one edge"); 1552 --i; 1553 } 1554 } else if (u->is_MergeMem()) { 1555 MergeMemNode* u_mm = u->as_MergeMem(); 1556 if (u_mm->memory_at(alias) == mem) { 1557 MergeMemNode* newmm = NULL; 1558 for (DUIterator_Fast jmax, j = u->fast_outs(jmax); j < jmax; j++) { 1559 Node* uu = u->fast_out(j); 1560 assert(!uu->is_MergeMem(), "chain of MergeMems?"); 1561 if (uu->is_Phi()) { 1562 if (should_process_phi(uu, alias, phase->C)) { 1563 Node* region = uu->in(0); 1564 int nb = 0; 1565 for (uint k = 1; k < uu->req(); k++) { 1566 if (uu->in(k) == u && phase->is_dominator(rep_ctrl, region->in(k))) { 1567 if (newmm == NULL) { 1568 newmm = clone_merge_mem(u, mem, alias, rep_proj, rep_ctrl, i, phase); 1569 } 1570 if (newmm != u) { 1571 phase->igvn().replace_input_of(uu, k, newmm); 1572 nb++; 1573 --jmax; 1574 } 1575 } 1576 } 1577 if (nb > 0) { 1578 --j; 1579 } 1580 } 1581 } else { 1582 if (rep_ctrl != uu && is_dominator(rep_ctrl, phase->ctrl_or_self(uu), replacement, uu, phase)) { 1583 if (newmm == NULL) { 1584 newmm = clone_merge_mem(u, mem, alias, rep_proj, rep_ctrl, i, phase); 1585 } 1586 if (newmm != u) { 1587 phase->igvn().replace_input_of(uu, uu->find_edge(u), newmm); 1588 --j, --jmax; 1589 } 1590 } 1591 } 1592 } 1593 } 1594 } else if (u->is_Phi()) { 1595 assert(u->bottom_type() == Type::MEMORY, "what else?"); 1596 Node* region = u->in(0); 1597 if (should_process_phi(u, alias, phase->C)) { 1598 bool replaced = false; 1599 for (uint j = 1; j < u->req(); j++) { 1600 if (u->in(j) == mem && phase->is_dominator(rep_ctrl, region->in(j))) { 1601 Node* nnew = rep_proj; 1602 if (u->adr_type() == TypePtr::BOTTOM) { 1603 if (mm == NULL) { 1604 mm = allocate_merge_mem(mem, alias, rep_proj, rep_ctrl, phase); 1605 } 1606 nnew = mm; 1607 } 1608 phase->igvn().replace_input_of(u, j, nnew); 1609 replaced = true; 1610 } 1611 } 1612 if (replaced) { 1613 --i; 1614 } 1615 1616 } 1617 } else if ((u->adr_type() == TypePtr::BOTTOM && u->Opcode() != Op_StrInflatedCopy) || 1618 u->adr_type() == NULL) { 1619 assert(u->adr_type() != NULL || 1620 u->Opcode() == Op_Rethrow || 1621 u->Opcode() == Op_Return || 1622 u->Opcode() == Op_SafePoint || 1623 (u->is_CallStaticJava() && u->as_CallStaticJava()->uncommon_trap_request() != 0) || 1624 (u->is_CallStaticJava() && u->as_CallStaticJava()->_entry_point == OptoRuntime::rethrow_stub()) || 1625 u->Opcode() == Op_CallLeaf, ""); 1626 if (is_dominator(rep_ctrl, phase->ctrl_or_self(u), replacement, u, phase)) { 1627 if (mm == NULL) { 1628 mm = allocate_merge_mem(mem, alias, rep_proj, rep_ctrl, phase); 1629 } 1630 phase->igvn().replace_input_of(u, u->find_edge(mem), mm); 1631 --i; 1632 } 1633 } else if (phase->C->get_alias_index(u->adr_type()) == alias) { 1634 if (is_dominator(rep_ctrl, phase->ctrl_or_self(u), replacement, u, phase)) { 1635 phase->igvn().replace_input_of(u, u->find_edge(mem), rep_proj); 1636 --i; 1637 } 1638 } 1639 } 1640 } 1641 } 1642 1643 Node* ShenandoahBarrierNode::no_branches(Node* c, Node* dom, bool allow_one_proj, PhaseIdealLoop* phase) { 1644 Node* iffproj = NULL; 1645 while (c != dom) { 1646 Node* next = phase->idom(c); 1647 assert(next->unique_ctrl_out() == c || c->is_Proj() || c->is_Region(), "multiple control flow out but no proj or region?"); 1648 if (c->is_Region()) { 1649 ResourceMark rm; 1650 Unique_Node_List wq; 1651 wq.push(c); 1652 for (uint i = 0; i < wq.size(); i++) { 1653 Node *n = wq.at(i); 1654 if (n->is_Region()) { 1655 for (uint j = 1; j < n->req(); j++) { 1656 if (n->in(j) != next) { 1657 wq.push(n->in(j)); 1658 } 1659 } 1660 } else { 1661 if (n->in(0) != next) { 1662 wq.push(n->in(0)); 1663 } 1664 } 1665 } 1666 for (DUIterator_Fast imax, i = next->fast_outs(imax); i < imax; i++) { 1667 Node* u = next->fast_out(i); 1668 if (u->is_CFG()) { 1669 if (!wq.member(u)) { 1670 return NodeSentinel; 1671 } 1672 } 1673 } 1674 1675 } else if (c->is_Proj()) { 1676 if (c->is_IfProj()) { 1677 if (c->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none) != NULL) { 1678 // continue; 1679 } else { 1680 if (!allow_one_proj) { 1681 return NodeSentinel; 1682 } 1683 if (iffproj == NULL) { 1684 iffproj = c; 1685 } else { 1686 return NodeSentinel; 1687 } 1688 } 1689 } else if (c->Opcode() == Op_JumpProj) { 1690 return NodeSentinel; // unsupported 1691 } else if (c->Opcode() == Op_CatchProj) { 1692 return NodeSentinel; // unsupported 1693 } else if (c->Opcode() == Op_CProj && next->Opcode() == Op_NeverBranch) { 1694 return NodeSentinel; // unsupported 1695 } else { 1696 assert(next->unique_ctrl_out() == c, "unsupported branch pattern"); 1697 } 1698 } 1699 c = next; 1700 } 1701 return iffproj; 1702 } 1703 1704 #ifdef ASSERT 1705 void ShenandoahWriteBarrierNode::memory_dominates_all_paths_helper(Node* c, Node* rep_ctrl, Unique_Node_List& controls, PhaseIdealLoop* phase) { 1706 const bool trace = false; 1707 if (trace) { tty->print("X control is"); c->dump(); } 1708 1709 uint start = controls.size(); 1710 controls.push(c); 1711 for (uint i = start; i < controls.size(); i++) { 1712 Node *n = controls.at(i); 1713 1714 if (trace) { tty->print("X from"); n->dump(); } 1715 1716 if (n == rep_ctrl) { 1717 continue; 1718 } 1719 1720 if (n->is_Proj()) { 1721 Node* n_dom = n->in(0); 1722 IdealLoopTree* n_dom_loop = phase->get_loop(n_dom); 1723 if (n->is_IfProj() && n_dom->outcnt() == 2) { 1724 n_dom_loop = phase->get_loop(n_dom->as_If()->proj_out(n->as_Proj()->_con == 0 ? 1 : 0)); 1725 } 1726 if (n_dom_loop != phase->ltree_root()) { 1727 Node* tail = n_dom_loop->tail(); 1728 if (tail->is_Region()) { 1729 for (uint j = 1; j < tail->req(); j++) { 1730 if (phase->is_dominator(n_dom, tail->in(j)) && !phase->is_dominator(n, tail->in(j))) { 1731 assert(phase->is_dominator(rep_ctrl, tail->in(j)), "why are we here?"); 1732 // entering loop from below, mark backedge 1733 if (trace) { tty->print("X pushing backedge"); tail->in(j)->dump(); } 1734 controls.push(tail->in(j)); 1735 //assert(n->in(0) == n_dom, "strange flow control"); 1736 } 1737 } 1738 } else if (phase->get_loop(n) != n_dom_loop && phase->is_dominator(n_dom, tail)) { 1739 // entering loop from below, mark backedge 1740 if (trace) { tty->print("X pushing backedge"); tail->dump(); } 1741 controls.push(tail); 1742 //assert(n->in(0) == n_dom, "strange flow control"); 1743 } 1744 } 1745 } 1746 1747 if (n->is_Loop()) { 1748 Node* c = n->in(LoopNode::EntryControl); 1749 if (trace) { tty->print("X pushing"); c->dump(); } 1750 controls.push(c); 1751 } else if (n->is_Region()) { 1752 for (uint i = 1; i < n->req(); i++) { 1753 Node* c = n->in(i); 1754 if (trace) { tty->print("X pushing"); c->dump(); } 1755 controls.push(c); 1756 } 1757 } else { 1758 Node* c = n->in(0); 1759 if (trace) { tty->print("X pushing"); c->dump(); } 1760 controls.push(c); 1761 } 1762 } 1763 } 1764 1765 bool ShenandoahWriteBarrierNode::memory_dominates_all_paths(Node* mem, Node* rep_ctrl, int alias, PhaseIdealLoop* phase) { 1766 const bool trace = false; 1767 if (trace) { 1768 tty->print("XXX mem is"); mem->dump(); 1769 tty->print("XXX rep ctrl is"); rep_ctrl->dump(); 1770 tty->print_cr("XXX alias is %d", alias); 1771 } 1772 ResourceMark rm; 1773 Unique_Node_List wq; 1774 Unique_Node_List controls; 1775 wq.push(mem); 1776 for (uint next = 0; next < wq.size(); next++) { 1777 Node *nn = wq.at(next); 1778 if (trace) { tty->print("XX from mem"); nn->dump(); } 1779 assert(nn->bottom_type() == Type::MEMORY, "memory only"); 1780 1781 if (nn->is_Phi()) { 1782 Node* r = nn->in(0); 1783 for (DUIterator_Fast jmax, j = r->fast_outs(jmax); j < jmax; j++) { 1784 Node* u = r->fast_out(j); 1785 if (u->is_Phi() && u->bottom_type() == Type::MEMORY && u != nn && 1786 (u->adr_type() == TypePtr::BOTTOM || phase->C->get_alias_index(u->adr_type()) == alias)) { 1787 if (trace) { tty->print("XX Next mem (other phi)"); u->dump(); } 1788 wq.push(u); 1789 } 1790 } 1791 } 1792 1793 for (DUIterator_Fast imax, i = nn->fast_outs(imax); i < imax; i++) { 1794 Node* use = nn->fast_out(i); 1795 1796 if (trace) { tty->print("XX use %p", use->adr_type()); use->dump(); } 1797 if (use->is_CFG()) { 1798 assert(use->in(TypeFunc::Memory) == nn, "bad cfg node"); 1799 Node* c = use->in(0); 1800 if (phase->is_dominator(rep_ctrl, c)) { 1801 memory_dominates_all_paths_helper(c, rep_ctrl, controls, phase); 1802 } else if (use->is_CallStaticJava() && use->as_CallStaticJava()->uncommon_trap_request() != 0 && c->is_Region()) { 1803 Node* region = c; 1804 if (trace) { tty->print("XX unc region"); region->dump(); } 1805 for (uint j = 1; j < region->req(); j++) { 1806 if (phase->is_dominator(rep_ctrl, region->in(j))) { 1807 if (trace) { tty->print("XX unc follows"); region->in(j)->dump(); } 1808 memory_dominates_all_paths_helper(region->in(j), rep_ctrl, controls, phase); 1809 } 1810 } 1811 } 1812 //continue; 1813 } else if (use->is_Phi()) { 1814 assert(use->bottom_type() == Type::MEMORY, "bad phi"); 1815 if ((use->adr_type() == TypePtr::BOTTOM /*&& !shenandoah_has_alias_phi(C, use, alias)*/) || 1816 phase->C->get_alias_index(use->adr_type()) == alias) { 1817 for (uint j = 1; j < use->req(); j++) { 1818 if (use->in(j) == nn) { 1819 Node* c = use->in(0)->in(j); 1820 if (phase->is_dominator(rep_ctrl, c)) { 1821 memory_dominates_all_paths_helper(c, rep_ctrl, controls, phase); 1822 } 1823 } 1824 } 1825 } 1826 // continue; 1827 } 1828 1829 if (use->is_MergeMem()) { 1830 if (use->as_MergeMem()->memory_at(alias) == nn) { 1831 if (trace) { tty->print("XX Next mem"); use->dump(); } 1832 // follow the memory edges 1833 wq.push(use); 1834 } 1835 } else if (use->is_Phi()) { 1836 assert(use->bottom_type() == Type::MEMORY, "bad phi"); 1837 if ((use->adr_type() == TypePtr::BOTTOM /*&& !shenandoah_has_alias_phi(C, use, alias)*/) || 1838 phase->C->get_alias_index(use->adr_type()) == alias) { 1839 if (trace) { tty->print("XX Next mem"); use->dump(); } 1840 // follow the memory edges 1841 wq.push(use); 1842 } 1843 } else if (use->bottom_type() == Type::MEMORY && 1844 (use->adr_type() == TypePtr::BOTTOM || phase->C->get_alias_index(use->adr_type()) == alias)) { 1845 if (trace) { tty->print("XX Next mem"); use->dump(); } 1846 // follow the memory edges 1847 wq.push(use); 1848 } else if ((use->is_SafePoint() || use->is_MemBar()) && 1849 (use->adr_type() == TypePtr::BOTTOM || phase->C->get_alias_index(use->adr_type()) == alias)) { 1850 for (DUIterator_Fast jmax, j = use->fast_outs(jmax); j < jmax; j++) { 1851 Node* u = use->fast_out(j); 1852 if (u->bottom_type() == Type::MEMORY) { 1853 if (trace) { tty->print("XX Next mem"); u->dump(); } 1854 // follow the memory edges 1855 wq.push(u); 1856 } 1857 } 1858 } else if (use->Opcode() == Op_ShenandoahWriteBarrier && phase->C->get_alias_index(use->adr_type()) == alias) { 1859 Node* m = use->find_out_with(Op_ShenandoahWBMemProj); 1860 if (m != NULL) { 1861 if (trace) { tty->print("XX Next mem"); m->dump(); } 1862 // follow the memory edges 1863 wq.push(m); 1864 } 1865 } 1866 } 1867 } 1868 1869 if (controls.size() == 0) { 1870 return false; 1871 } 1872 1873 for (uint i = 0; i < controls.size(); i++) { 1874 Node *n = controls.at(i); 1875 1876 if (trace) { tty->print("X checking"); n->dump(); } 1877 1878 if (n->unique_ctrl_out() != NULL) { 1879 continue; 1880 } 1881 1882 if (n->Opcode() == Op_NeverBranch) { 1883 Node* taken = n->as_Multi()->proj_out(0); 1884 if (!controls.member(taken)) { 1885 if (trace) { tty->print("X not seen"); taken->dump(); } 1886 return false; 1887 } 1888 continue; 1889 } 1890 1891 for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) { 1892 Node* u = n->fast_out(j); 1893 1894 if (u->is_CFG()) { 1895 if (!controls.member(u)) { 1896 if (u->is_Proj() && u->as_Proj()->is_uncommon_trap_proj(Deoptimization::Reason_none)) { 1897 if (trace) { tty->print("X not seen but unc"); u->dump(); } 1898 } else { 1899 Node* c = u; 1900 do { 1901 c = c->unique_ctrl_out(); 1902 } while (c != NULL && c->is_Region()); 1903 if (c != NULL && c->Opcode() == Op_Halt) { 1904 if (trace) { tty->print("X not seen but halt"); c->dump(); } 1905 } else { 1906 if (trace) { tty->print("X not seen"); u->dump(); } 1907 return false; 1908 } 1909 } 1910 } else { 1911 if (trace) { tty->print("X seen"); u->dump(); } 1912 } 1913 } 1914 } 1915 } 1916 return true; 1917 } 1918 #endif 1919 1920 static bool has_mem_phi(Compile* C, Node* region, int alias) { 1921 for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) { 1922 Node* use = region->fast_out(i); 1923 if (use->is_Phi() && use->bottom_type() == Type::MEMORY && 1924 (C->get_alias_index(use->adr_type()) == alias)) { 1925 return true; 1926 } 1927 } 1928 return false; 1929 } 1930 1931 bool ShenandoahWriteBarrierNode::fix_mem_phis_helper(Node* c, Node* mem, Node* mem_ctrl, Node* rep_ctrl, int alias, VectorSet& controls, GrowableArray<Node*>& regions, PhaseIdealLoop* phase) { 1932 const bool trace = false; 1933 Node_List wq; 1934 wq.push(c); 1935 1936 #ifdef ASSERT 1937 if (trace) { tty->print("YYY from"); c->dump(); } 1938 if (trace) { tty->print("YYY with mem"); mem->dump(); } 1939 #endif 1940 1941 while(wq.size() > 0) { 1942 c = wq.pop(); 1943 1944 while (!c->is_Region() || c->is_Loop()) { 1945 #ifdef ASSERT 1946 if (trace) { tty->print("YYY"); c->dump(); } 1947 #endif 1948 assert(c->is_CFG(), "node should be control node"); 1949 if (c == mem_ctrl || phase->is_dominator(c, rep_ctrl)) { 1950 c = NULL; 1951 break; 1952 } else if (c->is_Loop()) { 1953 c = c->in(LoopNode::EntryControl); 1954 } else { 1955 c = c->in(0); 1956 } 1957 } 1958 if (c == NULL) { 1959 continue; 1960 } 1961 1962 #ifdef ASSERT 1963 if (trace) { tty->print("YYY new region"); c->dump(); } 1964 #endif 1965 1966 bool has_phi = has_mem_phi(phase->C, c, alias); 1967 if (!has_phi) { 1968 1969 Node* m_ctrl = NULL; 1970 Node* m = dom_mem(mem, c, alias, m_ctrl, phase); 1971 if (m == NULL) { 1972 return false; 1973 } 1974 1975 #ifdef ASSERT 1976 if (trace) { tty->print("YYY mem "); m->dump(); } 1977 #endif 1978 1979 if (controls.test(c->_idx)) { 1980 int i; 1981 for (i = 0; i < regions.length() && regions.at(i) != c; i+=2) { 1982 // deliberately empty, rolling over the regions 1983 } 1984 assert(i < regions.length(), "missing region"); 1985 Node* prev_m = regions.at(i+1); 1986 if (prev_m == m) { 1987 continue; 1988 } 1989 #ifdef ASSERT 1990 if (trace) { tty->print("YYY prev mem "); prev_m->dump(); } 1991 #endif 1992 Node* prev_m_ctrl = phase->ctrl_or_self(prev_m); 1993 assert(is_dominator(m_ctrl, prev_m_ctrl, m, prev_m, phase) || 1994 is_dominator(prev_m_ctrl, m_ctrl, prev_m, m, phase), "one should dominate the other"); 1995 if (is_dominator(m_ctrl, prev_m_ctrl, m, prev_m, phase)) { 1996 continue; 1997 } 1998 #ifdef ASSERT 1999 if (trace) { tty->print("YYY Fixing "); c->dump(); } 2000 #endif 2001 regions.at_put(i+1, m); 2002 } else { 2003 #ifdef ASSERT 2004 if (trace) { tty->print("YYY Pushing "); c->dump(); } 2005 #endif 2006 regions.push(c); 2007 regions.push(m); 2008 } 2009 } else { 2010 continue; 2011 } 2012 2013 controls.set(c->_idx); 2014 2015 for (uint i = 1; i < c->req(); i++) { 2016 wq.push(c->in(i)); 2017 } 2018 } 2019 return true; 2020 } 2021 2022 2023 bool ShenandoahWriteBarrierNode::fix_mem_phis(Node* mem, Node* mem_ctrl, Node* rep_ctrl, int alias, PhaseIdealLoop* phase) { 2024 GrowableArray<Node*> regions; 2025 VectorSet controls(Thread::current()->resource_area()); 2026 const bool trace = false; 2027 2028 #ifdef ASSERT 2029 if (trace) { tty->print("YYY mem is "); mem->dump(); } 2030 if (trace) { tty->print("YYY mem ctrl is "); mem_ctrl->dump(); } 2031 if (trace) { tty->print("YYY rep ctrl is "); rep_ctrl->dump(); } 2032 if (trace) { tty->print_cr("YYY alias is %d", alias); } 2033 #endif 2034 2035 // Walk memory edges from mem until we hit a memory point where 2036 // control is known then follow the control up looking for regions 2037 // with no memory Phi for alias 2038 Unique_Node_List wq; 2039 wq.push(mem); 2040 2041 for (uint next = 0; next < wq.size(); next++) { 2042 Node *n = wq.at(next); 2043 #ifdef ASSERT 2044 if (trace) { tty->print("YYY from (2) "); n->dump(); } 2045 #endif 2046 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { 2047 Node* u = n->fast_out(i); 2048 #ifdef ASSERT 2049 if (trace) { tty->print("YYY processing "); u->dump(); } 2050 #endif 2051 if (u->is_Phi()) { 2052 assert(u->bottom_type() == Type::MEMORY, "strange memory graph"); 2053 if (should_process_phi(u, alias, phase->C)) { 2054 for (uint j = 1; j < u->req(); j++) { 2055 if (u->in(j) == n) { 2056 Node *c = u->in(0)->in(j); 2057 if (!fix_mem_phis_helper(c, n, mem_ctrl, rep_ctrl, alias, controls, regions, phase)) { 2058 return false; 2059 } 2060 } 2061 } 2062 } 2063 #ifdef ASSERT 2064 } else if (u->is_CallStaticJava() && u->as_CallStaticJava()->uncommon_trap_request() != 0) { 2065 if (!fix_mem_phis_helper(u->in(0), n, mem_ctrl, rep_ctrl, alias, controls, regions, phase)) { 2066 return false; 2067 } 2068 #endif 2069 } else if ((u->is_CFG() && u->adr_type() == TypePtr::BOTTOM) || u->Opcode() == Op_Rethrow || u->Opcode() == Op_Return) { 2070 if (!fix_mem_phis_helper(u->in(0), n, mem_ctrl, rep_ctrl, alias, controls, regions, phase)) { 2071 return false; 2072 } 2073 } else if (u->is_MergeMem() && u->as_MergeMem()->memory_at(alias) == n) { 2074 wq.push(u); 2075 } else if (u->Opcode() == Op_ShenandoahWriteBarrier && phase->C->get_alias_index(u->adr_type()) == alias) { 2076 Node* m = u->find_out_with(Op_ShenandoahWBMemProj); 2077 if (m != NULL) { 2078 wq.push(m); 2079 } 2080 } 2081 } 2082 } 2083 #ifdef ASSERT 2084 if (trace) { 2085 tty->print_cr("XXXXXXXXXXXXXXXXXXXX"); 2086 for (int i = 0; i < regions.length(); i++) { 2087 Node* r = regions.at(i); 2088 tty->print("%d", i); r->dump(); 2089 } 2090 tty->print_cr("XXXXXXXXXXXXXXXXXXXX"); 2091 } 2092 #endif 2093 2094 if (regions.length() == 0) { 2095 return true; 2096 } 2097 2098 { 2099 int i = 0; 2100 for (; i < regions.length(); i+=2) { 2101 Node* region = regions.at(i); 2102 bool has_phi = false; 2103 for (DUIterator_Fast jmax, j = region->fast_outs(jmax); j < jmax && !has_phi; j++) { 2104 Node* u = region->fast_out(j); 2105 if (u->is_Phi() && u->bottom_type() == Type::MEMORY && 2106 (u->adr_type() == TypePtr::BOTTOM || phase->C->get_alias_index(u->adr_type()) == alias)) { 2107 has_phi = true; 2108 } 2109 } 2110 if (!has_phi) { 2111 break; 2112 } 2113 } 2114 if (i == regions.length()) { 2115 return true; 2116 } 2117 } 2118 2119 // Try to restrict the update to path that post dominates rep_ctrl 2120 int k = 0; 2121 int start = 0; 2122 int end = 0; 2123 do { 2124 start = end; 2125 end = k; 2126 for (int i = end; i < regions.length(); i+=2) { 2127 Node* r = regions.at(i); 2128 int prev = k; 2129 for (uint j = 1; j < r->req() && prev == k; j++) { 2130 if (end == 0) { 2131 if (phase->is_dominator(rep_ctrl, r->in(j))) { 2132 Node* mem = regions.at(i+1); 2133 regions.at_put(i, regions.at(k)); 2134 regions.at_put(i+1, regions.at(k+1)); 2135 regions.at_put(k, r); 2136 regions.at_put(k+1, mem); 2137 k+=2; 2138 } 2139 } else { 2140 for (int l = start; l < end && prev == k; l+=2) { 2141 Node* r2 = regions.at(l); 2142 if (phase->is_dominator(r2, r->in(j))) { 2143 Node* mem = regions.at(i+1); 2144 regions.at_put(i, regions.at(k)); 2145 regions.at_put(i+1, regions.at(k+1)); 2146 regions.at_put(k, r); 2147 regions.at_put(k+1, mem); 2148 k+=2; 2149 } 2150 } 2151 } 2152 } 2153 } 2154 #ifdef ASSERT 2155 if (trace) { tty->print_cr("k = %d start = %d end = %d", k, start, end); } 2156 #endif 2157 } while(k != end); 2158 2159 #ifdef ASSERT 2160 if (end != regions.length()) { 2161 if (trace) { tty->print_cr("Compacting %d -> %d", regions.length(), end); } 2162 } 2163 #endif 2164 regions.trunc_to(end); 2165 2166 #ifdef ASSERT 2167 if (trace) { 2168 tty->print_cr("XXXXXXXXXXXXXXXXXXXX"); 2169 for (int i = 0; i < regions.length(); i++) { 2170 Node* r = regions.at(i); 2171 tty->print("%d", i); r->dump(); 2172 } 2173 tty->print_cr("XXXXXXXXXXXXXXXXXXXX"); 2174 } 2175 #endif 2176 2177 // Creating new phis must be done in post order 2178 while (regions.length() > 0) { 2179 int i = 0; 2180 for (; i < regions.length(); i+=2) { 2181 Node* r1 = regions.at(i); 2182 bool is_dom = false; 2183 for (int j = 0; j < regions.length() && !is_dom; j+=2) { 2184 if (i != j) { 2185 Node* r2 = regions.at(j); 2186 for (uint k = 1; k < r2->req() && !is_dom; k++) { 2187 if (phase->is_dominator(r1, r2->in(k))) { 2188 is_dom = true; 2189 } 2190 } 2191 } 2192 } 2193 if (!is_dom) { 2194 break; 2195 } 2196 } 2197 assert(i < regions.length(), "need one"); 2198 Node* r = regions.at(i); 2199 Node* m = regions.at(i+1); 2200 regions.delete_at(i+1); 2201 regions.delete_at(i); 2202 2203 if (!suitable_mem(m, NULL, NULL)) { 2204 return false; 2205 } 2206 Node* phi = PhiNode::make(r, m, Type::MEMORY, phase->C->get_adr_type(alias)); 2207 #ifdef ASSERT 2208 if (trace) { tty->print("YYY Adding new mem phi "); phi->dump(); } 2209 #endif 2210 phase->register_new_node(phi, r); 2211 2212 fix_memory_uses(m, phi, phi, r, phase->C->get_alias_index(phi->adr_type()), phase); 2213 assert(phi->outcnt() != 0, "new proj should have uses"); 2214 if (phi->outcnt() == 0) { 2215 phase->igvn().remove_dead_node(phi); 2216 } 2217 } 2218 2219 return true; 2220 } 2221 2222 Node* ShenandoahBarrierNode::dom_mem(Node* mem, Node*& mem_ctrl, Node* n, Node* rep_ctrl, int alias, PhaseIdealLoop* phase) { 2223 ResourceMark rm; 2224 VectorSet wq(Thread::current()->resource_area()); 2225 wq.set(mem->_idx); 2226 mem_ctrl = phase->get_ctrl(mem); 2227 while (!is_dominator(mem_ctrl, rep_ctrl, mem, n, phase)) { 2228 mem = next_mem(mem, alias); 2229 if (wq.test_set(mem->_idx)) { 2230 return NULL; // hit an unexpected loop 2231 } 2232 mem_ctrl = phase->ctrl_or_self(mem); 2233 } 2234 if (mem->is_MergeMem()) { 2235 mem = mem->as_MergeMem()->memory_at(alias); 2236 mem_ctrl = phase->ctrl_or_self(mem); 2237 } 2238 return mem; 2239 } 2240 2241 Node* ShenandoahBarrierNode::dom_mem(Node* mem, Node* ctrl, int alias, Node*& mem_ctrl, PhaseIdealLoop* phase) { 2242 ResourceMark rm; 2243 VectorSet wq(Thread::current()->resource_area()); 2244 wq.set(mem->_idx); 2245 mem_ctrl = phase->ctrl_or_self(mem); 2246 while (!phase->is_dominator(mem_ctrl, ctrl) || mem_ctrl == ctrl) { 2247 mem = next_mem(mem, alias); 2248 if (wq.test_set(mem->_idx)) { 2249 return NULL; 2250 } 2251 mem_ctrl = phase->ctrl_or_self(mem); 2252 } 2253 if (mem->is_MergeMem()) { 2254 mem = mem->as_MergeMem()->memory_at(alias); 2255 mem_ctrl = phase->ctrl_or_self(mem); 2256 } 2257 return mem; 2258 } 2259 2260 Node* ShenandoahBarrierNode::try_common(Node *n_ctrl, PhaseIdealLoop* phase) { 2261 if (!phase->C->has_irreducible_loop()) { 2262 // We look for a write barrier whose memory edge dominates n 2263 // Either the replacement write barrier dominates n or we have, 2264 // for instance: 2265 // if ( ) { 2266 // read barrier n 2267 // } else { 2268 // write barrier 2269 // } 2270 // in which case replacing n by the write barrier causes the write 2271 // barrier to move above the if() and the memory Phi that merges 2272 // the memory state for both branches must be updated so both 2273 // inputs become the write barrier's memory projection (and the 2274 // Phi is optimized out) otherwise we risk loosing a memory 2275 // dependency. 2276 // Once we find a replacement write barrier, the code below fixes 2277 // the memory graph in cases like the one above. 2278 Node* val = in(ValueIn); 2279 Node* val_ctrl = phase->get_ctrl(val); 2280 Node* n_proj = find_out_with(Op_ShenandoahWBMemProj); 2281 Node* replacement = NULL; 2282 int alias = phase->C->get_alias_index(adr_type()); 2283 Node* rep_ctrl = NULL; 2284 for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax && replacement == NULL; i++) { 2285 Node* u = val->fast_out(i); 2286 if (u != this && u->Opcode() == Op_ShenandoahWriteBarrier) { 2287 Node* u_mem = u->in(Memory); 2288 Node* u_proj = u->find_out_with(Op_ShenandoahWBMemProj); 2289 Node* u_ctrl = phase->get_ctrl(u); 2290 Node* u_mem_ctrl = phase->get_ctrl(u_mem); 2291 IdealLoopTree* n_loop = phase->get_loop(n_ctrl); 2292 IdealLoopTree* u_loop = phase->get_loop(u_ctrl); 2293 2294 Node* ctrl = phase->dom_lca(u_ctrl, n_ctrl); 2295 2296 if (ctrl->is_Proj() && 2297 ctrl->in(0)->is_Call() && 2298 ctrl->unique_ctrl_out() != NULL && 2299 ctrl->unique_ctrl_out()->Opcode() == Op_Catch && 2300 !phase->is_dominator(val_ctrl, ctrl->in(0)->in(0))) { 2301 continue; 2302 } 2303 2304 if (Opcode() == Op_ShenandoahWriteBarrier && u_proj == NULL && n_proj != NULL) { 2305 continue; 2306 } 2307 2308 IdealLoopTree* loop = phase->get_loop(ctrl); 2309 2310 // We don't want to move a write barrier in a loop 2311 // If the LCA is in a inner loop, try a control out of loop if possible 2312 bool loop_ok = true; 2313 while (!loop->is_member(u_loop) && (Opcode() != Op_ShenandoahWriteBarrier || !loop->is_member(n_loop))) { 2314 ctrl = phase->idom(ctrl); 2315 if (ctrl != val_ctrl && phase->is_dominator(ctrl, val_ctrl)) { 2316 loop_ok = false; 2317 break; 2318 } 2319 loop = phase->get_loop(ctrl); 2320 } 2321 2322 if (loop_ok) { 2323 if (ShenandoahDontIncreaseWBFreq) { 2324 Node* u_iffproj = no_branches(u_ctrl, ctrl, true, phase); 2325 if (Opcode() == Op_ShenandoahWriteBarrier) { 2326 Node* n_iffproj = no_branches(n_ctrl, ctrl, true, phase); 2327 if (u_iffproj == NULL || n_iffproj == NULL) { 2328 replacement = u; 2329 rep_ctrl = ctrl; 2330 } else if (u_iffproj != NodeSentinel && n_iffproj != NodeSentinel && u_iffproj->in(0) == n_iffproj->in(0)) { 2331 replacement = u; 2332 rep_ctrl = ctrl; 2333 } 2334 } else if (u_iffproj == NULL) { 2335 replacement = u; 2336 rep_ctrl = ctrl; 2337 } 2338 } else { 2339 replacement = u; 2340 rep_ctrl = ctrl; 2341 } 2342 } 2343 } 2344 } 2345 if (replacement != NULL) { 2346 if (rep_ctrl->is_Proj() && 2347 rep_ctrl->in(0)->is_Call() && 2348 rep_ctrl->unique_ctrl_out() != NULL && 2349 rep_ctrl->unique_ctrl_out()->Opcode() == Op_Catch) { 2350 rep_ctrl = rep_ctrl->in(0)->in(0); 2351 assert(phase->is_dominator(val_ctrl, rep_ctrl), "bad control"); 2352 } else { 2353 LoopNode* c = ShenandoahWriteBarrierNode::try_move_before_pre_loop(rep_ctrl, val_ctrl, phase); 2354 if (c != NULL) { 2355 rep_ctrl = ShenandoahWriteBarrierNode::move_above_predicates(c, val_ctrl, phase); 2356 } else { 2357 while (rep_ctrl->is_IfProj()) { 2358 CallStaticJavaNode* unc = rep_ctrl->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none); 2359 if (unc != NULL) { 2360 int req = unc->uncommon_trap_request(); 2361 Deoptimization::DeoptReason trap_reason = Deoptimization::trap_request_reason(req); 2362 if ((trap_reason == Deoptimization::Reason_loop_limit_check || 2363 trap_reason == Deoptimization::Reason_predicate) && phase->is_dominator(val_ctrl, rep_ctrl->in(0)->in(0))) { 2364 rep_ctrl = rep_ctrl->in(0)->in(0); 2365 continue; 2366 } 2367 } 2368 break; 2369 } 2370 } 2371 } 2372 2373 Node* mem = replacement->in(Memory); 2374 Node* old_mem = mem; 2375 Node* rep_proj = replacement->find_out_with(Op_ShenandoahWBMemProj); 2376 { 2377 Node* mem_ctrl = NULL; 2378 2379 mem = dom_mem(mem, mem_ctrl, this, rep_ctrl, alias, phase); 2380 if (mem == NULL) { 2381 return NULL; 2382 } 2383 2384 // Add a memory Phi for the slice of the write barrier to any 2385 // region that post dominates rep_ctrl and doesn't have one 2386 // already. 2387 if (rep_proj != NULL && !ShenandoahWriteBarrierNode::fix_mem_phis(mem, mem_ctrl, rep_ctrl, alias, phase)) { 2388 return NULL; 2389 } 2390 2391 assert(!ShenandoahVerifyOptoBarriers || ShenandoahWriteBarrierNode::memory_dominates_all_paths(mem, rep_ctrl, alias, phase), "can't fix the memory graph"); 2392 } 2393 assert(phase->igvn().type(mem) == Type::MEMORY, "not memory"); 2394 2395 if (rep_proj != NULL) { 2396 Node* old_mem = replacement->in(Memory); 2397 if (!suitable_mem(mem, old_mem, rep_proj)) { 2398 return NULL; 2399 } 2400 2401 if (replacement->in(Memory) != mem) { 2402 // tty->print("XXX setting memory of"); replacement->dump(); 2403 // tty->print("XXX to"); mem->dump(); 2404 for (DUIterator_Last imin, i = rep_proj->last_outs(imin); i >= imin; ) { 2405 Node* u = rep_proj->last_out(i); 2406 phase->igvn().rehash_node_delayed(u); 2407 int uses_found = u->replace_edge(rep_proj, old_mem); 2408 i -= uses_found; 2409 } 2410 phase->igvn().replace_input_of(replacement, Memory, mem); 2411 } 2412 phase->set_ctrl_and_loop(replacement, rep_ctrl); 2413 phase->igvn().replace_input_of(replacement, Control, rep_ctrl); 2414 2415 ShenandoahWriteBarrierNode::fix_memory_uses(mem, replacement, rep_proj, rep_ctrl, phase->C->get_alias_index(replacement->adr_type()), phase); 2416 assert(rep_proj->outcnt() != 0, "new proj should have uses"); 2417 } else { 2418 if (replacement->in(Memory) != mem) { 2419 phase->igvn()._worklist.push(replacement->in(Memory)); 2420 phase->igvn().replace_input_of(replacement, Memory, mem); 2421 } 2422 phase->set_ctrl_and_loop(replacement, rep_ctrl); 2423 phase->igvn().replace_input_of(replacement, Control, rep_ctrl); 2424 } 2425 if (Opcode() == Op_ShenandoahWriteBarrier) { 2426 if (n_proj != NULL) { 2427 phase->lazy_replace(n_proj, in(Memory)); 2428 } 2429 } 2430 phase->lazy_replace(this, replacement); 2431 if (rep_proj != NULL) { 2432 phase->set_ctrl_and_loop(rep_proj, rep_ctrl); 2433 } 2434 return replacement; 2435 } 2436 } 2437 2438 return NULL; 2439 } 2440 2441 static void disconnect_barrier_mem(Node* wb, PhaseIterGVN& igvn) { 2442 Node* mem_in = wb->in(ShenandoahBarrierNode::Memory); 2443 Node* proj = wb->find_out_with(Op_ShenandoahWBMemProj); 2444 2445 for (DUIterator_Last imin, i = proj->last_outs(imin); i >= imin; ) { 2446 Node* u = proj->last_out(i); 2447 igvn.rehash_node_delayed(u); 2448 int nb = u->replace_edge(proj, mem_in); 2449 assert(nb > 0, "no replacement?"); 2450 i -= nb; 2451 } 2452 } 2453 2454 Node* ShenandoahWriteBarrierNode::move_above_predicates(LoopNode* cl, Node* val_ctrl, PhaseIdealLoop* phase) { 2455 Node* entry = cl->skip_strip_mined()->in(LoopNode::EntryControl); 2456 Node* above_pred = phase->skip_loop_predicates(entry); 2457 Node* ctrl = entry; 2458 while (ctrl != above_pred) { 2459 Node* next = ctrl->in(0); 2460 if (!phase->is_dominator(val_ctrl, next)) { 2461 break; 2462 } 2463 ctrl = next; 2464 } 2465 return ctrl; 2466 } 2467 2468 Node* ShenandoahWriteBarrierNode::try_move_before_loop_helper(LoopNode* cl, Node* val_ctrl, Node* mem, PhaseIdealLoop* phase) { 2469 assert(cl->is_Loop(), "bad control"); 2470 Node* ctrl = move_above_predicates(cl, val_ctrl, phase); 2471 Node* mem_ctrl = NULL; 2472 int alias = phase->C->get_alias_index(adr_type()); 2473 mem = dom_mem(mem, mem_ctrl, this, ctrl, alias, phase); 2474 if (mem == NULL) { 2475 return NULL; 2476 } 2477 2478 Node* old_mem = in(Memory); 2479 Node* proj = find_out_with(Op_ShenandoahWBMemProj); 2480 if (old_mem != mem && !suitable_mem(mem, old_mem, proj)) { 2481 return NULL; 2482 } 2483 2484 assert(!ShenandoahVerifyOptoBarriers || memory_dominates_all_paths(mem, ctrl, alias, phase), "can't fix the memory graph"); 2485 phase->set_ctrl_and_loop(this, ctrl); 2486 phase->igvn().replace_input_of(this, Control, ctrl); 2487 if (old_mem != mem) { 2488 if (proj != NULL) { 2489 disconnect_barrier_mem(this, phase->igvn()); 2490 fix_memory_uses(mem, this, proj, ctrl, phase->C->get_alias_index(adr_type()), phase); 2491 assert(proj->outcnt() > 0, "disconnected write barrier"); 2492 } 2493 phase->igvn().replace_input_of(this, Memory, mem); 2494 } 2495 if (proj != NULL) { 2496 phase->set_ctrl_and_loop(proj, ctrl); 2497 } 2498 return this; 2499 } 2500 2501 LoopNode* ShenandoahWriteBarrierNode::try_move_before_pre_loop(Node* c, Node* val_ctrl, PhaseIdealLoop* phase) { 2502 // A write barrier between a pre and main loop can get in the way of 2503 // vectorization. Move it above the pre loop if possible 2504 CountedLoopNode* cl = NULL; 2505 if (c->is_IfFalse() && 2506 c->in(0)->is_CountedLoopEnd()) { 2507 cl = c->in(0)->as_CountedLoopEnd()->loopnode(); 2508 } else if (c->is_IfProj() && 2509 c->in(0)->is_If() && 2510 c->in(0)->in(0)->is_IfFalse() && 2511 c->in(0)->in(0)->in(0)->is_CountedLoopEnd()) { 2512 cl = c->in(0)->in(0)->in(0)->as_CountedLoopEnd()->loopnode(); 2513 } 2514 if (cl != NULL && 2515 cl->is_pre_loop() && 2516 val_ctrl != cl && 2517 phase->is_dominator(val_ctrl, cl)) { 2518 return cl; 2519 } 2520 return NULL; 2521 } 2522 2523 Node* ShenandoahWriteBarrierNode::try_move_before_loop(Node *n_ctrl, PhaseIdealLoop* phase) { 2524 IdealLoopTree *n_loop = phase->get_loop(n_ctrl); 2525 Node* val = in(ValueIn); 2526 Node* val_ctrl = phase->get_ctrl(val); 2527 if (n_loop != phase->ltree_root() && !n_loop->_irreducible) { 2528 IdealLoopTree *val_loop = phase->get_loop(val_ctrl); 2529 Node* mem = in(Memory); 2530 IdealLoopTree *mem_loop = phase->get_loop(phase->get_ctrl(mem)); 2531 if (!n_loop->is_member(val_loop) && 2532 n_loop->is_member(mem_loop)) { 2533 Node* n_loop_head = n_loop->_head; 2534 2535 if (n_loop_head->is_Loop()) { 2536 LoopNode* loop = n_loop_head->as_Loop(); 2537 if (n_loop_head->is_CountedLoop() && n_loop_head->as_CountedLoop()->is_main_loop()) { 2538 LoopNode* res = try_move_before_pre_loop(n_loop_head->in(LoopNode::EntryControl), val_ctrl, phase); 2539 if (res != NULL) { 2540 loop = res; 2541 } 2542 } 2543 2544 return try_move_before_loop_helper(loop, val_ctrl, mem, phase); 2545 } 2546 } 2547 } 2548 LoopNode* ctrl = try_move_before_pre_loop(in(0), val_ctrl, phase); 2549 if (ctrl != NULL) { 2550 return try_move_before_loop_helper(ctrl, val_ctrl, in(Memory), phase); 2551 } 2552 return NULL; 2553 } 2554 2555 void ShenandoahReadBarrierNode::try_move(Node *n_ctrl, PhaseIdealLoop* phase) { 2556 Node* mem = in(MemNode::Memory); 2557 int alias = phase->C->get_alias_index(adr_type()); 2558 const bool trace = false; 2559 2560 #ifdef ASSERT 2561 if (trace) { tty->print("Trying to move mem of"); dump(); } 2562 #endif 2563 2564 Node* new_mem = mem; 2565 2566 ResourceMark rm; 2567 VectorSet seen(Thread::current()->resource_area()); 2568 Node_List phis; 2569 2570 for (;;) { 2571 #ifdef ASSERT 2572 if (trace) { tty->print("Looking for dominator from"); mem->dump(); } 2573 #endif 2574 if (mem->is_Proj() && mem->in(0)->is_Start()) { 2575 if (new_mem != in(MemNode::Memory)) { 2576 #ifdef ASSERT 2577 if (trace) { tty->print("XXX Setting mem to"); new_mem->dump(); tty->print(" for "); dump(); } 2578 #endif 2579 phase->igvn().replace_input_of(this, MemNode::Memory, new_mem); 2580 } 2581 return; 2582 } 2583 2584 Node* candidate = mem; 2585 do { 2586 if (!is_independent(mem)) { 2587 if (trace) { tty->print_cr("Not independent"); } 2588 if (new_mem != in(MemNode::Memory)) { 2589 #ifdef ASSERT 2590 if (trace) { tty->print("XXX Setting mem to"); new_mem->dump(); tty->print(" for "); dump(); } 2591 #endif 2592 phase->igvn().replace_input_of(this, MemNode::Memory, new_mem); 2593 } 2594 return; 2595 } 2596 if (seen.test_set(mem->_idx)) { 2597 if (trace) { tty->print_cr("Already seen"); } 2598 ShouldNotReachHere(); 2599 // Strange graph 2600 if (new_mem != in(MemNode::Memory)) { 2601 #ifdef ASSERT 2602 if (trace) { tty->print("XXX Setting mem to"); new_mem->dump(); tty->print(" for "); dump(); } 2603 #endif 2604 phase->igvn().replace_input_of(this, MemNode::Memory, new_mem); 2605 } 2606 return; 2607 } 2608 if (mem->is_Phi()) { 2609 phis.push(mem); 2610 } 2611 mem = next_mem(mem, alias); 2612 if (mem->bottom_type() == Type::MEMORY) { 2613 candidate = mem; 2614 } 2615 assert(is_dominator(phase->ctrl_or_self(mem), n_ctrl, mem, this, phase) == phase->is_dominator(phase->ctrl_or_self(mem), n_ctrl), "strange dominator"); 2616 #ifdef ASSERT 2617 if (trace) { tty->print("Next mem is"); mem->dump(); } 2618 #endif 2619 } while (mem->bottom_type() != Type::MEMORY || !phase->is_dominator(phase->ctrl_or_self(mem), n_ctrl)); 2620 2621 assert(mem->bottom_type() == Type::MEMORY, "bad mem"); 2622 2623 bool not_dom = false; 2624 for (uint i = 0; i < phis.size() && !not_dom; i++) { 2625 Node* nn = phis.at(i); 2626 2627 #ifdef ASSERT 2628 if (trace) { tty->print("Looking from phi"); nn->dump(); } 2629 #endif 2630 assert(nn->is_Phi(), "phis only"); 2631 for (uint j = 2; j < nn->req() && !not_dom; j++) { 2632 Node* m = nn->in(j); 2633 #ifdef ASSERT 2634 if (trace) { tty->print("Input %d is", j); m->dump(); } 2635 #endif 2636 while (m != mem && !seen.test_set(m->_idx)) { 2637 if (is_dominator(phase->ctrl_or_self(m), phase->ctrl_or_self(mem), m, mem, phase)) { 2638 not_dom = true; 2639 // Scheduling anomaly 2640 #ifdef ASSERT 2641 if (trace) { tty->print("Giving up"); m->dump(); } 2642 #endif 2643 break; 2644 } 2645 if (!is_independent(m)) { 2646 if (trace) { tty->print_cr("Not independent"); } 2647 if (new_mem != in(MemNode::Memory)) { 2648 #ifdef ASSERT 2649 if (trace) { tty->print("XXX Setting mem to"); new_mem->dump(); tty->print(" for "); dump(); } 2650 #endif 2651 phase->igvn().replace_input_of(this, MemNode::Memory, new_mem); 2652 } 2653 return; 2654 } 2655 if (m->is_Phi()) { 2656 phis.push(m); 2657 } 2658 m = next_mem(m, alias); 2659 #ifdef ASSERT 2660 if (trace) { tty->print("Next mem is"); m->dump(); } 2661 #endif 2662 } 2663 } 2664 } 2665 if (!not_dom) { 2666 new_mem = mem; 2667 phis.clear(); 2668 } else { 2669 seen.Clear(); 2670 } 2671 } 2672 } 2673 2674 CallStaticJavaNode* ShenandoahWriteBarrierNode::pin_and_expand_null_check(PhaseIterGVN& igvn) { 2675 Node* val = in(ValueIn); 2676 2677 #ifdef ASSERT 2678 const Type* val_t = igvn.type(val); 2679 assert(val_t->meet(TypePtr::NULL_PTR) != val_t, "should be not null"); 2680 #endif 2681 2682 if (val->Opcode() == Op_CastPP && 2683 val->in(0)->Opcode() == Op_IfTrue && 2684 val->in(0)->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none) && 2685 val->in(0)->in(0)->is_If() && 2686 val->in(0)->in(0)->in(1)->Opcode() == Op_Bool && 2687 val->in(0)->in(0)->in(1)->as_Bool()->_test._test == BoolTest::ne && 2688 val->in(0)->in(0)->in(1)->in(1)->Opcode() == Op_CmpP && 2689 val->in(0)->in(0)->in(1)->in(1)->in(1) == val->in(1) && 2690 val->in(0)->in(0)->in(1)->in(1)->in(2)->bottom_type() == TypePtr::NULL_PTR) { 2691 assert(val->in(0)->in(0)->in(1)->in(1)->in(1) == val->in(1), ""); 2692 CallStaticJavaNode* unc = val->in(0)->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none); 2693 return unc; 2694 } 2695 return NULL; 2696 } 2697 2698 void ShenandoahWriteBarrierNode::pin_and_expand_move_barrier(PhaseIdealLoop* phase) { 2699 Node* unc = pin_and_expand_null_check(phase->igvn()); 2700 Node* val = in(ValueIn); 2701 2702 if (unc != NULL) { 2703 Node* ctrl = phase->get_ctrl(this); 2704 Node* unc_ctrl = val->in(0); 2705 2706 // Don't move write barrier in a loop 2707 IdealLoopTree* loop = phase->get_loop(ctrl); 2708 IdealLoopTree* unc_loop = phase->get_loop(unc_ctrl); 2709 2710 if (!unc_loop->is_member(loop)) { 2711 return; 2712 } 2713 2714 Node* branch = no_branches(ctrl, unc_ctrl, false, phase); 2715 assert(branch == NULL || branch == NodeSentinel, "was not looking for a branch"); 2716 if (branch == NodeSentinel) { 2717 return; 2718 } 2719 2720 Node* mem = in(Memory); 2721 Node* old_mem = mem; 2722 2723 Node* mem_ctrl = NULL; 2724 int alias = phase->C->get_alias_index(adr_type()); 2725 mem = dom_mem(mem, mem_ctrl, this, unc_ctrl, alias, phase); 2726 if (mem == NULL) { 2727 return; 2728 } 2729 2730 Node* proj = find_out_with(Op_ShenandoahWBMemProj); 2731 if (mem != old_mem && !fix_mem_phis(mem, mem_ctrl, unc_ctrl, alias, phase)) { 2732 return; 2733 } 2734 2735 assert(mem == old_mem || memory_dominates_all_paths(mem, unc_ctrl, alias, phase), "can't fix the memory graph"); 2736 phase->set_ctrl_and_loop(this, unc_ctrl); 2737 if (in(Control) != NULL) { 2738 phase->igvn().replace_input_of(this, Control, unc_ctrl); 2739 } 2740 disconnect_barrier_mem(this, phase->igvn()); 2741 fix_memory_uses(mem, this, proj, unc_ctrl, phase->C->get_alias_index(adr_type()), phase); 2742 assert(proj->outcnt() > 0, "disconnected write barrier"); 2743 phase->igvn().replace_input_of(this, Memory, mem); 2744 phase->set_ctrl_and_loop(proj, unc_ctrl); 2745 } 2746 } 2747 2748 void ShenandoahWriteBarrierNode::pin_and_expand_helper(PhaseIdealLoop* phase) { 2749 Node* val = in(ValueIn); 2750 Node* ctrl = phase->get_ctrl(this); 2751 // Replace all uses of barrier's input that are dominated by ctrl 2752 // with the value returned by the barrier: no need to keep both 2753 // live. 2754 for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) { 2755 Node* u = val->fast_out(i); 2756 if (u != this) { 2757 if (u->is_Phi()) { 2758 int nb = 0; 2759 for (uint j = 1; j < u->req(); j++) { 2760 if (u->in(j) == val) { 2761 Node* c = u->in(0)->in(j); 2762 if (phase->is_dominator(ctrl, c)) { 2763 phase->igvn().replace_input_of(u, j, this); 2764 nb++; 2765 } 2766 } 2767 } 2768 if (nb > 0) { 2769 imax -= nb; 2770 --i; 2771 } 2772 } else { 2773 Node* c = phase->ctrl_or_self(u); 2774 if (is_dominator(ctrl, c, this, u, phase)) { 2775 phase->igvn().rehash_node_delayed(u); 2776 int nb = u->replace_edge(val, this); 2777 assert(nb > 0, "no update?"); 2778 --i, imax -= nb; 2779 } 2780 } 2781 } 2782 } 2783 } 2784 2785 Node* ShenandoahWriteBarrierNode::pick_phi(Node* phi1, Node* phi2, Node_Stack& phis, VectorSet& visited, PhaseIdealLoop* phase) { 2786 assert(phis.size() == 0, "stack needs to be empty"); 2787 uint i = 1; 2788 int phi_dominates = -1; 2789 for (;;) { 2790 assert(phi1->req() == phi2->req(), "strange pair of phis"); 2791 assert(phis.size() % 2 == 0, ""); 2792 Node* in1 = phi1->in(i); 2793 Node* in2 = phi2->in(i); 2794 2795 if (in1->is_MergeMem()) { 2796 in1 = in1->as_MergeMem()->base_memory(); 2797 } 2798 if (in2->is_MergeMem()) { 2799 in2 = in2->as_MergeMem()->base_memory(); 2800 } 2801 2802 if (in1 == in2) { 2803 //continue 2804 } else if (in1->is_Phi() && in2->is_Phi() && in1->in(0) == in2->in(0)) { 2805 assert(!visited.test_set(in1->_idx), "no loop"); 2806 assert(!visited.test_set(in2->_idx), "no loop"); 2807 phis.push(phi1, i+1); 2808 phis.push(phi2, i+1); 2809 phi1 = in1; 2810 phi2 = in2; 2811 i = 1; 2812 } else { 2813 Node* in1_c = phase->get_ctrl(in1); 2814 Node* in2_c = phase->get_ctrl(in2); 2815 if (is_dominator(in1_c, in2_c, in1, in2, phase)) { 2816 assert(!is_dominator(in2_c, in1_c, in2, in1, phase), "one has to dominate the other"); 2817 assert(phi_dominates == -1 || phi_dominates == 1, "all inputs must dominate"); 2818 phi_dominates = 1; 2819 } else { 2820 assert(is_dominator(in2_c, in1_c, in2, in1, phase), "one must dominate the other"); 2821 assert(!is_dominator(in1_c, in2_c, in1, in2, phase), "one has to dominate the other"); 2822 assert(phi_dominates == -1 || phi_dominates == 2, "all inputs must dominate"); 2823 phi_dominates = 2; 2824 } 2825 } 2826 i++; 2827 2828 while (i >= phi1->req() && phis.size() > 0) { 2829 i = phis.index(); 2830 phi2 = phis.node(); 2831 phis.pop(); 2832 phi1 = phis.node(); 2833 phis.pop(); 2834 } 2835 2836 if (i >= phi1->req() && phis.size() == 0) { 2837 Node* phi = NULL; 2838 if (phi_dominates == 1) { 2839 return phi2; 2840 } else if (phi_dominates == 2) { 2841 return phi1; 2842 } else { 2843 return phi1; 2844 } 2845 } 2846 } 2847 return NULL; 2848 } 2849 2850 bool ShenandoahWriteBarrierNode::mem_is_valid(Node* m, Node* c, PhaseIdealLoop* phase) { 2851 return m != NULL && get_ctrl(m, phase) == c; 2852 } 2853 2854 Node* ShenandoahWriteBarrierNode::find_raw_mem(Node* ctrl, Node* n, const Node_List& memory_nodes, PhaseIdealLoop* phase) { 2855 assert(n == NULL || phase->ctrl_or_self(n) == ctrl, ""); 2856 Node* raw_mem = memory_nodes[ctrl->_idx]; 2857 Node* c = ctrl; 2858 while (!mem_is_valid(raw_mem, c, phase) && 2859 (!c->is_CatchProj() || raw_mem == NULL || c->in(0)->in(0)->in(0) != get_ctrl(raw_mem, phase))) { 2860 c = phase->idom(c); 2861 raw_mem = memory_nodes[c->_idx]; 2862 } 2863 if (n != NULL && mem_is_valid(raw_mem, c, phase)) { 2864 while (!is_dominator_same_ctrl(c, raw_mem, n, phase) && phase->ctrl_or_self(raw_mem) == ctrl) { 2865 raw_mem = next_mem(raw_mem, Compile::AliasIdxRaw); 2866 } 2867 if (raw_mem->is_MergeMem()) { 2868 raw_mem = raw_mem->as_MergeMem()->memory_at(Compile::AliasIdxRaw); 2869 } 2870 if (!mem_is_valid(raw_mem, c, phase)) { 2871 do { 2872 c = phase->idom(c); 2873 raw_mem = memory_nodes[c->_idx]; 2874 } while (!mem_is_valid(raw_mem, c, phase) && 2875 (!c->is_CatchProj() || raw_mem == NULL || c->in(0)->in(0)->in(0) != get_ctrl(raw_mem, phase))); 2876 } 2877 } 2878 assert(raw_mem->bottom_type() == Type::MEMORY, ""); 2879 return raw_mem; 2880 } 2881 2882 Node* ShenandoahWriteBarrierNode::find_bottom_mem(Node* ctrl, PhaseIdealLoop* phase) { 2883 Node* mem = NULL; 2884 Node* c = ctrl; 2885 do { 2886 if (c->is_Region()) { 2887 Node* phi_bottom = NULL; 2888 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) { 2889 Node* u = c->fast_out(i); 2890 if (u->is_Phi() && u->bottom_type() == Type::MEMORY) { 2891 if (u->adr_type() == TypePtr::BOTTOM) { 2892 if (phi_bottom != NULL) { 2893 phi_bottom = NodeSentinel; 2894 } else { 2895 phi_bottom = u; 2896 } 2897 } 2898 } 2899 } 2900 if (phi_bottom != NULL) { 2901 if (phi_bottom != NodeSentinel) { 2902 mem = phi_bottom; 2903 } else { 2904 Node* phi = NULL; 2905 ResourceMark rm; 2906 Node_Stack phis(0); 2907 VectorSet visited(Thread::current()->resource_area()); 2908 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) { 2909 Node* u = c->fast_out(i); 2910 if (u->is_Phi() && u->bottom_type() == Type::MEMORY && u->adr_type() == TypePtr::BOTTOM) { 2911 if (phi == NULL) { 2912 phi = u; 2913 } else { 2914 phi = pick_phi(phi, u, phis, visited, phase); 2915 } 2916 } 2917 } 2918 mem = phi; 2919 } 2920 } 2921 } else { 2922 if (c->is_Call() && c->as_Call()->_entry_point != OptoRuntime::rethrow_stub()) { 2923 CallProjections projs; 2924 c->as_Call()->extract_projections(&projs, true, false); 2925 if (projs.fallthrough_memproj != NULL) { 2926 if (projs.fallthrough_memproj->adr_type() == TypePtr::BOTTOM) { 2927 if (projs.catchall_memproj == NULL) { 2928 mem = projs.fallthrough_memproj; 2929 } else { 2930 if (phase->is_dominator(projs.fallthrough_catchproj, ctrl)) { 2931 mem = projs.fallthrough_memproj; 2932 } else { 2933 assert(phase->is_dominator(projs.catchall_catchproj, ctrl), "one proj must dominate barrier"); 2934 mem = projs.catchall_memproj; 2935 } 2936 } 2937 } 2938 } else { 2939 Node* proj = c->as_Call()->proj_out(TypeFunc::Memory); 2940 if (proj != NULL && 2941 proj->adr_type() == TypePtr::BOTTOM) { 2942 mem = proj; 2943 } 2944 } 2945 } else { 2946 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) { 2947 Node* u = c->fast_out(i); 2948 if (u->is_Proj() && 2949 u->bottom_type() == Type::MEMORY && 2950 u->adr_type() == TypePtr::BOTTOM) { 2951 assert(c->is_SafePoint() || c->is_MemBar() || c->is_Start(), ""); 2952 assert(mem == NULL, "only one proj"); 2953 mem = u; 2954 } 2955 } 2956 } 2957 } 2958 c = phase->idom(c); 2959 } while (mem == NULL); 2960 return mem; 2961 } 2962 2963 void ShenandoahWriteBarrierNode::follow_barrier_uses(Node* n, Node* ctrl, Unique_Node_List& uses, PhaseIdealLoop* phase) { 2964 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { 2965 Node* u = n->fast_out(i); 2966 if (!u->is_CFG() && phase->get_ctrl(u) == ctrl && (!u->is_Phi() || !u->in(0)->is_Loop() || u->in(LoopNode::LoopBackControl) != n)) { 2967 uses.push(u); 2968 } 2969 } 2970 } 2971 2972 Node* ShenandoahWriteBarrierNode::get_ctrl(Node* n, PhaseIdealLoop* phase) { 2973 Node* c = phase->get_ctrl(n); 2974 if (n->is_Proj() && n->in(0)->is_Call()) { 2975 assert(c == n->in(0), ""); 2976 CallNode* call = c->as_Call(); 2977 CallProjections projs; 2978 call->extract_projections(&projs, true, false); 2979 if (projs.catchall_memproj != NULL) { 2980 if (projs.fallthrough_memproj == n) { 2981 c = projs.fallthrough_catchproj; 2982 } else { 2983 assert(projs.catchall_memproj == n, ""); 2984 c = projs.catchall_catchproj; 2985 } 2986 } 2987 } 2988 return c; 2989 } 2990 2991 Node* ShenandoahWriteBarrierNode::ctrl_or_self(Node* n, PhaseIdealLoop* phase) { 2992 if (phase->has_ctrl(n)) 2993 return get_ctrl(n, phase); 2994 else { 2995 assert (n->is_CFG(), "must be a CFG node"); 2996 return n; 2997 } 2998 } 2999 3000 #ifdef ASSERT 3001 static bool has_never_branch(Node* root) { 3002 for (uint i = 1; i < root->req(); i++) { 3003 Node* in = root->in(i); 3004 if (in != NULL && in->Opcode() == Op_Halt && in->in(0)->is_Proj() && in->in(0)->in(0)->Opcode() == Op_NeverBranch) { 3005 return true; 3006 } 3007 } 3008 return false; 3009 } 3010 #endif 3011 3012 void ShenandoahWriteBarrierNode::collect_memory_nodes(int alias, Node_List& memory_nodes, PhaseIdealLoop* phase) { 3013 Node_Stack stack(0); 3014 VectorSet visited(Thread::current()->resource_area()); 3015 Node_List regions; 3016 3017 // Walk the raw memory graph and create a mapping from CFG node to 3018 // memory node. Exclude phis for now. 3019 stack.push(phase->C->root(), 1); 3020 do { 3021 Node* n = stack.node(); 3022 int opc = n->Opcode(); 3023 uint i = stack.index(); 3024 if (i < n->req()) { 3025 Node* mem = NULL; 3026 if (opc == Op_Root) { 3027 Node* in = n->in(i); 3028 int in_opc = in->Opcode(); 3029 if (in_opc == Op_Return || in_opc == Op_Rethrow) { 3030 mem = in->in(TypeFunc::Memory); 3031 } else if (in_opc == Op_Halt) { 3032 if (in->in(0)->is_Region()) { 3033 #ifdef ASSERT 3034 Node* r = in->in(0); 3035 for (uint j = 1; j < r->req(); j++) { 3036 assert(r->in(j)->is_Proj() && r->in(j)->in(0)->Opcode() == Op_NeverBranch, ""); 3037 } 3038 #endif 3039 } else { 3040 Node* proj = in->in(0); 3041 assert(proj->is_Proj(), ""); 3042 Node* in = proj->in(0); 3043 assert(in->is_CallStaticJava() || in->Opcode() == Op_NeverBranch || in->Opcode() == Op_Catch, ""); 3044 if (in->is_CallStaticJava()) { 3045 mem = in->in(TypeFunc::Memory); 3046 } else if (in->Opcode() == Op_Catch) { 3047 Node* call = in->in(0)->in(0); 3048 assert(call->is_Call(), ""); 3049 mem = call->in(TypeFunc::Memory); 3050 } 3051 } 3052 } else { 3053 #ifdef ASSERT 3054 n->dump(); 3055 in->dump(); 3056 #endif 3057 ShouldNotReachHere(); 3058 } 3059 } else { 3060 assert(n->is_Phi() && n->bottom_type() == Type::MEMORY, ""); 3061 assert(n->adr_type() == TypePtr::BOTTOM || phase->C->get_alias_index(n->adr_type()) == alias, ""); 3062 mem = n->in(i); 3063 } 3064 i++; 3065 stack.set_index(i); 3066 if (mem == NULL) { 3067 continue; 3068 } 3069 for (;;) { 3070 if (visited.test_set(mem->_idx) || mem->is_Start()) { 3071 break; 3072 } 3073 if (mem->is_Phi()) { 3074 stack.push(mem, 2); 3075 mem = mem->in(1); 3076 } else if (mem->is_Proj()) { 3077 stack.push(mem, mem->req()); 3078 mem = mem->in(0); 3079 } else if (mem->is_SafePoint() || mem->is_MemBar()) { 3080 mem = mem->in(TypeFunc::Memory); 3081 } else if (mem->is_MergeMem()) { 3082 mem = mem->as_MergeMem()->memory_at(alias); 3083 } else if (mem->is_Store() || mem->is_LoadStore() || mem->is_ClearArray()) { 3084 stack.push(mem, mem->req()); 3085 mem = mem->in(MemNode::Memory); 3086 } else { 3087 #ifdef ASSERT 3088 mem->dump(); 3089 #endif 3090 ShouldNotReachHere(); 3091 } 3092 } 3093 } else { 3094 if (n->is_Phi()) { 3095 // Nothing 3096 } else if (!n->is_Root()) { 3097 Node* c = get_ctrl(n, phase); 3098 memory_nodes.map(c->_idx, n); 3099 } 3100 stack.pop(); 3101 } 3102 } while(stack.is_nonempty()); 3103 3104 // Iterate over CFG nodes in rpo and propagate memory state to 3105 // compute memory state at regions, creating new phis if needed. 3106 Node_List rpo_list; 3107 visited.Clear(); 3108 phase->rpo(phase->C->root(), stack, visited, rpo_list); 3109 Node* root = rpo_list.pop(); 3110 assert(root == phase->C->root(), ""); 3111 3112 const bool trace = false; 3113 #ifdef ASSERT 3114 if (trace) { 3115 for (int i = rpo_list.size() - 1; i >= 0; i--) { 3116 Node* c = rpo_list.at(i); 3117 if (memory_nodes[c->_idx] != NULL) { 3118 tty->print("X %d", c->_idx); memory_nodes[c->_idx]->dump(); 3119 } 3120 } 3121 } 3122 #endif 3123 uint last = phase->C->unique(); 3124 3125 #ifdef ASSERT 3126 uint8_t max_depth = 0; 3127 for (LoopTreeIterator iter(phase->ltree_root()); !iter.done(); iter.next()) { 3128 IdealLoopTree* lpt = iter.current(); 3129 max_depth = MAX2(max_depth, lpt->_nest); 3130 } 3131 #endif 3132 3133 bool progress = true; 3134 int iteration = 0; 3135 Node_List dead_phis; 3136 while (progress) { 3137 progress = false; 3138 iteration++; 3139 assert(iteration <= 2+max_depth || phase->C->has_irreducible_loop(), ""); 3140 if (trace) { tty->print_cr("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"); } 3141 IdealLoopTree* last_updated_ilt = NULL; 3142 for (int i = rpo_list.size() - 1; i >= 0; i--) { 3143 Node* c = rpo_list.at(i); 3144 3145 Node* prev_mem = memory_nodes[c->_idx]; 3146 if (c->is_Region()) { 3147 Node* prev_region = regions[c->_idx]; 3148 Node* unique = NULL; 3149 for (uint j = 1; j < c->req() && unique != NodeSentinel; j++) { 3150 Node* m = memory_nodes[c->in(j)->_idx]; 3151 assert(m != NULL || (c->is_Loop() && j == LoopNode::LoopBackControl && iteration == 1) || phase->C->has_irreducible_loop() || has_never_branch(phase->C->root()), "expect memory state"); 3152 if (m != NULL) { 3153 if (m == prev_region && ((c->is_Loop() && j == LoopNode::LoopBackControl) || (prev_region->is_Phi() && prev_region->in(0) == c))) { 3154 assert(c->is_Loop() && j == LoopNode::LoopBackControl || phase->C->has_irreducible_loop(), ""); 3155 // continue 3156 } else if (unique == NULL) { 3157 unique = m; 3158 } else if (m == unique) { 3159 // continue 3160 } else { 3161 unique = NodeSentinel; 3162 } 3163 } 3164 } 3165 assert(unique != NULL, "empty phi???"); 3166 if (unique != NodeSentinel) { 3167 if (prev_region != NULL && prev_region->is_Phi() && prev_region->in(0) == c) { 3168 dead_phis.push(prev_region); 3169 } 3170 regions.map(c->_idx, unique); 3171 } else { 3172 Node* phi = NULL; 3173 if (prev_region != NULL && prev_region->is_Phi() && prev_region->in(0) == c && prev_region->_idx >= last) { 3174 phi = prev_region; 3175 for (uint k = 1; k < c->req(); k++) { 3176 Node* m = memory_nodes[c->in(k)->_idx]; 3177 assert(m != NULL, "expect memory state"); 3178 phi->set_req(k, m); 3179 } 3180 } else { 3181 for (DUIterator_Fast jmax, j = c->fast_outs(jmax); j < jmax && phi == NULL; j++) { 3182 Node* u = c->fast_out(j); 3183 if (u->is_Phi() && u->bottom_type() == Type::MEMORY && 3184 (u->adr_type() == TypePtr::BOTTOM || phase->C->get_alias_index(u->adr_type()) == alias)) { 3185 phi = u; 3186 for (uint k = 1; k < c->req() && phi != NULL; k++) { 3187 Node* m = memory_nodes[c->in(k)->_idx]; 3188 assert(m != NULL, "expect memory state"); 3189 if (u->in(k) != m) { 3190 phi = NULL; 3191 } 3192 } 3193 } 3194 } 3195 if (phi == NULL) { 3196 phi = new PhiNode(c, Type::MEMORY, phase->C->get_adr_type(alias)); 3197 for (uint k = 1; k < c->req(); k++) { 3198 Node* m = memory_nodes[c->in(k)->_idx]; 3199 assert(m != NULL, "expect memory state"); 3200 phi->init_req(k, m); 3201 } 3202 } 3203 } 3204 assert(phi != NULL, ""); 3205 regions.map(c->_idx, phi); 3206 } 3207 Node* current_region = regions[c->_idx]; 3208 if (current_region != prev_region) { 3209 progress = true; 3210 if (prev_region == prev_mem) { 3211 memory_nodes.map(c->_idx, current_region); 3212 } 3213 } 3214 } else if (prev_mem == NULL || prev_mem->is_Phi() || ctrl_or_self(prev_mem, phase) != c) { 3215 Node* m = memory_nodes[phase->idom(c)->_idx]; 3216 assert(m != NULL, "expect memory state"); 3217 if (m != prev_mem) { 3218 memory_nodes.map(c->_idx, m); 3219 progress = true; 3220 } 3221 } 3222 #ifdef ASSERT 3223 if (trace) { tty->print("X %d", c->_idx); memory_nodes[c->_idx]->dump(); } 3224 #endif 3225 } 3226 } 3227 3228 // Replace existing phi with computed memory state for that region 3229 // if different (could be a new phi or a dominating memory node if 3230 // that phi was found to be useless). 3231 while (dead_phis.size() > 0) { 3232 Node* n = dead_phis.pop(); 3233 n->replace_by(phase->C->top()); 3234 n->destruct(); 3235 } 3236 for (int i = rpo_list.size() - 1; i >= 0; i--) { 3237 Node* c = rpo_list.at(i); 3238 if (c->is_Region()) { 3239 Node* n = regions[c->_idx]; 3240 if (n->is_Phi() && n->_idx >= last && n->in(0) == c) { 3241 phase->register_new_node(n, c); 3242 } 3243 } 3244 } 3245 for (int i = rpo_list.size() - 1; i >= 0; i--) { 3246 Node* c = rpo_list.at(i); 3247 if (c->is_Region()) { 3248 Node* n = regions[c->_idx]; 3249 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) { 3250 Node* u = c->fast_out(i); 3251 if (u->is_Phi() && u->bottom_type() == Type::MEMORY && 3252 u != n) { 3253 if (u->adr_type() == TypePtr::BOTTOM) { 3254 fix_memory_uses(u, n, n, c, alias, phase); 3255 } else if (phase->C->get_alias_index(u->adr_type()) == alias) { 3256 phase->lazy_replace(u, n); 3257 --i; --imax; 3258 } 3259 } 3260 } 3261 } 3262 } 3263 } 3264 3265 void ShenandoahWriteBarrierNode::fix_raw_mem(Node* ctrl, Node* region, Node* raw_mem, Node* raw_mem_for_ctrl, Node* raw_mem_phi, 3266 Node_List& memory_nodes, Unique_Node_List& uses, PhaseIdealLoop* phase) { 3267 const bool trace = false; 3268 DEBUG_ONLY(if (trace) { tty->print("ZZZ control is"); ctrl->dump(); }); 3269 DEBUG_ONLY(if (trace) { tty->print("ZZZ mem is"); raw_mem->dump(); }); 3270 GrowableArray<Node*> phis; 3271 if (raw_mem_for_ctrl != raw_mem) { 3272 Node* old = raw_mem_for_ctrl; 3273 Node* prev = NULL; 3274 while (old != raw_mem) { 3275 assert(old->is_Store() || old->is_LoadStore() || old->is_ClearArray(), ""); 3276 prev = old; 3277 old = old->in(MemNode::Memory); 3278 } 3279 assert(prev != NULL, ""); 3280 memory_nodes.map(ctrl->_idx, raw_mem); 3281 memory_nodes.map(region->_idx, raw_mem_for_ctrl); 3282 phase->igvn().replace_input_of(prev, MemNode::Memory, raw_mem_phi); 3283 } else { 3284 memory_nodes.map(region->_idx, raw_mem_phi); 3285 uses.clear(); 3286 uses.push(region); 3287 for(uint next = 0; next < uses.size(); next++ ) { 3288 Node *n = uses.at(next); 3289 assert(n->is_CFG(), ""); 3290 DEBUG_ONLY(if (trace) { tty->print("ZZZ ctrl"); n->dump(); }); 3291 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { 3292 Node* u = n->fast_out(i); 3293 if (!u->is_Root() && u->is_CFG() && u != n) { 3294 Node* m = memory_nodes[u->_idx]; 3295 if (u->is_Region() && !has_mem_phi(phase->C, u, Compile::AliasIdxRaw)) { 3296 DEBUG_ONLY(if (trace) { tty->print("ZZZ region"); u->dump(); }); 3297 DEBUG_ONLY(if (trace && m != NULL) { tty->print("ZZZ mem"); m->dump(); }); 3298 3299 if (!mem_is_valid(m, u, phase) || !m->is_Phi()) { 3300 bool push = true; 3301 bool create_phi = true; 3302 if (phase->is_dominator(region, u)) { 3303 create_phi = false; 3304 } else if (!phase->C->has_irreducible_loop()) { 3305 IdealLoopTree* loop = phase->get_loop(ctrl); 3306 bool do_check = true; 3307 IdealLoopTree* l = loop; 3308 create_phi = false; 3309 while (l != phase->ltree_root()) { 3310 if (phase->is_dominator(l->_head, u) && phase->is_dominator(phase->idom(u), l->_head)) { 3311 create_phi = true; 3312 do_check = false; 3313 break; 3314 } 3315 l = l->_parent; 3316 } 3317 3318 if (do_check) { 3319 assert(!create_phi, ""); 3320 IdealLoopTree* u_loop = phase->get_loop(u); 3321 if (u_loop != phase->ltree_root() && u_loop->is_member(loop)) { 3322 Node* c = ctrl; 3323 while (!phase->is_dominator(c, u_loop->tail())) { 3324 c = phase->idom(c); 3325 } 3326 if (!phase->is_dominator(c, u)) { 3327 do_check = false; 3328 } 3329 } 3330 } 3331 3332 if (do_check && phase->is_dominator(phase->idom(u), region)) { 3333 create_phi = true; 3334 } 3335 } 3336 if (create_phi) { 3337 Node* phi = new PhiNode(u, Type::MEMORY, TypeRawPtr::BOTTOM); 3338 phase->register_new_node(phi, u); 3339 phis.push(phi); 3340 DEBUG_ONLY(if (trace) { tty->print("ZZZ new phi"); phi->dump(); }); 3341 if (!mem_is_valid(m, u, phase)) { 3342 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting mem"); phi->dump(); }); 3343 memory_nodes.map(u->_idx, phi); 3344 } else { 3345 DEBUG_ONLY(if (trace) { tty->print("ZZZ NOT setting mem"); m->dump(); }); 3346 for (;;) { 3347 assert(m->is_Mem() || m->is_LoadStore() || m->is_Proj() /*|| m->is_MergeMem()*/, ""); 3348 Node* next = NULL; 3349 if (m->is_Proj()) { 3350 next = m->in(0); 3351 } else { 3352 next = m->in(MemNode::Memory); 3353 } 3354 if (phase->get_ctrl(next) != u) { 3355 break; 3356 } 3357 if (next->is_MergeMem()) { 3358 assert(phase->get_ctrl(next->as_MergeMem()->memory_at(Compile::AliasIdxRaw)) != u, ""); 3359 break; 3360 } 3361 if (next->is_Phi()) { 3362 assert(next->adr_type() == TypePtr::BOTTOM && next->in(0) == u, ""); 3363 break; 3364 } 3365 m = next; 3366 } 3367 3368 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting to phi"); m->dump(); }); 3369 assert(m->is_Mem() || m->is_LoadStore(), ""); 3370 phase->igvn().replace_input_of(m, MemNode::Memory, phi); 3371 push = false; 3372 } 3373 } else { 3374 DEBUG_ONLY(if (trace) { tty->print("ZZZ skipping region"); u->dump(); }); 3375 } 3376 if (push) { 3377 uses.push(u); 3378 } 3379 } 3380 } else if (!mem_is_valid(m, u, phase)) { 3381 uses.push(u); 3382 } 3383 } 3384 } 3385 } 3386 for (int i = 0; i < phis.length(); i++) { 3387 Node* n = phis.at(i); 3388 Node* r = n->in(0); 3389 DEBUG_ONLY(if (trace) { tty->print("ZZZ fixing new phi"); n->dump(); }); 3390 for (uint j = 1; j < n->req(); j++) { 3391 Node* m = find_raw_mem(r->in(j), NULL, memory_nodes, phase); 3392 phase->igvn().replace_input_of(n, j, m); 3393 DEBUG_ONLY(if (trace) { tty->print("ZZZ fixing new phi: %d", j); m->dump(); }); 3394 } 3395 } 3396 } 3397 uint last = phase->C->unique(); 3398 MergeMemNode* mm = NULL; 3399 int alias = Compile::AliasIdxRaw; 3400 DEBUG_ONLY(if (trace) { tty->print("ZZZ raw mem is"); raw_mem->dump(); }); 3401 for (DUIterator i = raw_mem->outs(); raw_mem->has_out(i); i++) { 3402 Node* u = raw_mem->out(i); 3403 if (u->_idx < last) { 3404 if (u->is_Mem()) { 3405 if (phase->C->get_alias_index(u->adr_type()) == alias) { 3406 Node* m = find_raw_mem(phase->get_ctrl(u), u, memory_nodes, phase); 3407 if (m != raw_mem) { 3408 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting memory of use"); u->dump(); }); 3409 phase->igvn().replace_input_of(u, MemNode::Memory, m); 3410 --i; 3411 } 3412 } 3413 } else if (u->is_MergeMem()) { 3414 MergeMemNode* u_mm = u->as_MergeMem(); 3415 if (u_mm->memory_at(alias) == raw_mem) { 3416 MergeMemNode* newmm = NULL; 3417 for (DUIterator_Fast jmax, j = u->fast_outs(jmax); j < jmax; j++) { 3418 Node* uu = u->fast_out(j); 3419 assert(!uu->is_MergeMem(), "chain of MergeMems?"); 3420 if (uu->is_Phi()) { 3421 assert(uu->adr_type() == TypePtr::BOTTOM, ""); 3422 Node* region = uu->in(0); 3423 int nb = 0; 3424 for (uint k = 1; k < uu->req(); k++) { 3425 if (uu->in(k) == u) { 3426 Node* m = find_raw_mem(region->in(k), NULL, memory_nodes, phase); 3427 if (m != raw_mem) { 3428 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting memory of phi %d", k); uu->dump(); }); 3429 if (newmm == NULL || 1) { 3430 newmm = clone_merge_mem(u, raw_mem, alias, m, phase->ctrl_or_self(m), i, phase); 3431 } 3432 if (newmm != u) { 3433 phase->igvn().replace_input_of(uu, k, newmm); 3434 nb++; 3435 --jmax; 3436 } 3437 } 3438 } 3439 } 3440 if (nb > 0) { 3441 --j; 3442 } 3443 } else { 3444 Node* m = find_raw_mem(phase->ctrl_or_self(uu), uu, memory_nodes, phase); 3445 if (m != raw_mem) { 3446 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting memory of use"); uu->dump(); }); 3447 if (newmm == NULL || 1) { 3448 newmm = clone_merge_mem(u, raw_mem, alias, m, phase->ctrl_or_self(m), i, phase); 3449 } 3450 if (newmm != u) { 3451 phase->igvn().replace_input_of(uu, uu->find_edge(u), newmm); 3452 --j, --jmax; 3453 } 3454 } 3455 } 3456 } 3457 } 3458 } else if (u->is_Phi()) { 3459 assert(u->bottom_type() == Type::MEMORY, "what else?"); 3460 if (u->adr_type() == TypeRawPtr::BOTTOM || u->adr_type() == TypePtr::BOTTOM) { 3461 Node* region = u->in(0); 3462 bool replaced = false; 3463 for (uint j = 1; j < u->req(); j++) { 3464 if (u->in(j) == raw_mem) { 3465 Node* m = find_raw_mem(region->in(j), NULL, memory_nodes, phase); 3466 Node* nnew = m; 3467 if (m != raw_mem) { 3468 if (u->adr_type() == TypePtr::BOTTOM) { 3469 if (mm == NULL || 1) { 3470 mm = allocate_merge_mem(raw_mem, alias, m, phase->ctrl_or_self(m), phase); 3471 } 3472 nnew = mm; 3473 } 3474 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting memory of phi %d", j); u->dump(); }); 3475 phase->igvn().replace_input_of(u, j, nnew); 3476 replaced = true; 3477 } 3478 } 3479 } 3480 if (replaced) { 3481 --i; 3482 } 3483 } 3484 } else if ((u->adr_type() == TypePtr::BOTTOM && u->Opcode() != Op_StrInflatedCopy) || 3485 u->adr_type() == NULL) { 3486 assert(u->adr_type() != NULL || 3487 u->Opcode() == Op_Rethrow || 3488 u->Opcode() == Op_Return || 3489 u->Opcode() == Op_SafePoint || 3490 (u->is_CallStaticJava() && u->as_CallStaticJava()->uncommon_trap_request() != 0) || 3491 (u->is_CallStaticJava() && u->as_CallStaticJava()->_entry_point == OptoRuntime::rethrow_stub()) || 3492 u->Opcode() == Op_CallLeaf, ""); 3493 Node* m = find_raw_mem(phase->ctrl_or_self(u), u, memory_nodes, phase); 3494 if (m != raw_mem) { 3495 if (mm == NULL || 1) { 3496 mm = allocate_merge_mem(raw_mem, alias, m, phase->get_ctrl(m), phase); 3497 } 3498 phase->igvn().replace_input_of(u, u->find_edge(raw_mem), mm); 3499 --i; 3500 } 3501 } else if (phase->C->get_alias_index(u->adr_type()) == alias) { 3502 Node* m = find_raw_mem(phase->ctrl_or_self(u), u, memory_nodes, phase); 3503 if (m != raw_mem) { 3504 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting memory of use"); u->dump(); }); 3505 phase->igvn().replace_input_of(u, u->find_edge(raw_mem), m); 3506 --i; 3507 } 3508 } 3509 } 3510 } 3511 #ifdef ASSERT 3512 assert(raw_mem_phi->outcnt() > 0, ""); 3513 for (int i = 0; i < phis.length(); i++) { 3514 Node* n = phis.at(i); 3515 assert(n->outcnt() > 0, "new phi must have uses now"); 3516 } 3517 #endif 3518 } 3519 3520 void ShenandoahWriteBarrierNode::test_evacuation_in_progress(Node* ctrl, int alias, Node*& raw_mem, Node*& wb_mem, 3521 IfNode*& evacuation_iff, Node*& evac_in_progress, 3522 Node*& evac_not_in_progress, PhaseIdealLoop* phase) { 3523 IdealLoopTree *loop = phase->get_loop(ctrl); 3524 Node* thread = new ThreadLocalNode(); 3525 phase->register_new_node(thread, ctrl); 3526 Node* offset = phase->igvn().MakeConX(in_bytes(JavaThread::gc_state_offset())); 3527 phase->set_ctrl(offset, phase->C->root()); 3528 Node* gc_state_addr = new AddPNode(phase->C->top(), thread, offset); 3529 phase->register_new_node(gc_state_addr, ctrl); 3530 uint gc_state_idx = Compile::AliasIdxRaw; 3531 const TypePtr* gc_state_adr_type = NULL; // debug-mode-only argument 3532 debug_only(gc_state_adr_type = phase->C->get_adr_type(gc_state_idx)); 3533 3534 Node* gc_state = new LoadUBNode(ctrl, raw_mem, gc_state_addr, gc_state_adr_type, TypeInt::BYTE, MemNode::unordered); 3535 phase->register_new_node(gc_state, ctrl); 3536 3537 if (ShenandoahWriteBarrierMemBar) { 3538 Node* mb = MemBarNode::make(phase->C, Op_MemBarAcquire, Compile::AliasIdxRaw); 3539 mb->init_req(TypeFunc::Control, ctrl); 3540 mb->init_req(TypeFunc::Memory, raw_mem); 3541 phase->register_control(mb, loop, ctrl); 3542 Node* ctrl_proj = new ProjNode(mb,TypeFunc::Control); 3543 phase->register_control(ctrl_proj, loop, mb); 3544 raw_mem = new ProjNode(mb, TypeFunc::Memory); 3545 phase->register_new_node(raw_mem, mb); 3546 3547 mb = MemBarNode::make(phase->C, Op_MemBarAcquire, alias); 3548 mb->init_req(TypeFunc::Control, ctrl_proj); 3549 mb->init_req(TypeFunc::Memory, wb_mem); 3550 phase->register_control(mb, loop, ctrl_proj); 3551 ctrl_proj = new ProjNode(mb,TypeFunc::Control); 3552 phase->register_control(ctrl_proj, loop, mb); 3553 wb_mem = new ProjNode(mb,TypeFunc::Memory); 3554 phase->register_new_node(wb_mem, mb); 3555 3556 ctrl = ctrl_proj; 3557 } 3558 3559 Node* evacuation_in_progress = gc_state; 3560 evacuation_in_progress = new AndINode(gc_state, phase->igvn().intcon(ShenandoahHeap::EVACUATION)); 3561 phase->register_new_node(evacuation_in_progress, ctrl); 3562 Node* evacuation_in_progress_cmp = new CmpINode(evacuation_in_progress, phase->igvn().zerocon(T_INT)); 3563 phase->register_new_node(evacuation_in_progress_cmp, ctrl); 3564 Node* evacuation_in_progress_test = new BoolNode(evacuation_in_progress_cmp, BoolTest::ne); 3565 phase->register_new_node(evacuation_in_progress_test, ctrl); 3566 evacuation_iff = new IfNode(ctrl, evacuation_in_progress_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN); 3567 phase->register_control(evacuation_iff, loop, ctrl); 3568 3569 assert(is_evacuation_in_progress_test(evacuation_iff), "Should match the shape"); 3570 assert(is_gc_state_load(gc_state), "Should match the shape"); 3571 3572 evac_not_in_progress = new IfFalseNode(evacuation_iff); 3573 phase->register_control(evac_not_in_progress, loop, evacuation_iff); 3574 evac_in_progress = new IfTrueNode(evacuation_iff); 3575 phase->register_control(evac_in_progress, loop, evacuation_iff); 3576 } 3577 3578 void ShenandoahWriteBarrierNode::evacuation_not_in_progress_null_check(Node*& c, Node*& val, Node* unc_ctrl, Node*& unc_region, PhaseIdealLoop* phase) { 3579 if (unc_ctrl != NULL) { 3580 // Clone the null check in this branch to allow implicit null check 3581 IdealLoopTree *loop = phase->get_loop(c); 3582 Node* iff = unc_ctrl->in(0); 3583 assert(iff->is_If(), "broken"); 3584 Node* new_iff = iff->clone(); 3585 new_iff->set_req(0, c); 3586 phase->register_control(new_iff, loop, c); 3587 Node* iffalse = new IfFalseNode(new_iff->as_If()); 3588 phase->register_control(iffalse, loop, new_iff); 3589 Node* iftrue = new IfTrueNode(new_iff->as_If()); 3590 phase->register_control(iftrue, loop, new_iff); 3591 c = iftrue; 3592 unc_region = new RegionNode(3); 3593 unc_region->init_req(1, iffalse); 3594 const Type *t = phase->igvn().type(val); 3595 assert(val->Opcode() == Op_CastPP, "expect cast to non null here"); 3596 Node* uncasted_val = val->in(1); 3597 val = new CastPPNode(uncasted_val, t); 3598 val->init_req(0, c); 3599 phase->register_new_node(val, c); 3600 } 3601 } 3602 3603 void ShenandoahWriteBarrierNode::evacuation_not_in_progress(Node* c, Node* val, Node* unc_ctrl, Node* raw_mem, Node* wb_mem, Node* region, 3604 Node* val_phi, Node* mem_phi, Node* raw_mem_phi, Node*& unc_region, PhaseIdealLoop* phase) { 3605 evacuation_not_in_progress_null_check(c, val, unc_ctrl, unc_region, phase); 3606 region->init_req(1, c); 3607 if (ShenandoahWriteBarrierRB) { 3608 Node* rbfalse = new ShenandoahReadBarrierNode(c, wb_mem, val); 3609 phase->register_new_node(rbfalse, c); 3610 val_phi->init_req(1, rbfalse); 3611 } else { 3612 val_phi->init_req(1, val); 3613 } 3614 mem_phi->init_req(1, wb_mem); 3615 raw_mem_phi->init_req(1, raw_mem); 3616 } 3617 3618 void ShenandoahWriteBarrierNode::evacuation_in_progress_null_check(Node*& c, Node*& val, Node* evacuation_iff, Node* unc, Node* unc_ctrl, 3619 Node* unc_region, Unique_Node_List& uses, PhaseIdealLoop* phase) { 3620 if (unc != NULL) { 3621 // Clone the null check in this branch to allow implicit null check 3622 IdealLoopTree *loop = phase->get_loop(c); 3623 Node* iff = unc_ctrl->in(0); 3624 assert(iff->is_If(), "broken"); 3625 Node* new_iff = iff->clone(); 3626 new_iff->set_req(0, c); 3627 phase->register_control(new_iff, loop, c); 3628 Node* iffalse = new IfFalseNode(new_iff->as_If()); 3629 phase->register_control(iffalse, loop, new_iff); 3630 Node* iftrue = new IfTrueNode(new_iff->as_If()); 3631 phase->register_control(iftrue, loop, new_iff); 3632 c = iftrue; 3633 unc_region->init_req(2, iffalse); 3634 3635 Node* proj = iff->as_If()->proj_out(0); 3636 assert(proj != unc_ctrl, "bad projection"); 3637 Node* use = proj->unique_ctrl_out(); 3638 3639 assert(use == unc || use->is_Region(), "what else?"); 3640 3641 uses.clear(); 3642 if (use == unc) { 3643 phase->set_idom(use, unc_region, phase->dom_depth(unc_region)+1); 3644 for (uint i = 1; i < unc->req(); i++) { 3645 Node* n = unc->in(i); 3646 if (phase->has_ctrl(n) && phase->get_ctrl(n) == proj) { 3647 uses.push(n); 3648 } 3649 } 3650 } else { 3651 assert(use->is_Region(), "what else?"); 3652 uint idx = 1; 3653 for (; use->in(idx) != proj; idx++); 3654 for (DUIterator_Fast imax, i = use->fast_outs(imax); i < imax; i++) { 3655 Node* u = use->fast_out(i); 3656 if (u->is_Phi() && phase->get_ctrl(u->in(idx)) == proj) { 3657 uses.push(u->in(idx)); 3658 } 3659 } 3660 } 3661 for(uint next = 0; next < uses.size(); next++ ) { 3662 Node *n = uses.at(next); 3663 assert(phase->get_ctrl(n) == proj, "bad control"); 3664 phase->set_ctrl_and_loop(n, unc_region); 3665 if (n->in(0) == proj) { 3666 phase->igvn().replace_input_of(n, 0, unc_region); 3667 } 3668 for (uint i = 0; i < n->req(); i++) { 3669 Node* m = n->in(i); 3670 if (m != NULL && phase->has_ctrl(m) && phase->get_ctrl(m) == proj) { 3671 uses.push(m); 3672 } 3673 } 3674 } 3675 3676 phase->igvn().rehash_node_delayed(use); 3677 int nb = use->replace_edge(proj, unc_region); 3678 assert(nb == 1, "only use expected"); 3679 phase->register_control(unc_region, phase->ltree_root(), evacuation_iff); 3680 3681 phase->igvn().replace_input_of(iff, 1, phase->igvn().intcon(1)); 3682 const Type *t = phase->igvn().type(val); 3683 assert(val->Opcode() == Op_CastPP, "expect cast to non null here"); 3684 Node* uncasted_val = val->in(1); 3685 val = new CastPPNode(uncasted_val, t); 3686 val->init_req(0, c); 3687 phase->register_new_node(val, c); 3688 } 3689 } 3690 3691 void ShenandoahWriteBarrierNode::in_cset_fast_test(Node*& c, Node* rbtrue, Node* raw_mem, Node* wb_mem, Node* region, Node* val_phi, Node* mem_phi, 3692 Node* raw_mem_phi, PhaseIdealLoop* phase) { 3693 if (ShenandoahWriteBarrierCsetTestInIR) { 3694 IdealLoopTree *loop = phase->get_loop(c); 3695 Node* raw_rbtrue = new CastP2XNode(c, rbtrue); 3696 phase->register_new_node(raw_rbtrue, c); 3697 Node* cset_offset = new URShiftXNode(raw_rbtrue, phase->igvn().intcon(ShenandoahHeapRegion::region_size_bytes_shift_jint())); 3698 phase->register_new_node(cset_offset, c); 3699 Node* in_cset_fast_test_base_addr = phase->igvn().makecon(TypeRawPtr::make(ShenandoahHeap::in_cset_fast_test_addr())); 3700 phase->set_ctrl(in_cset_fast_test_base_addr, phase->C->root()); 3701 Node* in_cset_fast_test_adr = new AddPNode(phase->C->top(), in_cset_fast_test_base_addr, cset_offset); 3702 phase->register_new_node(in_cset_fast_test_adr, c); 3703 uint in_cset_fast_test_idx = Compile::AliasIdxRaw; 3704 const TypePtr* in_cset_fast_test_adr_type = NULL; // debug-mode-only argument 3705 debug_only(in_cset_fast_test_adr_type = phase->C->get_adr_type(in_cset_fast_test_idx)); 3706 Node* in_cset_fast_test_load = new LoadUBNode(c, raw_mem, in_cset_fast_test_adr, in_cset_fast_test_adr_type, TypeInt::BOOL, MemNode::unordered); 3707 phase->register_new_node(in_cset_fast_test_load, c); 3708 Node* in_cset_fast_test_cmp = new CmpINode(in_cset_fast_test_load, phase->igvn().zerocon(T_INT)); 3709 phase->register_new_node(in_cset_fast_test_cmp, c); 3710 Node* in_cset_fast_test_test = new BoolNode(in_cset_fast_test_cmp, BoolTest::ne); 3711 phase->register_new_node(in_cset_fast_test_test, c); 3712 IfNode* in_cset_fast_test_iff = new IfNode(c, in_cset_fast_test_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN); 3713 phase->register_control(in_cset_fast_test_iff, loop, c); 3714 3715 Node* in_cset_fast_test_success = new IfFalseNode(in_cset_fast_test_iff); 3716 phase->register_control(in_cset_fast_test_success, loop, in_cset_fast_test_iff); 3717 3718 region->init_req(3, in_cset_fast_test_success); 3719 val_phi->init_req(3, rbtrue); 3720 mem_phi->init_req(3, wb_mem); 3721 raw_mem_phi->init_req(3, raw_mem); 3722 3723 Node* in_cset_fast_test_failure = new IfTrueNode(in_cset_fast_test_iff); 3724 phase->register_control(in_cset_fast_test_failure, loop, in_cset_fast_test_iff); 3725 3726 c = in_cset_fast_test_failure; 3727 } 3728 } 3729 3730 void ShenandoahWriteBarrierNode::evacuation_in_progress(Node* c, Node* val, Node* evacuation_iff, Node* unc, Node* unc_ctrl, 3731 Node* raw_mem, Node* wb_mem, Node* region, Node* val_phi, Node* mem_phi, 3732 Node* raw_mem_phi, Node* unc_region, int alias, Unique_Node_List& uses, 3733 PhaseIdealLoop* phase) { 3734 evacuation_in_progress_null_check(c, val, evacuation_iff, unc, unc_ctrl, unc_region, uses, phase); 3735 3736 IdealLoopTree *loop = phase->get_loop(c); 3737 Node* rbtrue = new ShenandoahReadBarrierNode(c, wb_mem, val); 3738 phase->register_new_node(rbtrue, c); 3739 3740 Node* in_cset_fast_test_failure = NULL; 3741 in_cset_fast_test(c, rbtrue, raw_mem, wb_mem, region, val_phi, mem_phi, raw_mem_phi, phase); 3742 3743 // The slow path stub consumes and produces raw memory in addition 3744 // to the existing memory edges 3745 Node* base = find_bottom_mem(c, phase); 3746 3747 MergeMemNode* mm = MergeMemNode::make(base); 3748 mm->set_memory_at(alias, wb_mem); 3749 mm->set_memory_at(Compile::AliasIdxRaw, raw_mem); 3750 phase->register_new_node(mm, c); 3751 3752 Node* call = new CallLeafNoFPNode(OptoRuntime::shenandoah_write_barrier_Type(), StubRoutines::shenandoah_wb_C(), "shenandoah_write_barrier", TypeRawPtr::BOTTOM); 3753 call->init_req(TypeFunc::Control, c); 3754 call->init_req(TypeFunc::I_O, phase->C->top()); 3755 call->init_req(TypeFunc::Memory, mm); 3756 call->init_req(TypeFunc::FramePtr, phase->C->top()); 3757 call->init_req(TypeFunc::ReturnAdr, phase->C->top()); 3758 call->init_req(TypeFunc::Parms, rbtrue); 3759 phase->register_control(call, loop, c); 3760 Node* ctrl_proj = new ProjNode(call, TypeFunc::Control); 3761 phase->register_control(ctrl_proj, loop, call); 3762 Node* mem_proj = new ProjNode(call, TypeFunc::Memory); 3763 phase->register_new_node(mem_proj, call); 3764 Node* res_proj = new ProjNode(call, TypeFunc::Parms); 3765 phase->register_new_node(res_proj, call); 3766 Node* res = new CheckCastPPNode(ctrl_proj, res_proj, phase->igvn().type(val)->is_oopptr()->cast_to_nonconst()); 3767 phase->register_new_node(res, ctrl_proj); 3768 region->init_req(2, ctrl_proj); 3769 val_phi->init_req(2, res); 3770 mem_phi->init_req(2, mem_proj); 3771 raw_mem_phi->init_req(2, mem_proj); 3772 phase->register_control(region, loop, evacuation_iff); 3773 3774 } 3775 3776 void ShenandoahWriteBarrierNode::pin_and_expand(PhaseIdealLoop* phase) { 3777 const bool trace = false; 3778 3779 // Collect raw memory state at CFG points in the entire graph and 3780 // record it in memory_nodes. Optimize the raw memory graph in the 3781 // process. Optimizing the memory graph also makes the memory graph 3782 // simpler. 3783 Node_List memory_nodes; 3784 collect_memory_nodes(Compile::AliasIdxRaw, memory_nodes, phase); 3785 3786 // Let's try to common write barriers again 3787 for (;;) { 3788 bool progress = false; 3789 for (int i = phase->C->shenandoah_barriers_count(); i > 0; i--) { 3790 ShenandoahBarrierNode* wb = phase->C->shenandoah_barrier(i-1); 3791 Node* ctrl = phase->get_ctrl(wb); 3792 if (wb->try_common(ctrl, phase) != NULL) { 3793 progress = true; 3794 } 3795 } 3796 if (!progress) { 3797 break; 3798 } 3799 } 3800 3801 for (int i = 0; i < phase->C->shenandoah_barriers_count(); i++) { 3802 ShenandoahWriteBarrierNode* wb = phase->C->shenandoah_barrier(i); 3803 Node* ctrl = phase->get_ctrl(wb); 3804 3805 Node* val = wb->in(ValueIn); 3806 if (ctrl->is_Proj() && ctrl->in(0)->is_CallJava()) { 3807 assert(is_dominator(phase->get_ctrl(val), ctrl->in(0)->in(0), val, ctrl->in(0), phase), "can't move"); 3808 phase->set_ctrl(wb, ctrl->in(0)->in(0)); 3809 } else if (ctrl->is_CallRuntime()) { 3810 assert(is_dominator(phase->get_ctrl(val), ctrl->in(0), val, ctrl, phase), "can't move"); 3811 phase->set_ctrl(wb, ctrl->in(0)); 3812 } 3813 3814 assert(wb->Opcode() == Op_ShenandoahWriteBarrier, "only for write barriers"); 3815 // Look for a null check that dominates this barrier and move the 3816 // barrier right after the null check to enable implicit null 3817 // checks 3818 wb->pin_and_expand_move_barrier(phase); 3819 3820 ctrl = phase->get_ctrl(wb); 3821 wb->pin_and_expand_helper(phase); 3822 } 3823 3824 Unique_Node_List uses; 3825 Unique_Node_List uses_to_ignore; 3826 for (int i = phase->C->shenandoah_barriers_count(); i > 0; i--) { 3827 int cnt = phase->C->shenandoah_barriers_count(); 3828 ShenandoahWriteBarrierNode* wb = phase->C->shenandoah_barrier(i-1); 3829 3830 uint last = phase->C->unique(); 3831 Node* ctrl = phase->get_ctrl(wb); 3832 3833 Node* raw_mem = find_raw_mem(ctrl, wb, memory_nodes, phase); 3834 Node* init_raw_mem = raw_mem; 3835 Node* raw_mem_for_ctrl = find_raw_mem(ctrl, NULL, memory_nodes, phase); 3836 int alias = phase->C->get_alias_index(wb->adr_type()); 3837 Node* wb_mem = wb->in(Memory); 3838 3839 Node* val = wb->in(ValueIn); 3840 Node* wbproj = wb->find_out_with(Op_ShenandoahWBMemProj); 3841 IdealLoopTree *loop = phase->get_loop(ctrl); 3842 3843 assert(val->Opcode() != Op_ShenandoahWriteBarrier || phase->C->has_irreducible_loop(), "No chain of write barriers"); 3844 3845 CallStaticJavaNode* unc = wb->pin_and_expand_null_check(phase->igvn()); 3846 Node* unc_ctrl = NULL; 3847 if (unc != NULL) { 3848 if (val->in(0) != ctrl) { 3849 unc = NULL; 3850 } else { 3851 unc_ctrl = val->in(0); 3852 } 3853 } 3854 3855 Node* uncasted_val = val; 3856 if (unc != NULL) { 3857 uncasted_val = val->in(1); 3858 } 3859 3860 Node* evac_in_progress = NULL; 3861 Node* evac_not_in_progress = NULL; 3862 IfNode* evacuation_iff = NULL; 3863 test_evacuation_in_progress(ctrl, alias, raw_mem, wb_mem, evacuation_iff, evac_in_progress, evac_not_in_progress, phase); 3864 3865 Node* region = new RegionNode(4); 3866 Node* val_phi = new PhiNode(region, val->bottom_type()->is_oopptr()->cast_to_nonconst()); 3867 Node* mem_phi = PhiNode::make(region, wb_mem, Type::MEMORY, phase->C->alias_type(wb->adr_type())->adr_type()); 3868 Node* raw_mem_phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM); 3869 3870 Node* unc_region = NULL; 3871 evacuation_not_in_progress(evac_not_in_progress, val, unc_ctrl, raw_mem, wb_mem, 3872 region, val_phi, mem_phi, raw_mem_phi, unc_region, 3873 phase); 3874 3875 evacuation_in_progress(evac_in_progress, val, evacuation_iff, unc, unc_ctrl, 3876 raw_mem, wb_mem, region, val_phi, mem_phi, raw_mem_phi, 3877 unc_region, alias, uses, 3878 phase); 3879 Node* out_val = val_phi; 3880 phase->register_new_node(val_phi, region); 3881 phase->register_new_node(mem_phi, region); 3882 phase->register_new_node(raw_mem_phi, region); 3883 3884 // Update the control of all nodes that should be after the 3885 // barrier control flow 3886 uses.clear(); 3887 // Every node that is control dependent on the barrier's input 3888 // control will be after the expanded barrier. The raw memory (if 3889 // its memory is control dependent on the barrier's input control) 3890 // must stay above the barrier. 3891 uses_to_ignore.clear(); 3892 if (phase->has_ctrl(init_raw_mem) && phase->get_ctrl(init_raw_mem) == ctrl && !init_raw_mem->is_Phi()) { 3893 uses_to_ignore.push(init_raw_mem); 3894 } 3895 for (uint next = 0; next < uses_to_ignore.size(); next++) { 3896 Node *n = uses_to_ignore.at(next); 3897 for (uint i = 0; i < n->req(); i++) { 3898 Node* in = n->in(i); 3899 if (in != NULL && phase->has_ctrl(in) && phase->get_ctrl(in) == ctrl) { 3900 uses_to_ignore.push(in); 3901 } 3902 } 3903 } 3904 for (DUIterator_Fast imax, i = ctrl->fast_outs(imax); i < imax; i++) { 3905 Node* u = ctrl->fast_out(i); 3906 if (u->_idx < last && 3907 u != wb && 3908 !uses_to_ignore.member(u) && 3909 (u->in(0) != ctrl || (!u->is_Region() && !u->is_Phi())) && 3910 (ctrl->Opcode() != Op_CatchProj || u->Opcode() != Op_CreateEx)) { 3911 Node* old_c = phase->ctrl_or_self(u); 3912 Node* c = old_c; 3913 if (c != ctrl || 3914 is_dominator_same_ctrl(old_c, wb, u, phase) || 3915 u->is_g1_marking_load()) { 3916 phase->igvn().rehash_node_delayed(u); 3917 int nb = u->replace_edge(ctrl, region); 3918 if (u->is_CFG()) { 3919 if (phase->idom(u) == ctrl) { 3920 phase->set_idom(u, region, phase->dom_depth(region)); 3921 } 3922 } else if (phase->get_ctrl(u) == ctrl) { 3923 assert(u != init_raw_mem, "should leave input raw mem above the barrier"); 3924 uses.push(u); 3925 } 3926 assert(nb == 1, "more than 1 ctrl input?"); 3927 --i, imax -= nb; 3928 } 3929 } 3930 } 3931 3932 if (wbproj != NULL) { 3933 phase->igvn().replace_input_of(wbproj, 0, phase->C->top()); 3934 phase->lazy_replace(wbproj, mem_phi); 3935 } 3936 if (unc != NULL) { 3937 for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) { 3938 Node* u = val->fast_out(i); 3939 Node* c = phase->ctrl_or_self(u); 3940 if (u != wb && (c != ctrl || is_dominator_same_ctrl(c, wb, u, phase))) { 3941 phase->igvn().rehash_node_delayed(u); 3942 int nb = u->replace_edge(val, out_val); 3943 --i, imax -= nb; 3944 } 3945 } 3946 if (val->outcnt() == 0) { 3947 phase->lazy_update(val, out_val); 3948 phase->igvn()._worklist.push(val); 3949 } 3950 } 3951 phase->lazy_replace(wb, out_val); 3952 3953 follow_barrier_uses(mem_phi, ctrl, uses, phase); 3954 follow_barrier_uses(out_val, ctrl, uses, phase); 3955 3956 for(uint next = 0; next < uses.size(); next++ ) { 3957 Node *n = uses.at(next); 3958 assert(phase->get_ctrl(n) == ctrl, "bad control"); 3959 assert(n != init_raw_mem, "should leave input raw mem above the barrier"); 3960 phase->set_ctrl(n, region); 3961 follow_barrier_uses(n, ctrl, uses, phase); 3962 } 3963 3964 // The slow path call produces memory: hook the raw memory phi 3965 // from the expanded write barrier with the rest of the graph 3966 // which may require adding memory phis at every post dominated 3967 // region and at enclosing loop heads. Use the memory state 3968 // collected in memory_nodes to fix the memory graph. Update that 3969 // memory state as we go. 3970 fix_raw_mem(ctrl,region, init_raw_mem, raw_mem_for_ctrl, raw_mem_phi, memory_nodes, uses, phase); 3971 assert(phase->C->shenandoah_barriers_count() == cnt - 1, "not replaced"); 3972 } 3973 3974 assert(phase->C->shenandoah_barriers_count() == 0, "all write barrier nodes should have been replaced"); 3975 } 3976 3977 void ShenandoahWriteBarrierNode::move_evacuation_test_out_of_loop(IfNode* iff, PhaseIdealLoop* phase) { 3978 // move test and its mem barriers out of the loop 3979 assert(is_evacuation_in_progress_test(iff), "inconsistent"); 3980 3981 if (ShenandoahWriteBarrierMemBar) { 3982 IdealLoopTree *loop = phase->get_loop(iff); 3983 Node* loop_head = loop->_head; 3984 Node* entry_c = loop_head->in(LoopNode::EntryControl); 3985 IdealLoopTree *entry_loop = phase->get_loop(entry_c); 3986 3987 GrowableArray<Node*> new_mbs; 3988 Node* c = iff->in(0); 3989 MemBarNode* mb = NULL; 3990 do { 3991 Node* proj_ctrl = c; 3992 assert(c->is_Proj(), "proj expected"); 3993 mb = proj_ctrl->in(0)->as_MemBar(); 3994 c = c->in(0)->in(0); 3995 3996 Node* proj_mem = mb->proj_out(TypeFunc::Memory); 3997 3998 MemBarNode* new_mb = mb->clone()->as_MemBar();; 3999 Node* new_proj_ctrl = new ProjNode(new_mb,TypeFunc::Control); 4000 Node* new_proj_mem = new ProjNode(new_mb,TypeFunc::Memory); 4001 4002 int alias = phase->C->get_alias_index(mb->adr_type()); 4003 Node* mem_ctrl = NULL; 4004 Node* mem = dom_mem(mb, loop_head, alias, mem_ctrl, phase); 4005 new_mb->set_req(TypeFunc::Memory, mem); 4006 phase->register_new_node(new_proj_mem, new_mb); 4007 fix_memory_uses(mem, new_mb, new_proj_mem, entry_c, alias, phase); 4008 assert(new_proj_mem->outcnt() >= 1, "memory projection is disconnected"); 4009 new_mbs.push(new_proj_ctrl); 4010 } while (mb->adr_type() != TypeRawPtr::BOTTOM); 4011 4012 c = entry_c; 4013 for (int i = new_mbs.length()-1; i >= 0; i--) { 4014 Node* proj_ctrl = new_mbs.at(i); 4015 Node* mb = proj_ctrl->in(0); 4016 mb->set_req(0, c); 4017 phase->set_idom(mb, mb->in(0), phase->dom_depth(mb->in(0))); 4018 phase->set_idom(proj_ctrl, mb, phase->dom_depth(mb)); 4019 c = proj_ctrl; 4020 phase->register_control(mb, entry_loop, mb->in(0)); 4021 phase->register_control(proj_ctrl, entry_loop, mb); 4022 } 4023 phase->igvn().replace_input_of(loop_head, LoopNode::EntryControl, c); 4024 phase->set_idom(loop_head, c, phase->dom_depth(c)); 4025 //phase->recompute_dom_depth(); 4026 4027 Node* load = iff->in(1)->in(1)->in(1)->in(1); 4028 assert(load->Opcode() == Op_LoadUB, "inconsistent"); 4029 phase->igvn().replace_input_of(load, MemNode::Memory, new_mbs.at(new_mbs.length()-1)->in(0)->in(TypeFunc::Memory)); 4030 phase->igvn().replace_input_of(load, 0, entry_c); 4031 phase->set_ctrl_and_loop(load, entry_c); 4032 4033 c = iff->in(0); 4034 for (;;) { 4035 Node* next = c->in(0)->in(0); 4036 assert(c->is_Proj(), "proj expected"); 4037 Node* proj_ctrl = c; 4038 MemBarNode* mb = proj_ctrl->in(0)->as_MemBar(); 4039 Node* proj_mem = mb->proj_out(TypeFunc::Memory); 4040 Node* ctrl = mb->in(TypeFunc::Control); 4041 Node* mem = mb->in(TypeFunc::Memory); 4042 4043 phase->lazy_replace(proj_mem, mem); 4044 phase->lazy_replace(proj_ctrl, ctrl); 4045 phase->lazy_replace(mb, ctrl); 4046 loop->_body.yank(proj_ctrl); 4047 loop->_body.yank(proj_mem); 4048 loop->_body.yank(mb); 4049 if (mb->adr_type() == TypeRawPtr::BOTTOM) { 4050 break; 4051 } 4052 c = next; 4053 } 4054 4055 assert(phase->is_dominator(phase->get_ctrl(load->in(MemNode::Address)), entry_c), "address not out of loop?"); 4056 } else { 4057 IdealLoopTree *loop = phase->get_loop(iff); 4058 Node* loop_head = loop->_head; 4059 Node* entry_c = loop_head->in(LoopNode::EntryControl); 4060 4061 Node* load = iff->in(1)->in(1)->in(1); 4062 assert(load->Opcode() == Op_LoadUB, "inconsistent"); 4063 Node* mem_ctrl = NULL; 4064 } 4065 } 4066 4067 void ShenandoahWriteBarrierNode::backtoback_evacs(IfNode* iff, IfNode* dom_if, PhaseIdealLoop* phase) { 4068 if (!ShenandoahWriteBarrierMemBar) { 4069 return; 4070 } 4071 // move all mem barriers from this evac test to the dominating one, 4072 // removing duplicates in the process 4073 IdealLoopTree *loop = phase->get_loop(dom_if); 4074 Node* c1 = iff->in(0); 4075 Node* mb1 = NULL; 4076 GrowableArray<Node*> new_mbs; 4077 for(;;) { 4078 mb1 = c1->in(0); 4079 c1 = c1->in(0)->in(0); 4080 assert(mb1->Opcode() == Op_MemBarAcquire, "mem bar expected"); 4081 if (mb1->adr_type() == TypeRawPtr::BOTTOM) { 4082 phase->lazy_replace(mb1->as_MemBar()->proj_out(TypeFunc::Memory), mb1->in(TypeFunc::Memory)); 4083 break; 4084 } 4085 Node* c2 = dom_if->in(0); 4086 Node* mb2 = NULL; 4087 do { 4088 mb2 = c2->in(0); 4089 c2 = c2->in(0)->in(0); 4090 assert(mb2->Opcode() == Op_MemBarAcquire, "mem bar expected"); 4091 if (mb2->adr_type() == TypeRawPtr::BOTTOM) { 4092 // Barrier on same slice doesn't exist at dominating if: 4093 // move barrier up 4094 MemBarNode* mb = mb1->clone()->as_MemBar(); 4095 Node* proj_ctrl = new ProjNode(mb,TypeFunc::Control); 4096 Node* proj_mem = new ProjNode(mb,TypeFunc::Memory); 4097 int alias = phase->C->get_alias_index(mb->adr_type()); 4098 Node* mem_ctrl = NULL; 4099 Node* mem = dom_mem(mb1, dom_if->in(0), alias, mem_ctrl, phase); 4100 mb->set_req(TypeFunc::Memory, mem); 4101 phase->register_new_node(proj_mem, mb); 4102 fix_memory_uses(mem, mb, proj_mem, dom_if->in(0), alias, phase); 4103 assert(proj_mem->outcnt() >= 1, "memory projection is disconnected"); 4104 new_mbs.push(proj_ctrl); 4105 break; 4106 } 4107 } while (mb2->adr_type() != mb1->adr_type()); 4108 phase->lazy_replace(mb1->as_MemBar()->proj_out(TypeFunc::Memory), mb1->in(TypeFunc::Memory)); 4109 } 4110 if (new_mbs.length() > 0) { 4111 Node* c = dom_if->in(0); 4112 for (int i = 0; i < new_mbs.length(); i++) { 4113 Node* proj_ctrl = new_mbs.at(i); 4114 Node* mb = proj_ctrl->in(0); 4115 mb->set_req(0, c); 4116 phase->set_idom(mb, mb->in(0), phase->dom_depth(mb->in(0))); 4117 phase->set_idom(proj_ctrl, mb, phase->dom_depth(mb)); 4118 c = proj_ctrl; 4119 phase->register_control(mb, loop, mb->in(0)); 4120 phase->register_control(proj_ctrl, loop, mb); 4121 } 4122 phase->igvn().replace_input_of(dom_if, 0, c); 4123 phase->set_idom(dom_if, dom_if->in(0), phase->dom_depth(dom_if->in(0))); 4124 } 4125 Node* c = iff->in(0); 4126 for (;;) { 4127 Node* next = c->in(0)->in(0); 4128 assert(c->is_Proj(), "proj expected"); 4129 MemBarNode* mb = c->in(0)->as_MemBar(); 4130 Node* proj_ctrl = c; 4131 Node* ctrl = mb->in(TypeFunc::Control); 4132 4133 phase->lazy_replace(proj_ctrl, ctrl); 4134 phase->lazy_replace(mb, ctrl); 4135 if (mb->adr_type() == TypeRawPtr::BOTTOM) { 4136 break; 4137 } 4138 c = next; 4139 } 4140 } 4141 4142 void ShenandoahWriteBarrierNode::merge_back_to_back_evacuation_tests(Node* n, PhaseIdealLoop* phase) { 4143 if (phase->identical_backtoback_ifs(n)) { 4144 Node* n_ctrl = ShenandoahWriteBarrierNode::evacuation_in_progress_test_ctrl(n); 4145 if (phase->can_split_if(n_ctrl)) { 4146 IfNode* dom_if = phase->idom(n_ctrl)->as_If(); 4147 ShenandoahWriteBarrierNode::backtoback_evacs(n->as_If(), dom_if, phase); 4148 PhiNode* bolphi = PhiNode::make_blank(n_ctrl, n->in(1)); 4149 Node* proj_true = dom_if->proj_out(1); 4150 Node* proj_false = dom_if->proj_out(0); 4151 Node* con_true = phase->igvn().makecon(TypeInt::ONE); 4152 Node* con_false = phase->igvn().makecon(TypeInt::ZERO); 4153 4154 for (uint i = 1; i < n_ctrl->req(); i++) { 4155 if (phase->is_dominator(proj_true, n_ctrl->in(i))) { 4156 bolphi->init_req(i, con_true); 4157 } else { 4158 assert(phase->is_dominator(proj_false, n_ctrl->in(i)), "bad if"); 4159 bolphi->init_req(i, con_false); 4160 } 4161 } 4162 phase->register_new_node(bolphi, n_ctrl); 4163 phase->igvn().replace_input_of(n, 1, bolphi); 4164 phase->do_split_if(n); 4165 } 4166 } 4167 } 4168 4169 void ShenandoahWriteBarrierNode::optimize_after_expansion(const Node_List& evacuation_tests, const Node_List& gc_state_loads, Node_List &old_new, PhaseIdealLoop* phase) { 4170 bool progress; 4171 do { 4172 progress = false; 4173 for (uint i = 0; i < gc_state_loads.size(); i++) { 4174 Node* n = gc_state_loads.at(i); 4175 if (n->outcnt() != 0) { 4176 progress |= ShenandoahWriteBarrierNode::try_common_gc_state_load(n, phase); 4177 } 4178 } 4179 } while(progress); 4180 4181 for (uint i = 0; i < evacuation_tests.size(); i++) { 4182 Node* n = evacuation_tests.at(i); 4183 assert(is_evacuation_in_progress_test(n), "only evacuation test"); 4184 merge_back_to_back_evacuation_tests(n, phase); 4185 } 4186 if (!phase->C->major_progress()) { 4187 VectorSet seen(Thread::current()->resource_area()); 4188 for (uint i = 0; i < evacuation_tests.size(); i++) { 4189 Node* n = evacuation_tests.at(i); 4190 IdealLoopTree* loop = phase->get_loop(n); 4191 if (loop != phase->ltree_root() && 4192 loop->_child == NULL && 4193 !loop->_irreducible) { 4194 LoopNode* head = loop->_head->as_Loop(); 4195 if ((!head->is_CountedLoop() || head->as_CountedLoop()->is_main_loop() || head->as_CountedLoop()->is_normal_loop()) && 4196 !seen.test_set(head->_idx) && 4197 loop->policy_unswitching(phase)) { 4198 IfNode* iff = phase->find_unswitching_candidate(loop); 4199 if (iff != NULL && is_evacuation_in_progress_test(iff)) { 4200 if (head->is_strip_mined()) { 4201 head->verify_strip_mined(0); 4202 head->clear_strip_mined(); 4203 head->in(LoopNode::EntryControl)->as_Loop()->clear_strip_mined(); 4204 } 4205 phase->do_unswitching(loop, old_new); 4206 } 4207 } 4208 } 4209 } 4210 } 4211 } 4212 4213 #ifdef ASSERT 4214 void ShenandoahBarrierNode::verify_raw_mem(RootNode* root) { 4215 const bool trace = false; 4216 ResourceMark rm; 4217 Unique_Node_List nodes; 4218 Unique_Node_List controls; 4219 Unique_Node_List memories; 4220 4221 nodes.push(root); 4222 for (uint next = 0; next < nodes.size(); next++) { 4223 Node *n = nodes.at(next); 4224 if (n->Opcode() == Op_CallLeafNoFP && n->as_Call()->_entry_point == StubRoutines::shenandoah_wb_C()) { 4225 controls.push(n); 4226 if (trace) { tty->print("XXXXXX verifying"); n->dump(); } 4227 for (uint next2 = 0; next2 < controls.size(); next2++) { 4228 Node *m = controls.at(next2); 4229 if (!m->is_Loop() || controls.member(m->in(LoopNode::EntryControl)) || 1) { 4230 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) { 4231 Node* u = m->fast_out(i); 4232 if (u->is_CFG() && !u->is_Root()) { 4233 if (trace) { tty->print("XXXXXX pushing control"); u->dump(); } 4234 controls.push(u); 4235 } 4236 } 4237 } 4238 } 4239 memories.push(n->as_Call()->proj_out(TypeFunc::Memory)); 4240 for (uint next2 = 0; next2 < memories.size(); next2++) { 4241 Node *m = memories.at(next2); 4242 assert(m->bottom_type() == Type::MEMORY, ""); 4243 if (!m->is_Phi() || !m->in(0)->is_Loop() || controls.member(m->in(0)->in(LoopNode::EntryControl)) || 1) { 4244 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) { 4245 Node* u = m->fast_out(i); 4246 if (u->bottom_type() == Type::MEMORY && (u->is_Mem() || u->is_ClearArray())) { 4247 if (trace) { tty->print("XXXXXX pushing memory"); u->dump(); } 4248 memories.push(u); 4249 } else if (u->is_LoadStore()) { 4250 if (trace) { tty->print("XXXXXX pushing memory"); u->find_out_with(Op_SCMemProj)->dump(); } 4251 memories.push(u->find_out_with(Op_SCMemProj)); 4252 } else if (u->is_MergeMem() && u->as_MergeMem()->memory_at(Compile::AliasIdxRaw) == m) { 4253 if (trace) { tty->print("XXXXXX pushing memory"); u->dump(); } 4254 memories.push(u); 4255 } else if (u->is_Phi()) { 4256 assert(u->bottom_type() == Type::MEMORY, ""); 4257 if (u->adr_type() == TypeRawPtr::BOTTOM || u->adr_type() == TypePtr::BOTTOM) { 4258 assert(controls.member(u->in(0)), ""); 4259 if (trace) { tty->print("XXXXXX pushing memory"); u->dump(); } 4260 memories.push(u); 4261 } 4262 } else if (u->is_SafePoint() || u->is_MemBar()) { 4263 for (DUIterator_Fast jmax, j = u->fast_outs(jmax); j < jmax; j++) { 4264 Node* uu = u->fast_out(j); 4265 if (uu->bottom_type() == Type::MEMORY) { 4266 if (trace) { tty->print("XXXXXX pushing memory"); uu->dump(); } 4267 memories.push(uu); 4268 } 4269 } 4270 } 4271 } 4272 } 4273 } 4274 for (uint next2 = 0; next2 < controls.size(); next2++) { 4275 Node *m = controls.at(next2); 4276 if (m->is_Region()) { 4277 bool all_in = true; 4278 for (uint i = 1; i < m->req(); i++) { 4279 if (!controls.member(m->in(i))) { 4280 all_in = false; 4281 break; 4282 } 4283 } 4284 if (trace) { tty->print("XXX verifying %s", all_in ? "all in" : ""); m->dump(); } 4285 bool found_phi = false; 4286 for (DUIterator_Fast jmax, j = m->fast_outs(jmax); j < jmax && !found_phi; j++) { 4287 Node* u = m->fast_out(j); 4288 if (u->is_Phi() && memories.member(u)) { 4289 found_phi = true; 4290 for (uint i = 1; i < u->req() && found_phi; i++) { 4291 Node* k = u->in(i); 4292 if (memories.member(k) != controls.member(m->in(i))) { 4293 found_phi = false; 4294 } 4295 } 4296 } 4297 } 4298 assert(found_phi || all_in, ""); 4299 } 4300 } 4301 controls.clear(); 4302 memories.clear(); 4303 } 4304 for( uint i = 0; i < n->len(); ++i ) { 4305 Node *m = n->in(i); 4306 if (m != NULL) { 4307 nodes.push(m); 4308 } 4309 } 4310 } 4311 } 4312 #endif 4313 4314 static bool is_on_null_check_path(Block* b, Block* null_check_block) { 4315 if (null_check_block == NULL) { 4316 return false; 4317 } 4318 do { 4319 assert(null_check_block->_num_succs == 1, "only one succ on the path to unc"); 4320 if (b == null_check_block) { 4321 return true; 4322 } 4323 null_check_block = null_check_block->_succs[0]; 4324 } while(!null_check_block->head()->is_Root()); 4325 4326 return false; 4327 } 4328 4329 int PhaseCFG::replace_uses_with_shenandoah_barrier_helper(Node* n, Node* use, Node* val, Block* block, Block* null_check_block) { 4330 int nb = 0; 4331 Block* buse = get_block_for_node(use); 4332 if (is_on_null_check_path(buse, null_check_block)) { 4333 return 0; 4334 } 4335 if (use->is_Phi()) { 4336 for (uint j = 1; j < use->req(); j++) { 4337 if (use->in(j) == val) { 4338 Block* b = get_block_for_node(use->in(0)->in(j)); 4339 if ((block != b && block->dom_lca(b) == block) || 4340 block == b) { 4341 use->set_req(j, n); 4342 nb++; 4343 } 4344 } 4345 } 4346 } else { 4347 if ((block != buse && block->dom_lca(buse) == block) || 4348 (block == buse && !use->is_scheduled())) { 4349 // Let precedence edges alone (can confuse anti-dependence verification code) 4350 for (uint i = 0; i < use->req(); i++) { 4351 if (use->in(i) == val) { 4352 use->set_req(i, n); 4353 nb++; 4354 } 4355 } 4356 assert(nb > 0 || use->find_prec_edge(val) != -1, "no replacement?"); 4357 } 4358 } 4359 4360 return nb; 4361 } 4362 4363 void PhaseCFG::replace_uses_with_shenandoah_barrier(Node* n, Block* block, Node_List& worklist, GrowableArray<int>& ready_cnt, uint max_idx, uint& phi_cnt) { 4364 // Replace all uses of barrier's input that are dominated by the 4365 // barrier with the value returned by the barrier: no need to keep 4366 // both live. 4367 if (n->is_Mach() && n->as_Mach()->ideal_Opcode() == Op_ShenandoahReadBarrier) { 4368 MachNullCheckNode* null_check = NULL; 4369 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax && null_check == NULL; i++) { 4370 Node* use = n->fast_out(i); 4371 if (use->is_MachNullCheck()) { 4372 null_check = use->as_MachNullCheck(); 4373 } 4374 } 4375 Block* null_check_block = NULL; 4376 if (null_check != NULL) { 4377 Node* proj = null_check->find_out_with(Op_IfTrue); 4378 Node* head = proj->unique_out(); 4379 null_check_block = get_block_for_node(head); 4380 } 4381 4382 Node* val = n->in(ShenandoahBarrierNode::ValueIn); 4383 if (!val->bottom_type()->isa_narrowoop()) { 4384 for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) { 4385 Node* use = val->fast_out(i); 4386 if (use != n) { 4387 int nb = replace_uses_with_shenandoah_barrier_helper(n, use, val, block, null_check_block); 4388 if (nb > 0) { 4389 --i; imax -= nb; 4390 } 4391 } 4392 } 4393 } else { 4394 for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) { 4395 Node* u = val->fast_out(i); 4396 if (u->is_Mach() && u->as_Mach()->ideal_Opcode() == Op_DecodeN) { 4397 int projs = 0; 4398 for (DUIterator_Fast jmax, j = u->fast_outs(jmax); j < jmax; j++) { 4399 Node* uu = u->fast_out(j); 4400 assert(!uu->is_MachTemp(), ""); 4401 if (uu->is_MachProj() && uu->outcnt() == 0) { 4402 projs++; 4403 } else { 4404 int nb = replace_uses_with_shenandoah_barrier_helper(n, uu, u, block, null_check_block); 4405 if (nb > 0) { 4406 if (!u->is_scheduled()) { 4407 push_ready_nodes(n, uu, block, ready_cnt, worklist, max_idx, nb); 4408 } 4409 --j; jmax -= nb; 4410 } 4411 } 4412 } 4413 // The DecodeN may have gone dead 4414 if (u->outcnt() - projs == 0) { 4415 u->disconnect_inputs(NULL, C); 4416 Block* bu = get_block_for_node(u); 4417 unmap_node_from_block(u); 4418 if (bu == block) { 4419 if (u->is_scheduled()) { 4420 block->find_remove(u); 4421 phi_cnt--; 4422 } else { 4423 worklist.yank(u); 4424 block->remove_node(block->end_idx()-1); 4425 } 4426 } else { 4427 bu->find_remove(u); 4428 } 4429 for (DUIterator_Fast jmax, j = u->fast_outs(jmax); j < jmax; j++) { 4430 Node* uu = u->fast_out(j); 4431 assert(uu->is_MachProj() && uu->outcnt() == 0, ""); 4432 assert(bu == get_block_for_node(uu), ""); 4433 uu->disconnect_inputs(NULL, C); 4434 --j; --jmax; 4435 unmap_node_from_block(uu); 4436 if (bu == block) { 4437 if (u->is_scheduled()) { 4438 block->find_remove(uu); 4439 phi_cnt--; 4440 } else { 4441 worklist.yank(uu); 4442 block->remove_node(block->end_idx()-1); 4443 } 4444 } else { 4445 bu->find_remove(uu); 4446 } 4447 assert(uu->is_scheduled() == u->is_scheduled(), ""); 4448 } 4449 --i; --imax; 4450 } 4451 } 4452 } 4453 } 4454 } 4455 }