1 /* 2 * Copyright (c) 2015, Red Hat, Inc. and/or its affiliates. 3 * 4 * This code is free software; you can redistribute it and/or modify it 5 * under the terms of the GNU General Public License version 2 only, as 6 * published by the Free Software Foundation. 7 * 8 * This code is distributed in the hope that it will be useful, but WITHOUT 9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 11 * version 2 for more details (a copy is included in the LICENSE file that 12 * accompanied this code). 13 * 14 * You should have received a copy of the GNU General Public License version 15 * 2 along with this work; if not, write to the Free Software Foundation, 16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 17 * 18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 19 * or visit www.oracle.com if you need additional information or have any 20 * questions. 21 * 22 */ 23 24 #include "precompiled.hpp" 25 #include "gc/shenandoah/brooksPointer.hpp" 26 #include "gc/shenandoah/shenandoahHeap.hpp" 27 #include "gc/shenandoah/shenandoahHeapRegion.hpp" 28 #include "opto/arraycopynode.hpp" 29 #include "opto/block.hpp" 30 #include "opto/callnode.hpp" 31 #include "opto/castnode.hpp" 32 #include "opto/movenode.hpp" 33 #include "opto/phaseX.hpp" 34 #include "opto/rootnode.hpp" 35 #include "opto/runtime.hpp" 36 #include "opto/shenandoahSupport.hpp" 37 #include "opto/subnode.hpp" 38 39 Node* ShenandoahBarrierNode::skip_through_barrier(Node* n) { 40 if (n == NULL) { 41 return NULL; 42 } else if (n->is_ShenandoahBarrier()) { 43 return n->in(ValueIn); 44 } else if (n->is_Phi() && 45 n->req() == 3 && 46 n->in(1) != NULL && 47 n->in(1)->is_ShenandoahBarrier() && 48 n->in(2) != NULL && 49 n->in(2)->bottom_type() == TypePtr::NULL_PTR && 50 n->in(0) != NULL && 51 n->in(0)->in(1) != NULL && 52 n->in(0)->in(1)->is_IfProj() && 53 n->in(0)->in(2) != NULL && 54 n->in(0)->in(2)->is_IfProj() && 55 n->in(0)->in(1)->in(0) != NULL && 56 n->in(0)->in(1)->in(0) == n->in(0)->in(2)->in(0) && 57 n->in(1)->in(ValueIn)->Opcode() == Op_CastPP) { 58 Node* iff = n->in(0)->in(1)->in(0); 59 Node* res = n->in(1)->in(ValueIn)->in(1); 60 if (iff->is_If() && 61 iff->in(1) != NULL && 62 iff->in(1)->is_Bool() && 63 iff->in(1)->as_Bool()->_test._test == BoolTest::ne && 64 iff->in(1)->in(1) != NULL && 65 iff->in(1)->in(1)->Opcode() == Op_CmpP && 66 iff->in(1)->in(1)->in(1) != NULL && 67 iff->in(1)->in(1)->in(1) == res && 68 iff->in(1)->in(1)->in(2) != NULL && 69 iff->in(1)->in(1)->in(2)->bottom_type() == TypePtr::NULL_PTR) { 70 return res; 71 } 72 } 73 return n; 74 } 75 76 bool ShenandoahBarrierNode::needs_barrier(PhaseGVN* phase, ShenandoahBarrierNode* orig, Node* n, Node* rb_mem, bool allow_fromspace) { 77 Unique_Node_List visited; 78 return needs_barrier_impl(phase, orig, n, rb_mem, allow_fromspace, visited); 79 } 80 81 bool ShenandoahBarrierNode::needs_barrier_impl(PhaseGVN* phase, ShenandoahBarrierNode* orig, Node* n, Node* rb_mem, bool allow_fromspace, Unique_Node_List &visited) { 82 83 if (visited.member(n)) { 84 return false; // Been there. 85 } 86 visited.push(n); 87 88 if (n->is_Allocate()) { 89 // tty->print_cr("killed barrier for newly allocated object"); 90 return false; 91 } 92 93 if (n->is_CallJava() || n->Opcode() == Op_CallLeafNoFP) { 94 return true; 95 } 96 97 const Type* type = phase->type(n); 98 if (type == Type::TOP) { 99 return false; 100 } 101 if (type->make_ptr()->higher_equal(TypePtr::NULL_PTR)) { 102 // tty->print_cr("killed barrier for NULL object"); 103 return false; 104 } 105 if (type->make_oopptr() && type->make_oopptr()->const_oop() != NULL) { 106 // tty->print_cr("killed barrier for constant object"); 107 return ShenandoahBarriersForConst; 108 } 109 110 if (ShenandoahOptimizeStableFinals) { 111 const TypeAryPtr* ary = type->isa_aryptr(); 112 if (ary && ary->is_stable() && allow_fromspace) { 113 return false; 114 } 115 } 116 117 if (n->is_CheckCastPP() || n->is_ConstraintCast()) { 118 return needs_barrier_impl(phase, orig, n->in(1), rb_mem, allow_fromspace, visited); 119 } 120 if (n->is_Parm()) { 121 return true; 122 } 123 if (n->is_Proj()) { 124 return needs_barrier_impl(phase, orig, n->in(0), rb_mem, allow_fromspace, visited); 125 } 126 if (n->is_Phi()) { 127 bool need_barrier = false; 128 for (uint i = 1; i < n->req() && ! need_barrier; i++) { 129 Node* input = n->in(i); 130 if (input == NULL) { 131 need_barrier = true; // Phi not complete yet? 132 } else if (needs_barrier_impl(phase, orig, input, rb_mem, allow_fromspace, visited)) { 133 need_barrier = true; 134 } 135 } 136 return need_barrier; 137 } 138 if (n->is_CMove()) { 139 return needs_barrier_impl(phase, orig, n->in(CMoveNode::IfFalse), rb_mem, allow_fromspace, visited) || 140 needs_barrier_impl(phase, orig, n->in(CMoveNode::IfTrue ), rb_mem, allow_fromspace, visited); 141 } 142 if (n->Opcode() == Op_CreateEx) { 143 return true; 144 } 145 if (n->Opcode() == Op_ShenandoahWriteBarrier) { 146 // tty->print_cr("skipped barrier for chained write barrier object"); 147 return false; 148 } 149 if (n->Opcode() == Op_ShenandoahReadBarrier) { 150 if (rb_mem == n->in(Memory)) { 151 // tty->print_cr("Eliminated chained read barrier"); 152 return false; 153 } else { 154 return true; 155 } 156 } 157 158 if (n->Opcode() == Op_LoadP || 159 n->Opcode() == Op_LoadN || 160 n->Opcode() == Op_GetAndSetP || 161 n->Opcode() == Op_CompareAndExchangeP || 162 n->Opcode() == Op_GetAndSetN || 163 n->Opcode() == Op_CompareAndExchangeN) { 164 return true; 165 } 166 if (n->Opcode() == Op_DecodeN || 167 n->Opcode() == Op_EncodeP) { 168 return needs_barrier_impl(phase, orig, n->in(1), rb_mem, allow_fromspace, visited); 169 } 170 171 #ifdef ASSERT 172 tty->print("need barrier on?: "); n->dump(); 173 ShouldNotReachHere(); 174 #endif 175 return true; 176 } 177 178 /** 179 * In Shenandoah, we need barriers on acmp (and similar instructions that compare two 180 * oops) to avoid false negatives. If it compares a from-space and a to-space 181 * copy of an object, a regular acmp would return false, even though both are 182 * the same. The acmp barrier compares the two objects, and when they are 183 * *not equal* it does a read-barrier on both, and compares them again. When it 184 * failed because of different copies of the object, we know that the object 185 * must already have been evacuated (and therefore doesn't require a write-barrier). 186 */ 187 void ShenandoahBarrierNode::do_cmpp_if(GraphKit& kit, Node*& taken_branch, Node*& untaken_branch, Node*& taken_memory, Node*& untaken_memory) { 188 assert(taken_memory == NULL && untaken_memory == NULL, "unexpected memory inputs"); 189 if (!UseShenandoahGC || !ShenandoahAcmpBarrier || ShenandoahVerifyOptoBarriers) { 190 return; 191 } 192 if (taken_branch->is_top() || untaken_branch->is_top()) { 193 // one of the branches is known to be untaken 194 return; 195 } 196 assert(taken_branch->is_IfProj() && untaken_branch->is_IfProj(), "if projections only"); 197 assert(taken_branch->in(0) == untaken_branch->in(0), "should come from same if"); 198 IfNode* iff = taken_branch->in(0)->as_If(); 199 BoolNode* bol = iff->in(1)->as_Bool(); 200 Node* cmp = bol->in(1); 201 if (cmp->Opcode() != Op_CmpP) { 202 return; 203 } 204 Node* a = cmp->in(1); 205 Node* b = cmp->in(2); 206 const Type* a_type = kit.gvn().type(a); 207 const Type* b_type = kit.gvn().type(b); 208 if (a_type->higher_equal(TypePtr::NULL_PTR) || b_type->higher_equal(TypePtr::NULL_PTR)) { 209 // We know one arg is gonna be null. No need for barriers. 210 return; 211 } 212 213 const TypePtr* a_adr_type = ShenandoahBarrierNode::brooks_pointer_type(a_type); 214 const TypePtr* b_adr_type = ShenandoahBarrierNode::brooks_pointer_type(b_type); 215 if ((! ShenandoahBarrierNode::needs_barrier(&kit.gvn(), NULL, a, kit.memory(a_adr_type), false)) && 216 (! ShenandoahBarrierNode::needs_barrier(&kit.gvn(), NULL, b, kit.memory(b_adr_type), false))) { 217 // We know both args are in to-space already. No acmp barrier needed. 218 return; 219 } 220 221 Node* equal_path = iff->proj_out(true); 222 Node* not_equal_path = iff->proj_out(false); 223 224 if (bol->_test._test == BoolTest::ne) { 225 swap(equal_path, not_equal_path); 226 } 227 228 Node* init_equal_path = equal_path; 229 Node* init_not_equal_path = not_equal_path; 230 231 uint alias_a = kit.C->get_alias_index(a_adr_type); 232 uint alias_b = kit.C->get_alias_index(b_adr_type); 233 234 Node* equal_memory = NULL; 235 Node* not_equal_memory = NULL; 236 237 RegionNode* region = new RegionNode(3); 238 region->init_req(1, equal_path); 239 PhiNode* mem_phi = NULL; 240 if (alias_a == alias_b) { 241 mem_phi = PhiNode::make(region, kit.memory(alias_a), Type::MEMORY, kit.C->get_adr_type(alias_a)); 242 } else { 243 Node* mem = kit.reset_memory(); 244 mem_phi = PhiNode::make(region, mem, Type::MEMORY, TypePtr::BOTTOM); 245 kit.set_all_memory(mem); 246 } 247 248 kit.set_control(not_equal_path); 249 250 Node* mb = NULL; 251 if (alias_a == alias_b) { 252 Node* mem = kit.reset_memory(); 253 mb = MemBarNode::make(kit.C, Op_MemBarAcquire, alias_a); 254 mb->init_req(TypeFunc::Control, kit.control()); 255 mb->init_req(TypeFunc::Memory, mem); 256 Node* membar = kit.gvn().transform(mb); 257 kit.set_control(kit.gvn().transform(new ProjNode(membar, TypeFunc::Control))); 258 Node* newmem = kit.gvn().transform(new ProjNode(membar, TypeFunc::Memory)); 259 kit.set_all_memory(mem); 260 kit.set_memory(newmem, alias_a); 261 } else { 262 mb = kit.insert_mem_bar(Op_MemBarAcquire); 263 } 264 265 a = kit.shenandoah_read_barrier_acmp(a); 266 b = kit.shenandoah_read_barrier_acmp(b); 267 268 Node* cmp2 = kit.gvn().transform(new CmpPNode(a, b)); 269 Node* bol2 = bol->clone(); 270 bol2->set_req(1, cmp2); 271 bol2 = kit.gvn().transform(bol2); 272 Node* iff2 = iff->clone(); 273 iff2->set_req(0, kit.control()); 274 iff2->set_req(1, bol2); 275 kit.gvn().set_type(iff2, kit.gvn().type(iff)); 276 Node* equal_path2 = equal_path->clone(); 277 equal_path2->set_req(0, iff2); 278 equal_path2 = kit.gvn().transform(equal_path2); 279 Node* not_equal_path2 = not_equal_path->clone(); 280 not_equal_path2->set_req(0, iff2); 281 not_equal_path2 = kit.gvn().transform(not_equal_path2); 282 283 region->init_req(2, equal_path2); 284 not_equal_memory = kit.reset_memory(); 285 not_equal_path = not_equal_path2; 286 287 kit.set_all_memory(not_equal_memory); 288 289 if (alias_a == alias_b) { 290 mem_phi->init_req(2, kit.memory(alias_a)); 291 kit.set_memory(mem_phi, alias_a); 292 } else { 293 mem_phi->init_req(2, kit.reset_memory()); 294 } 295 296 kit.record_for_igvn(mem_phi); 297 kit.gvn().set_type(mem_phi, Type::MEMORY); 298 299 if (alias_a == alias_b) { 300 equal_memory = kit.reset_memory(); 301 } else { 302 equal_memory = mem_phi; 303 } 304 305 assert(kit.map()->memory() == NULL, "no live memory state"); 306 equal_path = kit.gvn().transform(region); 307 308 if (taken_branch == init_equal_path) { 309 assert(untaken_branch == init_not_equal_path, "inconsistent"); 310 taken_branch = equal_path; 311 untaken_branch = not_equal_path; 312 taken_memory = equal_memory; 313 untaken_memory = not_equal_memory; 314 } else { 315 assert(taken_branch == init_not_equal_path, "inconsistent"); 316 assert(untaken_branch == init_equal_path, "inconsistent"); 317 taken_branch = not_equal_path; 318 untaken_branch = equal_path; 319 taken_memory = not_equal_memory; 320 untaken_memory = equal_memory; 321 } 322 } 323 324 bool ShenandoahReadBarrierNode::dominates_memory_rb_impl(PhaseGVN* phase, 325 Node* b1, 326 Node* b2, 327 Node* current, 328 bool linear) { 329 ResourceMark rm; 330 VectorSet visited(Thread::current()->resource_area()); 331 Node_Stack phis(0); 332 333 for(int i = 0; i < 10; i++) { 334 if (current == NULL) { 335 return false; 336 } else if (visited.test_set(current->_idx) || current->is_top() || current == b1) { 337 current = NULL; 338 while (phis.is_nonempty() && current == NULL) { 339 uint idx = phis.index(); 340 Node* phi = phis.node(); 341 if (idx >= phi->req()) { 342 phis.pop(); 343 } else { 344 current = phi->in(idx); 345 phis.set_index(idx+1); 346 } 347 } 348 if (current == NULL) { 349 return true; 350 } 351 } else if (current == phase->C->immutable_memory()) { 352 return false; 353 } else if (current->isa_Phi()) { 354 if (!linear) { 355 return false; 356 } 357 phis.push(current, 2); 358 current = current->in(1); 359 } else if (current->Opcode() == Op_ShenandoahWriteBarrier) { 360 const Type* in_type = current->bottom_type(); 361 const Type* this_type = b2->bottom_type(); 362 if (is_independent(in_type, this_type)) { 363 current = current->in(Memory); 364 } else { 365 return false; 366 } 367 } else if (current->Opcode() == Op_ShenandoahWBMemProj) { 368 current = current->in(0); 369 } else if (current->is_Proj()) { 370 current = current->in(0); 371 } else if (current->is_Call()) { 372 return false; // TODO: Maybe improve by looking at the call's memory effects? 373 } else if (current->is_MemBar()) { 374 return false; // TODO: Do we need to stop at *any* membar? 375 } else if (current->is_MergeMem()) { 376 // if (true) return false; 377 // tty->print_cr("current == mergemem: "); current->dump(); 378 const TypePtr* adr_type = brooks_pointer_type(phase->type(b2)); 379 uint alias_idx = phase->C->get_alias_index(adr_type); 380 current = current->as_MergeMem()->memory_at(alias_idx); 381 } else { 382 // tty->print_cr("what else can we see here:"); 383 #ifdef ASSERT 384 current->dump(); 385 #endif 386 ShouldNotReachHere(); 387 return false; 388 } 389 } 390 return false; 391 } 392 393 bool ShenandoahReadBarrierNode::is_independent(Node* mem) { 394 if (mem->is_Phi() || mem->is_Proj() || mem->is_MergeMem()) { 395 return true; 396 } else if (mem->Opcode() == Op_ShenandoahWriteBarrier) { 397 const Type* mem_type = mem->bottom_type(); 398 const Type* this_type = bottom_type(); 399 if (is_independent(mem_type, this_type)) { 400 return true; 401 } else { 402 return false; 403 } 404 } else if (mem->is_Call() || mem->is_MemBar()) { 405 return false; 406 } 407 #ifdef ASSERT 408 mem->dump(); 409 #endif 410 ShouldNotReachHere(); 411 return true; 412 } 413 414 415 bool ShenandoahReadBarrierNode::dominates_memory_rb(PhaseGVN* phase, Node* b1, Node* b2, bool linear) { 416 return dominates_memory_rb_impl(phase, b1->in(Memory), b2, b2->in(Memory), linear); 417 } 418 419 bool ShenandoahReadBarrierNode::is_independent(const Type* in_type, const Type* this_type) { 420 assert(in_type->isa_oopptr(), "expect oop ptr"); 421 assert(this_type->isa_oopptr(), "expect oop ptr"); 422 /* 423 if ((! in_type->isa_oopptr()) || (! this_type->isa_oopptr())) { 424 #ifdef ASSERT 425 tty->print_cr("not oopptr"); 426 tty->print("in: "); in_type->dump(); tty->print_cr(" "); 427 tty->print("this: "); this_type->dump(); tty->print_cr(" "); 428 #endif 429 return false; 430 } 431 */ 432 433 ciKlass* in_kls = in_type->is_oopptr()->klass(); 434 ciKlass* this_kls = this_type->is_oopptr()->klass(); 435 if (in_kls != NULL && this_kls != NULL && 436 in_kls->is_loaded() && this_kls->is_loaded() && 437 (!in_kls->is_subclass_of(this_kls)) && 438 (!this_kls->is_subclass_of(in_kls))) { 439 #ifdef ASSERT 440 // tty->print_cr("independent: "); 441 // tty->print("in: "); in_kls->print(); tty->print_cr(" "); 442 // tty->print("this: "); this_kls->print(); tty->print_cr(" "); 443 #endif 444 return true; 445 } 446 #ifdef ASSERT 447 // tty->print_cr("possibly dependend?"); 448 // tty->print("in: "); in_type->dump(); tty->print_cr(" "); 449 // tty->print("this: "); this_type->dump(); tty->print_cr(" "); 450 #endif 451 return false; 452 } 453 454 Node* ShenandoahReadBarrierNode::Ideal(PhaseGVN *phase, bool can_reshape) { 455 456 if (! can_reshape) { 457 return NULL; 458 } 459 460 if (in(Memory) == phase->C->immutable_memory()) return NULL; 461 462 // If memory input is a MergeMem, take the appropriate slice out of it. 463 Node* mem_in = in(Memory); 464 if (mem_in->isa_MergeMem()) { 465 const TypePtr* adr_type = brooks_pointer_type(bottom_type()); 466 uint alias_idx = phase->C->get_alias_index(adr_type); 467 mem_in = mem_in->as_MergeMem()->memory_at(alias_idx); 468 set_req(Memory, mem_in); 469 return this; 470 } 471 472 Node* input = in(Memory); 473 if (input->Opcode() == Op_ShenandoahWBMemProj) { 474 Node* wb = input->in(0); 475 const Type* in_type = phase->type(wb); 476 // is_top() test not sufficient here: we can come here after CCP 477 // in a dead branch of the graph that has not yet been removed. 478 if (in_type == Type::TOP) return NULL; // Dead path. 479 assert(wb->Opcode() == Op_ShenandoahWriteBarrier, "expect write barrier"); 480 if (is_independent(in_type, _type)) { 481 phase->igvn_rehash_node_delayed(wb); 482 set_req(Memory, wb->in(Memory)); 483 if (can_reshape && input->outcnt() == 0) { 484 phase->is_IterGVN()->_worklist.push(input); 485 } 486 return this; 487 } 488 } 489 return NULL; 490 } 491 492 Node* ShenandoahWriteBarrierNode::Identity(PhaseGVN* phase) { 493 assert(in(0) != NULL, "should have control"); 494 PhaseIterGVN* igvn = phase->is_IterGVN(); 495 Node* mem_in = in(Memory); 496 Node* mem_proj = NULL; 497 498 if (igvn != NULL) { 499 mem_proj = find_out_with(Op_ShenandoahWBMemProj); 500 if (mem_proj == NULL || mem_in == mem_proj) { 501 return this; 502 } 503 } 504 505 Node* replacement = Identity_impl(phase); 506 if (igvn != NULL) { 507 if (replacement != NULL && replacement != this) { 508 igvn->replace_node(mem_proj, mem_in); 509 } 510 } 511 return replacement; 512 } 513 514 515 Node* ShenandoahWriteBarrierNode::Ideal(PhaseGVN *phase, bool can_reshape) { 516 assert(in(0) != NULL, "should have control"); 517 if (!can_reshape) { 518 return NULL; 519 } 520 521 PhaseIterGVN* igvn = phase->is_IterGVN(); 522 Node* mem_proj = find_out_with(Op_ShenandoahWBMemProj); 523 Node* mem_in = in(Memory); 524 525 if (mem_in == phase->C->immutable_memory()) return NULL; 526 527 if (mem_in->isa_MergeMem()) { 528 const TypePtr* adr_type = brooks_pointer_type(bottom_type()); 529 uint alias_idx = phase->C->get_alias_index(adr_type); 530 mem_in = mem_in->as_MergeMem()->memory_at(alias_idx); 531 set_req(Memory, mem_in); 532 return this; 533 } 534 535 return NULL; 536 } 537 538 bool ShenandoahWriteBarrierNode::expand(Compile* C, PhaseIterGVN& igvn, int& loop_opts_cnt) { 539 if (UseShenandoahGC && ShenandoahWriteBarrierToIR) { 540 if (C->shenandoah_barriers_count() > 0) { 541 bool attempt_more_loopopts = ShenandoahLoopOptsAfterExpansion && (C->shenandoah_barriers_count() > 1 || C->has_loops()); 542 C->clear_major_progress(); 543 PhaseIdealLoop ideal_loop(igvn, LoopOptsShenandoahExpand); 544 if (C->failing()) return false; 545 PhaseIdealLoop::verify(igvn); 546 DEBUG_ONLY(ShenandoahBarrierNode::verify_raw_mem(C->root());) 547 if (attempt_more_loopopts) { 548 C->set_major_progress(); 549 if (!C->optimize_loops(loop_opts_cnt, igvn, LoopOptsShenandoahPostExpand)) { 550 return false; 551 } 552 C->clear_major_progress(); 553 } 554 } 555 } 556 return true; 557 } 558 559 bool ShenandoahWriteBarrierNode::is_evacuation_in_progress_test(Node* iff) { 560 if (!UseShenandoahGC) { 561 return false; 562 } 563 assert(iff->is_If(), "bad input"); 564 if (iff->Opcode() != Op_If) { 565 return false; 566 } 567 Node* bol = iff->in(1); 568 if (!bol->is_Bool() || bol->as_Bool()->_test._test != BoolTest::ne) { 569 return false; 570 } 571 Node* cmp = bol->in(1); 572 if (cmp->Opcode() != Op_CmpI) { 573 return false; 574 } 575 Node* in1 = cmp->in(1); 576 Node* in2 = cmp->in(2); 577 if (in2->find_int_con(-1) != 0) { 578 return false; 579 } 580 if (in1->Opcode() != Op_AndI) { 581 return false; 582 } 583 in2 = in1->in(2); 584 if (in2->find_int_con(-1) != (ShenandoahHeap::EVACUATION | ShenandoahHeap::PARTIAL | ShenandoahHeap::TRAVERSAL)) { 585 return false; 586 } 587 in1 = in1->in(1); 588 589 return is_gc_state_load(in1); 590 } 591 592 bool ShenandoahWriteBarrierNode::is_gc_state_load(Node *n) { 593 if (!UseShenandoahGC) { 594 return false; 595 } 596 if (n->Opcode() != Op_LoadUB && n->Opcode() != Op_LoadB) { 597 return false; 598 } 599 Node* addp = n->in(MemNode::Address); 600 if (!addp->is_AddP()) { 601 return false; 602 } 603 Node* base = addp->in(AddPNode::Address); 604 Node* off = addp->in(AddPNode::Offset); 605 if (base->Opcode() != Op_ThreadLocal) { 606 return false; 607 } 608 if (off->find_intptr_t_con(-1) != in_bytes(JavaThread::gc_state_offset())) { 609 return false; 610 } 611 return true; 612 } 613 614 bool ShenandoahWriteBarrierNode::try_common_gc_state_load(Node *n, PhaseIdealLoop *phase) { 615 assert(is_gc_state_load(n), "inconsistent"); 616 Node* addp = n->in(MemNode::Address); 617 Node* dominator = NULL; 618 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) { 619 Node* u = addp->fast_out(i); 620 if (u != n && phase->is_dominator(u->in(0), n->in(0))) { 621 if (dominator == NULL) { 622 dominator = u; 623 } else { 624 if (phase->dom_depth(u->in(0)) < phase->dom_depth(dominator->in(0))) { 625 dominator = u; 626 } 627 } 628 } 629 } 630 if (dominator == NULL) { 631 return false; 632 } 633 ResourceMark rm; 634 Unique_Node_List wq; 635 wq.push(n->in(0)); 636 for (uint next = 0; next < wq.size(); next++) { 637 Node *m = wq.at(next); 638 if (m->is_SafePoint() && !m->is_CallLeaf()) { 639 return false; 640 } 641 if (m->is_Region()) { 642 for (uint i = 1; i < m->req(); i++) { 643 wq.push(m->in(i)); 644 } 645 } else { 646 wq.push(m->in(0)); 647 } 648 } 649 phase->igvn().replace_node(n, dominator); 650 651 return true; 652 } 653 654 Node* ShenandoahWriteBarrierNode::evacuation_in_progress_test_ctrl(Node* iff) { 655 assert(is_evacuation_in_progress_test(iff), "bad input"); 656 Node* c = iff; 657 if (ShenandoahWriteBarrierMemBar) { 658 do { 659 assert(c->in(0)->is_Proj() && c->in(0)->in(0)->is_MemBar(), "where's the mem bar?"); 660 c = c->in(0)->in(0); 661 } while (c->adr_type() != TypeRawPtr::BOTTOM); 662 } 663 return c->in(0); 664 } 665 666 bool ShenandoahBarrierNode::dominates_memory_impl(PhaseGVN* phase, 667 Node* b1, 668 Node* b2, 669 Node* current, 670 bool linear) { 671 ResourceMark rm; 672 VectorSet visited(Thread::current()->resource_area()); 673 Node_Stack phis(0); 674 675 676 for(int i = 0; i < 10; i++) { 677 if (current == NULL) { 678 return false; 679 } else if (visited.test_set(current->_idx) || current->is_top() || current == b1) { 680 current = NULL; 681 while (phis.is_nonempty() && current == NULL) { 682 uint idx = phis.index(); 683 Node* phi = phis.node(); 684 if (idx >= phi->req()) { 685 phis.pop(); 686 } else { 687 current = phi->in(idx); 688 phis.set_index(idx+1); 689 } 690 } 691 if (current == NULL) { 692 return true; 693 } 694 } else if (current == b2) { 695 return false; 696 } else if (current == phase->C->immutable_memory()) { 697 return false; 698 } else if (current->isa_Phi()) { 699 if (!linear) { 700 return false; 701 } 702 phis.push(current, 2); 703 current = current->in(1); 704 } else if (current->Opcode() == Op_ShenandoahWriteBarrier) { 705 current = current->in(Memory); 706 } else if (current->Opcode() == Op_ShenandoahWBMemProj) { 707 current = current->in(0); 708 } else if (current->is_Proj()) { 709 current = current->in(0); 710 } else if (current->is_Call()) { 711 current = current->in(TypeFunc::Memory); 712 } else if (current->is_MemBar()) { 713 current = current->in(TypeFunc::Memory); 714 } else if (current->is_MergeMem()) { 715 const TypePtr* adr_type = brooks_pointer_type(phase->type(b2)); 716 uint alias_idx = phase->C->get_alias_index(adr_type); 717 current = current->as_MergeMem()->memory_at(alias_idx); 718 } else { 719 #ifdef ASSERT 720 current->dump(); 721 #endif 722 ShouldNotReachHere(); 723 return false; 724 } 725 } 726 return false; 727 } 728 729 /** 730 * Determines if b1 dominates b2 through memory inputs. It returns true if: 731 * - b1 can be reached by following each branch in b2's memory input (through phis, etc) 732 * - or we get back to b2 (i.e. through a loop) without seeing b1 733 * In all other cases, (in particular, if we reach immutable_memory without having seen b1) 734 * we return false. 735 */ 736 bool ShenandoahBarrierNode::dominates_memory(PhaseGVN* phase, Node* b1, Node* b2, bool linear) { 737 return dominates_memory_impl(phase, b1, b2, b2->in(Memory), linear); 738 } 739 740 Node* ShenandoahBarrierNode::Identity_impl(PhaseGVN* phase) { 741 Node* n = in(ValueIn); 742 743 Node* rb_mem = Opcode() == Op_ShenandoahReadBarrier ? in(Memory) : NULL; 744 if (! needs_barrier(phase, this, n, rb_mem, _allow_fromspace)) { 745 return n; 746 } 747 748 // tty->print_cr("find sibling for: "); dump(2); 749 // Try to find a write barrier sibling with identical inputs that we can fold into. 750 for (DUIterator i = n->outs(); n->has_out(i); i++) { 751 Node* sibling = n->out(i); 752 if (sibling == this) { 753 continue; 754 } 755 /* 756 assert(sibling->Opcode() != Op_ShenandoahWriteBarrier || 757 Opcode() != Op_ShenandoahWriteBarrier || hash() == sibling->hash(), 758 "if this is a write barrier, then sibling can't be write barrier too"); 759 */ 760 if (sibling->Opcode() != Op_ShenandoahWriteBarrier) { 761 continue; 762 } 763 /* 764 if (sibling->outcnt() == 0) { 765 // Some dead node. 766 continue; 767 } 768 */ 769 assert(sibling->in(ValueIn) == in(ValueIn), "sanity"); 770 assert(sibling->Opcode() == Op_ShenandoahWriteBarrier, "sanity"); 771 // tty->print_cr("candidate: "); sibling->dump(); 772 773 if (dominates_memory(phase, sibling, this, phase->is_IterGVN() == NULL)) { 774 775 /* 776 tty->print_cr("matched barrier:"); 777 sibling->dump(); 778 tty->print_cr("for: "); 779 dump(); 780 */ 781 return sibling; 782 } 783 784 /* 785 tty->print_cr("couldn't match candidate:"); 786 sibling->dump(2); 787 */ 788 } 789 /* 790 tty->print_cr("couldn't match barrier to any:"); 791 dump(); 792 */ 793 return this; 794 } 795 796 #ifndef PRODUCT 797 void ShenandoahBarrierNode::dump_spec(outputStream *st) const { 798 const TypePtr* adr = adr_type(); 799 if (adr == NULL) { 800 return; 801 } 802 st->print(" @"); 803 adr->dump_on(st); 804 st->print(" ("); 805 Compile::current()->alias_type(adr)->adr_type()->dump_on(st); 806 st->print(") "); 807 } 808 #endif 809 810 Node* ShenandoahReadBarrierNode::Identity(PhaseGVN* phase) { 811 812 // if (true) return this; 813 814 // tty->print("optimizing rb: "); dump(); 815 Node* id = Identity_impl(phase); 816 817 if (id == this && phase->is_IterGVN()) { 818 Node* n = in(ValueIn); 819 // No success in super call. Try to combine identical read barriers. 820 for (DUIterator i = n->outs(); n->has_out(i); i++) { 821 Node* sibling = n->out(i); 822 if (sibling == this || sibling->Opcode() != Op_ShenandoahReadBarrier) { 823 continue; 824 } 825 assert(sibling->in(ValueIn) == in(ValueIn), "sanity"); 826 if (phase->is_IterGVN()->hash_find(sibling) && 827 sibling->bottom_type() == bottom_type() && 828 sibling->in(Control) == in(Control) && 829 dominates_memory_rb(phase, sibling, this, phase->is_IterGVN() == NULL)) { 830 /* 831 if (in(Memory) != sibling->in(Memory)) { 832 tty->print_cr("interesting rb-fold"); 833 dump(); 834 sibling->dump(); 835 } 836 */ 837 return sibling; 838 } 839 } 840 } 841 return id; 842 } 843 844 const Type* ShenandoahBarrierNode::Value(PhaseGVN* phase) const { 845 // Either input is TOP ==> the result is TOP 846 const Type *t1 = phase->type(in(Memory)); 847 if (t1 == Type::TOP) return Type::TOP; 848 const Type *t2 = phase->type(in(ValueIn)); 849 if( t2 == Type::TOP ) return Type::TOP; 850 851 Node* input = in(ValueIn); 852 const Type* type = phase->type(input)->is_oopptr()->cast_to_nonconst(); 853 return type; 854 } 855 856 uint ShenandoahBarrierNode::hash() const { 857 return TypeNode::hash() + _allow_fromspace; 858 } 859 860 uint ShenandoahBarrierNode::cmp(const Node& n) const { 861 return _allow_fromspace == ((ShenandoahBarrierNode&) n)._allow_fromspace 862 && TypeNode::cmp(n); 863 } 864 865 uint ShenandoahBarrierNode::size_of() const { 866 return sizeof(*this); 867 } 868 869 Node* ShenandoahWBMemProjNode::Identity(PhaseGVN* phase) { 870 871 Node* wb = in(0); 872 if (wb->is_top()) return phase->C->top(); // Dead path. 873 874 assert(wb->Opcode() == Op_ShenandoahWriteBarrier, "expect write barrier"); 875 PhaseIterGVN* igvn = phase->is_IterGVN(); 876 // We can't do the below unless the graph is fully constructed. 877 if (igvn == NULL) { 878 return this; 879 } 880 881 // If the mem projection has no barrier users, it's not needed anymore. 882 Unique_Node_List visited; 883 if (wb->outcnt() == 1) { 884 return wb->in(ShenandoahBarrierNode::Memory); 885 } 886 887 return this; 888 } 889 890 #ifdef ASSERT 891 bool ShenandoahBarrierNode::verify_helper(Node* in, Node_Stack& phis, VectorSet& visited, verify_type t, bool trace, Unique_Node_List& barriers_used) { 892 assert(phis.size() == 0, ""); 893 894 while (true) { 895 if (!in->bottom_type()->make_ptr()->make_oopptr()) { 896 if (trace) {tty->print_cr("Non oop");} 897 } else if (t == ShenandoahLoad && ShenandoahOptimizeStableFinals && 898 in->bottom_type()->make_ptr()->isa_aryptr() && 899 in->bottom_type()->make_ptr()->is_aryptr()->is_stable()) { 900 if (trace) {tty->print_cr("Stable array load");} 901 } else { 902 if (in->is_ConstraintCast()) { 903 in = in->in(1); 904 continue; 905 } else if (in->is_AddP()) { 906 assert(!in->in(AddPNode::Address)->is_top(), "no raw memory access"); 907 in = in->in(AddPNode::Address); 908 continue; 909 } else if (in->is_Con() && !ShenandoahBarriersForConst) { 910 if (trace) {tty->print("Found constant"); in->dump();} 911 } else if (in->is_ShenandoahBarrier()) { 912 if (t == ShenandoahStore && in->Opcode() != Op_ShenandoahWriteBarrier) { 913 return false; 914 } 915 barriers_used.push(in); 916 if (trace) {tty->print("Found barrier"); in->dump();} 917 } else if (in->is_Proj() && in->in(0)->is_Allocate()) { 918 if (trace) {tty->print("Found alloc"); in->in(0)->dump();} 919 } else if (in->is_Phi()) { 920 if (!visited.test_set(in->_idx)) { 921 if (trace) {tty->print("Pushed phi:"); in->dump();} 922 phis.push(in, 2); 923 in = in->in(1); 924 continue; 925 } 926 if (trace) {tty->print("Already seen phi:"); in->dump();} 927 } else if (in->Opcode() == Op_CMoveP || in->Opcode() == Op_CMoveN) { 928 if (!visited.test_set(in->_idx)) { 929 if (trace) {tty->print("Pushed cmovep:"); in->dump();} 930 phis.push(in, CMoveNode::IfTrue); 931 in = in->in(CMoveNode::IfFalse); 932 continue; 933 } 934 if (trace) {tty->print("Already seen cmovep:"); in->dump();} 935 } else if (in->Opcode() == Op_EncodeP || in->Opcode() == Op_DecodeN) { 936 in = in->in(1); 937 continue; 938 } else { 939 return false; 940 } 941 } 942 bool cont = false; 943 while (phis.is_nonempty()) { 944 uint idx = phis.index(); 945 Node* phi = phis.node(); 946 if (idx >= phi->req()) { 947 if (trace) {tty->print("Popped phi:"); phi->dump();} 948 phis.pop(); 949 continue; 950 } 951 if (trace) {tty->print("Next entry(%d) for phi:", idx); phi->dump();} 952 in = phi->in(idx); 953 phis.set_index(idx+1); 954 cont = true; 955 break; 956 } 957 if (!cont) { 958 break; 959 } 960 } 961 return true; 962 } 963 964 void ShenandoahBarrierNode::report_verify_failure(const char *msg, Node *n1, Node *n2) { 965 if (n1 != NULL) { 966 n1->dump(+10); 967 } 968 if (n2 != NULL) { 969 n2->dump(+10); 970 } 971 fatal("%s", msg); 972 } 973 974 void ShenandoahBarrierNode::verify(RootNode* root) { 975 ResourceMark rm; 976 Unique_Node_List wq; 977 GrowableArray<Node*> barriers; 978 Unique_Node_List barriers_used; 979 Node_Stack phis(0); 980 VectorSet visited(Thread::current()->resource_area()); 981 const bool trace = false; 982 const bool verify_no_useless_barrier = false; 983 984 wq.push(root); 985 for (uint next = 0; next < wq.size(); next++) { 986 Node *n = wq.at(next); 987 if (n->is_Load()) { 988 const bool trace = false; 989 if (trace) {tty->print("Verifying"); n->dump();} 990 if (n->Opcode() == Op_LoadRange || n->Opcode() == Op_LoadKlass || n->Opcode() == Op_LoadNKlass) { 991 if (trace) {tty->print_cr("Load range/klass");} 992 } else { 993 const TypePtr* adr_type = n->as_Load()->adr_type(); 994 995 if (adr_type->isa_oopptr() && adr_type->is_oopptr()->offset() == oopDesc::mark_offset_in_bytes()) { 996 if (trace) {tty->print_cr("Mark load");} 997 } else if (adr_type->isa_instptr() && 998 adr_type->is_instptr()->klass()->is_subtype_of(Compile::current()->env()->Reference_klass()) && 999 adr_type->is_instptr()->offset() == java_lang_ref_Reference::referent_offset) { 1000 if (trace) {tty->print_cr("Reference.get()");} 1001 } else { 1002 bool verify = true; 1003 if (adr_type->isa_instptr()) { 1004 const TypeInstPtr* tinst = adr_type->is_instptr(); 1005 ciKlass* k = tinst->klass(); 1006 assert(k->is_instance_klass(), ""); 1007 ciInstanceKlass* ik = (ciInstanceKlass*)k; 1008 int offset = adr_type->offset(); 1009 1010 if ((ik->debug_final_field_at(offset) && ShenandoahOptimizeInstanceFinals) || 1011 (ik->debug_stable_field_at(offset) && ShenandoahOptimizeStableFinals)) { 1012 if (trace) {tty->print_cr("Final/stable");} 1013 verify = false; 1014 } else if (k == ciEnv::current()->Class_klass() && 1015 tinst->const_oop() != NULL && 1016 tinst->offset() >= (ik->size_helper() * wordSize)) { 1017 ciInstanceKlass* k = tinst->const_oop()->as_instance()->java_lang_Class_klass()->as_instance_klass(); 1018 ciField* field = k->get_field_by_offset(tinst->offset(), true); 1019 if ((ShenandoahOptimizeStaticFinals && field->is_final()) || 1020 (ShenandoahOptimizeStableFinals && field->is_stable())) { 1021 verify = false; 1022 } 1023 } 1024 } 1025 1026 if (verify && !ShenandoahBarrierNode::verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahLoad, trace, barriers_used)) { 1027 report_verify_failure("Shenandoah verification: Load should have barriers", n); 1028 } 1029 } 1030 } 1031 } else if (n->is_Store()) { 1032 const bool trace = false; 1033 1034 if (trace) {tty->print("Verifying"); n->dump();} 1035 if (n->in(MemNode::ValueIn)->bottom_type()->make_oopptr()) { 1036 Node* adr = n->in(MemNode::Address); 1037 bool verify = true; 1038 1039 if (adr->is_AddP() && adr->in(AddPNode::Base)->is_top()) { 1040 adr = adr->in(AddPNode::Address); 1041 if (adr->is_AddP()) { 1042 assert(adr->in(AddPNode::Base)->is_top(), ""); 1043 adr = adr->in(AddPNode::Address); 1044 if (adr->Opcode() == Op_LoadP && 1045 adr->in(MemNode::Address)->in(AddPNode::Base)->is_top() && 1046 adr->in(MemNode::Address)->in(AddPNode::Address)->Opcode() == Op_ThreadLocal && 1047 adr->in(MemNode::Address)->in(AddPNode::Offset)->find_intptr_t_con(-1) == in_bytes(JavaThread::satb_mark_queue_offset() + 1048 SATBMarkQueue::byte_offset_of_buf())) { 1049 if (trace) {tty->print_cr("G1 prebarrier");} 1050 verify = false; 1051 } 1052 } 1053 } 1054 1055 if (verify && !ShenandoahBarrierNode::verify_helper(n->in(MemNode::ValueIn), phis, visited, ShenandoahValue, trace, barriers_used)) { 1056 report_verify_failure("Shenandoah verification: Store should have barriers", n); 1057 } 1058 } 1059 if (!ShenandoahBarrierNode::verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahStore, trace, barriers_used)) { 1060 report_verify_failure("Shenandoah verification: Store (address) should have barriers", n); 1061 } 1062 } else if (n->Opcode() == Op_CmpP) { 1063 const bool trace = false; 1064 1065 Node* in1 = n->in(1); 1066 Node* in2 = n->in(2); 1067 if (in1->bottom_type()->isa_oopptr()) { 1068 if (trace) {tty->print("Verifying"); n->dump();} 1069 1070 bool mark_inputs = false; 1071 if (in1->bottom_type() == TypePtr::NULL_PTR || in2->bottom_type() == TypePtr::NULL_PTR || 1072 ((in1->is_Con() || in2->is_Con()) && !ShenandoahBarriersForConst)) { 1073 if (trace) {tty->print_cr("Comparison against a constant");} 1074 mark_inputs = true; 1075 } else if ((in1->is_CheckCastPP() && in1->in(1)->is_Proj() && in1->in(1)->in(0)->is_Allocate()) || 1076 (in2->is_CheckCastPP() && in2->in(1)->is_Proj() && in2->in(1)->in(0)->is_Allocate())) { 1077 if (trace) {tty->print_cr("Comparison with newly alloc'ed object");} 1078 mark_inputs = true; 1079 } else { 1080 assert(in2->bottom_type()->isa_oopptr(), ""); 1081 1082 if (!ShenandoahBarrierNode::verify_helper(in1, phis, visited, ShenandoahStore, trace, barriers_used) || 1083 !ShenandoahBarrierNode::verify_helper(in2, phis, visited, ShenandoahStore, trace, barriers_used)) { 1084 report_verify_failure("Shenandoah verification: Cmp should have barriers", n); 1085 } 1086 } 1087 if (verify_no_useless_barrier && 1088 mark_inputs && 1089 (!ShenandoahBarrierNode::verify_helper(in1, phis, visited, ShenandoahValue, trace, barriers_used) || 1090 !ShenandoahBarrierNode::verify_helper(in2, phis, visited, ShenandoahValue, trace, barriers_used))) { 1091 phis.clear(); 1092 visited.Reset(); 1093 } 1094 } 1095 } else if (n->is_LoadStore()) { 1096 if (n->in(MemNode::ValueIn)->bottom_type()->isa_ptr() && 1097 !ShenandoahBarrierNode::verify_helper(n->in(MemNode::ValueIn), phis, visited, ShenandoahLoad, trace, barriers_used)) { 1098 report_verify_failure("Shenandoah verification: LoadStore (value) should have barriers", n); 1099 } 1100 1101 if (n->in(MemNode::Address)->bottom_type()->isa_oopptr() && !ShenandoahBarrierNode::verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahStore, trace, barriers_used)) { 1102 report_verify_failure("Shenandoah verification: LoadStore (address) should have barriers", n); 1103 } 1104 } else if (n->Opcode() == Op_CallLeafNoFP || n->Opcode() == Op_CallLeaf) { 1105 CallNode* call = n->as_Call(); 1106 1107 static struct { 1108 const char* name; 1109 struct { 1110 int pos; 1111 verify_type t; 1112 } args[6]; 1113 } calls[] = { 1114 "aescrypt_encryptBlock", 1115 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad }, 1116 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1117 "aescrypt_decryptBlock", 1118 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad }, 1119 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1120 "multiplyToLen", 1121 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad }, { TypeFunc::Parms+4, ShenandoahStore }, 1122 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1123 "squareToLen", 1124 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad }, { -1, ShenandoahNone}, 1125 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1126 "montgomery_multiply", 1127 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad }, 1128 { TypeFunc::Parms+6, ShenandoahStore }, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1129 "montgomery_square", 1130 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+5, ShenandoahStore }, 1131 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1132 "mulAdd", 1133 { { TypeFunc::Parms, ShenandoahStore }, { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone}, 1134 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1135 "vectorizedMismatch", 1136 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone}, 1137 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1138 "updateBytesCRC32", 1139 { { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone}, 1140 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1141 "updateBytesAdler32", 1142 { { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone}, 1143 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1144 "updateBytesCRC32C", 1145 { { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahLoad}, { -1, ShenandoahNone}, 1146 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1147 "counterMode_AESCrypt", 1148 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad }, 1149 { TypeFunc::Parms+3, ShenandoahStore }, { TypeFunc::Parms+5, ShenandoahStore }, { TypeFunc::Parms+6, ShenandoahStore } }, 1150 "cipherBlockChaining_encryptAESCrypt", 1151 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad }, 1152 { TypeFunc::Parms+3, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1153 "cipherBlockChaining_decryptAESCrypt", 1154 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad }, 1155 { TypeFunc::Parms+3, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1156 "shenandoah_clone_barrier", 1157 { { TypeFunc::Parms, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone}, 1158 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1159 "ghash_processBlocks", 1160 { { TypeFunc::Parms, ShenandoahStore }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad }, 1161 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1162 "sha1_implCompress", 1163 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone }, 1164 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1165 "sha256_implCompress", 1166 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone }, 1167 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1168 "sha512_implCompress", 1169 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone }, 1170 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1171 "sha1_implCompressMB", 1172 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone }, 1173 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1174 "sha256_implCompressMB", 1175 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone }, 1176 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1177 "sha512_implCompressMB", 1178 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone }, 1179 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1180 }; 1181 1182 if (call->is_call_to_arraycopystub()) { 1183 Node* dest = NULL; 1184 const TypeTuple* args = n->as_Call()->_tf->domain(); 1185 for (uint i = TypeFunc::Parms, j = 0; i < args->cnt(); i++) { 1186 if (args->field_at(i)->isa_ptr()) { 1187 j++; 1188 if (j == 2) { 1189 dest = n->in(i); 1190 break; 1191 } 1192 } 1193 } 1194 if (!ShenandoahBarrierNode::verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahLoad, trace, barriers_used) || 1195 !ShenandoahBarrierNode::verify_helper(dest, phis, visited, ShenandoahStore, trace, barriers_used)) { 1196 report_verify_failure("Shenandoah verification: ArrayCopy should have barriers", n); 1197 } 1198 } else if (strlen(call->_name) > 5 && 1199 !strcmp(call->_name + strlen(call->_name) - 5, "_fill")) { 1200 if (!ShenandoahBarrierNode::verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahStore, trace, barriers_used)) { 1201 report_verify_failure("Shenandoah verification: _fill should have barriers", n); 1202 } 1203 } else if (!strcmp(call->_name, "g1_wb_pre")) { 1204 // skip 1205 } else { 1206 const int calls_len = sizeof(calls) / sizeof(calls[0]); 1207 int i = 0; 1208 for (; i < calls_len; i++) { 1209 if (!strcmp(calls[i].name, call->_name)) { 1210 break; 1211 } 1212 } 1213 if (i != calls_len) { 1214 const uint args_len = sizeof(calls[0].args) / sizeof(calls[0].args[0]); 1215 for (uint j = 0; j < args_len; j++) { 1216 int pos = calls[i].args[j].pos; 1217 if (pos == -1) { 1218 break; 1219 } 1220 if (!ShenandoahBarrierNode::verify_helper(call->in(pos), phis, visited, calls[i].args[j].t, trace, barriers_used)) { 1221 report_verify_failure("Shenandoah verification: intrinsic calls should have barriers", n); 1222 } 1223 } 1224 for (uint j = TypeFunc::Parms; j < call->req(); j++) { 1225 if (call->in(j)->bottom_type()->make_ptr() && 1226 call->in(j)->bottom_type()->make_ptr()->isa_oopptr()) { 1227 uint k = 0; 1228 for (; k < args_len && calls[i].args[k].pos != (int)j; k++); 1229 if (k == args_len) { 1230 fatal("arg %d for call %s not covered", j, call->_name); 1231 } 1232 } 1233 } 1234 } else { 1235 for (uint j = TypeFunc::Parms; j < call->req(); j++) { 1236 if (call->in(j)->bottom_type()->make_ptr() && 1237 call->in(j)->bottom_type()->make_ptr()->isa_oopptr()) { 1238 fatal("%s not covered", call->_name); 1239 } 1240 } 1241 } 1242 } 1243 } else if (n->is_ShenandoahBarrier()) { 1244 assert(!barriers.contains(n), ""); 1245 assert(n->Opcode() != Op_ShenandoahWriteBarrier || n->find_out_with(Op_ShenandoahWBMemProj) != NULL, "bad shenandoah write barrier"); 1246 assert(n->Opcode() != Op_ShenandoahWriteBarrier || n->outcnt() > 1, "bad shenandoah write barrier"); 1247 barriers.push(n); 1248 } else if (n->is_AddP() 1249 || n->is_Phi() 1250 || n->is_ConstraintCast() 1251 || n->Opcode() == Op_Return 1252 || n->Opcode() == Op_CMoveP 1253 || n->Opcode() == Op_CMoveN 1254 || n->Opcode() == Op_Rethrow 1255 || n->is_MemBar() 1256 || n->Opcode() == Op_Conv2B 1257 || n->Opcode() == Op_SafePoint 1258 || n->is_CallJava() 1259 || n->Opcode() == Op_Unlock 1260 || n->Opcode() == Op_EncodeP 1261 || n->Opcode() == Op_DecodeN) { 1262 // nothing to do 1263 } else { 1264 static struct { 1265 int opcode; 1266 struct { 1267 int pos; 1268 verify_type t; 1269 } inputs[2]; 1270 } others[] = { 1271 Op_FastLock, 1272 { { 1, ShenandoahLoad }, { -1, ShenandoahNone} }, 1273 Op_Lock, 1274 { { TypeFunc::Parms, ShenandoahLoad }, { -1, ShenandoahNone} }, 1275 Op_ArrayCopy, 1276 { { ArrayCopyNode::Src, ShenandoahLoad }, { ArrayCopyNode::Dest, ShenandoahStore } }, 1277 Op_StrCompressedCopy, 1278 { { 2, ShenandoahLoad }, { 3, ShenandoahStore } }, 1279 Op_StrInflatedCopy, 1280 { { 2, ShenandoahLoad }, { 3, ShenandoahStore } }, 1281 Op_AryEq, 1282 { { 2, ShenandoahLoad }, { 3, ShenandoahLoad } }, 1283 Op_StrIndexOf, 1284 { { 2, ShenandoahLoad }, { 4, ShenandoahLoad } }, 1285 Op_StrComp, 1286 { { 2, ShenandoahLoad }, { 4, ShenandoahLoad } }, 1287 Op_StrEquals, 1288 { { 2, ShenandoahLoad }, { 3, ShenandoahLoad } }, 1289 Op_EncodeISOArray, 1290 { { 2, ShenandoahLoad }, { 3, ShenandoahStore } }, 1291 Op_HasNegatives, 1292 { { 2, ShenandoahLoad }, { -1, ShenandoahNone} }, 1293 Op_CastP2X, 1294 { { 1, ShenandoahLoad }, { -1, ShenandoahNone} }, 1295 Op_StrIndexOfChar, 1296 { { 2, ShenandoahLoad }, { -1, ShenandoahNone } }, 1297 }; 1298 1299 const int others_len = sizeof(others) / sizeof(others[0]); 1300 int i = 0; 1301 for (; i < others_len; i++) { 1302 if (others[i].opcode == n->Opcode()) { 1303 break; 1304 } 1305 } 1306 uint stop = n->is_Call() ? n->as_Call()->tf()->domain()->cnt() : n->req(); 1307 if (i != others_len) { 1308 const uint inputs_len = sizeof(others[0].inputs) / sizeof(others[0].inputs[0]); 1309 for (uint j = 0; j < inputs_len; j++) { 1310 int pos = others[i].inputs[j].pos; 1311 if (pos == -1) { 1312 break; 1313 } 1314 if (!ShenandoahBarrierNode::verify_helper(n->in(pos), phis, visited, others[i].inputs[j].t, trace, barriers_used)) { 1315 report_verify_failure("Shenandoah verification: intrinsic calls should have barriers", n); 1316 } 1317 } 1318 for (uint j = 1; j < stop; j++) { 1319 if (n->in(j) != NULL && n->in(j)->bottom_type()->make_ptr() && 1320 n->in(j)->bottom_type()->make_ptr()->make_oopptr()) { 1321 uint k = 0; 1322 for (; k < inputs_len && others[i].inputs[k].pos != (int)j; k++); 1323 if (k == inputs_len) { 1324 fatal("arg %d for node %s not covered", j, n->Name()); 1325 } 1326 } 1327 } 1328 } else { 1329 for (uint j = 1; j < stop; j++) { 1330 if (n->in(j) != NULL && n->in(j)->bottom_type()->make_ptr() && 1331 n->in(j)->bottom_type()->make_ptr()->make_oopptr()) { 1332 fatal("%s not covered", n->Name()); 1333 } 1334 } 1335 } 1336 } 1337 1338 if (n->is_SafePoint()) { 1339 SafePointNode* sfpt = n->as_SafePoint(); 1340 if (verify_no_useless_barrier && sfpt->jvms() != NULL) { 1341 for (uint i = sfpt->jvms()->scloff(); i < sfpt->jvms()->endoff(); i++) { 1342 if (!ShenandoahBarrierNode::verify_helper(sfpt->in(i), phis, visited, ShenandoahLoad, trace, barriers_used)) { 1343 phis.clear(); 1344 visited.Reset(); 1345 } 1346 } 1347 } 1348 } 1349 for( uint i = 0; i < n->len(); ++i ) { 1350 Node *m = n->in(i); 1351 if (m == NULL) continue; 1352 1353 // In most cases, inputs should be known to be non null. If it's 1354 // not the case, it could be a missing cast_not_null() in an 1355 // intrinsic or support might be needed in AddPNode::Ideal() to 1356 // avoid a NULL+offset input. 1357 if (!(n->is_Phi() || 1358 (n->is_SafePoint() && (!n->is_CallRuntime() || !strcmp(n->as_Call()->_name, "g1_wb_pre") || !strcmp(n->as_Call()->_name, "unsafe_arraycopy"))) || 1359 n->Opcode() == Op_CmpP || 1360 n->Opcode() == Op_CmpN || 1361 (n->Opcode() == Op_StoreP && i == StoreNode::ValueIn) || 1362 (n->Opcode() == Op_StoreN && i == StoreNode::ValueIn) || 1363 n->is_ConstraintCast() || 1364 n->Opcode() == Op_Return || 1365 n->Opcode() == Op_Conv2B || 1366 n->is_AddP() || 1367 n->Opcode() == Op_CMoveP || 1368 n->Opcode() == Op_CMoveN || 1369 n->Opcode() == Op_Rethrow || 1370 n->is_MemBar() || 1371 n->is_Mem() || 1372 n->Opcode() == Op_AryEq || 1373 n->Opcode() == Op_SCMemProj || 1374 n->Opcode() == Op_EncodeP || 1375 n->Opcode() == Op_DecodeN)) { 1376 if (m->bottom_type()->make_oopptr() && m->bottom_type()->make_oopptr()->meet(TypePtr::NULL_PTR) == m->bottom_type()) { 1377 report_verify_failure("Shenandoah verification: null input", n, m); 1378 } 1379 } 1380 1381 wq.push(m); 1382 } 1383 } 1384 1385 if (verify_no_useless_barrier) { 1386 for (int i = 0; i < barriers.length(); i++) { 1387 Node* n = barriers.at(i); 1388 if (!barriers_used.member(n)) { 1389 tty->print("XXX useless barrier"); n->dump(-2); 1390 ShouldNotReachHere(); 1391 } 1392 } 1393 } 1394 } 1395 #endif 1396 1397 MergeMemNode* ShenandoahWriteBarrierNode::allocate_merge_mem(Node* mem, int alias, Node* rep_proj, Node* rep_ctrl, PhaseIdealLoop* phase) { 1398 MergeMemNode* mm = MergeMemNode::make(mem); 1399 mm->set_memory_at(alias, rep_proj); 1400 phase->register_new_node(mm, rep_ctrl); 1401 return mm; 1402 } 1403 1404 MergeMemNode* ShenandoahWriteBarrierNode::clone_merge_mem(Node* u, Node* mem, int alias, Node* rep_proj, Node* rep_ctrl, DUIterator& i, PhaseIdealLoop* phase) { 1405 MergeMemNode* newmm = NULL; 1406 MergeMemNode* u_mm = u->as_MergeMem(); 1407 Node* c = phase->get_ctrl(u); 1408 if (phase->is_dominator(c, rep_ctrl)) { 1409 c = rep_ctrl; 1410 } else { 1411 assert(phase->is_dominator(rep_ctrl, c), "one must dominate the other"); 1412 } 1413 if (u->outcnt() == 1) { 1414 if (u->req() > (uint)alias && u->in(alias) == mem) { 1415 phase->igvn().replace_input_of(u, alias, rep_proj); 1416 --i; 1417 } else { 1418 phase->igvn().rehash_node_delayed(u); 1419 u_mm->set_memory_at(alias, rep_proj); 1420 } 1421 newmm = u_mm; 1422 phase->set_ctrl_and_loop(u, c); 1423 } else { 1424 // can't simply clone u and then change one of its input because 1425 // it adds and then removes an edge which messes with the 1426 // DUIterator 1427 newmm = MergeMemNode::make(u_mm->base_memory()); 1428 for (uint j = 0; j < u->req(); j++) { 1429 if (j < newmm->req()) { 1430 if (j == (uint)alias) { 1431 newmm->set_req(j, rep_proj); 1432 } else if (newmm->in(j) != u->in(j)) { 1433 newmm->set_req(j, u->in(j)); 1434 } 1435 } else if (j == (uint)alias) { 1436 newmm->add_req(rep_proj); 1437 } else { 1438 newmm->add_req(u->in(j)); 1439 } 1440 } 1441 if ((uint)alias >= u->req()) { 1442 newmm->set_memory_at(alias, rep_proj); 1443 } 1444 phase->register_new_node(newmm, c); 1445 } 1446 return newmm; 1447 } 1448 1449 bool ShenandoahWriteBarrierNode::should_process_phi(Node* phi, int alias, Compile* C) { 1450 if (phi->adr_type() == TypePtr::BOTTOM) { 1451 Node* region = phi->in(0); 1452 for (DUIterator_Fast jmax, j = region->fast_outs(jmax); j < jmax; j++) { 1453 Node* uu = region->fast_out(j); 1454 if (uu->is_Phi() && uu != phi && uu->bottom_type() == Type::MEMORY && C->get_alias_index(uu->adr_type()) == alias) { 1455 return false; 1456 } 1457 } 1458 return true; 1459 } 1460 return C->get_alias_index(phi->adr_type()) == alias; 1461 } 1462 1463 bool ShenandoahBarrierNode::is_dominator_same_ctrl(Node*c, Node* d, Node* n, PhaseIdealLoop* phase) { 1464 // That both nodes have the same control is not sufficient to prove 1465 // domination, verify that there's no path from d to n 1466 ResourceMark rm; 1467 Unique_Node_List wq; 1468 wq.push(d); 1469 for (uint next = 0; next < wq.size(); next++) { 1470 Node *m = wq.at(next); 1471 if (m == n) { 1472 return false; 1473 } 1474 if (m->is_Phi() && m->in(0)->is_Loop()) { 1475 assert(phase->ctrl_or_self(m->in(LoopNode::EntryControl)) != c, "following loop entry should lead to new control"); 1476 } else { 1477 for (uint i = 0; i < m->req(); i++) { 1478 if (m->in(i) != NULL && phase->ctrl_or_self(m->in(i)) == c) { 1479 wq.push(m->in(i)); 1480 } 1481 } 1482 } 1483 } 1484 return true; 1485 } 1486 1487 bool ShenandoahBarrierNode::is_dominator(Node *d_c, Node *n_c, Node* d, Node* n, PhaseIdealLoop* phase) { 1488 if (d_c != n_c) { 1489 return phase->is_dominator(d_c, n_c); 1490 } 1491 return is_dominator_same_ctrl(d_c, d, n, phase); 1492 } 1493 1494 Node* next_mem(Node* mem, int alias) { 1495 Node* res = NULL; 1496 if (mem->is_Proj()) { 1497 res = mem->in(0); 1498 } else if (mem->is_SafePoint() || mem->is_MemBar()) { 1499 res = mem->in(TypeFunc::Memory); 1500 } else if (mem->is_Phi()) { 1501 res = mem->in(1); 1502 } else if (mem->is_ShenandoahBarrier()) { 1503 res = mem->in(ShenandoahBarrierNode::Memory); 1504 } else if (mem->is_MergeMem()) { 1505 res = mem->as_MergeMem()->memory_at(alias); 1506 } else if (mem->is_Store() || mem->is_LoadStore() || mem->is_ClearArray()) { 1507 assert(alias = Compile::AliasIdxRaw, "following raw memory can't lead to a barrier"); 1508 res = mem->in(MemNode::Memory); 1509 } else { 1510 #ifdef ASSERT 1511 mem->dump(); 1512 #endif 1513 ShouldNotReachHere(); 1514 } 1515 return res; 1516 } 1517 1518 bool suitable_mem(Node* mem, Node* old_mem, Node* rep_proj) { 1519 for (DUIterator_Fast imax, i = mem->fast_outs(imax); i < imax; i++) { 1520 Node* u = mem->fast_out(i); 1521 if (u->is_MergeMem()) { 1522 if (u->has_out_with(Op_MergeMem)) { 1523 // too complicated for now 1524 return false; 1525 } 1526 if (old_mem == u && rep_proj->has_out_with(Op_MergeMem)) { 1527 return false; 1528 } 1529 } 1530 if (u->Opcode() == Op_Unlock && mem->is_Proj() && mem->in(0)->Opcode() == Op_MemBarReleaseLock) { 1531 // would require a merge mem between unlock and the 1532 // preceding membar. Would confuse logic that eliminates 1533 // lock/unlock nodes. 1534 return false; 1535 } 1536 } 1537 return true; 1538 } 1539 1540 void ShenandoahWriteBarrierNode::fix_memory_uses(Node* mem, Node* replacement, Node* rep_proj, Node* rep_ctrl, int alias, PhaseIdealLoop* phase) { 1541 uint last =phase-> C->unique(); 1542 MergeMemNode* mm = NULL; 1543 assert(mem->bottom_type() == Type::MEMORY, ""); 1544 for (DUIterator i = mem->outs(); mem->has_out(i); i++) { 1545 Node* u = mem->out(i); 1546 if (u != replacement && u->_idx < last) { 1547 if (u->is_ShenandoahBarrier() && alias != Compile::AliasIdxRaw) { 1548 if (phase->C->get_alias_index(u->adr_type()) == alias && is_dominator(rep_ctrl, phase->ctrl_or_self(u), replacement, u, phase)) { 1549 phase->igvn().replace_input_of(u, u->find_edge(mem), rep_proj); 1550 assert(u->find_edge(mem) == -1, "only one edge"); 1551 --i; 1552 } 1553 } else if (u->is_Mem()) { 1554 if (phase->C->get_alias_index(u->adr_type()) == alias && is_dominator(rep_ctrl, phase->ctrl_or_self(u), replacement, u, phase)) { 1555 assert(alias == Compile::AliasIdxRaw , "only raw memory can lead to a memory operation"); 1556 phase->igvn().replace_input_of(u, u->find_edge(mem), rep_proj); 1557 assert(u->find_edge(mem) == -1, "only one edge"); 1558 --i; 1559 } 1560 } else if (u->is_MergeMem()) { 1561 MergeMemNode* u_mm = u->as_MergeMem(); 1562 if (u_mm->memory_at(alias) == mem) { 1563 MergeMemNode* newmm = NULL; 1564 for (DUIterator_Fast jmax, j = u->fast_outs(jmax); j < jmax; j++) { 1565 Node* uu = u->fast_out(j); 1566 assert(!uu->is_MergeMem(), "chain of MergeMems?"); 1567 if (uu->is_Phi()) { 1568 if (should_process_phi(uu, alias, phase->C)) { 1569 Node* region = uu->in(0); 1570 int nb = 0; 1571 for (uint k = 1; k < uu->req(); k++) { 1572 if (uu->in(k) == u && phase->is_dominator(rep_ctrl, region->in(k))) { 1573 if (newmm == NULL) { 1574 newmm = clone_merge_mem(u, mem, alias, rep_proj, rep_ctrl, i, phase); 1575 } 1576 if (newmm != u) { 1577 phase->igvn().replace_input_of(uu, k, newmm); 1578 nb++; 1579 --jmax; 1580 } 1581 } 1582 } 1583 if (nb > 0) { 1584 --j; 1585 } 1586 } 1587 } else { 1588 if (rep_ctrl != uu && is_dominator(rep_ctrl, phase->ctrl_or_self(uu), replacement, uu, phase)) { 1589 if (newmm == NULL) { 1590 newmm = clone_merge_mem(u, mem, alias, rep_proj, rep_ctrl, i, phase); 1591 } 1592 if (newmm != u) { 1593 phase->igvn().replace_input_of(uu, uu->find_edge(u), newmm); 1594 --j, --jmax; 1595 } 1596 } 1597 } 1598 } 1599 } 1600 } else if (u->is_Phi()) { 1601 assert(u->bottom_type() == Type::MEMORY, "what else?"); 1602 Node* region = u->in(0); 1603 if (should_process_phi(u, alias, phase->C)) { 1604 bool replaced = false; 1605 for (uint j = 1; j < u->req(); j++) { 1606 if (u->in(j) == mem && phase->is_dominator(rep_ctrl, region->in(j))) { 1607 Node* nnew = rep_proj; 1608 if (u->adr_type() == TypePtr::BOTTOM) { 1609 if (mm == NULL) { 1610 mm = allocate_merge_mem(mem, alias, rep_proj, rep_ctrl, phase); 1611 } 1612 nnew = mm; 1613 } 1614 phase->igvn().replace_input_of(u, j, nnew); 1615 replaced = true; 1616 } 1617 } 1618 if (replaced) { 1619 --i; 1620 } 1621 1622 } 1623 } else if ((u->adr_type() == TypePtr::BOTTOM && u->Opcode() != Op_StrInflatedCopy) || 1624 u->adr_type() == NULL) { 1625 assert(u->adr_type() != NULL || 1626 u->Opcode() == Op_Rethrow || 1627 u->Opcode() == Op_Return || 1628 u->Opcode() == Op_SafePoint || 1629 (u->is_CallStaticJava() && u->as_CallStaticJava()->uncommon_trap_request() != 0) || 1630 (u->is_CallStaticJava() && u->as_CallStaticJava()->_entry_point == OptoRuntime::rethrow_stub()) || 1631 u->Opcode() == Op_CallLeaf, ""); 1632 if (is_dominator(rep_ctrl, phase->ctrl_or_self(u), replacement, u, phase)) { 1633 if (mm == NULL) { 1634 mm = allocate_merge_mem(mem, alias, rep_proj, rep_ctrl, phase); 1635 } 1636 phase->igvn().replace_input_of(u, u->find_edge(mem), mm); 1637 --i; 1638 } 1639 } else if (phase->C->get_alias_index(u->adr_type()) == alias) { 1640 if (is_dominator(rep_ctrl, phase->ctrl_or_self(u), replacement, u, phase)) { 1641 phase->igvn().replace_input_of(u, u->find_edge(mem), rep_proj); 1642 --i; 1643 } 1644 } 1645 } 1646 } 1647 } 1648 1649 Node* ShenandoahBarrierNode::no_branches(Node* c, Node* dom, bool allow_one_proj, PhaseIdealLoop* phase) { 1650 Node* iffproj = NULL; 1651 while (c != dom) { 1652 Node* next = phase->idom(c); 1653 assert(next->unique_ctrl_out() == c || c->is_Proj() || c->is_Region(), "multiple control flow out but no proj or region?"); 1654 if (c->is_Region()) { 1655 ResourceMark rm; 1656 Unique_Node_List wq; 1657 wq.push(c); 1658 for (uint i = 0; i < wq.size(); i++) { 1659 Node *n = wq.at(i); 1660 if (n->is_Region()) { 1661 for (uint j = 1; j < n->req(); j++) { 1662 if (n->in(j) != next) { 1663 wq.push(n->in(j)); 1664 } 1665 } 1666 } else { 1667 if (n->in(0) != next) { 1668 wq.push(n->in(0)); 1669 } 1670 } 1671 } 1672 for (DUIterator_Fast imax, i = next->fast_outs(imax); i < imax; i++) { 1673 Node* u = next->fast_out(i); 1674 if (u->is_CFG()) { 1675 if (!wq.member(u)) { 1676 return NodeSentinel; 1677 } 1678 } 1679 } 1680 1681 } else if (c->is_Proj()) { 1682 if (c->is_IfProj()) { 1683 if (c->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none) != NULL) { 1684 // continue; 1685 } else { 1686 if (!allow_one_proj) { 1687 return NodeSentinel; 1688 } 1689 if (iffproj == NULL) { 1690 iffproj = c; 1691 } else { 1692 return NodeSentinel; 1693 } 1694 } 1695 } else if (c->Opcode() == Op_JumpProj) { 1696 return NodeSentinel; // unsupported 1697 } else if (c->Opcode() == Op_CatchProj) { 1698 return NodeSentinel; // unsupported 1699 } else if (c->Opcode() == Op_CProj && next->Opcode() == Op_NeverBranch) { 1700 return NodeSentinel; // unsupported 1701 } else { 1702 assert(next->unique_ctrl_out() == c, "unsupported branch pattern"); 1703 } 1704 } 1705 c = next; 1706 } 1707 return iffproj; 1708 } 1709 1710 #ifdef ASSERT 1711 void ShenandoahWriteBarrierNode::memory_dominates_all_paths_helper(Node* c, Node* rep_ctrl, Unique_Node_List& controls, PhaseIdealLoop* phase) { 1712 const bool trace = false; 1713 if (trace) { tty->print("X control is"); c->dump(); } 1714 1715 uint start = controls.size(); 1716 controls.push(c); 1717 for (uint i = start; i < controls.size(); i++) { 1718 Node *n = controls.at(i); 1719 1720 if (trace) { tty->print("X from"); n->dump(); } 1721 1722 if (n == rep_ctrl) { 1723 continue; 1724 } 1725 1726 if (n->is_Proj()) { 1727 Node* n_dom = n->in(0); 1728 IdealLoopTree* n_dom_loop = phase->get_loop(n_dom); 1729 if (n->is_IfProj() && n_dom->outcnt() == 2) { 1730 n_dom_loop = phase->get_loop(n_dom->as_If()->proj_out(n->as_Proj()->_con == 0 ? 1 : 0)); 1731 } 1732 if (n_dom_loop != phase->ltree_root()) { 1733 Node* tail = n_dom_loop->tail(); 1734 if (tail->is_Region()) { 1735 for (uint j = 1; j < tail->req(); j++) { 1736 if (phase->is_dominator(n_dom, tail->in(j)) && !phase->is_dominator(n, tail->in(j))) { 1737 assert(phase->is_dominator(rep_ctrl, tail->in(j)), "why are we here?"); 1738 // entering loop from below, mark backedge 1739 if (trace) { tty->print("X pushing backedge"); tail->in(j)->dump(); } 1740 controls.push(tail->in(j)); 1741 //assert(n->in(0) == n_dom, "strange flow control"); 1742 } 1743 } 1744 } else if (phase->get_loop(n) != n_dom_loop && phase->is_dominator(n_dom, tail)) { 1745 // entering loop from below, mark backedge 1746 if (trace) { tty->print("X pushing backedge"); tail->dump(); } 1747 controls.push(tail); 1748 //assert(n->in(0) == n_dom, "strange flow control"); 1749 } 1750 } 1751 } 1752 1753 if (n->is_Loop()) { 1754 Node* c = n->in(LoopNode::EntryControl); 1755 if (trace) { tty->print("X pushing"); c->dump(); } 1756 controls.push(c); 1757 } else if (n->is_Region()) { 1758 for (uint i = 1; i < n->req(); i++) { 1759 Node* c = n->in(i); 1760 if (trace) { tty->print("X pushing"); c->dump(); } 1761 controls.push(c); 1762 } 1763 } else { 1764 Node* c = n->in(0); 1765 if (trace) { tty->print("X pushing"); c->dump(); } 1766 controls.push(c); 1767 } 1768 } 1769 } 1770 1771 bool ShenandoahWriteBarrierNode::memory_dominates_all_paths(Node* mem, Node* rep_ctrl, int alias, PhaseIdealLoop* phase) { 1772 const bool trace = false; 1773 if (trace) { 1774 tty->print("XXX mem is"); mem->dump(); 1775 tty->print("XXX rep ctrl is"); rep_ctrl->dump(); 1776 tty->print_cr("XXX alias is %d", alias); 1777 } 1778 ResourceMark rm; 1779 Unique_Node_List wq; 1780 Unique_Node_List controls; 1781 wq.push(mem); 1782 for (uint next = 0; next < wq.size(); next++) { 1783 Node *nn = wq.at(next); 1784 if (trace) { tty->print("XX from mem"); nn->dump(); } 1785 assert(nn->bottom_type() == Type::MEMORY, "memory only"); 1786 1787 if (nn->is_Phi()) { 1788 Node* r = nn->in(0); 1789 for (DUIterator_Fast jmax, j = r->fast_outs(jmax); j < jmax; j++) { 1790 Node* u = r->fast_out(j); 1791 if (u->is_Phi() && u->bottom_type() == Type::MEMORY && u != nn && 1792 (u->adr_type() == TypePtr::BOTTOM || phase->C->get_alias_index(u->adr_type()) == alias)) { 1793 if (trace) { tty->print("XX Next mem (other phi)"); u->dump(); } 1794 wq.push(u); 1795 } 1796 } 1797 } 1798 1799 for (DUIterator_Fast imax, i = nn->fast_outs(imax); i < imax; i++) { 1800 Node* use = nn->fast_out(i); 1801 1802 if (trace) { tty->print("XX use %p", use->adr_type()); use->dump(); } 1803 if (use->is_CFG()) { 1804 assert(use->in(TypeFunc::Memory) == nn, "bad cfg node"); 1805 Node* c = use->in(0); 1806 if (phase->is_dominator(rep_ctrl, c)) { 1807 memory_dominates_all_paths_helper(c, rep_ctrl, controls, phase); 1808 } else if (use->is_CallStaticJava() && use->as_CallStaticJava()->uncommon_trap_request() != 0 && c->is_Region()) { 1809 Node* region = c; 1810 if (trace) { tty->print("XX unc region"); region->dump(); } 1811 for (uint j = 1; j < region->req(); j++) { 1812 if (phase->is_dominator(rep_ctrl, region->in(j))) { 1813 if (trace) { tty->print("XX unc follows"); region->in(j)->dump(); } 1814 memory_dominates_all_paths_helper(region->in(j), rep_ctrl, controls, phase); 1815 } 1816 } 1817 } 1818 //continue; 1819 } else if (use->is_Phi()) { 1820 assert(use->bottom_type() == Type::MEMORY, "bad phi"); 1821 if ((use->adr_type() == TypePtr::BOTTOM /*&& !shenandoah_has_alias_phi(C, use, alias)*/) || 1822 phase->C->get_alias_index(use->adr_type()) == alias) { 1823 for (uint j = 1; j < use->req(); j++) { 1824 if (use->in(j) == nn) { 1825 Node* c = use->in(0)->in(j); 1826 if (phase->is_dominator(rep_ctrl, c)) { 1827 memory_dominates_all_paths_helper(c, rep_ctrl, controls, phase); 1828 } 1829 } 1830 } 1831 } 1832 // continue; 1833 } 1834 1835 if (use->is_MergeMem()) { 1836 if (use->as_MergeMem()->memory_at(alias) == nn) { 1837 if (trace) { tty->print("XX Next mem"); use->dump(); } 1838 // follow the memory edges 1839 wq.push(use); 1840 } 1841 } else if (use->is_Phi()) { 1842 assert(use->bottom_type() == Type::MEMORY, "bad phi"); 1843 if ((use->adr_type() == TypePtr::BOTTOM /*&& !shenandoah_has_alias_phi(C, use, alias)*/) || 1844 phase->C->get_alias_index(use->adr_type()) == alias) { 1845 if (trace) { tty->print("XX Next mem"); use->dump(); } 1846 // follow the memory edges 1847 wq.push(use); 1848 } 1849 } else if (use->bottom_type() == Type::MEMORY && 1850 (use->adr_type() == TypePtr::BOTTOM || phase->C->get_alias_index(use->adr_type()) == alias)) { 1851 if (trace) { tty->print("XX Next mem"); use->dump(); } 1852 // follow the memory edges 1853 wq.push(use); 1854 } else if ((use->is_SafePoint() || use->is_MemBar()) && 1855 (use->adr_type() == TypePtr::BOTTOM || phase->C->get_alias_index(use->adr_type()) == alias)) { 1856 for (DUIterator_Fast jmax, j = use->fast_outs(jmax); j < jmax; j++) { 1857 Node* u = use->fast_out(j); 1858 if (u->bottom_type() == Type::MEMORY) { 1859 if (trace) { tty->print("XX Next mem"); u->dump(); } 1860 // follow the memory edges 1861 wq.push(u); 1862 } 1863 } 1864 } else if (use->Opcode() == Op_ShenandoahWriteBarrier && phase->C->get_alias_index(use->adr_type()) == alias) { 1865 Node* m = use->find_out_with(Op_ShenandoahWBMemProj); 1866 if (m != NULL) { 1867 if (trace) { tty->print("XX Next mem"); m->dump(); } 1868 // follow the memory edges 1869 wq.push(m); 1870 } 1871 } 1872 } 1873 } 1874 1875 if (controls.size() == 0) { 1876 return false; 1877 } 1878 1879 for (uint i = 0; i < controls.size(); i++) { 1880 Node *n = controls.at(i); 1881 1882 if (trace) { tty->print("X checking"); n->dump(); } 1883 1884 if (n->unique_ctrl_out() != NULL) { 1885 continue; 1886 } 1887 1888 if (n->Opcode() == Op_NeverBranch) { 1889 Node* taken = n->as_Multi()->proj_out(0); 1890 if (!controls.member(taken)) { 1891 if (trace) { tty->print("X not seen"); taken->dump(); } 1892 return false; 1893 } 1894 continue; 1895 } 1896 1897 for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) { 1898 Node* u = n->fast_out(j); 1899 1900 if (u->is_CFG()) { 1901 if (!controls.member(u)) { 1902 if (u->is_Proj() && u->as_Proj()->is_uncommon_trap_proj(Deoptimization::Reason_none)) { 1903 if (trace) { tty->print("X not seen but unc"); u->dump(); } 1904 } else { 1905 Node* c = u; 1906 do { 1907 c = c->unique_ctrl_out(); 1908 } while (c != NULL && c->is_Region()); 1909 if (c != NULL && c->Opcode() == Op_Halt) { 1910 if (trace) { tty->print("X not seen but halt"); c->dump(); } 1911 } else { 1912 if (trace) { tty->print("X not seen"); u->dump(); } 1913 return false; 1914 } 1915 } 1916 } else { 1917 if (trace) { tty->print("X seen"); u->dump(); } 1918 } 1919 } 1920 } 1921 } 1922 return true; 1923 } 1924 #endif 1925 1926 static bool has_mem_phi(Compile* C, Node* region, int alias) { 1927 for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) { 1928 Node* use = region->fast_out(i); 1929 if (use->is_Phi() && use->bottom_type() == Type::MEMORY && 1930 (C->get_alias_index(use->adr_type()) == alias)) { 1931 return true; 1932 } 1933 } 1934 return false; 1935 } 1936 1937 bool ShenandoahWriteBarrierNode::fix_mem_phis_helper(Node* c, Node* mem, Node* mem_ctrl, Node* rep_ctrl, int alias, VectorSet& controls, GrowableArray<Node*>& regions, PhaseIdealLoop* phase) { 1938 const bool trace = false; 1939 Node_List wq; 1940 wq.push(c); 1941 1942 #ifdef ASSERT 1943 if (trace) { tty->print("YYY from"); c->dump(); } 1944 if (trace) { tty->print("YYY with mem"); mem->dump(); } 1945 #endif 1946 1947 while(wq.size() > 0) { 1948 c = wq.pop(); 1949 1950 while (!c->is_Region() || c->is_Loop()) { 1951 #ifdef ASSERT 1952 if (trace) { tty->print("YYY"); c->dump(); } 1953 #endif 1954 assert(c->is_CFG(), "node should be control node"); 1955 if (c == mem_ctrl || phase->is_dominator(c, rep_ctrl)) { 1956 c = NULL; 1957 break; 1958 } else if (c->is_Loop()) { 1959 c = c->in(LoopNode::EntryControl); 1960 } else { 1961 c = c->in(0); 1962 } 1963 } 1964 if (c == NULL) { 1965 continue; 1966 } 1967 1968 #ifdef ASSERT 1969 if (trace) { tty->print("YYY new region"); c->dump(); } 1970 #endif 1971 1972 bool has_phi = has_mem_phi(phase->C, c, alias); 1973 if (!has_phi) { 1974 1975 Node* m_ctrl = NULL; 1976 Node* m = dom_mem(mem, c, alias, m_ctrl, phase); 1977 if (m == NULL) { 1978 return false; 1979 } 1980 1981 #ifdef ASSERT 1982 if (trace) { tty->print("YYY mem "); m->dump(); } 1983 #endif 1984 1985 if (controls.test(c->_idx)) { 1986 int i; 1987 for (i = 0; i < regions.length() && regions.at(i) != c; i+=2) { 1988 // deliberately empty, rolling over the regions 1989 } 1990 assert(i < regions.length(), "missing region"); 1991 Node* prev_m = regions.at(i+1); 1992 if (prev_m == m) { 1993 continue; 1994 } 1995 #ifdef ASSERT 1996 if (trace) { tty->print("YYY prev mem "); prev_m->dump(); } 1997 #endif 1998 Node* prev_m_ctrl = phase->ctrl_or_self(prev_m); 1999 assert(is_dominator(m_ctrl, prev_m_ctrl, m, prev_m, phase) || 2000 is_dominator(prev_m_ctrl, m_ctrl, prev_m, m, phase), "one should dominate the other"); 2001 if (is_dominator(m_ctrl, prev_m_ctrl, m, prev_m, phase)) { 2002 continue; 2003 } 2004 #ifdef ASSERT 2005 if (trace) { tty->print("YYY Fixing "); c->dump(); } 2006 #endif 2007 regions.at_put(i+1, m); 2008 } else { 2009 #ifdef ASSERT 2010 if (trace) { tty->print("YYY Pushing "); c->dump(); } 2011 #endif 2012 regions.push(c); 2013 regions.push(m); 2014 } 2015 } else { 2016 continue; 2017 } 2018 2019 controls.set(c->_idx); 2020 2021 for (uint i = 1; i < c->req(); i++) { 2022 wq.push(c->in(i)); 2023 } 2024 } 2025 return true; 2026 } 2027 2028 2029 bool ShenandoahWriteBarrierNode::fix_mem_phis(Node* mem, Node* mem_ctrl, Node* rep_ctrl, int alias, PhaseIdealLoop* phase) { 2030 GrowableArray<Node*> regions; 2031 VectorSet controls(Thread::current()->resource_area()); 2032 const bool trace = false; 2033 2034 #ifdef ASSERT 2035 if (trace) { tty->print("YYY mem is "); mem->dump(); } 2036 if (trace) { tty->print("YYY mem ctrl is "); mem_ctrl->dump(); } 2037 if (trace) { tty->print("YYY rep ctrl is "); rep_ctrl->dump(); } 2038 if (trace) { tty->print_cr("YYY alias is %d", alias); } 2039 #endif 2040 2041 // Walk memory edges from mem until we hit a memory point where 2042 // control is known then follow the control up looking for regions 2043 // with no memory Phi for alias 2044 Unique_Node_List wq; 2045 wq.push(mem); 2046 2047 for (uint next = 0; next < wq.size(); next++) { 2048 Node *n = wq.at(next); 2049 #ifdef ASSERT 2050 if (trace) { tty->print("YYY from (2) "); n->dump(); } 2051 #endif 2052 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { 2053 Node* u = n->fast_out(i); 2054 #ifdef ASSERT 2055 if (trace) { tty->print("YYY processing "); u->dump(); } 2056 #endif 2057 if (u->is_Phi()) { 2058 assert(u->bottom_type() == Type::MEMORY, "strange memory graph"); 2059 if (should_process_phi(u, alias, phase->C)) { 2060 for (uint j = 1; j < u->req(); j++) { 2061 if (u->in(j) == n) { 2062 Node *c = u->in(0)->in(j); 2063 if (!fix_mem_phis_helper(c, n, mem_ctrl, rep_ctrl, alias, controls, regions, phase)) { 2064 return false; 2065 } 2066 } 2067 } 2068 } 2069 #ifdef ASSERT 2070 } else if (u->is_CallStaticJava() && u->as_CallStaticJava()->uncommon_trap_request() != 0) { 2071 if (!fix_mem_phis_helper(u->in(0), n, mem_ctrl, rep_ctrl, alias, controls, regions, phase)) { 2072 return false; 2073 } 2074 #endif 2075 } else if ((u->is_CFG() && u->adr_type() == TypePtr::BOTTOM) || u->Opcode() == Op_Rethrow || u->Opcode() == Op_Return) { 2076 if (!fix_mem_phis_helper(u->in(0), n, mem_ctrl, rep_ctrl, alias, controls, regions, phase)) { 2077 return false; 2078 } 2079 } else if (u->is_MergeMem() && u->as_MergeMem()->memory_at(alias) == n) { 2080 wq.push(u); 2081 } else if (u->Opcode() == Op_ShenandoahWriteBarrier && phase->C->get_alias_index(u->adr_type()) == alias) { 2082 Node* m = u->find_out_with(Op_ShenandoahWBMemProj); 2083 if (m != NULL) { 2084 wq.push(m); 2085 } 2086 } 2087 } 2088 } 2089 #ifdef ASSERT 2090 if (trace) { 2091 tty->print_cr("XXXXXXXXXXXXXXXXXXXX"); 2092 for (int i = 0; i < regions.length(); i++) { 2093 Node* r = regions.at(i); 2094 tty->print("%d", i); r->dump(); 2095 } 2096 tty->print_cr("XXXXXXXXXXXXXXXXXXXX"); 2097 } 2098 #endif 2099 2100 if (regions.length() == 0) { 2101 return true; 2102 } 2103 2104 { 2105 int i = 0; 2106 for (; i < regions.length(); i+=2) { 2107 Node* region = regions.at(i); 2108 bool has_phi = false; 2109 for (DUIterator_Fast jmax, j = region->fast_outs(jmax); j < jmax && !has_phi; j++) { 2110 Node* u = region->fast_out(j); 2111 if (u->is_Phi() && u->bottom_type() == Type::MEMORY && 2112 (u->adr_type() == TypePtr::BOTTOM || phase->C->get_alias_index(u->adr_type()) == alias)) { 2113 has_phi = true; 2114 } 2115 } 2116 if (!has_phi) { 2117 break; 2118 } 2119 } 2120 if (i == regions.length()) { 2121 return true; 2122 } 2123 } 2124 2125 // Try to restrict the update to path that post dominates rep_ctrl 2126 int k = 0; 2127 int start = 0; 2128 int end = 0; 2129 do { 2130 start = end; 2131 end = k; 2132 for (int i = end; i < regions.length(); i+=2) { 2133 Node* r = regions.at(i); 2134 int prev = k; 2135 for (uint j = 1; j < r->req() && prev == k; j++) { 2136 if (end == 0) { 2137 if (phase->is_dominator(rep_ctrl, r->in(j))) { 2138 Node* mem = regions.at(i+1); 2139 regions.at_put(i, regions.at(k)); 2140 regions.at_put(i+1, regions.at(k+1)); 2141 regions.at_put(k, r); 2142 regions.at_put(k+1, mem); 2143 k+=2; 2144 } 2145 } else { 2146 for (int l = start; l < end && prev == k; l+=2) { 2147 Node* r2 = regions.at(l); 2148 if (phase->is_dominator(r2, r->in(j))) { 2149 Node* mem = regions.at(i+1); 2150 regions.at_put(i, regions.at(k)); 2151 regions.at_put(i+1, regions.at(k+1)); 2152 regions.at_put(k, r); 2153 regions.at_put(k+1, mem); 2154 k+=2; 2155 } 2156 } 2157 } 2158 } 2159 } 2160 #ifdef ASSERT 2161 if (trace) { tty->print_cr("k = %d start = %d end = %d", k, start, end); } 2162 #endif 2163 } while(k != end); 2164 2165 #ifdef ASSERT 2166 if (end != regions.length()) { 2167 if (trace) { tty->print_cr("Compacting %d -> %d", regions.length(), end); } 2168 } 2169 #endif 2170 regions.trunc_to(end); 2171 2172 #ifdef ASSERT 2173 if (trace) { 2174 tty->print_cr("XXXXXXXXXXXXXXXXXXXX"); 2175 for (int i = 0; i < regions.length(); i++) { 2176 Node* r = regions.at(i); 2177 tty->print("%d", i); r->dump(); 2178 } 2179 tty->print_cr("XXXXXXXXXXXXXXXXXXXX"); 2180 } 2181 #endif 2182 2183 // Creating new phis must be done in post order 2184 while (regions.length() > 0) { 2185 int i = 0; 2186 for (; i < regions.length(); i+=2) { 2187 Node* r1 = regions.at(i); 2188 bool is_dom = false; 2189 for (int j = 0; j < regions.length() && !is_dom; j+=2) { 2190 if (i != j) { 2191 Node* r2 = regions.at(j); 2192 for (uint k = 1; k < r2->req() && !is_dom; k++) { 2193 if (phase->is_dominator(r1, r2->in(k))) { 2194 is_dom = true; 2195 } 2196 } 2197 } 2198 } 2199 if (!is_dom) { 2200 break; 2201 } 2202 } 2203 assert(i < regions.length(), "need one"); 2204 Node* r = regions.at(i); 2205 Node* m = regions.at(i+1); 2206 regions.delete_at(i+1); 2207 regions.delete_at(i); 2208 2209 if (!suitable_mem(m, NULL, NULL)) { 2210 return false; 2211 } 2212 Node* phi = PhiNode::make(r, m, Type::MEMORY, phase->C->get_adr_type(alias)); 2213 #ifdef ASSERT 2214 if (trace) { tty->print("YYY Adding new mem phi "); phi->dump(); } 2215 #endif 2216 phase->register_new_node(phi, r); 2217 2218 fix_memory_uses(m, phi, phi, r, phase->C->get_alias_index(phi->adr_type()), phase); 2219 assert(phi->outcnt() != 0, "new proj should have uses"); 2220 if (phi->outcnt() == 0) { 2221 phase->igvn().remove_dead_node(phi); 2222 } 2223 } 2224 2225 return true; 2226 } 2227 2228 Node* ShenandoahBarrierNode::dom_mem(Node* mem, Node*& mem_ctrl, Node* n, Node* rep_ctrl, int alias, PhaseIdealLoop* phase) { 2229 ResourceMark rm; 2230 VectorSet wq(Thread::current()->resource_area()); 2231 wq.set(mem->_idx); 2232 mem_ctrl = phase->get_ctrl(mem); 2233 while (!is_dominator(mem_ctrl, rep_ctrl, mem, n, phase)) { 2234 mem = next_mem(mem, alias); 2235 if (wq.test_set(mem->_idx)) { 2236 return NULL; // hit an unexpected loop 2237 } 2238 mem_ctrl = phase->ctrl_or_self(mem); 2239 } 2240 if (mem->is_MergeMem()) { 2241 mem = mem->as_MergeMem()->memory_at(alias); 2242 mem_ctrl = phase->ctrl_or_self(mem); 2243 } 2244 return mem; 2245 } 2246 2247 Node* ShenandoahBarrierNode::dom_mem(Node* mem, Node* ctrl, int alias, Node*& mem_ctrl, PhaseIdealLoop* phase) { 2248 ResourceMark rm; 2249 VectorSet wq(Thread::current()->resource_area()); 2250 wq.set(mem->_idx); 2251 mem_ctrl = phase->ctrl_or_self(mem); 2252 while (!phase->is_dominator(mem_ctrl, ctrl) || mem_ctrl == ctrl) { 2253 mem = next_mem(mem, alias); 2254 if (wq.test_set(mem->_idx)) { 2255 return NULL; 2256 } 2257 mem_ctrl = phase->ctrl_or_self(mem); 2258 } 2259 if (mem->is_MergeMem()) { 2260 mem = mem->as_MergeMem()->memory_at(alias); 2261 mem_ctrl = phase->ctrl_or_self(mem); 2262 } 2263 return mem; 2264 } 2265 2266 Node* ShenandoahBarrierNode::try_common(Node *n_ctrl, PhaseIdealLoop* phase) { 2267 if (!phase->C->has_irreducible_loop()) { 2268 // We look for a write barrier whose memory edge dominates n 2269 // Either the replacement write barrier dominates n or we have, 2270 // for instance: 2271 // if ( ) { 2272 // read barrier n 2273 // } else { 2274 // write barrier 2275 // } 2276 // in which case replacing n by the write barrier causes the write 2277 // barrier to move above the if() and the memory Phi that merges 2278 // the memory state for both branches must be updated so both 2279 // inputs become the write barrier's memory projection (and the 2280 // Phi is optimized out) otherwise we risk loosing a memory 2281 // dependency. 2282 // Once we find a replacement write barrier, the code below fixes 2283 // the memory graph in cases like the one above. 2284 Node* val = in(ValueIn); 2285 Node* val_ctrl = phase->get_ctrl(val); 2286 Node* n_proj = find_out_with(Op_ShenandoahWBMemProj); 2287 Node* replacement = NULL; 2288 int alias = phase->C->get_alias_index(adr_type()); 2289 Node* rep_ctrl = NULL; 2290 for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax && replacement == NULL; i++) { 2291 Node* u = val->fast_out(i); 2292 if (u != this && u->Opcode() == Op_ShenandoahWriteBarrier) { 2293 Node* u_mem = u->in(Memory); 2294 Node* u_proj = u->find_out_with(Op_ShenandoahWBMemProj); 2295 Node* u_ctrl = phase->get_ctrl(u); 2296 Node* u_mem_ctrl = phase->get_ctrl(u_mem); 2297 IdealLoopTree* n_loop = phase->get_loop(n_ctrl); 2298 IdealLoopTree* u_loop = phase->get_loop(u_ctrl); 2299 2300 Node* ctrl = phase->dom_lca(u_ctrl, n_ctrl); 2301 2302 if (ctrl->is_Proj() && 2303 ctrl->in(0)->is_Call() && 2304 ctrl->unique_ctrl_out() != NULL && 2305 ctrl->unique_ctrl_out()->Opcode() == Op_Catch && 2306 !phase->is_dominator(val_ctrl, ctrl->in(0)->in(0))) { 2307 continue; 2308 } 2309 2310 if (Opcode() == Op_ShenandoahWriteBarrier && u_proj == NULL && n_proj != NULL) { 2311 continue; 2312 } 2313 2314 IdealLoopTree* loop = phase->get_loop(ctrl); 2315 2316 // We don't want to move a write barrier in a loop 2317 // If the LCA is in a inner loop, try a control out of loop if possible 2318 bool loop_ok = true; 2319 while (!loop->is_member(u_loop) && (Opcode() != Op_ShenandoahWriteBarrier || !loop->is_member(n_loop))) { 2320 ctrl = phase->idom(ctrl); 2321 if (ctrl != val_ctrl && phase->is_dominator(ctrl, val_ctrl)) { 2322 loop_ok = false; 2323 break; 2324 } 2325 loop = phase->get_loop(ctrl); 2326 } 2327 2328 if (loop_ok) { 2329 if (ShenandoahDontIncreaseWBFreq) { 2330 Node* u_iffproj = no_branches(u_ctrl, ctrl, true, phase); 2331 if (Opcode() == Op_ShenandoahWriteBarrier) { 2332 Node* n_iffproj = no_branches(n_ctrl, ctrl, true, phase); 2333 if (u_iffproj == NULL || n_iffproj == NULL) { 2334 replacement = u; 2335 rep_ctrl = ctrl; 2336 } else if (u_iffproj != NodeSentinel && n_iffproj != NodeSentinel && u_iffproj->in(0) == n_iffproj->in(0)) { 2337 replacement = u; 2338 rep_ctrl = ctrl; 2339 } 2340 } else if (u_iffproj == NULL) { 2341 replacement = u; 2342 rep_ctrl = ctrl; 2343 } 2344 } else { 2345 replacement = u; 2346 rep_ctrl = ctrl; 2347 } 2348 } 2349 } 2350 } 2351 if (replacement != NULL) { 2352 if (rep_ctrl->is_Proj() && 2353 rep_ctrl->in(0)->is_Call() && 2354 rep_ctrl->unique_ctrl_out() != NULL && 2355 rep_ctrl->unique_ctrl_out()->Opcode() == Op_Catch) { 2356 rep_ctrl = rep_ctrl->in(0)->in(0); 2357 assert(phase->is_dominator(val_ctrl, rep_ctrl), "bad control"); 2358 } else { 2359 LoopNode* c = ShenandoahWriteBarrierNode::try_move_before_pre_loop(rep_ctrl, val_ctrl, phase); 2360 if (c != NULL) { 2361 rep_ctrl = ShenandoahWriteBarrierNode::move_above_predicates(c, val_ctrl, phase); 2362 } else { 2363 while (rep_ctrl->is_IfProj()) { 2364 CallStaticJavaNode* unc = rep_ctrl->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none); 2365 if (unc != NULL) { 2366 int req = unc->uncommon_trap_request(); 2367 Deoptimization::DeoptReason trap_reason = Deoptimization::trap_request_reason(req); 2368 if ((trap_reason == Deoptimization::Reason_loop_limit_check || 2369 trap_reason == Deoptimization::Reason_predicate || 2370 trap_reason == Deoptimization::Reason_profile_predicate) && 2371 phase->is_dominator(val_ctrl, rep_ctrl->in(0)->in(0))) { 2372 rep_ctrl = rep_ctrl->in(0)->in(0); 2373 continue; 2374 } 2375 } 2376 break; 2377 } 2378 } 2379 } 2380 2381 Node* mem = replacement->in(Memory); 2382 Node* old_mem = mem; 2383 Node* rep_proj = replacement->find_out_with(Op_ShenandoahWBMemProj); 2384 { 2385 Node* mem_ctrl = NULL; 2386 2387 mem = dom_mem(mem, mem_ctrl, this, rep_ctrl, alias, phase); 2388 if (mem == NULL) { 2389 return NULL; 2390 } 2391 2392 // Add a memory Phi for the slice of the write barrier to any 2393 // region that post dominates rep_ctrl and doesn't have one 2394 // already. 2395 if (rep_proj != NULL && !ShenandoahWriteBarrierNode::fix_mem_phis(mem, mem_ctrl, rep_ctrl, alias, phase)) { 2396 return NULL; 2397 } 2398 2399 assert(!ShenandoahVerifyOptoBarriers || ShenandoahWriteBarrierNode::memory_dominates_all_paths(mem, rep_ctrl, alias, phase), "can't fix the memory graph"); 2400 } 2401 assert(phase->igvn().type(mem) == Type::MEMORY, "not memory"); 2402 2403 if (rep_proj != NULL) { 2404 Node* old_mem = replacement->in(Memory); 2405 if (!suitable_mem(mem, old_mem, rep_proj)) { 2406 return NULL; 2407 } 2408 2409 if (replacement->in(Memory) != mem) { 2410 // tty->print("XXX setting memory of"); replacement->dump(); 2411 // tty->print("XXX to"); mem->dump(); 2412 for (DUIterator_Last imin, i = rep_proj->last_outs(imin); i >= imin; ) { 2413 Node* u = rep_proj->last_out(i); 2414 phase->igvn().rehash_node_delayed(u); 2415 int uses_found = u->replace_edge(rep_proj, old_mem); 2416 i -= uses_found; 2417 } 2418 phase->igvn().replace_input_of(replacement, Memory, mem); 2419 } 2420 phase->set_ctrl_and_loop(replacement, rep_ctrl); 2421 phase->igvn().replace_input_of(replacement, Control, rep_ctrl); 2422 2423 ShenandoahWriteBarrierNode::fix_memory_uses(mem, replacement, rep_proj, rep_ctrl, phase->C->get_alias_index(replacement->adr_type()), phase); 2424 assert(rep_proj->outcnt() != 0, "new proj should have uses"); 2425 } else { 2426 if (replacement->in(Memory) != mem) { 2427 phase->igvn()._worklist.push(replacement->in(Memory)); 2428 phase->igvn().replace_input_of(replacement, Memory, mem); 2429 } 2430 phase->set_ctrl_and_loop(replacement, rep_ctrl); 2431 phase->igvn().replace_input_of(replacement, Control, rep_ctrl); 2432 } 2433 if (Opcode() == Op_ShenandoahWriteBarrier) { 2434 if (n_proj != NULL) { 2435 phase->lazy_replace(n_proj, in(Memory)); 2436 } 2437 } 2438 phase->lazy_replace(this, replacement); 2439 if (rep_proj != NULL) { 2440 phase->set_ctrl_and_loop(rep_proj, rep_ctrl); 2441 } 2442 return replacement; 2443 } 2444 } 2445 2446 return NULL; 2447 } 2448 2449 static void disconnect_barrier_mem(Node* wb, PhaseIterGVN& igvn) { 2450 Node* mem_in = wb->in(ShenandoahBarrierNode::Memory); 2451 Node* proj = wb->find_out_with(Op_ShenandoahWBMemProj); 2452 2453 for (DUIterator_Last imin, i = proj->last_outs(imin); i >= imin; ) { 2454 Node* u = proj->last_out(i); 2455 igvn.rehash_node_delayed(u); 2456 int nb = u->replace_edge(proj, mem_in); 2457 assert(nb > 0, "no replacement?"); 2458 i -= nb; 2459 } 2460 } 2461 2462 Node* ShenandoahWriteBarrierNode::move_above_predicates(LoopNode* cl, Node* val_ctrl, PhaseIdealLoop* phase) { 2463 Node* entry = cl->skip_strip_mined()->in(LoopNode::EntryControl); 2464 Node* above_pred = phase->skip_all_loop_predicates(entry); 2465 Node* ctrl = entry; 2466 while (ctrl != above_pred) { 2467 Node* next = ctrl->in(0); 2468 if (!phase->is_dominator(val_ctrl, next)) { 2469 break; 2470 } 2471 ctrl = next; 2472 } 2473 return ctrl; 2474 } 2475 2476 Node* ShenandoahWriteBarrierNode::try_move_before_loop_helper(LoopNode* cl, Node* val_ctrl, Node* mem, PhaseIdealLoop* phase) { 2477 assert(cl->is_Loop(), "bad control"); 2478 Node* ctrl = move_above_predicates(cl, val_ctrl, phase); 2479 Node* mem_ctrl = NULL; 2480 int alias = phase->C->get_alias_index(adr_type()); 2481 mem = dom_mem(mem, mem_ctrl, this, ctrl, alias, phase); 2482 if (mem == NULL) { 2483 return NULL; 2484 } 2485 2486 Node* old_mem = in(Memory); 2487 Node* proj = find_out_with(Op_ShenandoahWBMemProj); 2488 if (old_mem != mem && !suitable_mem(mem, old_mem, proj)) { 2489 return NULL; 2490 } 2491 2492 assert(!ShenandoahVerifyOptoBarriers || memory_dominates_all_paths(mem, ctrl, alias, phase), "can't fix the memory graph"); 2493 phase->set_ctrl_and_loop(this, ctrl); 2494 phase->igvn().replace_input_of(this, Control, ctrl); 2495 if (old_mem != mem) { 2496 if (proj != NULL) { 2497 disconnect_barrier_mem(this, phase->igvn()); 2498 fix_memory_uses(mem, this, proj, ctrl, phase->C->get_alias_index(adr_type()), phase); 2499 assert(proj->outcnt() > 0, "disconnected write barrier"); 2500 } 2501 phase->igvn().replace_input_of(this, Memory, mem); 2502 } 2503 if (proj != NULL) { 2504 phase->set_ctrl_and_loop(proj, ctrl); 2505 } 2506 return this; 2507 } 2508 2509 LoopNode* ShenandoahWriteBarrierNode::try_move_before_pre_loop(Node* c, Node* val_ctrl, PhaseIdealLoop* phase) { 2510 // A write barrier between a pre and main loop can get in the way of 2511 // vectorization. Move it above the pre loop if possible 2512 CountedLoopNode* cl = NULL; 2513 if (c->is_IfFalse() && 2514 c->in(0)->is_CountedLoopEnd()) { 2515 cl = c->in(0)->as_CountedLoopEnd()->loopnode(); 2516 } else if (c->is_IfProj() && 2517 c->in(0)->is_If() && 2518 c->in(0)->in(0)->is_IfFalse() && 2519 c->in(0)->in(0)->in(0)->is_CountedLoopEnd()) { 2520 cl = c->in(0)->in(0)->in(0)->as_CountedLoopEnd()->loopnode(); 2521 } 2522 if (cl != NULL && 2523 cl->is_pre_loop() && 2524 val_ctrl != cl && 2525 phase->is_dominator(val_ctrl, cl)) { 2526 return cl; 2527 } 2528 return NULL; 2529 } 2530 2531 Node* ShenandoahWriteBarrierNode::try_move_before_loop(Node *n_ctrl, PhaseIdealLoop* phase) { 2532 IdealLoopTree *n_loop = phase->get_loop(n_ctrl); 2533 Node* val = in(ValueIn); 2534 Node* val_ctrl = phase->get_ctrl(val); 2535 if (n_loop != phase->ltree_root() && !n_loop->_irreducible) { 2536 IdealLoopTree *val_loop = phase->get_loop(val_ctrl); 2537 Node* mem = in(Memory); 2538 IdealLoopTree *mem_loop = phase->get_loop(phase->get_ctrl(mem)); 2539 if (!n_loop->is_member(val_loop) && 2540 n_loop->is_member(mem_loop)) { 2541 Node* n_loop_head = n_loop->_head; 2542 2543 if (n_loop_head->is_Loop()) { 2544 LoopNode* loop = n_loop_head->as_Loop(); 2545 if (n_loop_head->is_CountedLoop() && n_loop_head->as_CountedLoop()->is_main_loop()) { 2546 LoopNode* res = try_move_before_pre_loop(n_loop_head->in(LoopNode::EntryControl), val_ctrl, phase); 2547 if (res != NULL) { 2548 loop = res; 2549 } 2550 } 2551 2552 return try_move_before_loop_helper(loop, val_ctrl, mem, phase); 2553 } 2554 } 2555 } 2556 LoopNode* ctrl = try_move_before_pre_loop(in(0), val_ctrl, phase); 2557 if (ctrl != NULL) { 2558 return try_move_before_loop_helper(ctrl, val_ctrl, in(Memory), phase); 2559 } 2560 return NULL; 2561 } 2562 2563 void ShenandoahReadBarrierNode::try_move(Node *n_ctrl, PhaseIdealLoop* phase) { 2564 Node* mem = in(MemNode::Memory); 2565 int alias = phase->C->get_alias_index(adr_type()); 2566 const bool trace = false; 2567 2568 #ifdef ASSERT 2569 if (trace) { tty->print("Trying to move mem of"); dump(); } 2570 #endif 2571 2572 Node* new_mem = mem; 2573 2574 ResourceMark rm; 2575 VectorSet seen(Thread::current()->resource_area()); 2576 Node_List phis; 2577 2578 for (;;) { 2579 #ifdef ASSERT 2580 if (trace) { tty->print("Looking for dominator from"); mem->dump(); } 2581 #endif 2582 if (mem->is_Proj() && mem->in(0)->is_Start()) { 2583 if (new_mem != in(MemNode::Memory)) { 2584 #ifdef ASSERT 2585 if (trace) { tty->print("XXX Setting mem to"); new_mem->dump(); tty->print(" for "); dump(); } 2586 #endif 2587 phase->igvn().replace_input_of(this, MemNode::Memory, new_mem); 2588 } 2589 return; 2590 } 2591 2592 Node* candidate = mem; 2593 do { 2594 if (!is_independent(mem)) { 2595 if (trace) { tty->print_cr("Not independent"); } 2596 if (new_mem != in(MemNode::Memory)) { 2597 #ifdef ASSERT 2598 if (trace) { tty->print("XXX Setting mem to"); new_mem->dump(); tty->print(" for "); dump(); } 2599 #endif 2600 phase->igvn().replace_input_of(this, MemNode::Memory, new_mem); 2601 } 2602 return; 2603 } 2604 if (seen.test_set(mem->_idx)) { 2605 if (trace) { tty->print_cr("Already seen"); } 2606 ShouldNotReachHere(); 2607 // Strange graph 2608 if (new_mem != in(MemNode::Memory)) { 2609 #ifdef ASSERT 2610 if (trace) { tty->print("XXX Setting mem to"); new_mem->dump(); tty->print(" for "); dump(); } 2611 #endif 2612 phase->igvn().replace_input_of(this, MemNode::Memory, new_mem); 2613 } 2614 return; 2615 } 2616 if (mem->is_Phi()) { 2617 phis.push(mem); 2618 } 2619 mem = next_mem(mem, alias); 2620 if (mem->bottom_type() == Type::MEMORY) { 2621 candidate = mem; 2622 } 2623 assert(is_dominator(phase->ctrl_or_self(mem), n_ctrl, mem, this, phase) == phase->is_dominator(phase->ctrl_or_self(mem), n_ctrl), "strange dominator"); 2624 #ifdef ASSERT 2625 if (trace) { tty->print("Next mem is"); mem->dump(); } 2626 #endif 2627 } while (mem->bottom_type() != Type::MEMORY || !phase->is_dominator(phase->ctrl_or_self(mem), n_ctrl)); 2628 2629 assert(mem->bottom_type() == Type::MEMORY, "bad mem"); 2630 2631 bool not_dom = false; 2632 for (uint i = 0; i < phis.size() && !not_dom; i++) { 2633 Node* nn = phis.at(i); 2634 2635 #ifdef ASSERT 2636 if (trace) { tty->print("Looking from phi"); nn->dump(); } 2637 #endif 2638 assert(nn->is_Phi(), "phis only"); 2639 for (uint j = 2; j < nn->req() && !not_dom; j++) { 2640 Node* m = nn->in(j); 2641 #ifdef ASSERT 2642 if (trace) { tty->print("Input %d is", j); m->dump(); } 2643 #endif 2644 while (m != mem && !seen.test_set(m->_idx)) { 2645 if (is_dominator(phase->ctrl_or_self(m), phase->ctrl_or_self(mem), m, mem, phase)) { 2646 not_dom = true; 2647 // Scheduling anomaly 2648 #ifdef ASSERT 2649 if (trace) { tty->print("Giving up"); m->dump(); } 2650 #endif 2651 break; 2652 } 2653 if (!is_independent(m)) { 2654 if (trace) { tty->print_cr("Not independent"); } 2655 if (new_mem != in(MemNode::Memory)) { 2656 #ifdef ASSERT 2657 if (trace) { tty->print("XXX Setting mem to"); new_mem->dump(); tty->print(" for "); dump(); } 2658 #endif 2659 phase->igvn().replace_input_of(this, MemNode::Memory, new_mem); 2660 } 2661 return; 2662 } 2663 if (m->is_Phi()) { 2664 phis.push(m); 2665 } 2666 m = next_mem(m, alias); 2667 #ifdef ASSERT 2668 if (trace) { tty->print("Next mem is"); m->dump(); } 2669 #endif 2670 } 2671 } 2672 } 2673 if (!not_dom) { 2674 new_mem = mem; 2675 phis.clear(); 2676 } else { 2677 seen.Clear(); 2678 } 2679 } 2680 } 2681 2682 CallStaticJavaNode* ShenandoahWriteBarrierNode::pin_and_expand_null_check(PhaseIterGVN& igvn) { 2683 Node* val = in(ValueIn); 2684 2685 #ifdef ASSERT 2686 const Type* val_t = igvn.type(val); 2687 assert(val_t->meet(TypePtr::NULL_PTR) != val_t, "should be not null"); 2688 #endif 2689 2690 if (val->Opcode() == Op_CastPP && 2691 val->in(0)->Opcode() == Op_IfTrue && 2692 val->in(0)->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none) && 2693 val->in(0)->in(0)->is_If() && 2694 val->in(0)->in(0)->in(1)->Opcode() == Op_Bool && 2695 val->in(0)->in(0)->in(1)->as_Bool()->_test._test == BoolTest::ne && 2696 val->in(0)->in(0)->in(1)->in(1)->Opcode() == Op_CmpP && 2697 val->in(0)->in(0)->in(1)->in(1)->in(1) == val->in(1) && 2698 val->in(0)->in(0)->in(1)->in(1)->in(2)->bottom_type() == TypePtr::NULL_PTR) { 2699 assert(val->in(0)->in(0)->in(1)->in(1)->in(1) == val->in(1), ""); 2700 CallStaticJavaNode* unc = val->in(0)->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none); 2701 return unc; 2702 } 2703 return NULL; 2704 } 2705 2706 void ShenandoahWriteBarrierNode::pin_and_expand_move_barrier(PhaseIdealLoop* phase) { 2707 Node* unc = pin_and_expand_null_check(phase->igvn()); 2708 Node* val = in(ValueIn); 2709 2710 if (unc != NULL) { 2711 Node* ctrl = phase->get_ctrl(this); 2712 Node* unc_ctrl = val->in(0); 2713 2714 // Don't move write barrier in a loop 2715 IdealLoopTree* loop = phase->get_loop(ctrl); 2716 IdealLoopTree* unc_loop = phase->get_loop(unc_ctrl); 2717 2718 if (!unc_loop->is_member(loop)) { 2719 return; 2720 } 2721 2722 Node* branch = no_branches(ctrl, unc_ctrl, false, phase); 2723 assert(branch == NULL || branch == NodeSentinel, "was not looking for a branch"); 2724 if (branch == NodeSentinel) { 2725 return; 2726 } 2727 2728 Node* mem = in(Memory); 2729 Node* old_mem = mem; 2730 2731 Node* mem_ctrl = NULL; 2732 int alias = phase->C->get_alias_index(adr_type()); 2733 mem = dom_mem(mem, mem_ctrl, this, unc_ctrl, alias, phase); 2734 if (mem == NULL) { 2735 return; 2736 } 2737 2738 Node* proj = find_out_with(Op_ShenandoahWBMemProj); 2739 if (mem != old_mem && !fix_mem_phis(mem, mem_ctrl, unc_ctrl, alias, phase)) { 2740 return; 2741 } 2742 2743 assert(mem == old_mem || memory_dominates_all_paths(mem, unc_ctrl, alias, phase), "can't fix the memory graph"); 2744 phase->set_ctrl_and_loop(this, unc_ctrl); 2745 if (in(Control) != NULL) { 2746 phase->igvn().replace_input_of(this, Control, unc_ctrl); 2747 } 2748 disconnect_barrier_mem(this, phase->igvn()); 2749 fix_memory_uses(mem, this, proj, unc_ctrl, phase->C->get_alias_index(adr_type()), phase); 2750 assert(proj->outcnt() > 0, "disconnected write barrier"); 2751 phase->igvn().replace_input_of(this, Memory, mem); 2752 phase->set_ctrl_and_loop(proj, unc_ctrl); 2753 } 2754 } 2755 2756 void ShenandoahWriteBarrierNode::pin_and_expand_helper(PhaseIdealLoop* phase) { 2757 Node* val = in(ValueIn); 2758 Node* ctrl = phase->get_ctrl(this); 2759 // Replace all uses of barrier's input that are dominated by ctrl 2760 // with the value returned by the barrier: no need to keep both 2761 // live. 2762 for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) { 2763 Node* u = val->fast_out(i); 2764 if (u != this) { 2765 if (u->is_Phi()) { 2766 int nb = 0; 2767 for (uint j = 1; j < u->req(); j++) { 2768 if (u->in(j) == val) { 2769 Node* c = u->in(0)->in(j); 2770 if (phase->is_dominator(ctrl, c)) { 2771 phase->igvn().replace_input_of(u, j, this); 2772 nb++; 2773 } 2774 } 2775 } 2776 if (nb > 0) { 2777 imax -= nb; 2778 --i; 2779 } 2780 } else { 2781 Node* c = phase->ctrl_or_self(u); 2782 if (is_dominator(ctrl, c, this, u, phase)) { 2783 phase->igvn().rehash_node_delayed(u); 2784 int nb = u->replace_edge(val, this); 2785 assert(nb > 0, "no update?"); 2786 --i, imax -= nb; 2787 } 2788 } 2789 } 2790 } 2791 } 2792 2793 Node* ShenandoahWriteBarrierNode::pick_phi(Node* phi1, Node* phi2, Node_Stack& phis, VectorSet& visited, PhaseIdealLoop* phase) { 2794 assert(phis.size() == 0, "stack needs to be empty"); 2795 uint i = 1; 2796 int phi_dominates = -1; 2797 for (;;) { 2798 assert(phi1->req() == phi2->req(), "strange pair of phis"); 2799 assert(phis.size() % 2 == 0, ""); 2800 Node* in1 = phi1->in(i); 2801 Node* in2 = phi2->in(i); 2802 2803 if (in1->is_MergeMem()) { 2804 in1 = in1->as_MergeMem()->base_memory(); 2805 } 2806 if (in2->is_MergeMem()) { 2807 in2 = in2->as_MergeMem()->base_memory(); 2808 } 2809 2810 if (in1 == in2) { 2811 //continue 2812 } else if (in1->is_Phi() && in2->is_Phi() && in1->in(0) == in2->in(0)) { 2813 assert(!visited.test_set(in1->_idx), "no loop"); 2814 assert(!visited.test_set(in2->_idx), "no loop"); 2815 phis.push(phi1, i+1); 2816 phis.push(phi2, i+1); 2817 phi1 = in1; 2818 phi2 = in2; 2819 i = 1; 2820 } else { 2821 Node* in1_c = phase->get_ctrl(in1); 2822 Node* in2_c = phase->get_ctrl(in2); 2823 if (is_dominator(in1_c, in2_c, in1, in2, phase)) { 2824 assert(!is_dominator(in2_c, in1_c, in2, in1, phase), "one has to dominate the other"); 2825 assert(phi_dominates == -1 || phi_dominates == 1, "all inputs must dominate"); 2826 phi_dominates = 1; 2827 } else { 2828 assert(is_dominator(in2_c, in1_c, in2, in1, phase), "one must dominate the other"); 2829 assert(!is_dominator(in1_c, in2_c, in1, in2, phase), "one has to dominate the other"); 2830 assert(phi_dominates == -1 || phi_dominates == 2, "all inputs must dominate"); 2831 phi_dominates = 2; 2832 } 2833 } 2834 i++; 2835 2836 while (i >= phi1->req() && phis.size() > 0) { 2837 i = phis.index(); 2838 phi2 = phis.node(); 2839 phis.pop(); 2840 phi1 = phis.node(); 2841 phis.pop(); 2842 } 2843 2844 if (i >= phi1->req() && phis.size() == 0) { 2845 Node* phi = NULL; 2846 if (phi_dominates == 1) { 2847 return phi2; 2848 } else if (phi_dominates == 2) { 2849 return phi1; 2850 } else { 2851 return phi1; 2852 } 2853 } 2854 } 2855 return NULL; 2856 } 2857 2858 bool ShenandoahWriteBarrierNode::mem_is_valid(Node* m, Node* c, PhaseIdealLoop* phase) { 2859 return m != NULL && get_ctrl(m, phase) == c; 2860 } 2861 2862 Node* ShenandoahWriteBarrierNode::find_raw_mem(Node* ctrl, Node* n, const Node_List& memory_nodes, PhaseIdealLoop* phase) { 2863 assert(n == NULL || phase->ctrl_or_self(n) == ctrl, ""); 2864 Node* raw_mem = memory_nodes[ctrl->_idx]; 2865 Node* c = ctrl; 2866 while (!mem_is_valid(raw_mem, c, phase) && 2867 (!c->is_CatchProj() || raw_mem == NULL || c->in(0)->in(0)->in(0) != get_ctrl(raw_mem, phase))) { 2868 c = phase->idom(c); 2869 raw_mem = memory_nodes[c->_idx]; 2870 } 2871 if (n != NULL && mem_is_valid(raw_mem, c, phase)) { 2872 while (!is_dominator_same_ctrl(c, raw_mem, n, phase) && phase->ctrl_or_self(raw_mem) == ctrl) { 2873 raw_mem = next_mem(raw_mem, Compile::AliasIdxRaw); 2874 } 2875 if (raw_mem->is_MergeMem()) { 2876 raw_mem = raw_mem->as_MergeMem()->memory_at(Compile::AliasIdxRaw); 2877 } 2878 if (!mem_is_valid(raw_mem, c, phase)) { 2879 do { 2880 c = phase->idom(c); 2881 raw_mem = memory_nodes[c->_idx]; 2882 } while (!mem_is_valid(raw_mem, c, phase) && 2883 (!c->is_CatchProj() || raw_mem == NULL || c->in(0)->in(0)->in(0) != get_ctrl(raw_mem, phase))); 2884 } 2885 } 2886 assert(raw_mem->bottom_type() == Type::MEMORY, ""); 2887 return raw_mem; 2888 } 2889 2890 Node* ShenandoahWriteBarrierNode::find_bottom_mem(Node* ctrl, PhaseIdealLoop* phase) { 2891 Node* mem = NULL; 2892 Node* c = ctrl; 2893 do { 2894 if (c->is_Region()) { 2895 Node* phi_bottom = NULL; 2896 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) { 2897 Node* u = c->fast_out(i); 2898 if (u->is_Phi() && u->bottom_type() == Type::MEMORY) { 2899 if (u->adr_type() == TypePtr::BOTTOM) { 2900 if (phi_bottom != NULL) { 2901 phi_bottom = NodeSentinel; 2902 } else { 2903 phi_bottom = u; 2904 } 2905 } 2906 } 2907 } 2908 if (phi_bottom != NULL) { 2909 if (phi_bottom != NodeSentinel) { 2910 mem = phi_bottom; 2911 } else { 2912 Node* phi = NULL; 2913 ResourceMark rm; 2914 Node_Stack phis(0); 2915 VectorSet visited(Thread::current()->resource_area()); 2916 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) { 2917 Node* u = c->fast_out(i); 2918 if (u->is_Phi() && u->bottom_type() == Type::MEMORY && u->adr_type() == TypePtr::BOTTOM) { 2919 if (phi == NULL) { 2920 phi = u; 2921 } else { 2922 phi = pick_phi(phi, u, phis, visited, phase); 2923 } 2924 } 2925 } 2926 mem = phi; 2927 } 2928 } 2929 } else { 2930 if (c->is_Call() && c->as_Call()->adr_type() != NULL) { 2931 CallProjections projs; 2932 c->as_Call()->extract_projections(&projs, true, false); 2933 if (projs.fallthrough_memproj != NULL) { 2934 if (projs.fallthrough_memproj->adr_type() == TypePtr::BOTTOM) { 2935 if (projs.catchall_memproj == NULL) { 2936 mem = projs.fallthrough_memproj; 2937 } else { 2938 if (phase->is_dominator(projs.fallthrough_catchproj, ctrl)) { 2939 mem = projs.fallthrough_memproj; 2940 } else { 2941 assert(phase->is_dominator(projs.catchall_catchproj, ctrl), "one proj must dominate barrier"); 2942 mem = projs.catchall_memproj; 2943 } 2944 } 2945 } 2946 } else { 2947 Node* proj = c->as_Call()->proj_out(TypeFunc::Memory); 2948 if (proj != NULL && 2949 proj->adr_type() == TypePtr::BOTTOM) { 2950 mem = proj; 2951 } 2952 } 2953 } else { 2954 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) { 2955 Node* u = c->fast_out(i); 2956 if (u->is_Proj() && 2957 u->bottom_type() == Type::MEMORY && 2958 u->adr_type() == TypePtr::BOTTOM) { 2959 assert(c->is_SafePoint() || c->is_MemBar() || c->is_Start(), ""); 2960 assert(mem == NULL, "only one proj"); 2961 mem = u; 2962 } 2963 } 2964 assert(!c->is_Call() || c->as_Call()->adr_type() != NULL || mem == NULL, "no mem projection expected"); 2965 } 2966 } 2967 c = phase->idom(c); 2968 } while (mem == NULL); 2969 return mem; 2970 } 2971 2972 void ShenandoahWriteBarrierNode::follow_barrier_uses(Node* n, Node* ctrl, Unique_Node_List& uses, PhaseIdealLoop* phase) { 2973 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { 2974 Node* u = n->fast_out(i); 2975 if (!u->is_CFG() && phase->get_ctrl(u) == ctrl && (!u->is_Phi() || !u->in(0)->is_Loop() || u->in(LoopNode::LoopBackControl) != n)) { 2976 uses.push(u); 2977 } 2978 } 2979 } 2980 2981 Node* ShenandoahWriteBarrierNode::get_ctrl(Node* n, PhaseIdealLoop* phase) { 2982 Node* c = phase->get_ctrl(n); 2983 if (n->is_Proj() && n->in(0)->is_Call()) { 2984 assert(c == n->in(0), ""); 2985 CallNode* call = c->as_Call(); 2986 CallProjections projs; 2987 call->extract_projections(&projs, true, false); 2988 if (projs.catchall_memproj != NULL) { 2989 if (projs.fallthrough_memproj == n) { 2990 c = projs.fallthrough_catchproj; 2991 } else { 2992 assert(projs.catchall_memproj == n, ""); 2993 c = projs.catchall_catchproj; 2994 } 2995 } 2996 } 2997 return c; 2998 } 2999 3000 Node* ShenandoahWriteBarrierNode::ctrl_or_self(Node* n, PhaseIdealLoop* phase) { 3001 if (phase->has_ctrl(n)) 3002 return get_ctrl(n, phase); 3003 else { 3004 assert (n->is_CFG(), "must be a CFG node"); 3005 return n; 3006 } 3007 } 3008 3009 #ifdef ASSERT 3010 static bool has_never_branch(Node* root) { 3011 for (uint i = 1; i < root->req(); i++) { 3012 Node* in = root->in(i); 3013 if (in != NULL && in->Opcode() == Op_Halt && in->in(0)->is_Proj() && in->in(0)->in(0)->Opcode() == Op_NeverBranch) { 3014 return true; 3015 } 3016 } 3017 return false; 3018 } 3019 #endif 3020 3021 void ShenandoahWriteBarrierNode::collect_memory_nodes(int alias, Node_List& memory_nodes, PhaseIdealLoop* phase) { 3022 Node_Stack stack(0); 3023 VectorSet visited(Thread::current()->resource_area()); 3024 Node_List regions; 3025 3026 // Walk the raw memory graph and create a mapping from CFG node to 3027 // memory node. Exclude phis for now. 3028 stack.push(phase->C->root(), 1); 3029 do { 3030 Node* n = stack.node(); 3031 int opc = n->Opcode(); 3032 uint i = stack.index(); 3033 if (i < n->req()) { 3034 Node* mem = NULL; 3035 if (opc == Op_Root) { 3036 Node* in = n->in(i); 3037 int in_opc = in->Opcode(); 3038 if (in_opc == Op_Return || in_opc == Op_Rethrow) { 3039 mem = in->in(TypeFunc::Memory); 3040 } else if (in_opc == Op_Halt) { 3041 if (in->in(0)->is_Region()) { 3042 #ifdef ASSERT 3043 Node* r = in->in(0); 3044 for (uint j = 1; j < r->req(); j++) { 3045 assert(r->in(j)->is_Proj() && r->in(j)->in(0)->Opcode() == Op_NeverBranch, ""); 3046 } 3047 #endif 3048 } else { 3049 Node* proj = in->in(0); 3050 assert(proj->is_Proj(), ""); 3051 Node* in = proj->in(0); 3052 assert(in->is_CallStaticJava() || in->Opcode() == Op_NeverBranch || in->Opcode() == Op_Catch, ""); 3053 if (in->is_CallStaticJava()) { 3054 mem = in->in(TypeFunc::Memory); 3055 } else if (in->Opcode() == Op_Catch) { 3056 Node* call = in->in(0)->in(0); 3057 assert(call->is_Call(), ""); 3058 mem = call->in(TypeFunc::Memory); 3059 } 3060 } 3061 } else { 3062 #ifdef ASSERT 3063 n->dump(); 3064 in->dump(); 3065 #endif 3066 ShouldNotReachHere(); 3067 } 3068 } else { 3069 assert(n->is_Phi() && n->bottom_type() == Type::MEMORY, ""); 3070 assert(n->adr_type() == TypePtr::BOTTOM || phase->C->get_alias_index(n->adr_type()) == alias, ""); 3071 mem = n->in(i); 3072 } 3073 i++; 3074 stack.set_index(i); 3075 if (mem == NULL) { 3076 continue; 3077 } 3078 for (;;) { 3079 if (visited.test_set(mem->_idx) || mem->is_Start()) { 3080 break; 3081 } 3082 if (mem->is_Phi()) { 3083 stack.push(mem, 2); 3084 mem = mem->in(1); 3085 } else if (mem->is_Proj()) { 3086 stack.push(mem, mem->req()); 3087 mem = mem->in(0); 3088 } else if (mem->is_SafePoint() || mem->is_MemBar()) { 3089 mem = mem->in(TypeFunc::Memory); 3090 } else if (mem->is_MergeMem()) { 3091 mem = mem->as_MergeMem()->memory_at(alias); 3092 } else if (mem->is_Store() || mem->is_LoadStore() || mem->is_ClearArray()) { 3093 stack.push(mem, mem->req()); 3094 mem = mem->in(MemNode::Memory); 3095 } else { 3096 #ifdef ASSERT 3097 mem->dump(); 3098 #endif 3099 ShouldNotReachHere(); 3100 } 3101 } 3102 } else { 3103 if (n->is_Phi()) { 3104 // Nothing 3105 } else if (!n->is_Root()) { 3106 Node* c = get_ctrl(n, phase); 3107 memory_nodes.map(c->_idx, n); 3108 } 3109 stack.pop(); 3110 } 3111 } while(stack.is_nonempty()); 3112 3113 // Iterate over CFG nodes in rpo and propagate memory state to 3114 // compute memory state at regions, creating new phis if needed. 3115 Node_List rpo_list; 3116 visited.Clear(); 3117 phase->rpo(phase->C->root(), stack, visited, rpo_list); 3118 Node* root = rpo_list.pop(); 3119 assert(root == phase->C->root(), ""); 3120 3121 const bool trace = false; 3122 #ifdef ASSERT 3123 if (trace) { 3124 for (int i = rpo_list.size() - 1; i >= 0; i--) { 3125 Node* c = rpo_list.at(i); 3126 if (memory_nodes[c->_idx] != NULL) { 3127 tty->print("X %d", c->_idx); memory_nodes[c->_idx]->dump(); 3128 } 3129 } 3130 } 3131 #endif 3132 uint last = phase->C->unique(); 3133 3134 #ifdef ASSERT 3135 uint8_t max_depth = 0; 3136 for (LoopTreeIterator iter(phase->ltree_root()); !iter.done(); iter.next()) { 3137 IdealLoopTree* lpt = iter.current(); 3138 max_depth = MAX2(max_depth, lpt->_nest); 3139 } 3140 #endif 3141 3142 bool progress = true; 3143 int iteration = 0; 3144 Node_List dead_phis; 3145 while (progress) { 3146 progress = false; 3147 iteration++; 3148 assert(iteration <= 2+max_depth || phase->C->has_irreducible_loop(), ""); 3149 if (trace) { tty->print_cr("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"); } 3150 IdealLoopTree* last_updated_ilt = NULL; 3151 for (int i = rpo_list.size() - 1; i >= 0; i--) { 3152 Node* c = rpo_list.at(i); 3153 3154 Node* prev_mem = memory_nodes[c->_idx]; 3155 if (c->is_Region()) { 3156 Node* prev_region = regions[c->_idx]; 3157 Node* unique = NULL; 3158 for (uint j = 1; j < c->req() && unique != NodeSentinel; j++) { 3159 Node* m = memory_nodes[c->in(j)->_idx]; 3160 assert(m != NULL || (c->is_Loop() && j == LoopNode::LoopBackControl && iteration == 1) || phase->C->has_irreducible_loop() || has_never_branch(phase->C->root()), "expect memory state"); 3161 if (m != NULL) { 3162 if (m == prev_region && ((c->is_Loop() && j == LoopNode::LoopBackControl) || (prev_region->is_Phi() && prev_region->in(0) == c))) { 3163 assert(c->is_Loop() && j == LoopNode::LoopBackControl || phase->C->has_irreducible_loop(), ""); 3164 // continue 3165 } else if (unique == NULL) { 3166 unique = m; 3167 } else if (m == unique) { 3168 // continue 3169 } else { 3170 unique = NodeSentinel; 3171 } 3172 } 3173 } 3174 assert(unique != NULL, "empty phi???"); 3175 if (unique != NodeSentinel) { 3176 if (prev_region != NULL && prev_region->is_Phi() && prev_region->in(0) == c) { 3177 dead_phis.push(prev_region); 3178 } 3179 regions.map(c->_idx, unique); 3180 } else { 3181 Node* phi = NULL; 3182 if (prev_region != NULL && prev_region->is_Phi() && prev_region->in(0) == c && prev_region->_idx >= last) { 3183 phi = prev_region; 3184 for (uint k = 1; k < c->req(); k++) { 3185 Node* m = memory_nodes[c->in(k)->_idx]; 3186 assert(m != NULL, "expect memory state"); 3187 phi->set_req(k, m); 3188 } 3189 } else { 3190 for (DUIterator_Fast jmax, j = c->fast_outs(jmax); j < jmax && phi == NULL; j++) { 3191 Node* u = c->fast_out(j); 3192 if (u->is_Phi() && u->bottom_type() == Type::MEMORY && 3193 (u->adr_type() == TypePtr::BOTTOM || phase->C->get_alias_index(u->adr_type()) == alias)) { 3194 phi = u; 3195 for (uint k = 1; k < c->req() && phi != NULL; k++) { 3196 Node* m = memory_nodes[c->in(k)->_idx]; 3197 assert(m != NULL, "expect memory state"); 3198 if (u->in(k) != m) { 3199 phi = NULL; 3200 } 3201 } 3202 } 3203 } 3204 if (phi == NULL) { 3205 phi = new PhiNode(c, Type::MEMORY, phase->C->get_adr_type(alias)); 3206 for (uint k = 1; k < c->req(); k++) { 3207 Node* m = memory_nodes[c->in(k)->_idx]; 3208 assert(m != NULL, "expect memory state"); 3209 phi->init_req(k, m); 3210 } 3211 } 3212 } 3213 assert(phi != NULL, ""); 3214 regions.map(c->_idx, phi); 3215 } 3216 Node* current_region = regions[c->_idx]; 3217 if (current_region != prev_region) { 3218 progress = true; 3219 if (prev_region == prev_mem) { 3220 memory_nodes.map(c->_idx, current_region); 3221 } 3222 } 3223 } else if (prev_mem == NULL || prev_mem->is_Phi() || ctrl_or_self(prev_mem, phase) != c) { 3224 Node* m = memory_nodes[phase->idom(c)->_idx]; 3225 assert(m != NULL, "expect memory state"); 3226 if (m != prev_mem) { 3227 memory_nodes.map(c->_idx, m); 3228 progress = true; 3229 } 3230 } 3231 #ifdef ASSERT 3232 if (trace) { tty->print("X %d", c->_idx); memory_nodes[c->_idx]->dump(); } 3233 #endif 3234 } 3235 } 3236 3237 // Replace existing phi with computed memory state for that region 3238 // if different (could be a new phi or a dominating memory node if 3239 // that phi was found to be useless). 3240 while (dead_phis.size() > 0) { 3241 Node* n = dead_phis.pop(); 3242 n->replace_by(phase->C->top()); 3243 n->destruct(); 3244 } 3245 for (int i = rpo_list.size() - 1; i >= 0; i--) { 3246 Node* c = rpo_list.at(i); 3247 if (c->is_Region()) { 3248 Node* n = regions[c->_idx]; 3249 if (n->is_Phi() && n->_idx >= last && n->in(0) == c) { 3250 phase->register_new_node(n, c); 3251 } 3252 } 3253 } 3254 for (int i = rpo_list.size() - 1; i >= 0; i--) { 3255 Node* c = rpo_list.at(i); 3256 if (c->is_Region()) { 3257 Node* n = regions[c->_idx]; 3258 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) { 3259 Node* u = c->fast_out(i); 3260 if (u->is_Phi() && u->bottom_type() == Type::MEMORY && 3261 u != n) { 3262 if (u->adr_type() == TypePtr::BOTTOM) { 3263 fix_memory_uses(u, n, n, c, alias, phase); 3264 } else if (phase->C->get_alias_index(u->adr_type()) == alias) { 3265 phase->lazy_replace(u, n); 3266 --i; --imax; 3267 } 3268 } 3269 } 3270 } 3271 } 3272 } 3273 3274 void ShenandoahWriteBarrierNode::fix_raw_mem(Node* ctrl, Node* region, Node* raw_mem, Node* raw_mem_for_ctrl, Node* raw_mem_phi, 3275 Node_List& memory_nodes, Unique_Node_List& uses, PhaseIdealLoop* phase) { 3276 const bool trace = false; 3277 DEBUG_ONLY(if (trace) { tty->print("ZZZ control is"); ctrl->dump(); }); 3278 DEBUG_ONLY(if (trace) { tty->print("ZZZ mem is"); raw_mem->dump(); }); 3279 GrowableArray<Node*> phis; 3280 if (raw_mem_for_ctrl != raw_mem) { 3281 Node* old = raw_mem_for_ctrl; 3282 Node* prev = NULL; 3283 while (old != raw_mem) { 3284 assert(old->is_Store() || old->is_LoadStore() || old->is_ClearArray(), ""); 3285 prev = old; 3286 old = old->in(MemNode::Memory); 3287 } 3288 assert(prev != NULL, ""); 3289 memory_nodes.map(ctrl->_idx, raw_mem); 3290 memory_nodes.map(region->_idx, raw_mem_for_ctrl); 3291 phase->igvn().replace_input_of(prev, MemNode::Memory, raw_mem_phi); 3292 } else { 3293 memory_nodes.map(region->_idx, raw_mem_phi); 3294 uses.clear(); 3295 uses.push(region); 3296 for(uint next = 0; next < uses.size(); next++ ) { 3297 Node *n = uses.at(next); 3298 assert(n->is_CFG(), ""); 3299 DEBUG_ONLY(if (trace) { tty->print("ZZZ ctrl"); n->dump(); }); 3300 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { 3301 Node* u = n->fast_out(i); 3302 if (!u->is_Root() && u->is_CFG() && u != n) { 3303 Node* m = memory_nodes[u->_idx]; 3304 if (u->is_Region() && !has_mem_phi(phase->C, u, Compile::AliasIdxRaw)) { 3305 DEBUG_ONLY(if (trace) { tty->print("ZZZ region"); u->dump(); }); 3306 DEBUG_ONLY(if (trace && m != NULL) { tty->print("ZZZ mem"); m->dump(); }); 3307 3308 if (!mem_is_valid(m, u, phase) || !m->is_Phi()) { 3309 bool push = true; 3310 bool create_phi = true; 3311 if (phase->is_dominator(region, u)) { 3312 create_phi = false; 3313 } else if (!phase->C->has_irreducible_loop()) { 3314 IdealLoopTree* loop = phase->get_loop(ctrl); 3315 bool do_check = true; 3316 IdealLoopTree* l = loop; 3317 create_phi = false; 3318 while (l != phase->ltree_root()) { 3319 if (phase->is_dominator(l->_head, u) && phase->is_dominator(phase->idom(u), l->_head)) { 3320 create_phi = true; 3321 do_check = false; 3322 break; 3323 } 3324 l = l->_parent; 3325 } 3326 3327 if (do_check) { 3328 assert(!create_phi, ""); 3329 IdealLoopTree* u_loop = phase->get_loop(u); 3330 if (u_loop != phase->ltree_root() && u_loop->is_member(loop)) { 3331 Node* c = ctrl; 3332 while (!phase->is_dominator(c, u_loop->tail())) { 3333 c = phase->idom(c); 3334 } 3335 if (!phase->is_dominator(c, u)) { 3336 do_check = false; 3337 } 3338 } 3339 } 3340 3341 if (do_check && phase->is_dominator(phase->idom(u), region)) { 3342 create_phi = true; 3343 } 3344 } 3345 if (create_phi) { 3346 Node* phi = new PhiNode(u, Type::MEMORY, TypeRawPtr::BOTTOM); 3347 phase->register_new_node(phi, u); 3348 phis.push(phi); 3349 DEBUG_ONLY(if (trace) { tty->print("ZZZ new phi"); phi->dump(); }); 3350 if (!mem_is_valid(m, u, phase)) { 3351 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting mem"); phi->dump(); }); 3352 memory_nodes.map(u->_idx, phi); 3353 } else { 3354 DEBUG_ONLY(if (trace) { tty->print("ZZZ NOT setting mem"); m->dump(); }); 3355 for (;;) { 3356 assert(m->is_Mem() || m->is_LoadStore() || m->is_Proj() /*|| m->is_MergeMem()*/, ""); 3357 Node* next = NULL; 3358 if (m->is_Proj()) { 3359 next = m->in(0); 3360 } else { 3361 next = m->in(MemNode::Memory); 3362 } 3363 if (phase->get_ctrl(next) != u) { 3364 break; 3365 } 3366 if (next->is_MergeMem()) { 3367 assert(phase->get_ctrl(next->as_MergeMem()->memory_at(Compile::AliasIdxRaw)) != u, ""); 3368 break; 3369 } 3370 if (next->is_Phi()) { 3371 assert(next->adr_type() == TypePtr::BOTTOM && next->in(0) == u, ""); 3372 break; 3373 } 3374 m = next; 3375 } 3376 3377 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting to phi"); m->dump(); }); 3378 assert(m->is_Mem() || m->is_LoadStore(), ""); 3379 phase->igvn().replace_input_of(m, MemNode::Memory, phi); 3380 push = false; 3381 } 3382 } else { 3383 DEBUG_ONLY(if (trace) { tty->print("ZZZ skipping region"); u->dump(); }); 3384 } 3385 if (push) { 3386 uses.push(u); 3387 } 3388 } 3389 } else if (!mem_is_valid(m, u, phase)) { 3390 uses.push(u); 3391 } 3392 } 3393 } 3394 } 3395 for (int i = 0; i < phis.length(); i++) { 3396 Node* n = phis.at(i); 3397 Node* r = n->in(0); 3398 DEBUG_ONLY(if (trace) { tty->print("ZZZ fixing new phi"); n->dump(); }); 3399 for (uint j = 1; j < n->req(); j++) { 3400 Node* m = find_raw_mem(r->in(j), NULL, memory_nodes, phase); 3401 phase->igvn().replace_input_of(n, j, m); 3402 DEBUG_ONLY(if (trace) { tty->print("ZZZ fixing new phi: %d", j); m->dump(); }); 3403 } 3404 } 3405 } 3406 uint last = phase->C->unique(); 3407 MergeMemNode* mm = NULL; 3408 int alias = Compile::AliasIdxRaw; 3409 DEBUG_ONLY(if (trace) { tty->print("ZZZ raw mem is"); raw_mem->dump(); }); 3410 for (DUIterator i = raw_mem->outs(); raw_mem->has_out(i); i++) { 3411 Node* u = raw_mem->out(i); 3412 if (u->_idx < last) { 3413 if (u->is_Mem()) { 3414 if (phase->C->get_alias_index(u->adr_type()) == alias) { 3415 Node* m = find_raw_mem(phase->get_ctrl(u), u, memory_nodes, phase); 3416 if (m != raw_mem) { 3417 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting memory of use"); u->dump(); }); 3418 phase->igvn().replace_input_of(u, MemNode::Memory, m); 3419 --i; 3420 } 3421 } 3422 } else if (u->is_MergeMem()) { 3423 MergeMemNode* u_mm = u->as_MergeMem(); 3424 if (u_mm->memory_at(alias) == raw_mem) { 3425 MergeMemNode* newmm = NULL; 3426 for (DUIterator_Fast jmax, j = u->fast_outs(jmax); j < jmax; j++) { 3427 Node* uu = u->fast_out(j); 3428 assert(!uu->is_MergeMem(), "chain of MergeMems?"); 3429 if (uu->is_Phi()) { 3430 assert(uu->adr_type() == TypePtr::BOTTOM, ""); 3431 Node* region = uu->in(0); 3432 int nb = 0; 3433 for (uint k = 1; k < uu->req(); k++) { 3434 if (uu->in(k) == u) { 3435 Node* m = find_raw_mem(region->in(k), NULL, memory_nodes, phase); 3436 if (m != raw_mem) { 3437 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting memory of phi %d", k); uu->dump(); }); 3438 if (newmm == NULL || 1) { 3439 newmm = clone_merge_mem(u, raw_mem, alias, m, phase->ctrl_or_self(m), i, phase); 3440 } 3441 if (newmm != u) { 3442 phase->igvn().replace_input_of(uu, k, newmm); 3443 nb++; 3444 --jmax; 3445 } 3446 } 3447 } 3448 } 3449 if (nb > 0) { 3450 --j; 3451 } 3452 } else { 3453 Node* m = find_raw_mem(phase->ctrl_or_self(uu), uu, memory_nodes, phase); 3454 if (m != raw_mem) { 3455 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting memory of use"); uu->dump(); }); 3456 if (newmm == NULL || 1) { 3457 newmm = clone_merge_mem(u, raw_mem, alias, m, phase->ctrl_or_self(m), i, phase); 3458 } 3459 if (newmm != u) { 3460 phase->igvn().replace_input_of(uu, uu->find_edge(u), newmm); 3461 --j, --jmax; 3462 } 3463 } 3464 } 3465 } 3466 } 3467 } else if (u->is_Phi()) { 3468 assert(u->bottom_type() == Type::MEMORY, "what else?"); 3469 if (u->adr_type() == TypeRawPtr::BOTTOM || u->adr_type() == TypePtr::BOTTOM) { 3470 Node* region = u->in(0); 3471 bool replaced = false; 3472 for (uint j = 1; j < u->req(); j++) { 3473 if (u->in(j) == raw_mem) { 3474 Node* m = find_raw_mem(region->in(j), NULL, memory_nodes, phase); 3475 Node* nnew = m; 3476 if (m != raw_mem) { 3477 if (u->adr_type() == TypePtr::BOTTOM) { 3478 if (mm == NULL || 1) { 3479 mm = allocate_merge_mem(raw_mem, alias, m, phase->ctrl_or_self(m), phase); 3480 } 3481 nnew = mm; 3482 } 3483 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting memory of phi %d", j); u->dump(); }); 3484 phase->igvn().replace_input_of(u, j, nnew); 3485 replaced = true; 3486 } 3487 } 3488 } 3489 if (replaced) { 3490 --i; 3491 } 3492 } 3493 } else if ((u->adr_type() == TypePtr::BOTTOM && u->Opcode() != Op_StrInflatedCopy) || 3494 u->adr_type() == NULL) { 3495 assert(u->adr_type() != NULL || 3496 u->Opcode() == Op_Rethrow || 3497 u->Opcode() == Op_Return || 3498 u->Opcode() == Op_SafePoint || 3499 (u->is_CallStaticJava() && u->as_CallStaticJava()->uncommon_trap_request() != 0) || 3500 (u->is_CallStaticJava() && u->as_CallStaticJava()->_entry_point == OptoRuntime::rethrow_stub()) || 3501 u->Opcode() == Op_CallLeaf, ""); 3502 Node* m = find_raw_mem(phase->ctrl_or_self(u), u, memory_nodes, phase); 3503 if (m != raw_mem) { 3504 if (mm == NULL || 1) { 3505 mm = allocate_merge_mem(raw_mem, alias, m, phase->get_ctrl(m), phase); 3506 } 3507 phase->igvn().replace_input_of(u, u->find_edge(raw_mem), mm); 3508 --i; 3509 } 3510 } else if (phase->C->get_alias_index(u->adr_type()) == alias) { 3511 Node* m = find_raw_mem(phase->ctrl_or_self(u), u, memory_nodes, phase); 3512 if (m != raw_mem) { 3513 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting memory of use"); u->dump(); }); 3514 phase->igvn().replace_input_of(u, u->find_edge(raw_mem), m); 3515 --i; 3516 } 3517 } 3518 } 3519 } 3520 #ifdef ASSERT 3521 assert(raw_mem_phi->outcnt() > 0, ""); 3522 for (int i = 0; i < phis.length(); i++) { 3523 Node* n = phis.at(i); 3524 assert(n->outcnt() > 0, "new phi must have uses now"); 3525 } 3526 #endif 3527 } 3528 3529 void ShenandoahWriteBarrierNode::test_evacuation_in_progress(Node* ctrl, int alias, Node*& raw_mem, Node*& wb_mem, 3530 IfNode*& evacuation_iff, Node*& evac_in_progress, 3531 Node*& evac_not_in_progress, PhaseIdealLoop* phase) { 3532 IdealLoopTree *loop = phase->get_loop(ctrl); 3533 Node* thread = new ThreadLocalNode(); 3534 phase->register_new_node(thread, ctrl); 3535 Node* offset = phase->igvn().MakeConX(in_bytes(JavaThread::gc_state_offset())); 3536 phase->set_ctrl(offset, phase->C->root()); 3537 Node* gc_state_addr = new AddPNode(phase->C->top(), thread, offset); 3538 phase->register_new_node(gc_state_addr, ctrl); 3539 uint gc_state_idx = Compile::AliasIdxRaw; 3540 const TypePtr* gc_state_adr_type = NULL; // debug-mode-only argument 3541 debug_only(gc_state_adr_type = phase->C->get_adr_type(gc_state_idx)); 3542 3543 Node* gc_state = new LoadUBNode(ctrl, raw_mem, gc_state_addr, gc_state_adr_type, TypeInt::BYTE, MemNode::unordered); 3544 phase->register_new_node(gc_state, ctrl); 3545 3546 if (ShenandoahWriteBarrierMemBar) { 3547 Node* mb = MemBarNode::make(phase->C, Op_MemBarAcquire, Compile::AliasIdxRaw); 3548 mb->init_req(TypeFunc::Control, ctrl); 3549 mb->init_req(TypeFunc::Memory, raw_mem); 3550 phase->register_control(mb, loop, ctrl); 3551 Node* ctrl_proj = new ProjNode(mb,TypeFunc::Control); 3552 phase->register_control(ctrl_proj, loop, mb); 3553 raw_mem = new ProjNode(mb, TypeFunc::Memory); 3554 phase->register_new_node(raw_mem, mb); 3555 3556 mb = MemBarNode::make(phase->C, Op_MemBarAcquire, alias); 3557 mb->init_req(TypeFunc::Control, ctrl_proj); 3558 mb->init_req(TypeFunc::Memory, wb_mem); 3559 phase->register_control(mb, loop, ctrl_proj); 3560 ctrl_proj = new ProjNode(mb,TypeFunc::Control); 3561 phase->register_control(ctrl_proj, loop, mb); 3562 wb_mem = new ProjNode(mb,TypeFunc::Memory); 3563 phase->register_new_node(wb_mem, mb); 3564 3565 ctrl = ctrl_proj; 3566 } 3567 3568 Node* evacuation_in_progress = new AndINode(gc_state, phase->igvn().intcon(ShenandoahHeap::EVACUATION | ShenandoahHeap::PARTIAL | ShenandoahHeap::TRAVERSAL)); 3569 phase->register_new_node(evacuation_in_progress, ctrl); 3570 Node* evacuation_in_progress_cmp = new CmpINode(evacuation_in_progress, phase->igvn().zerocon(T_INT)); 3571 phase->register_new_node(evacuation_in_progress_cmp, ctrl); 3572 Node* evacuation_in_progress_test = new BoolNode(evacuation_in_progress_cmp, BoolTest::ne); 3573 phase->register_new_node(evacuation_in_progress_test, ctrl); 3574 evacuation_iff = new IfNode(ctrl, evacuation_in_progress_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN); 3575 phase->register_control(evacuation_iff, loop, ctrl); 3576 3577 assert(is_evacuation_in_progress_test(evacuation_iff), "Should match the shape"); 3578 assert(is_gc_state_load(gc_state), "Should match the shape"); 3579 3580 evac_not_in_progress = new IfFalseNode(evacuation_iff); 3581 phase->register_control(evac_not_in_progress, loop, evacuation_iff); 3582 evac_in_progress = new IfTrueNode(evacuation_iff); 3583 phase->register_control(evac_in_progress, loop, evacuation_iff); 3584 } 3585 3586 void ShenandoahWriteBarrierNode::evacuation_not_in_progress_null_check(Node*& c, Node*& val, Node* unc_ctrl, Node*& unc_region, PhaseIdealLoop* phase) { 3587 if (unc_ctrl != NULL) { 3588 // Clone the null check in this branch to allow implicit null check 3589 IdealLoopTree *loop = phase->get_loop(c); 3590 Node* iff = unc_ctrl->in(0); 3591 assert(iff->is_If(), "broken"); 3592 Node* new_iff = iff->clone(); 3593 new_iff->set_req(0, c); 3594 phase->register_control(new_iff, loop, c); 3595 Node* iffalse = new IfFalseNode(new_iff->as_If()); 3596 phase->register_control(iffalse, loop, new_iff); 3597 Node* iftrue = new IfTrueNode(new_iff->as_If()); 3598 phase->register_control(iftrue, loop, new_iff); 3599 c = iftrue; 3600 unc_region = new RegionNode(3); 3601 unc_region->init_req(1, iffalse); 3602 const Type *t = phase->igvn().type(val); 3603 assert(val->Opcode() == Op_CastPP, "expect cast to non null here"); 3604 Node* uncasted_val = val->in(1); 3605 val = new CastPPNode(uncasted_val, t); 3606 val->init_req(0, c); 3607 phase->register_new_node(val, c); 3608 } 3609 } 3610 3611 void ShenandoahWriteBarrierNode::evacuation_not_in_progress(Node* c, Node* val, Node* unc_ctrl, Node* raw_mem, Node* wb_mem, Node* region, 3612 Node* val_phi, Node* mem_phi, Node* raw_mem_phi, Node*& unc_region, PhaseIdealLoop* phase) { 3613 evacuation_not_in_progress_null_check(c, val, unc_ctrl, unc_region, phase); 3614 region->init_req(1, c); 3615 if (ShenandoahWriteBarrierRB) { 3616 Node* rbfalse = new ShenandoahReadBarrierNode(c, wb_mem, val); 3617 phase->register_new_node(rbfalse, c); 3618 val_phi->init_req(1, rbfalse); 3619 } else { 3620 val_phi->init_req(1, val); 3621 } 3622 mem_phi->init_req(1, wb_mem); 3623 raw_mem_phi->init_req(1, raw_mem); 3624 } 3625 3626 void ShenandoahWriteBarrierNode::evacuation_in_progress_null_check(Node*& c, Node*& val, Node* evacuation_iff, Node* unc, Node* unc_ctrl, 3627 Node* unc_region, Unique_Node_List& uses, PhaseIdealLoop* phase) { 3628 if (unc != NULL) { 3629 // Clone the null check in this branch to allow implicit null check 3630 IdealLoopTree *loop = phase->get_loop(c); 3631 Node* iff = unc_ctrl->in(0); 3632 assert(iff->is_If(), "broken"); 3633 Node* new_iff = iff->clone(); 3634 new_iff->set_req(0, c); 3635 phase->register_control(new_iff, loop, c); 3636 Node* iffalse = new IfFalseNode(new_iff->as_If()); 3637 phase->register_control(iffalse, loop, new_iff); 3638 Node* iftrue = new IfTrueNode(new_iff->as_If()); 3639 phase->register_control(iftrue, loop, new_iff); 3640 c = iftrue; 3641 unc_region->init_req(2, iffalse); 3642 3643 Node* proj = iff->as_If()->proj_out(0); 3644 assert(proj != unc_ctrl, "bad projection"); 3645 Node* use = proj->unique_ctrl_out(); 3646 3647 assert(use == unc || use->is_Region(), "what else?"); 3648 3649 uses.clear(); 3650 if (use == unc) { 3651 phase->set_idom(use, unc_region, phase->dom_depth(unc_region)+1); 3652 for (uint i = 1; i < unc->req(); i++) { 3653 Node* n = unc->in(i); 3654 if (phase->has_ctrl(n) && phase->get_ctrl(n) == proj) { 3655 uses.push(n); 3656 } 3657 } 3658 } else { 3659 assert(use->is_Region(), "what else?"); 3660 uint idx = 1; 3661 for (; use->in(idx) != proj; idx++); 3662 for (DUIterator_Fast imax, i = use->fast_outs(imax); i < imax; i++) { 3663 Node* u = use->fast_out(i); 3664 if (u->is_Phi() && phase->get_ctrl(u->in(idx)) == proj) { 3665 uses.push(u->in(idx)); 3666 } 3667 } 3668 } 3669 for(uint next = 0; next < uses.size(); next++ ) { 3670 Node *n = uses.at(next); 3671 assert(phase->get_ctrl(n) == proj, "bad control"); 3672 phase->set_ctrl_and_loop(n, unc_region); 3673 if (n->in(0) == proj) { 3674 phase->igvn().replace_input_of(n, 0, unc_region); 3675 } 3676 for (uint i = 0; i < n->req(); i++) { 3677 Node* m = n->in(i); 3678 if (m != NULL && phase->has_ctrl(m) && phase->get_ctrl(m) == proj) { 3679 uses.push(m); 3680 } 3681 } 3682 } 3683 3684 phase->igvn().rehash_node_delayed(use); 3685 int nb = use->replace_edge(proj, unc_region); 3686 assert(nb == 1, "only use expected"); 3687 phase->register_control(unc_region, phase->ltree_root(), evacuation_iff); 3688 3689 phase->igvn().replace_input_of(iff, 1, phase->igvn().intcon(1)); 3690 const Type *t = phase->igvn().type(val); 3691 assert(val->Opcode() == Op_CastPP, "expect cast to non null here"); 3692 Node* uncasted_val = val->in(1); 3693 val = new CastPPNode(uncasted_val, t); 3694 val->init_req(0, c); 3695 phase->register_new_node(val, c); 3696 } 3697 } 3698 3699 void ShenandoahWriteBarrierNode::in_cset_fast_test(Node*& c, Node* rbtrue, Node* raw_mem, Node* wb_mem, Node* region, Node* val_phi, Node* mem_phi, 3700 Node* raw_mem_phi, PhaseIdealLoop* phase) { 3701 if (ShenandoahWriteBarrierCsetTestInIR) { 3702 IdealLoopTree *loop = phase->get_loop(c); 3703 Node* raw_rbtrue = new CastP2XNode(c, rbtrue); 3704 phase->register_new_node(raw_rbtrue, c); 3705 Node* cset_offset = new URShiftXNode(raw_rbtrue, phase->igvn().intcon(ShenandoahHeapRegion::region_size_bytes_shift_jint())); 3706 phase->register_new_node(cset_offset, c); 3707 Node* in_cset_fast_test_base_addr = phase->igvn().makecon(TypeRawPtr::make(ShenandoahHeap::in_cset_fast_test_addr())); 3708 phase->set_ctrl(in_cset_fast_test_base_addr, phase->C->root()); 3709 Node* in_cset_fast_test_adr = new AddPNode(phase->C->top(), in_cset_fast_test_base_addr, cset_offset); 3710 phase->register_new_node(in_cset_fast_test_adr, c); 3711 uint in_cset_fast_test_idx = Compile::AliasIdxRaw; 3712 const TypePtr* in_cset_fast_test_adr_type = NULL; // debug-mode-only argument 3713 debug_only(in_cset_fast_test_adr_type = phase->C->get_adr_type(in_cset_fast_test_idx)); 3714 Node* in_cset_fast_test_load = new LoadUBNode(c, raw_mem, in_cset_fast_test_adr, in_cset_fast_test_adr_type, TypeInt::BOOL, MemNode::unordered); 3715 phase->register_new_node(in_cset_fast_test_load, c); 3716 Node* in_cset_fast_test_cmp = new CmpINode(in_cset_fast_test_load, phase->igvn().zerocon(T_INT)); 3717 phase->register_new_node(in_cset_fast_test_cmp, c); 3718 Node* in_cset_fast_test_test = new BoolNode(in_cset_fast_test_cmp, BoolTest::ne); 3719 phase->register_new_node(in_cset_fast_test_test, c); 3720 IfNode* in_cset_fast_test_iff = new IfNode(c, in_cset_fast_test_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN); 3721 phase->register_control(in_cset_fast_test_iff, loop, c); 3722 3723 Node* in_cset_fast_test_success = new IfFalseNode(in_cset_fast_test_iff); 3724 phase->register_control(in_cset_fast_test_success, loop, in_cset_fast_test_iff); 3725 3726 region->init_req(3, in_cset_fast_test_success); 3727 val_phi->init_req(3, rbtrue); 3728 mem_phi->init_req(3, wb_mem); 3729 raw_mem_phi->init_req(3, raw_mem); 3730 3731 Node* in_cset_fast_test_failure = new IfTrueNode(in_cset_fast_test_iff); 3732 phase->register_control(in_cset_fast_test_failure, loop, in_cset_fast_test_iff); 3733 3734 c = in_cset_fast_test_failure; 3735 } 3736 } 3737 3738 void ShenandoahWriteBarrierNode::evacuation_in_progress(Node* c, Node* val, Node* evacuation_iff, Node* unc, Node* unc_ctrl, 3739 Node* raw_mem, Node* wb_mem, Node* region, Node* val_phi, Node* mem_phi, 3740 Node* raw_mem_phi, Node* unc_region, int alias, Unique_Node_List& uses, 3741 PhaseIdealLoop* phase) { 3742 evacuation_in_progress_null_check(c, val, evacuation_iff, unc, unc_ctrl, unc_region, uses, phase); 3743 3744 IdealLoopTree *loop = phase->get_loop(c); 3745 Node* rbtrue = new ShenandoahReadBarrierNode(c, wb_mem, val); 3746 phase->register_new_node(rbtrue, c); 3747 3748 Node* in_cset_fast_test_failure = NULL; 3749 in_cset_fast_test(c, rbtrue, raw_mem, wb_mem, region, val_phi, mem_phi, raw_mem_phi, phase); 3750 3751 // The slow path stub consumes and produces raw memory in addition 3752 // to the existing memory edges 3753 Node* base = find_bottom_mem(c, phase); 3754 3755 MergeMemNode* mm = MergeMemNode::make(base); 3756 mm->set_memory_at(alias, wb_mem); 3757 mm->set_memory_at(Compile::AliasIdxRaw, raw_mem); 3758 phase->register_new_node(mm, c); 3759 3760 Node* call = new CallLeafNoFPNode(OptoRuntime::shenandoah_write_barrier_Type(), StubRoutines::shenandoah_wb_C(), "shenandoah_write_barrier", TypeRawPtr::BOTTOM); 3761 call->init_req(TypeFunc::Control, c); 3762 call->init_req(TypeFunc::I_O, phase->C->top()); 3763 call->init_req(TypeFunc::Memory, mm); 3764 call->init_req(TypeFunc::FramePtr, phase->C->top()); 3765 call->init_req(TypeFunc::ReturnAdr, phase->C->top()); 3766 call->init_req(TypeFunc::Parms, rbtrue); 3767 phase->register_control(call, loop, c); 3768 Node* ctrl_proj = new ProjNode(call, TypeFunc::Control); 3769 phase->register_control(ctrl_proj, loop, call); 3770 Node* mem_proj = new ProjNode(call, TypeFunc::Memory); 3771 phase->register_new_node(mem_proj, call); 3772 Node* res_proj = new ProjNode(call, TypeFunc::Parms); 3773 phase->register_new_node(res_proj, call); 3774 Node* res = new CheckCastPPNode(ctrl_proj, res_proj, phase->igvn().type(val)->is_oopptr()->cast_to_nonconst()); 3775 phase->register_new_node(res, ctrl_proj); 3776 region->init_req(2, ctrl_proj); 3777 val_phi->init_req(2, res); 3778 mem_phi->init_req(2, mem_proj); 3779 raw_mem_phi->init_req(2, mem_proj); 3780 phase->register_control(region, loop, evacuation_iff); 3781 3782 } 3783 3784 void ShenandoahWriteBarrierNode::pin_and_expand(PhaseIdealLoop* phase) { 3785 const bool trace = false; 3786 3787 // Collect raw memory state at CFG points in the entire graph and 3788 // record it in memory_nodes. Optimize the raw memory graph in the 3789 // process. Optimizing the memory graph also makes the memory graph 3790 // simpler. 3791 Node_List memory_nodes; 3792 collect_memory_nodes(Compile::AliasIdxRaw, memory_nodes, phase); 3793 3794 // Let's try to common write barriers again 3795 for (;;) { 3796 bool progress = false; 3797 for (int i = phase->C->shenandoah_barriers_count(); i > 0; i--) { 3798 ShenandoahBarrierNode* wb = phase->C->shenandoah_barrier(i-1); 3799 Node* ctrl = phase->get_ctrl(wb); 3800 if (wb->try_common(ctrl, phase) != NULL) { 3801 progress = true; 3802 } 3803 } 3804 if (!progress) { 3805 break; 3806 } 3807 } 3808 3809 for (int i = 0; i < phase->C->shenandoah_barriers_count(); i++) { 3810 ShenandoahWriteBarrierNode* wb = phase->C->shenandoah_barrier(i); 3811 Node* ctrl = phase->get_ctrl(wb); 3812 3813 Node* val = wb->in(ValueIn); 3814 if (ctrl->is_Proj() && ctrl->in(0)->is_CallJava()) { 3815 assert(is_dominator(phase->get_ctrl(val), ctrl->in(0)->in(0), val, ctrl->in(0), phase), "can't move"); 3816 phase->set_ctrl(wb, ctrl->in(0)->in(0)); 3817 } else if (ctrl->is_CallRuntime()) { 3818 assert(is_dominator(phase->get_ctrl(val), ctrl->in(0), val, ctrl, phase), "can't move"); 3819 phase->set_ctrl(wb, ctrl->in(0)); 3820 } 3821 3822 assert(wb->Opcode() == Op_ShenandoahWriteBarrier, "only for write barriers"); 3823 // Look for a null check that dominates this barrier and move the 3824 // barrier right after the null check to enable implicit null 3825 // checks 3826 wb->pin_and_expand_move_barrier(phase); 3827 3828 ctrl = phase->get_ctrl(wb); 3829 wb->pin_and_expand_helper(phase); 3830 } 3831 3832 Unique_Node_List uses; 3833 Unique_Node_List uses_to_ignore; 3834 for (int i = phase->C->shenandoah_barriers_count(); i > 0; i--) { 3835 int cnt = phase->C->shenandoah_barriers_count(); 3836 ShenandoahWriteBarrierNode* wb = phase->C->shenandoah_barrier(i-1); 3837 3838 uint last = phase->C->unique(); 3839 Node* ctrl = phase->get_ctrl(wb); 3840 3841 Node* raw_mem = find_raw_mem(ctrl, wb, memory_nodes, phase); 3842 Node* init_raw_mem = raw_mem; 3843 Node* raw_mem_for_ctrl = find_raw_mem(ctrl, NULL, memory_nodes, phase); 3844 int alias = phase->C->get_alias_index(wb->adr_type()); 3845 Node* wb_mem = wb->in(Memory); 3846 3847 Node* val = wb->in(ValueIn); 3848 Node* wbproj = wb->find_out_with(Op_ShenandoahWBMemProj); 3849 IdealLoopTree *loop = phase->get_loop(ctrl); 3850 3851 assert(val->Opcode() != Op_ShenandoahWriteBarrier || phase->C->has_irreducible_loop(), "No chain of write barriers"); 3852 3853 CallStaticJavaNode* unc = wb->pin_and_expand_null_check(phase->igvn()); 3854 Node* unc_ctrl = NULL; 3855 if (unc != NULL) { 3856 if (val->in(0) != ctrl) { 3857 unc = NULL; 3858 } else { 3859 unc_ctrl = val->in(0); 3860 } 3861 } 3862 3863 Node* uncasted_val = val; 3864 if (unc != NULL) { 3865 uncasted_val = val->in(1); 3866 } 3867 3868 Node* evac_in_progress = NULL; 3869 Node* evac_not_in_progress = NULL; 3870 IfNode* evacuation_iff = NULL; 3871 test_evacuation_in_progress(ctrl, alias, raw_mem, wb_mem, evacuation_iff, evac_in_progress, evac_not_in_progress, phase); 3872 3873 Node* region = new RegionNode(4); 3874 Node* val_phi = new PhiNode(region, val->bottom_type()->is_oopptr()->cast_to_nonconst()); 3875 Node* mem_phi = PhiNode::make(region, wb_mem, Type::MEMORY, phase->C->alias_type(wb->adr_type())->adr_type()); 3876 Node* raw_mem_phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM); 3877 3878 Node* unc_region = NULL; 3879 evacuation_not_in_progress(evac_not_in_progress, val, unc_ctrl, raw_mem, wb_mem, 3880 region, val_phi, mem_phi, raw_mem_phi, unc_region, 3881 phase); 3882 3883 evacuation_in_progress(evac_in_progress, val, evacuation_iff, unc, unc_ctrl, 3884 raw_mem, wb_mem, region, val_phi, mem_phi, raw_mem_phi, 3885 unc_region, alias, uses, 3886 phase); 3887 Node* out_val = val_phi; 3888 phase->register_new_node(val_phi, region); 3889 phase->register_new_node(mem_phi, region); 3890 phase->register_new_node(raw_mem_phi, region); 3891 3892 // Update the control of all nodes that should be after the 3893 // barrier control flow 3894 uses.clear(); 3895 // Every node that is control dependent on the barrier's input 3896 // control will be after the expanded barrier. The raw memory (if 3897 // its memory is control dependent on the barrier's input control) 3898 // must stay above the barrier. 3899 uses_to_ignore.clear(); 3900 if (phase->has_ctrl(init_raw_mem) && phase->get_ctrl(init_raw_mem) == ctrl && !init_raw_mem->is_Phi()) { 3901 uses_to_ignore.push(init_raw_mem); 3902 } 3903 for (uint next = 0; next < uses_to_ignore.size(); next++) { 3904 Node *n = uses_to_ignore.at(next); 3905 for (uint i = 0; i < n->req(); i++) { 3906 Node* in = n->in(i); 3907 if (in != NULL && phase->has_ctrl(in) && phase->get_ctrl(in) == ctrl) { 3908 uses_to_ignore.push(in); 3909 } 3910 } 3911 } 3912 for (DUIterator_Fast imax, i = ctrl->fast_outs(imax); i < imax; i++) { 3913 Node* u = ctrl->fast_out(i); 3914 if (u->_idx < last && 3915 u != wb && 3916 !uses_to_ignore.member(u) && 3917 (u->in(0) != ctrl || (!u->is_Region() && !u->is_Phi())) && 3918 (ctrl->Opcode() != Op_CatchProj || u->Opcode() != Op_CreateEx)) { 3919 Node* old_c = phase->ctrl_or_self(u); 3920 Node* c = old_c; 3921 if (c != ctrl || 3922 is_dominator_same_ctrl(old_c, wb, u, phase) || 3923 u->is_g1_marking_load()) { 3924 phase->igvn().rehash_node_delayed(u); 3925 int nb = u->replace_edge(ctrl, region); 3926 if (u->is_CFG()) { 3927 if (phase->idom(u) == ctrl) { 3928 phase->set_idom(u, region, phase->dom_depth(region)); 3929 } 3930 } else if (phase->get_ctrl(u) == ctrl) { 3931 assert(u != init_raw_mem, "should leave input raw mem above the barrier"); 3932 uses.push(u); 3933 } 3934 assert(nb == 1, "more than 1 ctrl input?"); 3935 --i, imax -= nb; 3936 } 3937 } 3938 } 3939 3940 if (wbproj != NULL) { 3941 phase->igvn().replace_input_of(wbproj, 0, phase->C->top()); 3942 phase->lazy_replace(wbproj, mem_phi); 3943 } 3944 if (unc != NULL) { 3945 for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) { 3946 Node* u = val->fast_out(i); 3947 Node* c = phase->ctrl_or_self(u); 3948 if (u != wb && (c != ctrl || is_dominator_same_ctrl(c, wb, u, phase))) { 3949 phase->igvn().rehash_node_delayed(u); 3950 int nb = u->replace_edge(val, out_val); 3951 --i, imax -= nb; 3952 } 3953 } 3954 if (val->outcnt() == 0) { 3955 phase->lazy_update(val, out_val); 3956 phase->igvn()._worklist.push(val); 3957 } 3958 } 3959 phase->lazy_replace(wb, out_val); 3960 3961 follow_barrier_uses(mem_phi, ctrl, uses, phase); 3962 follow_barrier_uses(out_val, ctrl, uses, phase); 3963 3964 for(uint next = 0; next < uses.size(); next++ ) { 3965 Node *n = uses.at(next); 3966 assert(phase->get_ctrl(n) == ctrl, "bad control"); 3967 assert(n != init_raw_mem, "should leave input raw mem above the barrier"); 3968 phase->set_ctrl(n, region); 3969 follow_barrier_uses(n, ctrl, uses, phase); 3970 } 3971 3972 // The slow path call produces memory: hook the raw memory phi 3973 // from the expanded write barrier with the rest of the graph 3974 // which may require adding memory phis at every post dominated 3975 // region and at enclosing loop heads. Use the memory state 3976 // collected in memory_nodes to fix the memory graph. Update that 3977 // memory state as we go. 3978 fix_raw_mem(ctrl,region, init_raw_mem, raw_mem_for_ctrl, raw_mem_phi, memory_nodes, uses, phase); 3979 assert(phase->C->shenandoah_barriers_count() == cnt - 1, "not replaced"); 3980 } 3981 3982 assert(phase->C->shenandoah_barriers_count() == 0, "all write barrier nodes should have been replaced"); 3983 } 3984 3985 void ShenandoahWriteBarrierNode::move_evacuation_test_out_of_loop(IfNode* iff, PhaseIdealLoop* phase) { 3986 // move test and its mem barriers out of the loop 3987 assert(is_evacuation_in_progress_test(iff), "inconsistent"); 3988 3989 if (ShenandoahWriteBarrierMemBar) { 3990 IdealLoopTree *loop = phase->get_loop(iff); 3991 Node* loop_head = loop->_head; 3992 Node* entry_c = loop_head->in(LoopNode::EntryControl); 3993 IdealLoopTree *entry_loop = phase->get_loop(entry_c); 3994 3995 GrowableArray<Node*> new_mbs; 3996 Node* c = iff->in(0); 3997 MemBarNode* mb = NULL; 3998 do { 3999 Node* proj_ctrl = c; 4000 assert(c->is_Proj(), "proj expected"); 4001 mb = proj_ctrl->in(0)->as_MemBar(); 4002 c = c->in(0)->in(0); 4003 4004 Node* proj_mem = mb->proj_out(TypeFunc::Memory); 4005 4006 MemBarNode* new_mb = mb->clone()->as_MemBar();; 4007 Node* new_proj_ctrl = new ProjNode(new_mb,TypeFunc::Control); 4008 Node* new_proj_mem = new ProjNode(new_mb,TypeFunc::Memory); 4009 4010 int alias = phase->C->get_alias_index(mb->adr_type()); 4011 Node* mem_ctrl = NULL; 4012 Node* mem = dom_mem(mb, loop_head, alias, mem_ctrl, phase); 4013 new_mb->set_req(TypeFunc::Memory, mem); 4014 phase->register_new_node(new_proj_mem, new_mb); 4015 fix_memory_uses(mem, new_mb, new_proj_mem, entry_c, alias, phase); 4016 assert(new_proj_mem->outcnt() >= 1, "memory projection is disconnected"); 4017 new_mbs.push(new_proj_ctrl); 4018 } while (mb->adr_type() != TypeRawPtr::BOTTOM); 4019 4020 c = entry_c; 4021 for (int i = new_mbs.length()-1; i >= 0; i--) { 4022 Node* proj_ctrl = new_mbs.at(i); 4023 Node* mb = proj_ctrl->in(0); 4024 mb->set_req(0, c); 4025 phase->set_idom(mb, mb->in(0), phase->dom_depth(mb->in(0))); 4026 phase->set_idom(proj_ctrl, mb, phase->dom_depth(mb)); 4027 c = proj_ctrl; 4028 phase->register_control(mb, entry_loop, mb->in(0)); 4029 phase->register_control(proj_ctrl, entry_loop, mb); 4030 } 4031 phase->igvn().replace_input_of(loop_head, LoopNode::EntryControl, c); 4032 phase->set_idom(loop_head, c, phase->dom_depth(c)); 4033 4034 Node* load = iff->in(1)->in(1)->in(1)->in(1); 4035 assert(load->Opcode() == Op_LoadUB, "inconsistent"); 4036 phase->igvn().replace_input_of(load, MemNode::Memory, new_mbs.at(new_mbs.length()-1)->in(0)->in(TypeFunc::Memory)); 4037 phase->igvn().replace_input_of(load, 0, entry_c); 4038 phase->set_ctrl_and_loop(load, entry_c); 4039 4040 c = iff->in(0); 4041 for (;;) { 4042 Node* next = c->in(0)->in(0); 4043 assert(c->is_Proj(), "proj expected"); 4044 Node* proj_ctrl = c; 4045 MemBarNode* mb = proj_ctrl->in(0)->as_MemBar(); 4046 Node* proj_mem = mb->proj_out(TypeFunc::Memory); 4047 Node* ctrl = mb->in(TypeFunc::Control); 4048 Node* mem = mb->in(TypeFunc::Memory); 4049 4050 phase->lazy_replace(proj_mem, mem); 4051 phase->lazy_replace(proj_ctrl, ctrl); 4052 phase->lazy_replace(mb, ctrl); 4053 loop->_body.yank(proj_ctrl); 4054 loop->_body.yank(proj_mem); 4055 loop->_body.yank(mb); 4056 if (mb->adr_type() == TypeRawPtr::BOTTOM) { 4057 break; 4058 } 4059 c = next; 4060 } 4061 4062 assert(phase->is_dominator(phase->get_ctrl(load->in(MemNode::Address)), entry_c), "address not out of loop?"); 4063 } else { 4064 IdealLoopTree *loop = phase->get_loop(iff); 4065 Node* loop_head = loop->_head; 4066 Node* entry_c = loop_head->in(LoopNode::EntryControl); 4067 4068 Node* load = iff->in(1)->in(1)->in(1)->in(1); 4069 assert(load->Opcode() == Op_LoadUB, "inconsistent"); 4070 Node* mem_ctrl = NULL; 4071 Node* mem = dom_mem(load->in(MemNode::Memory), loop_head, Compile::AliasIdxRaw, mem_ctrl, phase); 4072 phase->igvn().replace_input_of(load, MemNode::Memory, mem); 4073 phase->igvn().replace_input_of(load, 0, entry_c); 4074 phase->set_ctrl_and_loop(load, entry_c); 4075 } 4076 } 4077 4078 void ShenandoahWriteBarrierNode::backtoback_evacs(IfNode* iff, IfNode* dom_if, PhaseIdealLoop* phase) { 4079 if (!ShenandoahWriteBarrierMemBar) { 4080 return; 4081 } 4082 // move all mem barriers from this evac test to the dominating one, 4083 // removing duplicates in the process 4084 IdealLoopTree *loop = phase->get_loop(dom_if); 4085 Node* c1 = iff->in(0); 4086 Node* mb1 = NULL; 4087 GrowableArray<Node*> new_mbs; 4088 for(;;) { 4089 mb1 = c1->in(0); 4090 c1 = c1->in(0)->in(0); 4091 assert(mb1->Opcode() == Op_MemBarAcquire, "mem bar expected"); 4092 if (mb1->adr_type() == TypeRawPtr::BOTTOM) { 4093 phase->lazy_replace(mb1->as_MemBar()->proj_out(TypeFunc::Memory), mb1->in(TypeFunc::Memory)); 4094 break; 4095 } 4096 Node* c2 = dom_if->in(0); 4097 Node* mb2 = NULL; 4098 do { 4099 mb2 = c2->in(0); 4100 c2 = c2->in(0)->in(0); 4101 assert(mb2->Opcode() == Op_MemBarAcquire, "mem bar expected"); 4102 if (mb2->adr_type() == TypeRawPtr::BOTTOM) { 4103 // Barrier on same slice doesn't exist at dominating if: 4104 // move barrier up 4105 MemBarNode* mb = mb1->clone()->as_MemBar(); 4106 Node* proj_ctrl = new ProjNode(mb,TypeFunc::Control); 4107 Node* proj_mem = new ProjNode(mb,TypeFunc::Memory); 4108 int alias = phase->C->get_alias_index(mb->adr_type()); 4109 Node* mem_ctrl = NULL; 4110 Node* mem = dom_mem(mb1, dom_if->in(0), alias, mem_ctrl, phase); 4111 mb->set_req(TypeFunc::Memory, mem); 4112 phase->register_new_node(proj_mem, mb); 4113 fix_memory_uses(mem, mb, proj_mem, dom_if->in(0), alias, phase); 4114 assert(proj_mem->outcnt() >= 1, "memory projection is disconnected"); 4115 new_mbs.push(proj_ctrl); 4116 break; 4117 } 4118 } while (mb2->adr_type() != mb1->adr_type()); 4119 phase->lazy_replace(mb1->as_MemBar()->proj_out(TypeFunc::Memory), mb1->in(TypeFunc::Memory)); 4120 } 4121 if (new_mbs.length() > 0) { 4122 Node* c = dom_if->in(0); 4123 for (int i = 0; i < new_mbs.length(); i++) { 4124 Node* proj_ctrl = new_mbs.at(i); 4125 Node* mb = proj_ctrl->in(0); 4126 mb->set_req(0, c); 4127 phase->set_idom(mb, mb->in(0), phase->dom_depth(mb->in(0))); 4128 phase->set_idom(proj_ctrl, mb, phase->dom_depth(mb)); 4129 c = proj_ctrl; 4130 phase->register_control(mb, loop, mb->in(0)); 4131 phase->register_control(proj_ctrl, loop, mb); 4132 } 4133 phase->igvn().replace_input_of(dom_if, 0, c); 4134 phase->set_idom(dom_if, dom_if->in(0), phase->dom_depth(dom_if->in(0))); 4135 } 4136 Node* c = iff->in(0); 4137 for (;;) { 4138 Node* next = c->in(0)->in(0); 4139 assert(c->is_Proj(), "proj expected"); 4140 MemBarNode* mb = c->in(0)->as_MemBar(); 4141 Node* proj_ctrl = c; 4142 Node* ctrl = mb->in(TypeFunc::Control); 4143 4144 phase->lazy_replace(proj_ctrl, ctrl); 4145 phase->lazy_replace(mb, ctrl); 4146 if (mb->adr_type() == TypeRawPtr::BOTTOM) { 4147 break; 4148 } 4149 c = next; 4150 } 4151 } 4152 4153 void ShenandoahWriteBarrierNode::merge_back_to_back_evacuation_tests(Node* n, PhaseIdealLoop* phase) { 4154 if (phase->identical_backtoback_ifs(n)) { 4155 Node* n_ctrl = ShenandoahWriteBarrierNode::evacuation_in_progress_test_ctrl(n); 4156 if (phase->can_split_if(n_ctrl)) { 4157 IfNode* dom_if = phase->idom(n_ctrl)->as_If(); 4158 ShenandoahWriteBarrierNode::backtoback_evacs(n->as_If(), dom_if, phase); 4159 PhiNode* bolphi = PhiNode::make_blank(n_ctrl, n->in(1)); 4160 Node* proj_true = dom_if->proj_out(1); 4161 Node* proj_false = dom_if->proj_out(0); 4162 Node* con_true = phase->igvn().makecon(TypeInt::ONE); 4163 Node* con_false = phase->igvn().makecon(TypeInt::ZERO); 4164 4165 for (uint i = 1; i < n_ctrl->req(); i++) { 4166 if (phase->is_dominator(proj_true, n_ctrl->in(i))) { 4167 bolphi->init_req(i, con_true); 4168 } else { 4169 assert(phase->is_dominator(proj_false, n_ctrl->in(i)), "bad if"); 4170 bolphi->init_req(i, con_false); 4171 } 4172 } 4173 phase->register_new_node(bolphi, n_ctrl); 4174 phase->igvn().replace_input_of(n, 1, bolphi); 4175 phase->do_split_if(n); 4176 } 4177 } 4178 } 4179 4180 void ShenandoahWriteBarrierNode::optimize_after_expansion(const Node_List& evacuation_tests, const Node_List& gc_state_loads, Node_List &old_new, PhaseIdealLoop* phase) { 4181 bool progress; 4182 do { 4183 progress = false; 4184 for (uint i = 0; i < gc_state_loads.size(); i++) { 4185 Node* n = gc_state_loads.at(i); 4186 if (n->outcnt() != 0) { 4187 progress |= ShenandoahWriteBarrierNode::try_common_gc_state_load(n, phase); 4188 } 4189 } 4190 } while(progress); 4191 4192 for (uint i = 0; i < evacuation_tests.size(); i++) { 4193 Node* n = evacuation_tests.at(i); 4194 assert(is_evacuation_in_progress_test(n), "only evacuation test"); 4195 merge_back_to_back_evacuation_tests(n, phase); 4196 } 4197 if (!phase->C->major_progress()) { 4198 VectorSet seen(Thread::current()->resource_area()); 4199 for (uint i = 0; i < evacuation_tests.size(); i++) { 4200 Node* n = evacuation_tests.at(i); 4201 IdealLoopTree* loop = phase->get_loop(n); 4202 if (loop != phase->ltree_root() && 4203 loop->_child == NULL && 4204 !loop->_irreducible) { 4205 LoopNode* head = loop->_head->as_Loop(); 4206 if ((!head->is_CountedLoop() || head->as_CountedLoop()->is_main_loop() || head->as_CountedLoop()->is_normal_loop()) && 4207 !seen.test_set(head->_idx) && 4208 loop->policy_unswitching(phase)) { 4209 IfNode* iff = phase->find_unswitching_candidate(loop); 4210 if (iff != NULL && is_evacuation_in_progress_test(iff)) { 4211 if (head->is_strip_mined()) { 4212 head->verify_strip_mined(0); 4213 OuterStripMinedLoopNode* outer = head->as_CountedLoop()->outer_loop(); 4214 OuterStripMinedLoopEndNode* le = head->outer_loop_end(); 4215 Node* new_outer = new LoopNode(outer->in(LoopNode::EntryControl), outer->in(LoopNode::LoopBackControl)); 4216 phase->register_control(new_outer, phase->get_loop(outer), outer->in(LoopNode::EntryControl)); 4217 Node* new_le = new IfNode(le->in(0), le->in(1), le->_prob, le->_fcnt); 4218 phase->register_control(new_le, phase->get_loop(le), le->in(0)); 4219 phase->lazy_replace(outer, new_outer); 4220 phase->lazy_replace(le, new_le); 4221 head->clear_strip_mined(); 4222 } 4223 phase->do_unswitching(loop, old_new); 4224 } 4225 } 4226 } 4227 } 4228 } 4229 } 4230 4231 #ifdef ASSERT 4232 void ShenandoahBarrierNode::verify_raw_mem(RootNode* root) { 4233 const bool trace = false; 4234 ResourceMark rm; 4235 Unique_Node_List nodes; 4236 Unique_Node_List controls; 4237 Unique_Node_List memories; 4238 4239 nodes.push(root); 4240 for (uint next = 0; next < nodes.size(); next++) { 4241 Node *n = nodes.at(next); 4242 if (n->Opcode() == Op_CallLeafNoFP && n->as_Call()->_entry_point == StubRoutines::shenandoah_wb_C()) { 4243 controls.push(n); 4244 if (trace) { tty->print("XXXXXX verifying"); n->dump(); } 4245 for (uint next2 = 0; next2 < controls.size(); next2++) { 4246 Node *m = controls.at(next2); 4247 if (!m->is_Loop() || controls.member(m->in(LoopNode::EntryControl)) || 1) { 4248 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) { 4249 Node* u = m->fast_out(i); 4250 if (u->is_CFG() && !u->is_Root()) { 4251 if (trace) { tty->print("XXXXXX pushing control"); u->dump(); } 4252 controls.push(u); 4253 } 4254 } 4255 } 4256 } 4257 memories.push(n->as_Call()->proj_out(TypeFunc::Memory)); 4258 for (uint next2 = 0; next2 < memories.size(); next2++) { 4259 Node *m = memories.at(next2); 4260 assert(m->bottom_type() == Type::MEMORY, ""); 4261 if (!m->is_Phi() || !m->in(0)->is_Loop() || controls.member(m->in(0)->in(LoopNode::EntryControl)) || 1) { 4262 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) { 4263 Node* u = m->fast_out(i); 4264 if (u->bottom_type() == Type::MEMORY && (u->is_Mem() || u->is_ClearArray())) { 4265 if (trace) { tty->print("XXXXXX pushing memory"); u->dump(); } 4266 memories.push(u); 4267 } else if (u->is_LoadStore()) { 4268 if (trace) { tty->print("XXXXXX pushing memory"); u->find_out_with(Op_SCMemProj)->dump(); } 4269 memories.push(u->find_out_with(Op_SCMemProj)); 4270 } else if (u->is_MergeMem() && u->as_MergeMem()->memory_at(Compile::AliasIdxRaw) == m) { 4271 if (trace) { tty->print("XXXXXX pushing memory"); u->dump(); } 4272 memories.push(u); 4273 } else if (u->is_Phi()) { 4274 assert(u->bottom_type() == Type::MEMORY, ""); 4275 if (u->adr_type() == TypeRawPtr::BOTTOM || u->adr_type() == TypePtr::BOTTOM) { 4276 assert(controls.member(u->in(0)), ""); 4277 if (trace) { tty->print("XXXXXX pushing memory"); u->dump(); } 4278 memories.push(u); 4279 } 4280 } else if (u->is_SafePoint() || u->is_MemBar()) { 4281 for (DUIterator_Fast jmax, j = u->fast_outs(jmax); j < jmax; j++) { 4282 Node* uu = u->fast_out(j); 4283 if (uu->bottom_type() == Type::MEMORY) { 4284 if (trace) { tty->print("XXXXXX pushing memory"); uu->dump(); } 4285 memories.push(uu); 4286 } 4287 } 4288 } 4289 } 4290 } 4291 } 4292 for (uint next2 = 0; next2 < controls.size(); next2++) { 4293 Node *m = controls.at(next2); 4294 if (m->is_Region()) { 4295 bool all_in = true; 4296 for (uint i = 1; i < m->req(); i++) { 4297 if (!controls.member(m->in(i))) { 4298 all_in = false; 4299 break; 4300 } 4301 } 4302 if (trace) { tty->print("XXX verifying %s", all_in ? "all in" : ""); m->dump(); } 4303 bool found_phi = false; 4304 for (DUIterator_Fast jmax, j = m->fast_outs(jmax); j < jmax && !found_phi; j++) { 4305 Node* u = m->fast_out(j); 4306 if (u->is_Phi() && memories.member(u)) { 4307 found_phi = true; 4308 for (uint i = 1; i < u->req() && found_phi; i++) { 4309 Node* k = u->in(i); 4310 if (memories.member(k) != controls.member(m->in(i))) { 4311 found_phi = false; 4312 } 4313 } 4314 } 4315 } 4316 assert(found_phi || all_in, ""); 4317 } 4318 } 4319 controls.clear(); 4320 memories.clear(); 4321 } 4322 for( uint i = 0; i < n->len(); ++i ) { 4323 Node *m = n->in(i); 4324 if (m != NULL) { 4325 nodes.push(m); 4326 } 4327 } 4328 } 4329 } 4330 #endif 4331 4332 static bool is_on_null_check_path(Block* b, Block* null_check_block) { 4333 if (null_check_block == NULL) { 4334 return false; 4335 } 4336 do { 4337 assert(null_check_block->_num_succs == 1, "only one succ on the path to unc"); 4338 if (b == null_check_block) { 4339 return true; 4340 } 4341 null_check_block = null_check_block->_succs[0]; 4342 } while(!null_check_block->head()->is_Root()); 4343 4344 return false; 4345 } 4346 4347 int PhaseCFG::replace_uses_with_shenandoah_barrier_helper(Node* n, Node* use, Node* val, Block* block, Block* null_check_block) { 4348 int nb = 0; 4349 Block* buse = get_block_for_node(use); 4350 if (is_on_null_check_path(buse, null_check_block)) { 4351 return 0; 4352 } 4353 if (use->is_Phi()) { 4354 for (uint j = 1; j < use->req(); j++) { 4355 if (use->in(j) == val) { 4356 Block* b = get_block_for_node(use->in(0)->in(j)); 4357 if ((block != b && block->dom_lca(b) == block) || 4358 block == b) { 4359 use->set_req(j, n); 4360 nb++; 4361 } 4362 } 4363 } 4364 } else { 4365 if ((block != buse && block->dom_lca(buse) == block) || 4366 (block == buse && !use->is_scheduled())) { 4367 // Let precedence edges alone (can confuse anti-dependence verification code) 4368 for (uint i = 0; i < use->req(); i++) { 4369 if (use->in(i) == val) { 4370 use->set_req(i, n); 4371 nb++; 4372 } 4373 } 4374 assert(nb > 0 || use->find_prec_edge(val) != -1, "no replacement?"); 4375 } 4376 } 4377 4378 return nb; 4379 } 4380 4381 void PhaseCFG::replace_uses_with_shenandoah_barrier(Node* n, Block* block, Node_List& worklist, GrowableArray<int>& ready_cnt, uint max_idx, uint& phi_cnt) { 4382 // Replace all uses of barrier's input that are dominated by the 4383 // barrier with the value returned by the barrier: no need to keep 4384 // both live. 4385 if (n->is_Mach() && n->as_Mach()->ideal_Opcode() == Op_ShenandoahReadBarrier) { 4386 MachNullCheckNode* null_check = NULL; 4387 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax && null_check == NULL; i++) { 4388 Node* use = n->fast_out(i); 4389 if (use->is_MachNullCheck()) { 4390 null_check = use->as_MachNullCheck(); 4391 } 4392 } 4393 Block* null_check_block = NULL; 4394 if (null_check != NULL) { 4395 Node* proj = null_check->find_out_with(Op_IfTrue); 4396 Node* head = proj->unique_out(); 4397 null_check_block = get_block_for_node(head); 4398 } 4399 4400 Node* val = n->in(ShenandoahBarrierNode::ValueIn); 4401 if (!val->bottom_type()->isa_narrowoop()) { 4402 for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) { 4403 Node* use = val->fast_out(i); 4404 if (use != n) { 4405 int nb = replace_uses_with_shenandoah_barrier_helper(n, use, val, block, null_check_block); 4406 if (nb > 0) { 4407 --i; imax -= nb; 4408 } 4409 } 4410 } 4411 } else { 4412 for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) { 4413 Node* u = val->fast_out(i); 4414 if (u->is_Mach() && u->as_Mach()->ideal_Opcode() == Op_DecodeN) { 4415 int projs = 0; 4416 for (DUIterator_Fast jmax, j = u->fast_outs(jmax); j < jmax; j++) { 4417 Node* uu = u->fast_out(j); 4418 assert(!uu->is_MachTemp(), ""); 4419 if (uu->is_MachProj() && uu->outcnt() == 0) { 4420 projs++; 4421 } else { 4422 int nb = replace_uses_with_shenandoah_barrier_helper(n, uu, u, block, null_check_block); 4423 if (nb > 0) { 4424 if (!u->is_scheduled()) { 4425 push_ready_nodes(n, uu, block, ready_cnt, worklist, max_idx, nb); 4426 } 4427 --j; jmax -= nb; 4428 } 4429 } 4430 } 4431 // The DecodeN may have gone dead 4432 if (u->outcnt() - projs == 0) { 4433 u->disconnect_inputs(NULL, C); 4434 Block* bu = get_block_for_node(u); 4435 unmap_node_from_block(u); 4436 if (bu == block) { 4437 if (u->is_scheduled()) { 4438 block->find_remove(u); 4439 phi_cnt--; 4440 } else { 4441 worklist.yank(u); 4442 block->remove_node(block->end_idx()-1); 4443 } 4444 } else { 4445 bu->find_remove(u); 4446 } 4447 for (DUIterator_Fast jmax, j = u->fast_outs(jmax); j < jmax; j++) { 4448 Node* uu = u->fast_out(j); 4449 assert(uu->is_MachProj() && uu->outcnt() == 0, ""); 4450 assert(bu == get_block_for_node(uu), ""); 4451 uu->disconnect_inputs(NULL, C); 4452 --j; --jmax; 4453 unmap_node_from_block(uu); 4454 if (bu == block) { 4455 if (u->is_scheduled()) { 4456 block->find_remove(uu); 4457 phi_cnt--; 4458 } else { 4459 worklist.yank(uu); 4460 block->remove_node(block->end_idx()-1); 4461 } 4462 } else { 4463 bu->find_remove(uu); 4464 } 4465 assert(uu->is_scheduled() == u->is_scheduled(), ""); 4466 } 4467 --i; --imax; 4468 } 4469 } 4470 } 4471 } 4472 } 4473 }