1 /* 2 * Copyright (c) 2015, Red Hat, Inc. and/or its affiliates. 3 * 4 * This code is free software; you can redistribute it and/or modify it 5 * under the terms of the GNU General Public License version 2 only, as 6 * published by the Free Software Foundation. 7 * 8 * This code is distributed in the hope that it will be useful, but WITHOUT 9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 11 * version 2 for more details (a copy is included in the LICENSE file that 12 * accompanied this code). 13 * 14 * You should have received a copy of the GNU General Public License version 15 * 2 along with this work; if not, write to the Free Software Foundation, 16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 17 * 18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 19 * or visit www.oracle.com if you need additional information or have any 20 * questions. 21 * 22 */ 23 24 #include "precompiled.hpp" 25 #include "gc/shenandoah/brooksPointer.hpp" 26 #include "gc/shenandoah/shenandoahHeap.hpp" 27 #include "gc/shenandoah/shenandoahHeapRegion.hpp" 28 #include "opto/arraycopynode.hpp" 29 #include "opto/block.hpp" 30 #include "opto/callnode.hpp" 31 #include "opto/castnode.hpp" 32 #include "opto/movenode.hpp" 33 #include "opto/phaseX.hpp" 34 #include "opto/rootnode.hpp" 35 #include "opto/runtime.hpp" 36 #include "opto/shenandoahSupport.hpp" 37 #include "opto/subnode.hpp" 38 39 Node* ShenandoahBarrierNode::skip_through_barrier(Node* n) { 40 if (n == NULL) { 41 return NULL; 42 } else if (n->is_ShenandoahBarrier()) { 43 return n->in(ValueIn); 44 } else if (n->is_Phi() && 45 n->req() == 3 && 46 n->in(1) != NULL && 47 n->in(1)->is_ShenandoahBarrier() && 48 n->in(2) != NULL && 49 n->in(2)->bottom_type() == TypePtr::NULL_PTR && 50 n->in(0) != NULL && 51 n->in(0)->in(1) != NULL && 52 n->in(0)->in(1)->is_IfProj() && 53 n->in(0)->in(2) != NULL && 54 n->in(0)->in(2)->is_IfProj() && 55 n->in(0)->in(1)->in(0) != NULL && 56 n->in(0)->in(1)->in(0) == n->in(0)->in(2)->in(0) && 57 n->in(1)->in(ValueIn)->Opcode() == Op_CastPP) { 58 Node* iff = n->in(0)->in(1)->in(0); 59 Node* res = n->in(1)->in(ValueIn)->in(1); 60 if (iff->is_If() && 61 iff->in(1) != NULL && 62 iff->in(1)->is_Bool() && 63 iff->in(1)->as_Bool()->_test._test == BoolTest::ne && 64 iff->in(1)->in(1) != NULL && 65 iff->in(1)->in(1)->Opcode() == Op_CmpP && 66 iff->in(1)->in(1)->in(1) != NULL && 67 iff->in(1)->in(1)->in(1) == res && 68 iff->in(1)->in(1)->in(2) != NULL && 69 iff->in(1)->in(1)->in(2)->bottom_type() == TypePtr::NULL_PTR) { 70 return res; 71 } 72 } 73 return n; 74 } 75 76 bool ShenandoahBarrierNode::needs_barrier(PhaseGVN* phase, ShenandoahBarrierNode* orig, Node* n, Node* rb_mem, bool allow_fromspace) { 77 Unique_Node_List visited; 78 return needs_barrier_impl(phase, orig, n, rb_mem, allow_fromspace, visited); 79 } 80 81 bool ShenandoahBarrierNode::needs_barrier_impl(PhaseGVN* phase, ShenandoahBarrierNode* orig, Node* n, Node* rb_mem, bool allow_fromspace, Unique_Node_List &visited) { 82 83 if (visited.member(n)) { 84 return false; // Been there. 85 } 86 visited.push(n); 87 88 if (n->is_Allocate()) { 89 // tty->print_cr("killed barrier for newly allocated object"); 90 return false; 91 } 92 93 if (n->is_CallJava() || n->Opcode() == Op_CallLeafNoFP) { 94 return true; 95 } 96 97 const Type* type = phase->type(n); 98 if (type == Type::TOP) { 99 return false; 100 } 101 if (type->make_ptr()->higher_equal(TypePtr::NULL_PTR)) { 102 // tty->print_cr("killed barrier for NULL object"); 103 return false; 104 } 105 if (type->make_oopptr() && type->make_oopptr()->const_oop() != NULL) { 106 // tty->print_cr("killed barrier for constant object"); 107 return ShenandoahBarriersForConst; 108 } 109 110 if (ShenandoahOptimizeStableFinals) { 111 const TypeAryPtr* ary = type->isa_aryptr(); 112 if (ary && ary->is_stable() && allow_fromspace) { 113 return false; 114 } 115 } 116 117 if (n->is_CheckCastPP() || n->is_ConstraintCast()) { 118 return needs_barrier_impl(phase, orig, n->in(1), rb_mem, allow_fromspace, visited); 119 } 120 if (n->is_Parm()) { 121 return true; 122 } 123 if (n->is_Proj()) { 124 return needs_barrier_impl(phase, orig, n->in(0), rb_mem, allow_fromspace, visited); 125 } 126 if (n->is_Phi()) { 127 bool need_barrier = false; 128 for (uint i = 1; i < n->req() && ! need_barrier; i++) { 129 Node* input = n->in(i); 130 if (input == NULL) { 131 need_barrier = true; // Phi not complete yet? 132 } else if (needs_barrier_impl(phase, orig, input, rb_mem, allow_fromspace, visited)) { 133 need_barrier = true; 134 } 135 } 136 return need_barrier; 137 } 138 if (n->is_CMove()) { 139 return needs_barrier_impl(phase, orig, n->in(CMoveNode::IfFalse), rb_mem, allow_fromspace, visited) || 140 needs_barrier_impl(phase, orig, n->in(CMoveNode::IfTrue ), rb_mem, allow_fromspace, visited); 141 } 142 if (n->Opcode() == Op_CreateEx) { 143 return true; 144 } 145 if (n->Opcode() == Op_ShenandoahWriteBarrier) { 146 // tty->print_cr("skipped barrier for chained write barrier object"); 147 return false; 148 } 149 if (n->Opcode() == Op_ShenandoahReadBarrier) { 150 if (rb_mem == n->in(Memory)) { 151 // tty->print_cr("Eliminated chained read barrier"); 152 return false; 153 } else { 154 return true; 155 } 156 } 157 158 if (n->Opcode() == Op_LoadP || 159 n->Opcode() == Op_LoadN || 160 n->Opcode() == Op_GetAndSetP || 161 n->Opcode() == Op_CompareAndExchangeP || 162 n->Opcode() == Op_GetAndSetN || 163 n->Opcode() == Op_CompareAndExchangeN) { 164 return true; 165 } 166 if (n->Opcode() == Op_DecodeN || 167 n->Opcode() == Op_EncodeP) { 168 return needs_barrier_impl(phase, orig, n->in(1), rb_mem, allow_fromspace, visited); 169 } 170 171 #ifdef ASSERT 172 tty->print("need barrier on?: "); n->dump(); 173 ShouldNotReachHere(); 174 #endif 175 return true; 176 } 177 178 /** 179 * In Shenandoah, we need barriers on acmp (and similar instructions that compare two 180 * oops) to avoid false negatives. If it compares a from-space and a to-space 181 * copy of an object, a regular acmp would return false, even though both are 182 * the same. The acmp barrier compares the two objects, and when they are 183 * *not equal* it does a read-barrier on both, and compares them again. When it 184 * failed because of different copies of the object, we know that the object 185 * must already have been evacuated (and therefore doesn't require a write-barrier). 186 */ 187 void ShenandoahBarrierNode::do_cmpp_if(GraphKit& kit, Node*& taken_branch, Node*& untaken_branch, Node*& taken_memory, Node*& untaken_memory) { 188 assert(taken_memory == NULL && untaken_memory == NULL, "unexpected memory inputs"); 189 if (!UseShenandoahGC || !ShenandoahAcmpBarrier || ShenandoahVerifyOptoBarriers) { 190 return; 191 } 192 if (taken_branch->is_top() || untaken_branch->is_top()) { 193 // one of the branches is known to be untaken 194 return; 195 } 196 assert(taken_branch->is_IfProj() && untaken_branch->is_IfProj(), "if projections only"); 197 assert(taken_branch->in(0) == untaken_branch->in(0), "should come from same if"); 198 IfNode* iff = taken_branch->in(0)->as_If(); 199 BoolNode* bol = iff->in(1)->as_Bool(); 200 Node* cmp = bol->in(1); 201 if (cmp->Opcode() != Op_CmpP) { 202 return; 203 } 204 Node* a = cmp->in(1); 205 Node* b = cmp->in(2); 206 const Type* a_type = kit.gvn().type(a); 207 const Type* b_type = kit.gvn().type(b); 208 if (a_type->higher_equal(TypePtr::NULL_PTR) || b_type->higher_equal(TypePtr::NULL_PTR)) { 209 // We know one arg is gonna be null. No need for barriers. 210 return; 211 } 212 213 const TypePtr* a_adr_type = ShenandoahBarrierNode::brooks_pointer_type(a_type); 214 const TypePtr* b_adr_type = ShenandoahBarrierNode::brooks_pointer_type(b_type); 215 if ((! ShenandoahBarrierNode::needs_barrier(&kit.gvn(), NULL, a, kit.memory(a_adr_type), false)) && 216 (! ShenandoahBarrierNode::needs_barrier(&kit.gvn(), NULL, b, kit.memory(b_adr_type), false))) { 217 // We know both args are in to-space already. No acmp barrier needed. 218 return; 219 } 220 221 Node* equal_path = iff->proj_out(true); 222 Node* not_equal_path = iff->proj_out(false); 223 224 if (bol->_test._test == BoolTest::ne) { 225 swap(equal_path, not_equal_path); 226 } 227 228 Node* init_equal_path = equal_path; 229 Node* init_not_equal_path = not_equal_path; 230 231 uint alias_a = kit.C->get_alias_index(a_adr_type); 232 uint alias_b = kit.C->get_alias_index(b_adr_type); 233 234 Node* equal_memory = NULL; 235 Node* not_equal_memory = NULL; 236 237 RegionNode* region = new RegionNode(3); 238 region->init_req(1, equal_path); 239 PhiNode* mem_phi = NULL; 240 if (alias_a == alias_b) { 241 mem_phi = PhiNode::make(region, kit.memory(alias_a), Type::MEMORY, kit.C->get_adr_type(alias_a)); 242 } else { 243 Node* mem = kit.reset_memory(); 244 mem_phi = PhiNode::make(region, mem, Type::MEMORY, TypePtr::BOTTOM); 245 kit.set_all_memory(mem); 246 } 247 248 kit.set_control(not_equal_path); 249 250 Node* mb = NULL; 251 if (alias_a == alias_b) { 252 Node* mem = kit.reset_memory(); 253 mb = MemBarNode::make(kit.C, Op_MemBarAcquire, alias_a); 254 mb->init_req(TypeFunc::Control, kit.control()); 255 mb->init_req(TypeFunc::Memory, mem); 256 Node* membar = kit.gvn().transform(mb); 257 kit.set_control(kit.gvn().transform(new ProjNode(membar, TypeFunc::Control))); 258 Node* newmem = kit.gvn().transform(new ProjNode(membar, TypeFunc::Memory)); 259 kit.set_all_memory(mem); 260 kit.set_memory(newmem, alias_a); 261 } else { 262 mb = kit.insert_mem_bar(Op_MemBarAcquire); 263 } 264 265 a = kit.shenandoah_read_barrier_acmp(a); 266 b = kit.shenandoah_read_barrier_acmp(b); 267 268 Node* cmp2 = kit.gvn().transform(new CmpPNode(a, b)); 269 Node* bol2 = bol->clone(); 270 bol2->set_req(1, cmp2); 271 bol2 = kit.gvn().transform(bol2); 272 Node* iff2 = iff->clone(); 273 iff2->set_req(0, kit.control()); 274 iff2->set_req(1, bol2); 275 kit.gvn().set_type(iff2, kit.gvn().type(iff)); 276 Node* equal_path2 = equal_path->clone(); 277 equal_path2->set_req(0, iff2); 278 equal_path2 = kit.gvn().transform(equal_path2); 279 Node* not_equal_path2 = not_equal_path->clone(); 280 not_equal_path2->set_req(0, iff2); 281 not_equal_path2 = kit.gvn().transform(not_equal_path2); 282 283 region->init_req(2, equal_path2); 284 not_equal_memory = kit.reset_memory(); 285 not_equal_path = not_equal_path2; 286 287 kit.set_all_memory(not_equal_memory); 288 289 if (alias_a == alias_b) { 290 mem_phi->init_req(2, kit.memory(alias_a)); 291 kit.set_memory(mem_phi, alias_a); 292 } else { 293 mem_phi->init_req(2, kit.reset_memory()); 294 } 295 296 kit.record_for_igvn(mem_phi); 297 kit.gvn().set_type(mem_phi, Type::MEMORY); 298 299 if (alias_a == alias_b) { 300 equal_memory = kit.reset_memory(); 301 } else { 302 equal_memory = mem_phi; 303 } 304 305 assert(kit.map()->memory() == NULL, "no live memory state"); 306 equal_path = kit.gvn().transform(region); 307 308 if (taken_branch == init_equal_path) { 309 assert(untaken_branch == init_not_equal_path, "inconsistent"); 310 taken_branch = equal_path; 311 untaken_branch = not_equal_path; 312 taken_memory = equal_memory; 313 untaken_memory = not_equal_memory; 314 } else { 315 assert(taken_branch == init_not_equal_path, "inconsistent"); 316 assert(untaken_branch == init_equal_path, "inconsistent"); 317 taken_branch = not_equal_path; 318 untaken_branch = equal_path; 319 taken_memory = not_equal_memory; 320 untaken_memory = equal_memory; 321 } 322 } 323 324 bool ShenandoahReadBarrierNode::dominates_memory_rb_impl(PhaseGVN* phase, 325 Node* b1, 326 Node* b2, 327 Node* current, 328 bool linear) { 329 ResourceMark rm; 330 VectorSet visited(Thread::current()->resource_area()); 331 Node_Stack phis(0); 332 333 for(int i = 0; i < 10; i++) { 334 if (current == NULL) { 335 return false; 336 } else if (visited.test_set(current->_idx) || current->is_top() || current == b1) { 337 current = NULL; 338 while (phis.is_nonempty() && current == NULL) { 339 uint idx = phis.index(); 340 Node* phi = phis.node(); 341 if (idx >= phi->req()) { 342 phis.pop(); 343 } else { 344 current = phi->in(idx); 345 phis.set_index(idx+1); 346 } 347 } 348 if (current == NULL) { 349 return true; 350 } 351 } else if (current == phase->C->immutable_memory()) { 352 return false; 353 } else if (current->isa_Phi()) { 354 if (!linear) { 355 return false; 356 } 357 phis.push(current, 2); 358 current = current->in(1); 359 } else if (current->Opcode() == Op_ShenandoahWriteBarrier) { 360 const Type* in_type = current->bottom_type(); 361 const Type* this_type = b2->bottom_type(); 362 if (is_independent(in_type, this_type)) { 363 current = current->in(Memory); 364 } else { 365 return false; 366 } 367 } else if (current->Opcode() == Op_ShenandoahWBMemProj) { 368 current = current->in(0); 369 } else if (current->is_Proj()) { 370 current = current->in(0); 371 } else if (current->is_Call()) { 372 return false; // TODO: Maybe improve by looking at the call's memory effects? 373 } else if (current->is_MemBar()) { 374 return false; // TODO: Do we need to stop at *any* membar? 375 } else if (current->is_MergeMem()) { 376 // if (true) return false; 377 // tty->print_cr("current == mergemem: "); current->dump(); 378 const TypePtr* adr_type = brooks_pointer_type(phase->type(b2)); 379 uint alias_idx = phase->C->get_alias_index(adr_type); 380 current = current->as_MergeMem()->memory_at(alias_idx); 381 } else { 382 // tty->print_cr("what else can we see here:"); 383 #ifdef ASSERT 384 current->dump(); 385 #endif 386 ShouldNotReachHere(); 387 return false; 388 } 389 } 390 return false; 391 } 392 393 bool ShenandoahReadBarrierNode::is_independent(Node* mem) { 394 if (mem->is_Phi() || mem->is_Proj() || mem->is_MergeMem()) { 395 return true; 396 } else if (mem->Opcode() == Op_ShenandoahWriteBarrier) { 397 const Type* mem_type = mem->bottom_type(); 398 const Type* this_type = bottom_type(); 399 if (is_independent(mem_type, this_type)) { 400 return true; 401 } else { 402 return false; 403 } 404 } else if (mem->is_Call() || mem->is_MemBar()) { 405 return false; 406 } 407 #ifdef ASSERT 408 mem->dump(); 409 #endif 410 ShouldNotReachHere(); 411 return true; 412 } 413 414 415 bool ShenandoahReadBarrierNode::dominates_memory_rb(PhaseGVN* phase, Node* b1, Node* b2, bool linear) { 416 return dominates_memory_rb_impl(phase, b1->in(Memory), b2, b2->in(Memory), linear); 417 } 418 419 bool ShenandoahReadBarrierNode::is_independent(const Type* in_type, const Type* this_type) { 420 assert(in_type->isa_oopptr(), "expect oop ptr"); 421 assert(this_type->isa_oopptr(), "expect oop ptr"); 422 /* 423 if ((! in_type->isa_oopptr()) || (! this_type->isa_oopptr())) { 424 #ifdef ASSERT 425 tty->print_cr("not oopptr"); 426 tty->print("in: "); in_type->dump(); tty->print_cr(" "); 427 tty->print("this: "); this_type->dump(); tty->print_cr(" "); 428 #endif 429 return false; 430 } 431 */ 432 433 ciKlass* in_kls = in_type->is_oopptr()->klass(); 434 ciKlass* this_kls = this_type->is_oopptr()->klass(); 435 if (in_kls != NULL && this_kls != NULL && 436 in_kls->is_loaded() && this_kls->is_loaded() && 437 (!in_kls->is_subclass_of(this_kls)) && 438 (!this_kls->is_subclass_of(in_kls))) { 439 #ifdef ASSERT 440 // tty->print_cr("independent: "); 441 // tty->print("in: "); in_kls->print(); tty->print_cr(" "); 442 // tty->print("this: "); this_kls->print(); tty->print_cr(" "); 443 #endif 444 return true; 445 } 446 #ifdef ASSERT 447 // tty->print_cr("possibly dependend?"); 448 // tty->print("in: "); in_type->dump(); tty->print_cr(" "); 449 // tty->print("this: "); this_type->dump(); tty->print_cr(" "); 450 #endif 451 return false; 452 } 453 454 Node* ShenandoahReadBarrierNode::Ideal(PhaseGVN *phase, bool can_reshape) { 455 456 if (! can_reshape) { 457 return NULL; 458 } 459 460 if (in(Memory) == phase->C->immutable_memory()) return NULL; 461 462 // If memory input is a MergeMem, take the appropriate slice out of it. 463 Node* mem_in = in(Memory); 464 if (mem_in->isa_MergeMem()) { 465 const TypePtr* adr_type = brooks_pointer_type(bottom_type()); 466 uint alias_idx = phase->C->get_alias_index(adr_type); 467 mem_in = mem_in->as_MergeMem()->memory_at(alias_idx); 468 set_req(Memory, mem_in); 469 return this; 470 } 471 472 Node* input = in(Memory); 473 if (input->Opcode() == Op_ShenandoahWBMemProj) { 474 Node* wb = input->in(0); 475 const Type* in_type = phase->type(wb); 476 // is_top() test not sufficient here: we can come here after CCP 477 // in a dead branch of the graph that has not yet been removed. 478 if (in_type == Type::TOP) return NULL; // Dead path. 479 assert(wb->Opcode() == Op_ShenandoahWriteBarrier, "expect write barrier"); 480 if (is_independent(in_type, _type)) { 481 phase->igvn_rehash_node_delayed(wb); 482 set_req(Memory, wb->in(Memory)); 483 if (can_reshape && input->outcnt() == 0) { 484 phase->is_IterGVN()->_worklist.push(input); 485 } 486 return this; 487 } 488 } 489 return NULL; 490 } 491 492 Node* ShenandoahWriteBarrierNode::Identity(PhaseGVN* phase) { 493 assert(in(0) != NULL, "should have control"); 494 PhaseIterGVN* igvn = phase->is_IterGVN(); 495 Node* mem_in = in(Memory); 496 Node* mem_proj = NULL; 497 498 if (igvn != NULL) { 499 mem_proj = find_out_with(Op_ShenandoahWBMemProj); 500 if (mem_proj == NULL || mem_in == mem_proj) { 501 return this; 502 } 503 } 504 505 Node* replacement = Identity_impl(phase); 506 if (igvn != NULL) { 507 if (replacement != NULL && replacement != this) { 508 igvn->replace_node(mem_proj, mem_in); 509 } 510 } 511 return replacement; 512 } 513 514 515 Node* ShenandoahWriteBarrierNode::Ideal(PhaseGVN *phase, bool can_reshape) { 516 assert(in(0) != NULL, "should have control"); 517 if (!can_reshape) { 518 return NULL; 519 } 520 521 PhaseIterGVN* igvn = phase->is_IterGVN(); 522 Node* mem_proj = find_out_with(Op_ShenandoahWBMemProj); 523 Node* mem_in = in(Memory); 524 525 if (mem_in == phase->C->immutable_memory()) return NULL; 526 527 if (mem_in->isa_MergeMem()) { 528 const TypePtr* adr_type = brooks_pointer_type(bottom_type()); 529 uint alias_idx = phase->C->get_alias_index(adr_type); 530 mem_in = mem_in->as_MergeMem()->memory_at(alias_idx); 531 set_req(Memory, mem_in); 532 return this; 533 } 534 535 return NULL; 536 } 537 538 bool ShenandoahWriteBarrierNode::expand(Compile* C, PhaseIterGVN& igvn, int& loop_opts_cnt) { 539 if (UseShenandoahGC && ShenandoahWriteBarrierToIR) { 540 if (C->shenandoah_barriers_count() > 0) { 541 bool attempt_more_loopopts = ShenandoahLoopOptsAfterExpansion && (C->shenandoah_barriers_count() > 1 || C->has_loops()); 542 C->clear_major_progress(); 543 PhaseIdealLoop ideal_loop(igvn, LoopOptsShenandoahExpand); 544 if (C->failing()) return false; 545 PhaseIdealLoop::verify(igvn); 546 DEBUG_ONLY(ShenandoahBarrierNode::verify_raw_mem(C->root());) 547 if (attempt_more_loopopts) { 548 C->set_major_progress(); 549 if (!C->optimize_loops(loop_opts_cnt, igvn, LoopOptsShenandoahPostExpand)) { 550 return false; 551 } 552 C->clear_major_progress(); 553 } 554 } 555 } 556 return true; 557 } 558 559 bool ShenandoahWriteBarrierNode::is_evacuation_in_progress_test(Node* iff) { 560 if (!UseShenandoahGC) { 561 return false; 562 } 563 assert(iff->is_If(), "bad input"); 564 if (iff->Opcode() != Op_If) { 565 return false; 566 } 567 Node* bol = iff->in(1); 568 if (!bol->is_Bool() || bol->as_Bool()->_test._test != BoolTest::ne) { 569 return false; 570 } 571 Node* cmp = bol->in(1); 572 if (cmp->Opcode() != Op_CmpI) { 573 return false; 574 } 575 Node* in1 = cmp->in(1); 576 Node* in2 = cmp->in(2); 577 if (in2->find_int_con(-1) != 0) { 578 return false; 579 } 580 if (in1->Opcode() != Op_AndI) { 581 return false; 582 } 583 in2 = in1->in(2); 584 if (in2->find_int_con(-1) != (ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL)) { 585 return false; 586 } 587 in1 = in1->in(1); 588 589 return is_gc_state_load(in1); 590 } 591 592 bool ShenandoahWriteBarrierNode::is_gc_state_load(Node *n) { 593 if (!UseShenandoahGC) { 594 return false; 595 } 596 if (n->Opcode() != Op_LoadUB && n->Opcode() != Op_LoadB) { 597 return false; 598 } 599 Node* addp = n->in(MemNode::Address); 600 if (!addp->is_AddP()) { 601 return false; 602 } 603 Node* base = addp->in(AddPNode::Address); 604 Node* off = addp->in(AddPNode::Offset); 605 if (base->Opcode() != Op_ThreadLocal) { 606 return false; 607 } 608 if (off->find_intptr_t_con(-1) != in_bytes(ShenandoahThreadLocalData::gc_state_offset())) { 609 return false; 610 } 611 return true; 612 } 613 614 bool ShenandoahWriteBarrierNode::try_common_gc_state_load(Node *n, PhaseIdealLoop *phase) { 615 assert(is_gc_state_load(n), "inconsistent"); 616 Node* addp = n->in(MemNode::Address); 617 Node* dominator = NULL; 618 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) { 619 Node* u = addp->fast_out(i); 620 if (u != n && phase->is_dominator(u->in(0), n->in(0))) { 621 if (dominator == NULL) { 622 dominator = u; 623 } else { 624 if (phase->dom_depth(u->in(0)) < phase->dom_depth(dominator->in(0))) { 625 dominator = u; 626 } 627 } 628 } 629 } 630 if (dominator == NULL) { 631 return false; 632 } 633 ResourceMark rm; 634 Unique_Node_List wq; 635 wq.push(n->in(0)); 636 for (uint next = 0; next < wq.size(); next++) { 637 Node *m = wq.at(next); 638 if (m->is_SafePoint() && !m->is_CallLeaf()) { 639 return false; 640 } 641 if (m->is_Region()) { 642 for (uint i = 1; i < m->req(); i++) { 643 wq.push(m->in(i)); 644 } 645 } else { 646 wq.push(m->in(0)); 647 } 648 } 649 phase->igvn().replace_node(n, dominator); 650 651 return true; 652 } 653 654 Node* ShenandoahWriteBarrierNode::evacuation_in_progress_test_ctrl(Node* iff) { 655 assert(is_evacuation_in_progress_test(iff), "bad input"); 656 return iff->in(0); 657 } 658 659 bool ShenandoahBarrierNode::dominates_memory_impl(PhaseGVN* phase, 660 Node* b1, 661 Node* b2, 662 Node* current, 663 bool linear) { 664 ResourceMark rm; 665 VectorSet visited(Thread::current()->resource_area()); 666 Node_Stack phis(0); 667 668 669 for(int i = 0; i < 10; i++) { 670 if (current == NULL) { 671 return false; 672 } else if (visited.test_set(current->_idx) || current->is_top() || current == b1) { 673 current = NULL; 674 while (phis.is_nonempty() && current == NULL) { 675 uint idx = phis.index(); 676 Node* phi = phis.node(); 677 if (idx >= phi->req()) { 678 phis.pop(); 679 } else { 680 current = phi->in(idx); 681 phis.set_index(idx+1); 682 } 683 } 684 if (current == NULL) { 685 return true; 686 } 687 } else if (current == b2) { 688 return false; 689 } else if (current == phase->C->immutable_memory()) { 690 return false; 691 } else if (current->isa_Phi()) { 692 if (!linear) { 693 return false; 694 } 695 phis.push(current, 2); 696 current = current->in(1); 697 } else if (current->Opcode() == Op_ShenandoahWriteBarrier) { 698 current = current->in(Memory); 699 } else if (current->Opcode() == Op_ShenandoahWBMemProj) { 700 current = current->in(0); 701 } else if (current->is_Proj()) { 702 current = current->in(0); 703 } else if (current->is_Call()) { 704 current = current->in(TypeFunc::Memory); 705 } else if (current->is_MemBar()) { 706 current = current->in(TypeFunc::Memory); 707 } else if (current->is_MergeMem()) { 708 const TypePtr* adr_type = brooks_pointer_type(phase->type(b2)); 709 uint alias_idx = phase->C->get_alias_index(adr_type); 710 current = current->as_MergeMem()->memory_at(alias_idx); 711 } else { 712 #ifdef ASSERT 713 current->dump(); 714 #endif 715 ShouldNotReachHere(); 716 return false; 717 } 718 } 719 return false; 720 } 721 722 /** 723 * Determines if b1 dominates b2 through memory inputs. It returns true if: 724 * - b1 can be reached by following each branch in b2's memory input (through phis, etc) 725 * - or we get back to b2 (i.e. through a loop) without seeing b1 726 * In all other cases, (in particular, if we reach immutable_memory without having seen b1) 727 * we return false. 728 */ 729 bool ShenandoahBarrierNode::dominates_memory(PhaseGVN* phase, Node* b1, Node* b2, bool linear) { 730 return dominates_memory_impl(phase, b1, b2, b2->in(Memory), linear); 731 } 732 733 Node* ShenandoahBarrierNode::Identity_impl(PhaseGVN* phase) { 734 Node* n = in(ValueIn); 735 736 Node* rb_mem = Opcode() == Op_ShenandoahReadBarrier ? in(Memory) : NULL; 737 if (! needs_barrier(phase, this, n, rb_mem, _allow_fromspace)) { 738 return n; 739 } 740 741 // tty->print_cr("find sibling for: "); dump(2); 742 // Try to find a write barrier sibling with identical inputs that we can fold into. 743 for (DUIterator i = n->outs(); n->has_out(i); i++) { 744 Node* sibling = n->out(i); 745 if (sibling == this) { 746 continue; 747 } 748 /* 749 assert(sibling->Opcode() != Op_ShenandoahWriteBarrier || 750 Opcode() != Op_ShenandoahWriteBarrier || hash() == sibling->hash(), 751 "if this is a write barrier, then sibling can't be write barrier too"); 752 */ 753 if (sibling->Opcode() != Op_ShenandoahWriteBarrier) { 754 continue; 755 } 756 /* 757 if (sibling->outcnt() == 0) { 758 // Some dead node. 759 continue; 760 } 761 */ 762 assert(sibling->in(ValueIn) == in(ValueIn), "sanity"); 763 assert(sibling->Opcode() == Op_ShenandoahWriteBarrier, "sanity"); 764 // tty->print_cr("candidate: "); sibling->dump(); 765 766 if (dominates_memory(phase, sibling, this, phase->is_IterGVN() == NULL)) { 767 768 /* 769 tty->print_cr("matched barrier:"); 770 sibling->dump(); 771 tty->print_cr("for: "); 772 dump(); 773 */ 774 return sibling; 775 } 776 777 /* 778 tty->print_cr("couldn't match candidate:"); 779 sibling->dump(2); 780 */ 781 } 782 /* 783 tty->print_cr("couldn't match barrier to any:"); 784 dump(); 785 */ 786 return this; 787 } 788 789 #ifndef PRODUCT 790 void ShenandoahBarrierNode::dump_spec(outputStream *st) const { 791 const TypePtr* adr = adr_type(); 792 if (adr == NULL) { 793 return; 794 } 795 st->print(" @"); 796 adr->dump_on(st); 797 st->print(" ("); 798 Compile::current()->alias_type(adr)->adr_type()->dump_on(st); 799 st->print(") "); 800 } 801 #endif 802 803 Node* ShenandoahReadBarrierNode::Identity(PhaseGVN* phase) { 804 805 // if (true) return this; 806 807 // tty->print("optimizing rb: "); dump(); 808 Node* id = Identity_impl(phase); 809 810 if (id == this && phase->is_IterGVN()) { 811 Node* n = in(ValueIn); 812 // No success in super call. Try to combine identical read barriers. 813 for (DUIterator i = n->outs(); n->has_out(i); i++) { 814 Node* sibling = n->out(i); 815 if (sibling == this || sibling->Opcode() != Op_ShenandoahReadBarrier) { 816 continue; 817 } 818 assert(sibling->in(ValueIn) == in(ValueIn), "sanity"); 819 if (phase->is_IterGVN()->hash_find(sibling) && 820 sibling->bottom_type() == bottom_type() && 821 sibling->in(Control) == in(Control) && 822 dominates_memory_rb(phase, sibling, this, phase->is_IterGVN() == NULL)) { 823 /* 824 if (in(Memory) != sibling->in(Memory)) { 825 tty->print_cr("interesting rb-fold"); 826 dump(); 827 sibling->dump(); 828 } 829 */ 830 return sibling; 831 } 832 } 833 } 834 return id; 835 } 836 837 const Type* ShenandoahBarrierNode::Value(PhaseGVN* phase) const { 838 // Either input is TOP ==> the result is TOP 839 const Type *t1 = phase->type(in(Memory)); 840 if (t1 == Type::TOP) return Type::TOP; 841 const Type *t2 = phase->type(in(ValueIn)); 842 if( t2 == Type::TOP ) return Type::TOP; 843 844 Node* input = in(ValueIn); 845 const Type* type = phase->type(input)->is_oopptr()->cast_to_nonconst(); 846 return type; 847 } 848 849 uint ShenandoahBarrierNode::hash() const { 850 return TypeNode::hash() + _allow_fromspace; 851 } 852 853 uint ShenandoahBarrierNode::cmp(const Node& n) const { 854 return _allow_fromspace == ((ShenandoahBarrierNode&) n)._allow_fromspace 855 && TypeNode::cmp(n); 856 } 857 858 uint ShenandoahBarrierNode::size_of() const { 859 return sizeof(*this); 860 } 861 862 Node* ShenandoahWBMemProjNode::Identity(PhaseGVN* phase) { 863 864 Node* wb = in(0); 865 if (wb->is_top()) return phase->C->top(); // Dead path. 866 867 assert(wb->Opcode() == Op_ShenandoahWriteBarrier, "expect write barrier"); 868 PhaseIterGVN* igvn = phase->is_IterGVN(); 869 // We can't do the below unless the graph is fully constructed. 870 if (igvn == NULL) { 871 return this; 872 } 873 874 // If the mem projection has no barrier users, it's not needed anymore. 875 Unique_Node_List visited; 876 if (wb->outcnt() == 1) { 877 return wb->in(ShenandoahBarrierNode::Memory); 878 } 879 880 return this; 881 } 882 883 #ifdef ASSERT 884 bool ShenandoahBarrierNode::verify_helper(Node* in, Node_Stack& phis, VectorSet& visited, verify_type t, bool trace, Unique_Node_List& barriers_used) { 885 assert(phis.size() == 0, ""); 886 887 while (true) { 888 if (!in->bottom_type()->make_ptr()->make_oopptr()) { 889 if (trace) {tty->print_cr("Non oop");} 890 } else if (t == ShenandoahLoad && ShenandoahOptimizeStableFinals && 891 in->bottom_type()->make_ptr()->isa_aryptr() && 892 in->bottom_type()->make_ptr()->is_aryptr()->is_stable()) { 893 if (trace) {tty->print_cr("Stable array load");} 894 } else { 895 if (in->is_ConstraintCast()) { 896 in = in->in(1); 897 continue; 898 } else if (in->is_AddP()) { 899 assert(!in->in(AddPNode::Address)->is_top(), "no raw memory access"); 900 in = in->in(AddPNode::Address); 901 continue; 902 } else if (in->is_Con() && !ShenandoahBarriersForConst) { 903 if (trace) {tty->print("Found constant"); in->dump();} 904 } else if (in->is_ShenandoahBarrier()) { 905 if (t == ShenandoahStore && in->Opcode() != Op_ShenandoahWriteBarrier) { 906 return false; 907 } 908 barriers_used.push(in); 909 if (trace) {tty->print("Found barrier"); in->dump();} 910 } else if (in->is_Proj() && in->in(0)->is_Allocate()) { 911 if (trace) {tty->print("Found alloc"); in->in(0)->dump();} 912 } else if (in->is_Phi()) { 913 if (!visited.test_set(in->_idx)) { 914 if (trace) {tty->print("Pushed phi:"); in->dump();} 915 phis.push(in, 2); 916 in = in->in(1); 917 continue; 918 } 919 if (trace) {tty->print("Already seen phi:"); in->dump();} 920 } else if (in->Opcode() == Op_CMoveP || in->Opcode() == Op_CMoveN) { 921 if (!visited.test_set(in->_idx)) { 922 if (trace) {tty->print("Pushed cmovep:"); in->dump();} 923 phis.push(in, CMoveNode::IfTrue); 924 in = in->in(CMoveNode::IfFalse); 925 continue; 926 } 927 if (trace) {tty->print("Already seen cmovep:"); in->dump();} 928 } else if (in->Opcode() == Op_EncodeP || in->Opcode() == Op_DecodeN) { 929 in = in->in(1); 930 continue; 931 } else { 932 return false; 933 } 934 } 935 bool cont = false; 936 while (phis.is_nonempty()) { 937 uint idx = phis.index(); 938 Node* phi = phis.node(); 939 if (idx >= phi->req()) { 940 if (trace) {tty->print("Popped phi:"); phi->dump();} 941 phis.pop(); 942 continue; 943 } 944 if (trace) {tty->print("Next entry(%d) for phi:", idx); phi->dump();} 945 in = phi->in(idx); 946 phis.set_index(idx+1); 947 cont = true; 948 break; 949 } 950 if (!cont) { 951 break; 952 } 953 } 954 return true; 955 } 956 957 void ShenandoahBarrierNode::report_verify_failure(const char *msg, Node *n1, Node *n2) { 958 if (n1 != NULL) { 959 n1->dump(+10); 960 } 961 if (n2 != NULL) { 962 n2->dump(+10); 963 } 964 fatal("%s", msg); 965 } 966 967 void ShenandoahBarrierNode::verify(RootNode* root) { 968 ResourceMark rm; 969 Unique_Node_List wq; 970 GrowableArray<Node*> barriers; 971 Unique_Node_List barriers_used; 972 Node_Stack phis(0); 973 VectorSet visited(Thread::current()->resource_area()); 974 const bool trace = false; 975 const bool verify_no_useless_barrier = false; 976 977 wq.push(root); 978 for (uint next = 0; next < wq.size(); next++) { 979 Node *n = wq.at(next); 980 if (n->is_Load()) { 981 const bool trace = false; 982 if (trace) {tty->print("Verifying"); n->dump();} 983 if (n->Opcode() == Op_LoadRange || n->Opcode() == Op_LoadKlass || n->Opcode() == Op_LoadNKlass) { 984 if (trace) {tty->print_cr("Load range/klass");} 985 } else { 986 const TypePtr* adr_type = n->as_Load()->adr_type(); 987 988 if (adr_type->isa_oopptr() && adr_type->is_oopptr()->offset() == oopDesc::mark_offset_in_bytes()) { 989 if (trace) {tty->print_cr("Mark load");} 990 } else if (adr_type->isa_instptr() && 991 adr_type->is_instptr()->klass()->is_subtype_of(Compile::current()->env()->Reference_klass()) && 992 adr_type->is_instptr()->offset() == java_lang_ref_Reference::referent_offset) { 993 if (trace) {tty->print_cr("Reference.get()");} 994 } else { 995 bool verify = true; 996 if (adr_type->isa_instptr()) { 997 const TypeInstPtr* tinst = adr_type->is_instptr(); 998 ciKlass* k = tinst->klass(); 999 assert(k->is_instance_klass(), ""); 1000 ciInstanceKlass* ik = (ciInstanceKlass*)k; 1001 int offset = adr_type->offset(); 1002 1003 if ((ik->debug_final_field_at(offset) && ShenandoahOptimizeInstanceFinals) || 1004 (ik->debug_stable_field_at(offset) && ShenandoahOptimizeStableFinals)) { 1005 if (trace) {tty->print_cr("Final/stable");} 1006 verify = false; 1007 } else if (k == ciEnv::current()->Class_klass() && 1008 tinst->const_oop() != NULL && 1009 tinst->offset() >= (ik->size_helper() * wordSize)) { 1010 ciInstanceKlass* k = tinst->const_oop()->as_instance()->java_lang_Class_klass()->as_instance_klass(); 1011 ciField* field = k->get_field_by_offset(tinst->offset(), true); 1012 if ((ShenandoahOptimizeStaticFinals && field->is_final()) || 1013 (ShenandoahOptimizeStableFinals && field->is_stable())) { 1014 verify = false; 1015 } 1016 } 1017 } 1018 1019 if (verify && !ShenandoahBarrierNode::verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahLoad, trace, barriers_used)) { 1020 report_verify_failure("Shenandoah verification: Load should have barriers", n); 1021 } 1022 } 1023 } 1024 } else if (n->is_Store()) { 1025 const bool trace = false; 1026 1027 if (trace) {tty->print("Verifying"); n->dump();} 1028 if (n->in(MemNode::ValueIn)->bottom_type()->make_oopptr()) { 1029 Node* adr = n->in(MemNode::Address); 1030 bool verify = true; 1031 1032 if (adr->is_AddP() && adr->in(AddPNode::Base)->is_top()) { 1033 adr = adr->in(AddPNode::Address); 1034 if (adr->is_AddP()) { 1035 assert(adr->in(AddPNode::Base)->is_top(), ""); 1036 adr = adr->in(AddPNode::Address); 1037 if (adr->Opcode() == Op_LoadP && 1038 adr->in(MemNode::Address)->in(AddPNode::Base)->is_top() && 1039 adr->in(MemNode::Address)->in(AddPNode::Address)->Opcode() == Op_ThreadLocal && 1040 adr->in(MemNode::Address)->in(AddPNode::Offset)->find_intptr_t_con(-1) == in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset())) { 1041 if (trace) {tty->print_cr("SATB prebarrier");} 1042 verify = false; 1043 } 1044 } 1045 } 1046 1047 if (verify && !ShenandoahBarrierNode::verify_helper(n->in(MemNode::ValueIn), phis, visited, ShenandoahValue, trace, barriers_used)) { 1048 report_verify_failure("Shenandoah verification: Store should have barriers", n); 1049 } 1050 } 1051 if (!ShenandoahBarrierNode::verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahStore, trace, barriers_used)) { 1052 report_verify_failure("Shenandoah verification: Store (address) should have barriers", n); 1053 } 1054 } else if (n->Opcode() == Op_CmpP) { 1055 const bool trace = false; 1056 1057 Node* in1 = n->in(1); 1058 Node* in2 = n->in(2); 1059 if (in1->bottom_type()->isa_oopptr()) { 1060 if (trace) {tty->print("Verifying"); n->dump();} 1061 1062 bool mark_inputs = false; 1063 if (in1->bottom_type() == TypePtr::NULL_PTR || in2->bottom_type() == TypePtr::NULL_PTR || 1064 ((in1->is_Con() || in2->is_Con()) && !ShenandoahBarriersForConst)) { 1065 if (trace) {tty->print_cr("Comparison against a constant");} 1066 mark_inputs = true; 1067 } else if ((in1->is_CheckCastPP() && in1->in(1)->is_Proj() && in1->in(1)->in(0)->is_Allocate()) || 1068 (in2->is_CheckCastPP() && in2->in(1)->is_Proj() && in2->in(1)->in(0)->is_Allocate())) { 1069 if (trace) {tty->print_cr("Comparison with newly alloc'ed object");} 1070 mark_inputs = true; 1071 } else { 1072 assert(in2->bottom_type()->isa_oopptr(), ""); 1073 1074 if (!ShenandoahBarrierNode::verify_helper(in1, phis, visited, ShenandoahStore, trace, barriers_used) || 1075 !ShenandoahBarrierNode::verify_helper(in2, phis, visited, ShenandoahStore, trace, barriers_used)) { 1076 report_verify_failure("Shenandoah verification: Cmp should have barriers", n); 1077 } 1078 } 1079 if (verify_no_useless_barrier && 1080 mark_inputs && 1081 (!ShenandoahBarrierNode::verify_helper(in1, phis, visited, ShenandoahValue, trace, barriers_used) || 1082 !ShenandoahBarrierNode::verify_helper(in2, phis, visited, ShenandoahValue, trace, barriers_used))) { 1083 phis.clear(); 1084 visited.Reset(); 1085 } 1086 } 1087 } else if (n->is_LoadStore()) { 1088 if (n->in(MemNode::ValueIn)->bottom_type()->isa_ptr() && 1089 !ShenandoahBarrierNode::verify_helper(n->in(MemNode::ValueIn), phis, visited, ShenandoahLoad, trace, barriers_used)) { 1090 report_verify_failure("Shenandoah verification: LoadStore (value) should have barriers", n); 1091 } 1092 1093 if (n->in(MemNode::Address)->bottom_type()->isa_oopptr() && !ShenandoahBarrierNode::verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahStore, trace, barriers_used)) { 1094 report_verify_failure("Shenandoah verification: LoadStore (address) should have barriers", n); 1095 } 1096 } else if (n->Opcode() == Op_CallLeafNoFP || n->Opcode() == Op_CallLeaf) { 1097 CallNode* call = n->as_Call(); 1098 1099 static struct { 1100 const char* name; 1101 struct { 1102 int pos; 1103 verify_type t; 1104 } args[6]; 1105 } calls[] = { 1106 "aescrypt_encryptBlock", 1107 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad }, 1108 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1109 "aescrypt_decryptBlock", 1110 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad }, 1111 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1112 "multiplyToLen", 1113 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad }, { TypeFunc::Parms+4, ShenandoahStore }, 1114 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1115 "squareToLen", 1116 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad }, { -1, ShenandoahNone}, 1117 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1118 "montgomery_multiply", 1119 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad }, 1120 { TypeFunc::Parms+6, ShenandoahStore }, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1121 "montgomery_square", 1122 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+5, ShenandoahStore }, 1123 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1124 "mulAdd", 1125 { { TypeFunc::Parms, ShenandoahStore }, { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone}, 1126 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1127 "vectorizedMismatch", 1128 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone}, 1129 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1130 "updateBytesCRC32", 1131 { { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone}, 1132 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1133 "updateBytesAdler32", 1134 { { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone}, 1135 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1136 "updateBytesCRC32C", 1137 { { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahLoad}, { -1, ShenandoahNone}, 1138 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1139 "counterMode_AESCrypt", 1140 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad }, 1141 { TypeFunc::Parms+3, ShenandoahStore }, { TypeFunc::Parms+5, ShenandoahStore }, { TypeFunc::Parms+6, ShenandoahStore } }, 1142 "cipherBlockChaining_encryptAESCrypt", 1143 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad }, 1144 { TypeFunc::Parms+3, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1145 "cipherBlockChaining_decryptAESCrypt", 1146 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad }, 1147 { TypeFunc::Parms+3, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1148 "shenandoah_clone_barrier", 1149 { { TypeFunc::Parms, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone}, 1150 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1151 "ghash_processBlocks", 1152 { { TypeFunc::Parms, ShenandoahStore }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad }, 1153 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1154 "sha1_implCompress", 1155 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone }, 1156 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1157 "sha256_implCompress", 1158 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone }, 1159 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1160 "sha512_implCompress", 1161 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone }, 1162 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1163 "sha1_implCompressMB", 1164 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone }, 1165 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1166 "sha256_implCompressMB", 1167 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone }, 1168 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1169 "sha512_implCompressMB", 1170 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone }, 1171 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 1172 }; 1173 1174 if (call->is_call_to_arraycopystub()) { 1175 Node* dest = NULL; 1176 const TypeTuple* args = n->as_Call()->_tf->domain(); 1177 for (uint i = TypeFunc::Parms, j = 0; i < args->cnt(); i++) { 1178 if (args->field_at(i)->isa_ptr()) { 1179 j++; 1180 if (j == 2) { 1181 dest = n->in(i); 1182 break; 1183 } 1184 } 1185 } 1186 if (!ShenandoahBarrierNode::verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahLoad, trace, barriers_used) || 1187 !ShenandoahBarrierNode::verify_helper(dest, phis, visited, ShenandoahStore, trace, barriers_used)) { 1188 report_verify_failure("Shenandoah verification: ArrayCopy should have barriers", n); 1189 } 1190 } else if (strlen(call->_name) > 5 && 1191 !strcmp(call->_name + strlen(call->_name) - 5, "_fill")) { 1192 if (!ShenandoahBarrierNode::verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahStore, trace, barriers_used)) { 1193 report_verify_failure("Shenandoah verification: _fill should have barriers", n); 1194 } 1195 } else if (!strcmp(call->_name, "g1_wb_pre")) { 1196 // skip 1197 } else { 1198 const int calls_len = sizeof(calls) / sizeof(calls[0]); 1199 int i = 0; 1200 for (; i < calls_len; i++) { 1201 if (!strcmp(calls[i].name, call->_name)) { 1202 break; 1203 } 1204 } 1205 if (i != calls_len) { 1206 const uint args_len = sizeof(calls[0].args) / sizeof(calls[0].args[0]); 1207 for (uint j = 0; j < args_len; j++) { 1208 int pos = calls[i].args[j].pos; 1209 if (pos == -1) { 1210 break; 1211 } 1212 if (!ShenandoahBarrierNode::verify_helper(call->in(pos), phis, visited, calls[i].args[j].t, trace, barriers_used)) { 1213 report_verify_failure("Shenandoah verification: intrinsic calls should have barriers", n); 1214 } 1215 } 1216 for (uint j = TypeFunc::Parms; j < call->req(); j++) { 1217 if (call->in(j)->bottom_type()->make_ptr() && 1218 call->in(j)->bottom_type()->make_ptr()->isa_oopptr()) { 1219 uint k = 0; 1220 for (; k < args_len && calls[i].args[k].pos != (int)j; k++); 1221 if (k == args_len) { 1222 fatal("arg %d for call %s not covered", j, call->_name); 1223 } 1224 } 1225 } 1226 } else { 1227 for (uint j = TypeFunc::Parms; j < call->req(); j++) { 1228 if (call->in(j)->bottom_type()->make_ptr() && 1229 call->in(j)->bottom_type()->make_ptr()->isa_oopptr()) { 1230 fatal("%s not covered", call->_name); 1231 } 1232 } 1233 } 1234 } 1235 } else if (n->is_ShenandoahBarrier()) { 1236 assert(!barriers.contains(n), ""); 1237 assert(n->Opcode() != Op_ShenandoahWriteBarrier || n->find_out_with(Op_ShenandoahWBMemProj) != NULL, "bad shenandoah write barrier"); 1238 assert(n->Opcode() != Op_ShenandoahWriteBarrier || n->outcnt() > 1, "bad shenandoah write barrier"); 1239 barriers.push(n); 1240 } else if (n->is_AddP() 1241 || n->is_Phi() 1242 || n->is_ConstraintCast() 1243 || n->Opcode() == Op_Return 1244 || n->Opcode() == Op_CMoveP 1245 || n->Opcode() == Op_CMoveN 1246 || n->Opcode() == Op_Rethrow 1247 || n->is_MemBar() 1248 || n->Opcode() == Op_Conv2B 1249 || n->Opcode() == Op_SafePoint 1250 || n->is_CallJava() 1251 || n->Opcode() == Op_Unlock 1252 || n->Opcode() == Op_EncodeP 1253 || n->Opcode() == Op_DecodeN) { 1254 // nothing to do 1255 } else { 1256 static struct { 1257 int opcode; 1258 struct { 1259 int pos; 1260 verify_type t; 1261 } inputs[2]; 1262 } others[] = { 1263 Op_FastLock, 1264 { { 1, ShenandoahLoad }, { -1, ShenandoahNone} }, 1265 Op_Lock, 1266 { { TypeFunc::Parms, ShenandoahLoad }, { -1, ShenandoahNone} }, 1267 Op_ArrayCopy, 1268 { { ArrayCopyNode::Src, ShenandoahLoad }, { ArrayCopyNode::Dest, ShenandoahStore } }, 1269 Op_StrCompressedCopy, 1270 { { 2, ShenandoahLoad }, { 3, ShenandoahStore } }, 1271 Op_StrInflatedCopy, 1272 { { 2, ShenandoahLoad }, { 3, ShenandoahStore } }, 1273 Op_AryEq, 1274 { { 2, ShenandoahLoad }, { 3, ShenandoahLoad } }, 1275 Op_StrIndexOf, 1276 { { 2, ShenandoahLoad }, { 4, ShenandoahLoad } }, 1277 Op_StrComp, 1278 { { 2, ShenandoahLoad }, { 4, ShenandoahLoad } }, 1279 Op_StrEquals, 1280 { { 2, ShenandoahLoad }, { 3, ShenandoahLoad } }, 1281 Op_EncodeISOArray, 1282 { { 2, ShenandoahLoad }, { 3, ShenandoahStore } }, 1283 Op_HasNegatives, 1284 { { 2, ShenandoahLoad }, { -1, ShenandoahNone} }, 1285 Op_CastP2X, 1286 { { 1, ShenandoahLoad }, { -1, ShenandoahNone} }, 1287 Op_StrIndexOfChar, 1288 { { 2, ShenandoahLoad }, { -1, ShenandoahNone } }, 1289 }; 1290 1291 const int others_len = sizeof(others) / sizeof(others[0]); 1292 int i = 0; 1293 for (; i < others_len; i++) { 1294 if (others[i].opcode == n->Opcode()) { 1295 break; 1296 } 1297 } 1298 uint stop = n->is_Call() ? n->as_Call()->tf()->domain()->cnt() : n->req(); 1299 if (i != others_len) { 1300 const uint inputs_len = sizeof(others[0].inputs) / sizeof(others[0].inputs[0]); 1301 for (uint j = 0; j < inputs_len; j++) { 1302 int pos = others[i].inputs[j].pos; 1303 if (pos == -1) { 1304 break; 1305 } 1306 if (!ShenandoahBarrierNode::verify_helper(n->in(pos), phis, visited, others[i].inputs[j].t, trace, barriers_used)) { 1307 report_verify_failure("Shenandoah verification: intrinsic calls should have barriers", n); 1308 } 1309 } 1310 for (uint j = 1; j < stop; j++) { 1311 if (n->in(j) != NULL && n->in(j)->bottom_type()->make_ptr() && 1312 n->in(j)->bottom_type()->make_ptr()->make_oopptr()) { 1313 uint k = 0; 1314 for (; k < inputs_len && others[i].inputs[k].pos != (int)j; k++); 1315 if (k == inputs_len) { 1316 fatal("arg %d for node %s not covered", j, n->Name()); 1317 } 1318 } 1319 } 1320 } else { 1321 for (uint j = 1; j < stop; j++) { 1322 if (n->in(j) != NULL && n->in(j)->bottom_type()->make_ptr() && 1323 n->in(j)->bottom_type()->make_ptr()->make_oopptr()) { 1324 fatal("%s not covered", n->Name()); 1325 } 1326 } 1327 } 1328 } 1329 1330 if (n->is_SafePoint()) { 1331 SafePointNode* sfpt = n->as_SafePoint(); 1332 if (verify_no_useless_barrier && sfpt->jvms() != NULL) { 1333 for (uint i = sfpt->jvms()->scloff(); i < sfpt->jvms()->endoff(); i++) { 1334 if (!ShenandoahBarrierNode::verify_helper(sfpt->in(i), phis, visited, ShenandoahLoad, trace, barriers_used)) { 1335 phis.clear(); 1336 visited.Reset(); 1337 } 1338 } 1339 } 1340 } 1341 for( uint i = 0; i < n->len(); ++i ) { 1342 Node *m = n->in(i); 1343 if (m == NULL) continue; 1344 1345 // In most cases, inputs should be known to be non null. If it's 1346 // not the case, it could be a missing cast_not_null() in an 1347 // intrinsic or support might be needed in AddPNode::Ideal() to 1348 // avoid a NULL+offset input. 1349 if (!(n->is_Phi() || 1350 (n->is_SafePoint() && (!n->is_CallRuntime() || !strcmp(n->as_Call()->_name, "g1_wb_pre") || !strcmp(n->as_Call()->_name, "unsafe_arraycopy"))) || 1351 n->Opcode() == Op_CmpP || 1352 n->Opcode() == Op_CmpN || 1353 (n->Opcode() == Op_StoreP && i == StoreNode::ValueIn) || 1354 (n->Opcode() == Op_StoreN && i == StoreNode::ValueIn) || 1355 n->is_ConstraintCast() || 1356 n->Opcode() == Op_Return || 1357 n->Opcode() == Op_Conv2B || 1358 n->is_AddP() || 1359 n->Opcode() == Op_CMoveP || 1360 n->Opcode() == Op_CMoveN || 1361 n->Opcode() == Op_Rethrow || 1362 n->is_MemBar() || 1363 n->is_Mem() || 1364 n->Opcode() == Op_AryEq || 1365 n->Opcode() == Op_SCMemProj || 1366 n->Opcode() == Op_EncodeP || 1367 n->Opcode() == Op_DecodeN)) { 1368 if (m->bottom_type()->make_oopptr() && m->bottom_type()->make_oopptr()->meet(TypePtr::NULL_PTR) == m->bottom_type()) { 1369 report_verify_failure("Shenandoah verification: null input", n, m); 1370 } 1371 } 1372 1373 wq.push(m); 1374 } 1375 } 1376 1377 if (verify_no_useless_barrier) { 1378 for (int i = 0; i < barriers.length(); i++) { 1379 Node* n = barriers.at(i); 1380 if (!barriers_used.member(n)) { 1381 tty->print("XXX useless barrier"); n->dump(-2); 1382 ShouldNotReachHere(); 1383 } 1384 } 1385 } 1386 } 1387 #endif 1388 1389 MergeMemNode* ShenandoahWriteBarrierNode::allocate_merge_mem(Node* mem, int alias, Node* rep_proj, Node* rep_ctrl, PhaseIdealLoop* phase) { 1390 MergeMemNode* mm = MergeMemNode::make(mem); 1391 mm->set_memory_at(alias, rep_proj); 1392 phase->register_new_node(mm, rep_ctrl); 1393 return mm; 1394 } 1395 1396 MergeMemNode* ShenandoahWriteBarrierNode::clone_merge_mem(Node* u, Node* mem, int alias, Node* rep_proj, Node* rep_ctrl, DUIterator& i, PhaseIdealLoop* phase) { 1397 MergeMemNode* newmm = NULL; 1398 MergeMemNode* u_mm = u->as_MergeMem(); 1399 Node* c = phase->get_ctrl(u); 1400 if (phase->is_dominator(c, rep_ctrl)) { 1401 c = rep_ctrl; 1402 } else { 1403 assert(phase->is_dominator(rep_ctrl, c), "one must dominate the other"); 1404 } 1405 if (u->outcnt() == 1) { 1406 if (u->req() > (uint)alias && u->in(alias) == mem) { 1407 phase->igvn().replace_input_of(u, alias, rep_proj); 1408 --i; 1409 } else { 1410 phase->igvn().rehash_node_delayed(u); 1411 u_mm->set_memory_at(alias, rep_proj); 1412 } 1413 newmm = u_mm; 1414 phase->set_ctrl_and_loop(u, c); 1415 } else { 1416 // can't simply clone u and then change one of its input because 1417 // it adds and then removes an edge which messes with the 1418 // DUIterator 1419 newmm = MergeMemNode::make(u_mm->base_memory()); 1420 for (uint j = 0; j < u->req(); j++) { 1421 if (j < newmm->req()) { 1422 if (j == (uint)alias) { 1423 newmm->set_req(j, rep_proj); 1424 } else if (newmm->in(j) != u->in(j)) { 1425 newmm->set_req(j, u->in(j)); 1426 } 1427 } else if (j == (uint)alias) { 1428 newmm->add_req(rep_proj); 1429 } else { 1430 newmm->add_req(u->in(j)); 1431 } 1432 } 1433 if ((uint)alias >= u->req()) { 1434 newmm->set_memory_at(alias, rep_proj); 1435 } 1436 phase->register_new_node(newmm, c); 1437 } 1438 return newmm; 1439 } 1440 1441 bool ShenandoahWriteBarrierNode::should_process_phi(Node* phi, int alias, Compile* C) { 1442 if (phi->adr_type() == TypePtr::BOTTOM) { 1443 Node* region = phi->in(0); 1444 for (DUIterator_Fast jmax, j = region->fast_outs(jmax); j < jmax; j++) { 1445 Node* uu = region->fast_out(j); 1446 if (uu->is_Phi() && uu != phi && uu->bottom_type() == Type::MEMORY && C->get_alias_index(uu->adr_type()) == alias) { 1447 return false; 1448 } 1449 } 1450 return true; 1451 } 1452 return C->get_alias_index(phi->adr_type()) == alias; 1453 } 1454 1455 bool ShenandoahBarrierNode::is_dominator_same_ctrl(Node*c, Node* d, Node* n, PhaseIdealLoop* phase) { 1456 // That both nodes have the same control is not sufficient to prove 1457 // domination, verify that there's no path from d to n 1458 ResourceMark rm; 1459 Unique_Node_List wq; 1460 wq.push(d); 1461 for (uint next = 0; next < wq.size(); next++) { 1462 Node *m = wq.at(next); 1463 if (m == n) { 1464 return false; 1465 } 1466 if (m->is_Phi() && m->in(0)->is_Loop()) { 1467 assert(phase->ctrl_or_self(m->in(LoopNode::EntryControl)) != c, "following loop entry should lead to new control"); 1468 } else { 1469 for (uint i = 0; i < m->req(); i++) { 1470 if (m->in(i) != NULL && phase->ctrl_or_self(m->in(i)) == c) { 1471 wq.push(m->in(i)); 1472 } 1473 } 1474 } 1475 } 1476 return true; 1477 } 1478 1479 bool ShenandoahBarrierNode::is_dominator(Node *d_c, Node *n_c, Node* d, Node* n, PhaseIdealLoop* phase) { 1480 if (d_c != n_c) { 1481 return phase->is_dominator(d_c, n_c); 1482 } 1483 return is_dominator_same_ctrl(d_c, d, n, phase); 1484 } 1485 1486 Node* next_mem(Node* mem, int alias) { 1487 Node* res = NULL; 1488 if (mem->is_Proj()) { 1489 res = mem->in(0); 1490 } else if (mem->is_SafePoint() || mem->is_MemBar()) { 1491 res = mem->in(TypeFunc::Memory); 1492 } else if (mem->is_Phi()) { 1493 res = mem->in(1); 1494 } else if (mem->is_ShenandoahBarrier()) { 1495 res = mem->in(ShenandoahBarrierNode::Memory); 1496 } else if (mem->is_MergeMem()) { 1497 res = mem->as_MergeMem()->memory_at(alias); 1498 } else if (mem->is_Store() || mem->is_LoadStore() || mem->is_ClearArray()) { 1499 assert(alias = Compile::AliasIdxRaw, "following raw memory can't lead to a barrier"); 1500 res = mem->in(MemNode::Memory); 1501 } else { 1502 #ifdef ASSERT 1503 mem->dump(); 1504 #endif 1505 ShouldNotReachHere(); 1506 } 1507 return res; 1508 } 1509 1510 bool suitable_mem(Node* mem, Node* old_mem, Node* rep_proj) { 1511 for (DUIterator_Fast imax, i = mem->fast_outs(imax); i < imax; i++) { 1512 Node* u = mem->fast_out(i); 1513 if (u->is_MergeMem()) { 1514 if (u->has_out_with(Op_MergeMem)) { 1515 // too complicated for now 1516 return false; 1517 } 1518 if (old_mem == u && rep_proj->has_out_with(Op_MergeMem)) { 1519 return false; 1520 } 1521 } 1522 if (u->Opcode() == Op_Unlock && mem->is_Proj() && mem->in(0)->Opcode() == Op_MemBarReleaseLock) { 1523 // would require a merge mem between unlock and the 1524 // preceding membar. Would confuse logic that eliminates 1525 // lock/unlock nodes. 1526 return false; 1527 } 1528 } 1529 return true; 1530 } 1531 1532 void ShenandoahWriteBarrierNode::fix_memory_uses(Node* mem, Node* replacement, Node* rep_proj, Node* rep_ctrl, int alias, PhaseIdealLoop* phase) { 1533 uint last =phase-> C->unique(); 1534 MergeMemNode* mm = NULL; 1535 assert(mem->bottom_type() == Type::MEMORY, ""); 1536 for (DUIterator i = mem->outs(); mem->has_out(i); i++) { 1537 Node* u = mem->out(i); 1538 if (u != replacement && u->_idx < last) { 1539 if (u->is_ShenandoahBarrier() && alias != Compile::AliasIdxRaw) { 1540 if (phase->C->get_alias_index(u->adr_type()) == alias && is_dominator(rep_ctrl, phase->ctrl_or_self(u), replacement, u, phase)) { 1541 phase->igvn().replace_input_of(u, u->find_edge(mem), rep_proj); 1542 assert(u->find_edge(mem) == -1, "only one edge"); 1543 --i; 1544 } 1545 } else if (u->is_Mem()) { 1546 if (phase->C->get_alias_index(u->adr_type()) == alias && is_dominator(rep_ctrl, phase->ctrl_or_self(u), replacement, u, phase)) { 1547 assert(alias == Compile::AliasIdxRaw , "only raw memory can lead to a memory operation"); 1548 phase->igvn().replace_input_of(u, u->find_edge(mem), rep_proj); 1549 assert(u->find_edge(mem) == -1, "only one edge"); 1550 --i; 1551 } 1552 } else if (u->is_MergeMem()) { 1553 MergeMemNode* u_mm = u->as_MergeMem(); 1554 if (u_mm->memory_at(alias) == mem) { 1555 MergeMemNode* newmm = NULL; 1556 for (DUIterator_Fast jmax, j = u->fast_outs(jmax); j < jmax; j++) { 1557 Node* uu = u->fast_out(j); 1558 assert(!uu->is_MergeMem(), "chain of MergeMems?"); 1559 if (uu->is_Phi()) { 1560 if (should_process_phi(uu, alias, phase->C)) { 1561 Node* region = uu->in(0); 1562 int nb = 0; 1563 for (uint k = 1; k < uu->req(); k++) { 1564 if (uu->in(k) == u && phase->is_dominator(rep_ctrl, region->in(k))) { 1565 if (newmm == NULL) { 1566 newmm = clone_merge_mem(u, mem, alias, rep_proj, rep_ctrl, i, phase); 1567 } 1568 if (newmm != u) { 1569 phase->igvn().replace_input_of(uu, k, newmm); 1570 nb++; 1571 --jmax; 1572 } 1573 } 1574 } 1575 if (nb > 0) { 1576 --j; 1577 } 1578 } 1579 } else { 1580 if (rep_ctrl != uu && is_dominator(rep_ctrl, phase->ctrl_or_self(uu), replacement, uu, phase)) { 1581 if (newmm == NULL) { 1582 newmm = clone_merge_mem(u, mem, alias, rep_proj, rep_ctrl, i, phase); 1583 } 1584 if (newmm != u) { 1585 phase->igvn().replace_input_of(uu, uu->find_edge(u), newmm); 1586 --j, --jmax; 1587 } 1588 } 1589 } 1590 } 1591 } 1592 } else if (u->is_Phi()) { 1593 assert(u->bottom_type() == Type::MEMORY, "what else?"); 1594 Node* region = u->in(0); 1595 if (should_process_phi(u, alias, phase->C)) { 1596 bool replaced = false; 1597 for (uint j = 1; j < u->req(); j++) { 1598 if (u->in(j) == mem && phase->is_dominator(rep_ctrl, region->in(j))) { 1599 Node* nnew = rep_proj; 1600 if (u->adr_type() == TypePtr::BOTTOM) { 1601 if (mm == NULL) { 1602 mm = allocate_merge_mem(mem, alias, rep_proj, rep_ctrl, phase); 1603 } 1604 nnew = mm; 1605 } 1606 phase->igvn().replace_input_of(u, j, nnew); 1607 replaced = true; 1608 } 1609 } 1610 if (replaced) { 1611 --i; 1612 } 1613 1614 } 1615 } else if ((u->adr_type() == TypePtr::BOTTOM && u->Opcode() != Op_StrInflatedCopy) || 1616 u->adr_type() == NULL) { 1617 assert(u->adr_type() != NULL || 1618 u->Opcode() == Op_Rethrow || 1619 u->Opcode() == Op_Return || 1620 u->Opcode() == Op_SafePoint || 1621 (u->is_CallStaticJava() && u->as_CallStaticJava()->uncommon_trap_request() != 0) || 1622 (u->is_CallStaticJava() && u->as_CallStaticJava()->_entry_point == OptoRuntime::rethrow_stub()) || 1623 u->Opcode() == Op_CallLeaf, ""); 1624 if (is_dominator(rep_ctrl, phase->ctrl_or_self(u), replacement, u, phase)) { 1625 if (mm == NULL) { 1626 mm = allocate_merge_mem(mem, alias, rep_proj, rep_ctrl, phase); 1627 } 1628 phase->igvn().replace_input_of(u, u->find_edge(mem), mm); 1629 --i; 1630 } 1631 } else if (phase->C->get_alias_index(u->adr_type()) == alias) { 1632 if (is_dominator(rep_ctrl, phase->ctrl_or_self(u), replacement, u, phase)) { 1633 phase->igvn().replace_input_of(u, u->find_edge(mem), rep_proj); 1634 --i; 1635 } 1636 } 1637 } 1638 } 1639 } 1640 1641 Node* ShenandoahBarrierNode::no_branches(Node* c, Node* dom, bool allow_one_proj, PhaseIdealLoop* phase) { 1642 Node* iffproj = NULL; 1643 while (c != dom) { 1644 Node* next = phase->idom(c); 1645 assert(next->unique_ctrl_out() == c || c->is_Proj() || c->is_Region(), "multiple control flow out but no proj or region?"); 1646 if (c->is_Region()) { 1647 ResourceMark rm; 1648 Unique_Node_List wq; 1649 wq.push(c); 1650 for (uint i = 0; i < wq.size(); i++) { 1651 Node *n = wq.at(i); 1652 if (n->is_Region()) { 1653 for (uint j = 1; j < n->req(); j++) { 1654 if (n->in(j) != next) { 1655 wq.push(n->in(j)); 1656 } 1657 } 1658 } else { 1659 if (n->in(0) != next) { 1660 wq.push(n->in(0)); 1661 } 1662 } 1663 } 1664 for (DUIterator_Fast imax, i = next->fast_outs(imax); i < imax; i++) { 1665 Node* u = next->fast_out(i); 1666 if (u->is_CFG()) { 1667 if (!wq.member(u)) { 1668 return NodeSentinel; 1669 } 1670 } 1671 } 1672 1673 } else if (c->is_Proj()) { 1674 if (c->is_IfProj()) { 1675 if (c->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none) != NULL) { 1676 // continue; 1677 } else { 1678 if (!allow_one_proj) { 1679 return NodeSentinel; 1680 } 1681 if (iffproj == NULL) { 1682 iffproj = c; 1683 } else { 1684 return NodeSentinel; 1685 } 1686 } 1687 } else if (c->Opcode() == Op_JumpProj) { 1688 return NodeSentinel; // unsupported 1689 } else if (c->Opcode() == Op_CatchProj) { 1690 return NodeSentinel; // unsupported 1691 } else if (c->Opcode() == Op_CProj && next->Opcode() == Op_NeverBranch) { 1692 return NodeSentinel; // unsupported 1693 } else { 1694 assert(next->unique_ctrl_out() == c, "unsupported branch pattern"); 1695 } 1696 } 1697 c = next; 1698 } 1699 return iffproj; 1700 } 1701 1702 #ifdef ASSERT 1703 void ShenandoahWriteBarrierNode::memory_dominates_all_paths_helper(Node* c, Node* rep_ctrl, Unique_Node_List& controls, PhaseIdealLoop* phase) { 1704 const bool trace = false; 1705 if (trace) { tty->print("X control is"); c->dump(); } 1706 1707 uint start = controls.size(); 1708 controls.push(c); 1709 for (uint i = start; i < controls.size(); i++) { 1710 Node *n = controls.at(i); 1711 1712 if (trace) { tty->print("X from"); n->dump(); } 1713 1714 if (n == rep_ctrl) { 1715 continue; 1716 } 1717 1718 if (n->is_Proj()) { 1719 Node* n_dom = n->in(0); 1720 IdealLoopTree* n_dom_loop = phase->get_loop(n_dom); 1721 if (n->is_IfProj() && n_dom->outcnt() == 2) { 1722 n_dom_loop = phase->get_loop(n_dom->as_If()->proj_out(n->as_Proj()->_con == 0 ? 1 : 0)); 1723 } 1724 if (n_dom_loop != phase->ltree_root()) { 1725 Node* tail = n_dom_loop->tail(); 1726 if (tail->is_Region()) { 1727 for (uint j = 1; j < tail->req(); j++) { 1728 if (phase->is_dominator(n_dom, tail->in(j)) && !phase->is_dominator(n, tail->in(j))) { 1729 assert(phase->is_dominator(rep_ctrl, tail->in(j)), "why are we here?"); 1730 // entering loop from below, mark backedge 1731 if (trace) { tty->print("X pushing backedge"); tail->in(j)->dump(); } 1732 controls.push(tail->in(j)); 1733 //assert(n->in(0) == n_dom, "strange flow control"); 1734 } 1735 } 1736 } else if (phase->get_loop(n) != n_dom_loop && phase->is_dominator(n_dom, tail)) { 1737 // entering loop from below, mark backedge 1738 if (trace) { tty->print("X pushing backedge"); tail->dump(); } 1739 controls.push(tail); 1740 //assert(n->in(0) == n_dom, "strange flow control"); 1741 } 1742 } 1743 } 1744 1745 if (n->is_Loop()) { 1746 Node* c = n->in(LoopNode::EntryControl); 1747 if (trace) { tty->print("X pushing"); c->dump(); } 1748 controls.push(c); 1749 } else if (n->is_Region()) { 1750 for (uint i = 1; i < n->req(); i++) { 1751 Node* c = n->in(i); 1752 if (trace) { tty->print("X pushing"); c->dump(); } 1753 controls.push(c); 1754 } 1755 } else { 1756 Node* c = n->in(0); 1757 if (trace) { tty->print("X pushing"); c->dump(); } 1758 controls.push(c); 1759 } 1760 } 1761 } 1762 1763 bool ShenandoahWriteBarrierNode::memory_dominates_all_paths(Node* mem, Node* rep_ctrl, int alias, PhaseIdealLoop* phase) { 1764 const bool trace = false; 1765 if (trace) { 1766 tty->print("XXX mem is"); mem->dump(); 1767 tty->print("XXX rep ctrl is"); rep_ctrl->dump(); 1768 tty->print_cr("XXX alias is %d", alias); 1769 } 1770 ResourceMark rm; 1771 Unique_Node_List wq; 1772 Unique_Node_List controls; 1773 wq.push(mem); 1774 for (uint next = 0; next < wq.size(); next++) { 1775 Node *nn = wq.at(next); 1776 if (trace) { tty->print("XX from mem"); nn->dump(); } 1777 assert(nn->bottom_type() == Type::MEMORY, "memory only"); 1778 1779 if (nn->is_Phi()) { 1780 Node* r = nn->in(0); 1781 for (DUIterator_Fast jmax, j = r->fast_outs(jmax); j < jmax; j++) { 1782 Node* u = r->fast_out(j); 1783 if (u->is_Phi() && u->bottom_type() == Type::MEMORY && u != nn && 1784 (u->adr_type() == TypePtr::BOTTOM || phase->C->get_alias_index(u->adr_type()) == alias)) { 1785 if (trace) { tty->print("XX Next mem (other phi)"); u->dump(); } 1786 wq.push(u); 1787 } 1788 } 1789 } 1790 1791 for (DUIterator_Fast imax, i = nn->fast_outs(imax); i < imax; i++) { 1792 Node* use = nn->fast_out(i); 1793 1794 if (trace) { tty->print("XX use %p", use->adr_type()); use->dump(); } 1795 if (use->is_CFG()) { 1796 assert(use->in(TypeFunc::Memory) == nn, "bad cfg node"); 1797 Node* c = use->in(0); 1798 if (phase->is_dominator(rep_ctrl, c)) { 1799 memory_dominates_all_paths_helper(c, rep_ctrl, controls, phase); 1800 } else if (use->is_CallStaticJava() && use->as_CallStaticJava()->uncommon_trap_request() != 0 && c->is_Region()) { 1801 Node* region = c; 1802 if (trace) { tty->print("XX unc region"); region->dump(); } 1803 for (uint j = 1; j < region->req(); j++) { 1804 if (phase->is_dominator(rep_ctrl, region->in(j))) { 1805 if (trace) { tty->print("XX unc follows"); region->in(j)->dump(); } 1806 memory_dominates_all_paths_helper(region->in(j), rep_ctrl, controls, phase); 1807 } 1808 } 1809 } 1810 //continue; 1811 } else if (use->is_Phi()) { 1812 assert(use->bottom_type() == Type::MEMORY, "bad phi"); 1813 if ((use->adr_type() == TypePtr::BOTTOM /*&& !shenandoah_has_alias_phi(C, use, alias)*/) || 1814 phase->C->get_alias_index(use->adr_type()) == alias) { 1815 for (uint j = 1; j < use->req(); j++) { 1816 if (use->in(j) == nn) { 1817 Node* c = use->in(0)->in(j); 1818 if (phase->is_dominator(rep_ctrl, c)) { 1819 memory_dominates_all_paths_helper(c, rep_ctrl, controls, phase); 1820 } 1821 } 1822 } 1823 } 1824 // continue; 1825 } 1826 1827 if (use->is_MergeMem()) { 1828 if (use->as_MergeMem()->memory_at(alias) == nn) { 1829 if (trace) { tty->print("XX Next mem"); use->dump(); } 1830 // follow the memory edges 1831 wq.push(use); 1832 } 1833 } else if (use->is_Phi()) { 1834 assert(use->bottom_type() == Type::MEMORY, "bad phi"); 1835 if ((use->adr_type() == TypePtr::BOTTOM /*&& !shenandoah_has_alias_phi(C, use, alias)*/) || 1836 phase->C->get_alias_index(use->adr_type()) == alias) { 1837 if (trace) { tty->print("XX Next mem"); use->dump(); } 1838 // follow the memory edges 1839 wq.push(use); 1840 } 1841 } else if (use->bottom_type() == Type::MEMORY && 1842 (use->adr_type() == TypePtr::BOTTOM || phase->C->get_alias_index(use->adr_type()) == alias)) { 1843 if (trace) { tty->print("XX Next mem"); use->dump(); } 1844 // follow the memory edges 1845 wq.push(use); 1846 } else if ((use->is_SafePoint() || use->is_MemBar()) && 1847 (use->adr_type() == TypePtr::BOTTOM || phase->C->get_alias_index(use->adr_type()) == alias)) { 1848 for (DUIterator_Fast jmax, j = use->fast_outs(jmax); j < jmax; j++) { 1849 Node* u = use->fast_out(j); 1850 if (u->bottom_type() == Type::MEMORY) { 1851 if (trace) { tty->print("XX Next mem"); u->dump(); } 1852 // follow the memory edges 1853 wq.push(u); 1854 } 1855 } 1856 } else if (use->Opcode() == Op_ShenandoahWriteBarrier && phase->C->get_alias_index(use->adr_type()) == alias) { 1857 Node* m = use->find_out_with(Op_ShenandoahWBMemProj); 1858 if (m != NULL) { 1859 if (trace) { tty->print("XX Next mem"); m->dump(); } 1860 // follow the memory edges 1861 wq.push(m); 1862 } 1863 } 1864 } 1865 } 1866 1867 if (controls.size() == 0) { 1868 return false; 1869 } 1870 1871 for (uint i = 0; i < controls.size(); i++) { 1872 Node *n = controls.at(i); 1873 1874 if (trace) { tty->print("X checking"); n->dump(); } 1875 1876 if (n->unique_ctrl_out() != NULL) { 1877 continue; 1878 } 1879 1880 if (n->Opcode() == Op_NeverBranch) { 1881 Node* taken = n->as_Multi()->proj_out(0); 1882 if (!controls.member(taken)) { 1883 if (trace) { tty->print("X not seen"); taken->dump(); } 1884 return false; 1885 } 1886 continue; 1887 } 1888 1889 for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) { 1890 Node* u = n->fast_out(j); 1891 1892 if (u->is_CFG()) { 1893 if (!controls.member(u)) { 1894 if (u->is_Proj() && u->as_Proj()->is_uncommon_trap_proj(Deoptimization::Reason_none)) { 1895 if (trace) { tty->print("X not seen but unc"); u->dump(); } 1896 } else { 1897 Node* c = u; 1898 do { 1899 c = c->unique_ctrl_out(); 1900 } while (c != NULL && c->is_Region()); 1901 if (c != NULL && c->Opcode() == Op_Halt) { 1902 if (trace) { tty->print("X not seen but halt"); c->dump(); } 1903 } else { 1904 if (trace) { tty->print("X not seen"); u->dump(); } 1905 return false; 1906 } 1907 } 1908 } else { 1909 if (trace) { tty->print("X seen"); u->dump(); } 1910 } 1911 } 1912 } 1913 } 1914 return true; 1915 } 1916 #endif 1917 1918 static bool has_mem_phi(Compile* C, Node* region, int alias) { 1919 for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) { 1920 Node* use = region->fast_out(i); 1921 if (use->is_Phi() && use->bottom_type() == Type::MEMORY && 1922 (C->get_alias_index(use->adr_type()) == alias)) { 1923 return true; 1924 } 1925 } 1926 return false; 1927 } 1928 1929 bool ShenandoahWriteBarrierNode::fix_mem_phis_helper(Node* c, Node* mem, Node* mem_ctrl, Node* rep_ctrl, int alias, VectorSet& controls, GrowableArray<Node*>& regions, PhaseIdealLoop* phase) { 1930 const bool trace = false; 1931 Node_List wq; 1932 wq.push(c); 1933 1934 #ifdef ASSERT 1935 if (trace) { tty->print("YYY from"); c->dump(); } 1936 if (trace) { tty->print("YYY with mem"); mem->dump(); } 1937 #endif 1938 1939 while(wq.size() > 0) { 1940 c = wq.pop(); 1941 1942 while (!c->is_Region() || c->is_Loop()) { 1943 #ifdef ASSERT 1944 if (trace) { tty->print("YYY"); c->dump(); } 1945 #endif 1946 assert(c->is_CFG(), "node should be control node"); 1947 if (c == mem_ctrl || phase->is_dominator(c, rep_ctrl)) { 1948 c = NULL; 1949 break; 1950 } else if (c->is_Loop()) { 1951 c = c->in(LoopNode::EntryControl); 1952 } else { 1953 c = c->in(0); 1954 } 1955 } 1956 if (c == NULL) { 1957 continue; 1958 } 1959 1960 #ifdef ASSERT 1961 if (trace) { tty->print("YYY new region"); c->dump(); } 1962 #endif 1963 1964 bool has_phi = has_mem_phi(phase->C, c, alias); 1965 if (!has_phi) { 1966 1967 Node* m_ctrl = NULL; 1968 Node* m = dom_mem(mem, c, alias, m_ctrl, phase); 1969 if (m == NULL) { 1970 return false; 1971 } 1972 1973 #ifdef ASSERT 1974 if (trace) { tty->print("YYY mem "); m->dump(); } 1975 #endif 1976 1977 if (controls.test(c->_idx)) { 1978 int i; 1979 for (i = 0; i < regions.length() && regions.at(i) != c; i+=2) { 1980 // deliberately empty, rolling over the regions 1981 } 1982 assert(i < regions.length(), "missing region"); 1983 Node* prev_m = regions.at(i+1); 1984 if (prev_m == m) { 1985 continue; 1986 } 1987 #ifdef ASSERT 1988 if (trace) { tty->print("YYY prev mem "); prev_m->dump(); } 1989 #endif 1990 Node* prev_m_ctrl = phase->ctrl_or_self(prev_m); 1991 assert(is_dominator(m_ctrl, prev_m_ctrl, m, prev_m, phase) || 1992 is_dominator(prev_m_ctrl, m_ctrl, prev_m, m, phase), "one should dominate the other"); 1993 if (is_dominator(m_ctrl, prev_m_ctrl, m, prev_m, phase)) { 1994 continue; 1995 } 1996 #ifdef ASSERT 1997 if (trace) { tty->print("YYY Fixing "); c->dump(); } 1998 #endif 1999 regions.at_put(i+1, m); 2000 } else { 2001 #ifdef ASSERT 2002 if (trace) { tty->print("YYY Pushing "); c->dump(); } 2003 #endif 2004 regions.push(c); 2005 regions.push(m); 2006 } 2007 } else { 2008 continue; 2009 } 2010 2011 controls.set(c->_idx); 2012 2013 for (uint i = 1; i < c->req(); i++) { 2014 wq.push(c->in(i)); 2015 } 2016 } 2017 return true; 2018 } 2019 2020 2021 bool ShenandoahWriteBarrierNode::fix_mem_phis(Node* mem, Node* mem_ctrl, Node* rep_ctrl, int alias, PhaseIdealLoop* phase) { 2022 GrowableArray<Node*> regions; 2023 VectorSet controls(Thread::current()->resource_area()); 2024 const bool trace = false; 2025 2026 #ifdef ASSERT 2027 if (trace) { tty->print("YYY mem is "); mem->dump(); } 2028 if (trace) { tty->print("YYY mem ctrl is "); mem_ctrl->dump(); } 2029 if (trace) { tty->print("YYY rep ctrl is "); rep_ctrl->dump(); } 2030 if (trace) { tty->print_cr("YYY alias is %d", alias); } 2031 #endif 2032 2033 // Walk memory edges from mem until we hit a memory point where 2034 // control is known then follow the control up looking for regions 2035 // with no memory Phi for alias 2036 Unique_Node_List wq; 2037 wq.push(mem); 2038 2039 for (uint next = 0; next < wq.size(); next++) { 2040 Node *n = wq.at(next); 2041 #ifdef ASSERT 2042 if (trace) { tty->print("YYY from (2) "); n->dump(); } 2043 #endif 2044 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { 2045 Node* u = n->fast_out(i); 2046 #ifdef ASSERT 2047 if (trace) { tty->print("YYY processing "); u->dump(); } 2048 #endif 2049 if (u->is_Phi()) { 2050 assert(u->bottom_type() == Type::MEMORY, "strange memory graph"); 2051 if (should_process_phi(u, alias, phase->C)) { 2052 for (uint j = 1; j < u->req(); j++) { 2053 if (u->in(j) == n) { 2054 Node *c = u->in(0)->in(j); 2055 if (!fix_mem_phis_helper(c, n, mem_ctrl, rep_ctrl, alias, controls, regions, phase)) { 2056 return false; 2057 } 2058 } 2059 } 2060 } 2061 #ifdef ASSERT 2062 } else if (u->is_CallStaticJava() && u->as_CallStaticJava()->uncommon_trap_request() != 0) { 2063 if (!fix_mem_phis_helper(u->in(0), n, mem_ctrl, rep_ctrl, alias, controls, regions, phase)) { 2064 return false; 2065 } 2066 #endif 2067 } else if ((u->is_CFG() && u->adr_type() == TypePtr::BOTTOM) || u->Opcode() == Op_Rethrow || u->Opcode() == Op_Return) { 2068 if (!fix_mem_phis_helper(u->in(0), n, mem_ctrl, rep_ctrl, alias, controls, regions, phase)) { 2069 return false; 2070 } 2071 } else if (u->is_MergeMem() && u->as_MergeMem()->memory_at(alias) == n) { 2072 wq.push(u); 2073 } else if (u->Opcode() == Op_ShenandoahWriteBarrier && phase->C->get_alias_index(u->adr_type()) == alias) { 2074 Node* m = u->find_out_with(Op_ShenandoahWBMemProj); 2075 if (m != NULL) { 2076 wq.push(m); 2077 } 2078 } 2079 } 2080 } 2081 #ifdef ASSERT 2082 if (trace) { 2083 tty->print_cr("XXXXXXXXXXXXXXXXXXXX"); 2084 for (int i = 0; i < regions.length(); i++) { 2085 Node* r = regions.at(i); 2086 tty->print("%d", i); r->dump(); 2087 } 2088 tty->print_cr("XXXXXXXXXXXXXXXXXXXX"); 2089 } 2090 #endif 2091 2092 if (regions.length() == 0) { 2093 return true; 2094 } 2095 2096 { 2097 int i = 0; 2098 for (; i < regions.length(); i+=2) { 2099 Node* region = regions.at(i); 2100 bool has_phi = false; 2101 for (DUIterator_Fast jmax, j = region->fast_outs(jmax); j < jmax && !has_phi; j++) { 2102 Node* u = region->fast_out(j); 2103 if (u->is_Phi() && u->bottom_type() == Type::MEMORY && 2104 (u->adr_type() == TypePtr::BOTTOM || phase->C->get_alias_index(u->adr_type()) == alias)) { 2105 has_phi = true; 2106 } 2107 } 2108 if (!has_phi) { 2109 break; 2110 } 2111 } 2112 if (i == regions.length()) { 2113 return true; 2114 } 2115 } 2116 2117 // Try to restrict the update to path that post dominates rep_ctrl 2118 int k = 0; 2119 int start = 0; 2120 int end = 0; 2121 do { 2122 start = end; 2123 end = k; 2124 for (int i = end; i < regions.length(); i+=2) { 2125 Node* r = regions.at(i); 2126 int prev = k; 2127 for (uint j = 1; j < r->req() && prev == k; j++) { 2128 if (end == 0) { 2129 if (phase->is_dominator(rep_ctrl, r->in(j))) { 2130 Node* mem = regions.at(i+1); 2131 regions.at_put(i, regions.at(k)); 2132 regions.at_put(i+1, regions.at(k+1)); 2133 regions.at_put(k, r); 2134 regions.at_put(k+1, mem); 2135 k+=2; 2136 } 2137 } else { 2138 for (int l = start; l < end && prev == k; l+=2) { 2139 Node* r2 = regions.at(l); 2140 if (phase->is_dominator(r2, r->in(j))) { 2141 Node* mem = regions.at(i+1); 2142 regions.at_put(i, regions.at(k)); 2143 regions.at_put(i+1, regions.at(k+1)); 2144 regions.at_put(k, r); 2145 regions.at_put(k+1, mem); 2146 k+=2; 2147 } 2148 } 2149 } 2150 } 2151 } 2152 #ifdef ASSERT 2153 if (trace) { tty->print_cr("k = %d start = %d end = %d", k, start, end); } 2154 #endif 2155 } while(k != end); 2156 2157 #ifdef ASSERT 2158 if (end != regions.length()) { 2159 if (trace) { tty->print_cr("Compacting %d -> %d", regions.length(), end); } 2160 } 2161 #endif 2162 regions.trunc_to(end); 2163 2164 #ifdef ASSERT 2165 if (trace) { 2166 tty->print_cr("XXXXXXXXXXXXXXXXXXXX"); 2167 for (int i = 0; i < regions.length(); i++) { 2168 Node* r = regions.at(i); 2169 tty->print("%d", i); r->dump(); 2170 } 2171 tty->print_cr("XXXXXXXXXXXXXXXXXXXX"); 2172 } 2173 #endif 2174 2175 // Creating new phis must be done in post order 2176 while (regions.length() > 0) { 2177 int i = 0; 2178 for (; i < regions.length(); i+=2) { 2179 Node* r1 = regions.at(i); 2180 bool is_dom = false; 2181 for (int j = 0; j < regions.length() && !is_dom; j+=2) { 2182 if (i != j) { 2183 Node* r2 = regions.at(j); 2184 for (uint k = 1; k < r2->req() && !is_dom; k++) { 2185 if (phase->is_dominator(r1, r2->in(k))) { 2186 is_dom = true; 2187 } 2188 } 2189 } 2190 } 2191 if (!is_dom) { 2192 break; 2193 } 2194 } 2195 assert(i < regions.length(), "need one"); 2196 Node* r = regions.at(i); 2197 Node* m = regions.at(i+1); 2198 regions.delete_at(i+1); 2199 regions.delete_at(i); 2200 2201 if (!suitable_mem(m, NULL, NULL)) { 2202 return false; 2203 } 2204 Node* phi = PhiNode::make(r, m, Type::MEMORY, phase->C->get_adr_type(alias)); 2205 #ifdef ASSERT 2206 if (trace) { tty->print("YYY Adding new mem phi "); phi->dump(); } 2207 #endif 2208 phase->register_new_node(phi, r); 2209 2210 fix_memory_uses(m, phi, phi, r, phase->C->get_alias_index(phi->adr_type()), phase); 2211 assert(phi->outcnt() != 0, "new proj should have uses"); 2212 if (phi->outcnt() == 0) { 2213 phase->igvn().remove_dead_node(phi); 2214 } 2215 } 2216 2217 return true; 2218 } 2219 2220 Node* ShenandoahBarrierNode::dom_mem(Node* mem, Node*& mem_ctrl, Node* n, Node* rep_ctrl, int alias, PhaseIdealLoop* phase) { 2221 ResourceMark rm; 2222 VectorSet wq(Thread::current()->resource_area()); 2223 wq.set(mem->_idx); 2224 mem_ctrl = phase->get_ctrl(mem); 2225 while (!is_dominator(mem_ctrl, rep_ctrl, mem, n, phase)) { 2226 mem = next_mem(mem, alias); 2227 if (wq.test_set(mem->_idx)) { 2228 return NULL; // hit an unexpected loop 2229 } 2230 mem_ctrl = phase->ctrl_or_self(mem); 2231 } 2232 if (mem->is_MergeMem()) { 2233 mem = mem->as_MergeMem()->memory_at(alias); 2234 mem_ctrl = phase->ctrl_or_self(mem); 2235 } 2236 return mem; 2237 } 2238 2239 Node* ShenandoahBarrierNode::dom_mem(Node* mem, Node* ctrl, int alias, Node*& mem_ctrl, PhaseIdealLoop* phase) { 2240 ResourceMark rm; 2241 VectorSet wq(Thread::current()->resource_area()); 2242 wq.set(mem->_idx); 2243 mem_ctrl = phase->ctrl_or_self(mem); 2244 while (!phase->is_dominator(mem_ctrl, ctrl) || mem_ctrl == ctrl) { 2245 mem = next_mem(mem, alias); 2246 if (wq.test_set(mem->_idx)) { 2247 return NULL; 2248 } 2249 mem_ctrl = phase->ctrl_or_self(mem); 2250 } 2251 if (mem->is_MergeMem()) { 2252 mem = mem->as_MergeMem()->memory_at(alias); 2253 mem_ctrl = phase->ctrl_or_self(mem); 2254 } 2255 return mem; 2256 } 2257 2258 Node* ShenandoahBarrierNode::try_common(Node *n_ctrl, PhaseIdealLoop* phase) { 2259 if (!phase->C->has_irreducible_loop()) { 2260 // We look for a write barrier whose memory edge dominates n 2261 // Either the replacement write barrier dominates n or we have, 2262 // for instance: 2263 // if ( ) { 2264 // read barrier n 2265 // } else { 2266 // write barrier 2267 // } 2268 // in which case replacing n by the write barrier causes the write 2269 // barrier to move above the if() and the memory Phi that merges 2270 // the memory state for both branches must be updated so both 2271 // inputs become the write barrier's memory projection (and the 2272 // Phi is optimized out) otherwise we risk loosing a memory 2273 // dependency. 2274 // Once we find a replacement write barrier, the code below fixes 2275 // the memory graph in cases like the one above. 2276 Node* val = in(ValueIn); 2277 Node* val_ctrl = phase->get_ctrl(val); 2278 Node* n_proj = find_out_with(Op_ShenandoahWBMemProj); 2279 Node* replacement = NULL; 2280 int alias = phase->C->get_alias_index(adr_type()); 2281 Node* rep_ctrl = NULL; 2282 for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax && replacement == NULL; i++) { 2283 Node* u = val->fast_out(i); 2284 if (u != this && u->Opcode() == Op_ShenandoahWriteBarrier) { 2285 Node* u_mem = u->in(Memory); 2286 Node* u_proj = u->find_out_with(Op_ShenandoahWBMemProj); 2287 Node* u_ctrl = phase->get_ctrl(u); 2288 Node* u_mem_ctrl = phase->get_ctrl(u_mem); 2289 IdealLoopTree* n_loop = phase->get_loop(n_ctrl); 2290 IdealLoopTree* u_loop = phase->get_loop(u_ctrl); 2291 2292 Node* ctrl = phase->dom_lca(u_ctrl, n_ctrl); 2293 2294 if (ctrl->is_Proj() && 2295 ctrl->in(0)->is_Call() && 2296 ctrl->unique_ctrl_out() != NULL && 2297 ctrl->unique_ctrl_out()->Opcode() == Op_Catch && 2298 !phase->is_dominator(val_ctrl, ctrl->in(0)->in(0))) { 2299 continue; 2300 } 2301 2302 if (Opcode() == Op_ShenandoahWriteBarrier && u_proj == NULL && n_proj != NULL) { 2303 continue; 2304 } 2305 2306 IdealLoopTree* loop = phase->get_loop(ctrl); 2307 2308 // We don't want to move a write barrier in a loop 2309 // If the LCA is in a inner loop, try a control out of loop if possible 2310 bool loop_ok = true; 2311 while (!loop->is_member(u_loop) && (Opcode() != Op_ShenandoahWriteBarrier || !loop->is_member(n_loop))) { 2312 ctrl = phase->idom(ctrl); 2313 if (ctrl != val_ctrl && phase->is_dominator(ctrl, val_ctrl)) { 2314 loop_ok = false; 2315 break; 2316 } 2317 loop = phase->get_loop(ctrl); 2318 } 2319 2320 if (loop_ok) { 2321 if (ShenandoahDontIncreaseWBFreq) { 2322 Node* u_iffproj = no_branches(u_ctrl, ctrl, true, phase); 2323 if (Opcode() == Op_ShenandoahWriteBarrier) { 2324 Node* n_iffproj = no_branches(n_ctrl, ctrl, true, phase); 2325 if (u_iffproj == NULL || n_iffproj == NULL) { 2326 replacement = u; 2327 rep_ctrl = ctrl; 2328 } else if (u_iffproj != NodeSentinel && n_iffproj != NodeSentinel && u_iffproj->in(0) == n_iffproj->in(0)) { 2329 replacement = u; 2330 rep_ctrl = ctrl; 2331 } 2332 } else if (u_iffproj == NULL) { 2333 replacement = u; 2334 rep_ctrl = ctrl; 2335 } 2336 } else { 2337 replacement = u; 2338 rep_ctrl = ctrl; 2339 } 2340 } 2341 } 2342 } 2343 if (replacement != NULL) { 2344 if (rep_ctrl->is_Proj() && 2345 rep_ctrl->in(0)->is_Call() && 2346 rep_ctrl->unique_ctrl_out() != NULL && 2347 rep_ctrl->unique_ctrl_out()->Opcode() == Op_Catch) { 2348 rep_ctrl = rep_ctrl->in(0)->in(0); 2349 assert(phase->is_dominator(val_ctrl, rep_ctrl), "bad control"); 2350 } else { 2351 LoopNode* c = ShenandoahWriteBarrierNode::try_move_before_pre_loop(rep_ctrl, val_ctrl, phase); 2352 if (c != NULL) { 2353 rep_ctrl = ShenandoahWriteBarrierNode::move_above_predicates(c, val_ctrl, phase); 2354 } else { 2355 while (rep_ctrl->is_IfProj()) { 2356 CallStaticJavaNode* unc = rep_ctrl->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none); 2357 if (unc != NULL) { 2358 int req = unc->uncommon_trap_request(); 2359 Deoptimization::DeoptReason trap_reason = Deoptimization::trap_request_reason(req); 2360 if ((trap_reason == Deoptimization::Reason_loop_limit_check || 2361 trap_reason == Deoptimization::Reason_predicate || 2362 trap_reason == Deoptimization::Reason_profile_predicate) && 2363 phase->is_dominator(val_ctrl, rep_ctrl->in(0)->in(0))) { 2364 rep_ctrl = rep_ctrl->in(0)->in(0); 2365 continue; 2366 } 2367 } 2368 break; 2369 } 2370 } 2371 } 2372 2373 Node* mem = replacement->in(Memory); 2374 Node* old_mem = mem; 2375 Node* rep_proj = replacement->find_out_with(Op_ShenandoahWBMemProj); 2376 { 2377 Node* mem_ctrl = NULL; 2378 2379 mem = dom_mem(mem, mem_ctrl, this, rep_ctrl, alias, phase); 2380 if (mem == NULL) { 2381 return NULL; 2382 } 2383 2384 // Add a memory Phi for the slice of the write barrier to any 2385 // region that post dominates rep_ctrl and doesn't have one 2386 // already. 2387 if (rep_proj != NULL && !ShenandoahWriteBarrierNode::fix_mem_phis(mem, mem_ctrl, rep_ctrl, alias, phase)) { 2388 return NULL; 2389 } 2390 2391 assert(!ShenandoahVerifyOptoBarriers || ShenandoahWriteBarrierNode::memory_dominates_all_paths(mem, rep_ctrl, alias, phase), "can't fix the memory graph"); 2392 } 2393 assert(phase->igvn().type(mem) == Type::MEMORY, "not memory"); 2394 2395 if (rep_proj != NULL) { 2396 Node* old_mem = replacement->in(Memory); 2397 if (!suitable_mem(mem, old_mem, rep_proj)) { 2398 return NULL; 2399 } 2400 2401 if (replacement->in(Memory) != mem) { 2402 // tty->print("XXX setting memory of"); replacement->dump(); 2403 // tty->print("XXX to"); mem->dump(); 2404 for (DUIterator_Last imin, i = rep_proj->last_outs(imin); i >= imin; ) { 2405 Node* u = rep_proj->last_out(i); 2406 phase->igvn().rehash_node_delayed(u); 2407 int uses_found = u->replace_edge(rep_proj, old_mem); 2408 i -= uses_found; 2409 } 2410 phase->igvn().replace_input_of(replacement, Memory, mem); 2411 } 2412 phase->set_ctrl_and_loop(replacement, rep_ctrl); 2413 phase->igvn().replace_input_of(replacement, Control, rep_ctrl); 2414 2415 ShenandoahWriteBarrierNode::fix_memory_uses(mem, replacement, rep_proj, rep_ctrl, phase->C->get_alias_index(replacement->adr_type()), phase); 2416 assert(rep_proj->outcnt() != 0, "new proj should have uses"); 2417 } else { 2418 if (replacement->in(Memory) != mem) { 2419 phase->igvn()._worklist.push(replacement->in(Memory)); 2420 phase->igvn().replace_input_of(replacement, Memory, mem); 2421 } 2422 phase->set_ctrl_and_loop(replacement, rep_ctrl); 2423 phase->igvn().replace_input_of(replacement, Control, rep_ctrl); 2424 } 2425 if (Opcode() == Op_ShenandoahWriteBarrier) { 2426 if (n_proj != NULL) { 2427 phase->lazy_replace(n_proj, in(Memory)); 2428 } 2429 } 2430 phase->lazy_replace(this, replacement); 2431 if (rep_proj != NULL) { 2432 phase->set_ctrl_and_loop(rep_proj, rep_ctrl); 2433 } 2434 return replacement; 2435 } 2436 } 2437 2438 return NULL; 2439 } 2440 2441 const TypePtr* ShenandoahBarrierNode::fix_addp_type(const TypePtr* res, Node* base) { 2442 if (UseShenandoahGC && ShenandoahBarriersForConst) { 2443 // With barriers on constant oops, if a field being accessed is a 2444 // static field, correct alias analysis requires that we look 2445 // beyond the barriers (that hide the constant) to find the actual 2446 // java class mirror constant. 2447 const TypeInstPtr* ti = res->isa_instptr(); 2448 if (ti != NULL && 2449 ti->const_oop() == NULL && 2450 ti->klass() == ciEnv::current()->Class_klass() && 2451 ti->offset() >= (ti->klass()->as_instance_klass()->size_helper() * wordSize)) { 2452 ResourceMark rm; 2453 Unique_Node_List wq; 2454 ciObject* const_oop = NULL; 2455 wq.push(base); 2456 for (uint i = 0; i < wq.size(); i++) { 2457 Node *n = wq.at(i); 2458 if (n->is_ShenandoahBarrier() || 2459 (n->is_Mach() && n->as_Mach()->ideal_Opcode() == Op_ShenandoahReadBarrier)) { 2460 Node* m = n->in(ShenandoahBarrierNode::ValueIn); 2461 if (m != NULL) { 2462 wq.push(m); 2463 } 2464 } else if (n->is_Phi()) { 2465 for (uint j = 1; j < n->req(); j++) { 2466 Node* m = n->in(j); 2467 if (m != NULL) { 2468 wq.push(m); 2469 } 2470 } 2471 } else if (n->is_ConstraintCast() || (n->is_Mach() && n->as_Mach()->ideal_Opcode() == Op_CheckCastPP)) { 2472 Node* m = n->in(1); 2473 if (m != NULL) { 2474 wq.push(m); 2475 } 2476 } else { 2477 const TypeInstPtr* tn = n->bottom_type()->isa_instptr(); 2478 if (tn != NULL) { 2479 if (tn->const_oop() != NULL) { 2480 if (const_oop == NULL) { 2481 const_oop = tn->const_oop(); 2482 } else if (const_oop != tn->const_oop()) { 2483 const_oop = NULL; 2484 break; 2485 } 2486 } else { 2487 if (n->is_Proj()) { 2488 if (n->in(0)->Opcode() == Op_CallLeafNoFP) { 2489 if (n->in(0)->as_Call()->entry_point() != StubRoutines::shenandoah_wb_C()) { 2490 const_oop = NULL; 2491 break; 2492 } 2493 } else if (n->in(0)->is_MachCallLeaf()) { 2494 if (n->in(0)->as_MachCall()->entry_point() != StubRoutines::shenandoah_wb_C()) { 2495 const_oop = NULL; 2496 break; 2497 } 2498 } 2499 } else { 2500 fatal("2 different static fields being accessed with a single AddP"); 2501 const_oop = NULL; 2502 break; 2503 } 2504 } 2505 } else { 2506 assert(n->bottom_type() == Type::TOP, "not an instance ptr?"); 2507 } 2508 } 2509 } 2510 if (const_oop != NULL) { 2511 res = ti->cast_to_const(const_oop); 2512 } 2513 } 2514 } 2515 return res; 2516 } 2517 2518 static void disconnect_barrier_mem(Node* wb, PhaseIterGVN& igvn) { 2519 Node* mem_in = wb->in(ShenandoahBarrierNode::Memory); 2520 Node* proj = wb->find_out_with(Op_ShenandoahWBMemProj); 2521 2522 for (DUIterator_Last imin, i = proj->last_outs(imin); i >= imin; ) { 2523 Node* u = proj->last_out(i); 2524 igvn.rehash_node_delayed(u); 2525 int nb = u->replace_edge(proj, mem_in); 2526 assert(nb > 0, "no replacement?"); 2527 i -= nb; 2528 } 2529 } 2530 2531 Node* ShenandoahWriteBarrierNode::move_above_predicates(LoopNode* cl, Node* val_ctrl, PhaseIdealLoop* phase) { 2532 Node* entry = cl->skip_strip_mined()->in(LoopNode::EntryControl); 2533 Node* above_pred = phase->skip_all_loop_predicates(entry); 2534 Node* ctrl = entry; 2535 while (ctrl != above_pred) { 2536 Node* next = ctrl->in(0); 2537 if (!phase->is_dominator(val_ctrl, next)) { 2538 break; 2539 } 2540 ctrl = next; 2541 } 2542 return ctrl; 2543 } 2544 2545 Node* ShenandoahWriteBarrierNode::try_move_before_loop_helper(LoopNode* cl, Node* val_ctrl, Node* mem, PhaseIdealLoop* phase) { 2546 assert(cl->is_Loop(), "bad control"); 2547 Node* ctrl = move_above_predicates(cl, val_ctrl, phase); 2548 Node* mem_ctrl = NULL; 2549 int alias = phase->C->get_alias_index(adr_type()); 2550 mem = dom_mem(mem, mem_ctrl, this, ctrl, alias, phase); 2551 if (mem == NULL) { 2552 return NULL; 2553 } 2554 2555 Node* old_mem = in(Memory); 2556 Node* proj = find_out_with(Op_ShenandoahWBMemProj); 2557 if (old_mem != mem && !suitable_mem(mem, old_mem, proj)) { 2558 return NULL; 2559 } 2560 2561 assert(!ShenandoahVerifyOptoBarriers || memory_dominates_all_paths(mem, ctrl, alias, phase), "can't fix the memory graph"); 2562 phase->set_ctrl_and_loop(this, ctrl); 2563 phase->igvn().replace_input_of(this, Control, ctrl); 2564 if (old_mem != mem) { 2565 if (proj != NULL) { 2566 disconnect_barrier_mem(this, phase->igvn()); 2567 fix_memory_uses(mem, this, proj, ctrl, phase->C->get_alias_index(adr_type()), phase); 2568 assert(proj->outcnt() > 0, "disconnected write barrier"); 2569 } 2570 phase->igvn().replace_input_of(this, Memory, mem); 2571 } 2572 if (proj != NULL) { 2573 phase->set_ctrl_and_loop(proj, ctrl); 2574 } 2575 return this; 2576 } 2577 2578 LoopNode* ShenandoahWriteBarrierNode::try_move_before_pre_loop(Node* c, Node* val_ctrl, PhaseIdealLoop* phase) { 2579 // A write barrier between a pre and main loop can get in the way of 2580 // vectorization. Move it above the pre loop if possible 2581 CountedLoopNode* cl = NULL; 2582 if (c->is_IfFalse() && 2583 c->in(0)->is_CountedLoopEnd()) { 2584 cl = c->in(0)->as_CountedLoopEnd()->loopnode(); 2585 } else if (c->is_IfProj() && 2586 c->in(0)->is_If() && 2587 c->in(0)->in(0)->is_IfFalse() && 2588 c->in(0)->in(0)->in(0)->is_CountedLoopEnd()) { 2589 cl = c->in(0)->in(0)->in(0)->as_CountedLoopEnd()->loopnode(); 2590 } 2591 if (cl != NULL && 2592 cl->is_pre_loop() && 2593 val_ctrl != cl && 2594 phase->is_dominator(val_ctrl, cl)) { 2595 return cl; 2596 } 2597 return NULL; 2598 } 2599 2600 Node* ShenandoahWriteBarrierNode::try_move_before_loop(Node *n_ctrl, PhaseIdealLoop* phase) { 2601 IdealLoopTree *n_loop = phase->get_loop(n_ctrl); 2602 Node* val = in(ValueIn); 2603 Node* val_ctrl = phase->get_ctrl(val); 2604 if (n_loop != phase->ltree_root() && !n_loop->_irreducible) { 2605 IdealLoopTree *val_loop = phase->get_loop(val_ctrl); 2606 Node* mem = in(Memory); 2607 IdealLoopTree *mem_loop = phase->get_loop(phase->get_ctrl(mem)); 2608 if (!n_loop->is_member(val_loop) && 2609 n_loop->is_member(mem_loop)) { 2610 Node* n_loop_head = n_loop->_head; 2611 2612 if (n_loop_head->is_Loop()) { 2613 LoopNode* loop = n_loop_head->as_Loop(); 2614 if (n_loop_head->is_CountedLoop() && n_loop_head->as_CountedLoop()->is_main_loop()) { 2615 LoopNode* res = try_move_before_pre_loop(n_loop_head->in(LoopNode::EntryControl), val_ctrl, phase); 2616 if (res != NULL) { 2617 loop = res; 2618 } 2619 } 2620 2621 return try_move_before_loop_helper(loop, val_ctrl, mem, phase); 2622 } 2623 } 2624 } 2625 LoopNode* ctrl = try_move_before_pre_loop(in(0), val_ctrl, phase); 2626 if (ctrl != NULL) { 2627 return try_move_before_loop_helper(ctrl, val_ctrl, in(Memory), phase); 2628 } 2629 return NULL; 2630 } 2631 2632 void ShenandoahReadBarrierNode::try_move(Node *n_ctrl, PhaseIdealLoop* phase) { 2633 Node* mem = in(MemNode::Memory); 2634 int alias = phase->C->get_alias_index(adr_type()); 2635 const bool trace = false; 2636 2637 #ifdef ASSERT 2638 if (trace) { tty->print("Trying to move mem of"); dump(); } 2639 #endif 2640 2641 Node* new_mem = mem; 2642 2643 ResourceMark rm; 2644 VectorSet seen(Thread::current()->resource_area()); 2645 Node_List phis; 2646 2647 for (;;) { 2648 #ifdef ASSERT 2649 if (trace) { tty->print("Looking for dominator from"); mem->dump(); } 2650 #endif 2651 if (mem->is_Proj() && mem->in(0)->is_Start()) { 2652 if (new_mem != in(MemNode::Memory)) { 2653 #ifdef ASSERT 2654 if (trace) { tty->print("XXX Setting mem to"); new_mem->dump(); tty->print(" for "); dump(); } 2655 #endif 2656 phase->igvn().replace_input_of(this, MemNode::Memory, new_mem); 2657 } 2658 return; 2659 } 2660 2661 Node* candidate = mem; 2662 do { 2663 if (!is_independent(mem)) { 2664 if (trace) { tty->print_cr("Not independent"); } 2665 if (new_mem != in(MemNode::Memory)) { 2666 #ifdef ASSERT 2667 if (trace) { tty->print("XXX Setting mem to"); new_mem->dump(); tty->print(" for "); dump(); } 2668 #endif 2669 phase->igvn().replace_input_of(this, MemNode::Memory, new_mem); 2670 } 2671 return; 2672 } 2673 if (seen.test_set(mem->_idx)) { 2674 if (trace) { tty->print_cr("Already seen"); } 2675 ShouldNotReachHere(); 2676 // Strange graph 2677 if (new_mem != in(MemNode::Memory)) { 2678 #ifdef ASSERT 2679 if (trace) { tty->print("XXX Setting mem to"); new_mem->dump(); tty->print(" for "); dump(); } 2680 #endif 2681 phase->igvn().replace_input_of(this, MemNode::Memory, new_mem); 2682 } 2683 return; 2684 } 2685 if (mem->is_Phi()) { 2686 phis.push(mem); 2687 } 2688 mem = next_mem(mem, alias); 2689 if (mem->bottom_type() == Type::MEMORY) { 2690 candidate = mem; 2691 } 2692 assert(is_dominator(phase->ctrl_or_self(mem), n_ctrl, mem, this, phase) == phase->is_dominator(phase->ctrl_or_self(mem), n_ctrl), "strange dominator"); 2693 #ifdef ASSERT 2694 if (trace) { tty->print("Next mem is"); mem->dump(); } 2695 #endif 2696 } while (mem->bottom_type() != Type::MEMORY || !phase->is_dominator(phase->ctrl_or_self(mem), n_ctrl)); 2697 2698 assert(mem->bottom_type() == Type::MEMORY, "bad mem"); 2699 2700 bool not_dom = false; 2701 for (uint i = 0; i < phis.size() && !not_dom; i++) { 2702 Node* nn = phis.at(i); 2703 2704 #ifdef ASSERT 2705 if (trace) { tty->print("Looking from phi"); nn->dump(); } 2706 #endif 2707 assert(nn->is_Phi(), "phis only"); 2708 for (uint j = 2; j < nn->req() && !not_dom; j++) { 2709 Node* m = nn->in(j); 2710 #ifdef ASSERT 2711 if (trace) { tty->print("Input %d is", j); m->dump(); } 2712 #endif 2713 while (m != mem && !seen.test_set(m->_idx)) { 2714 if (is_dominator(phase->ctrl_or_self(m), phase->ctrl_or_self(mem), m, mem, phase)) { 2715 not_dom = true; 2716 // Scheduling anomaly 2717 #ifdef ASSERT 2718 if (trace) { tty->print("Giving up"); m->dump(); } 2719 #endif 2720 break; 2721 } 2722 if (!is_independent(m)) { 2723 if (trace) { tty->print_cr("Not independent"); } 2724 if (new_mem != in(MemNode::Memory)) { 2725 #ifdef ASSERT 2726 if (trace) { tty->print("XXX Setting mem to"); new_mem->dump(); tty->print(" for "); dump(); } 2727 #endif 2728 phase->igvn().replace_input_of(this, MemNode::Memory, new_mem); 2729 } 2730 return; 2731 } 2732 if (m->is_Phi()) { 2733 phis.push(m); 2734 } 2735 m = next_mem(m, alias); 2736 #ifdef ASSERT 2737 if (trace) { tty->print("Next mem is"); m->dump(); } 2738 #endif 2739 } 2740 } 2741 } 2742 if (!not_dom) { 2743 new_mem = mem; 2744 phis.clear(); 2745 } else { 2746 seen.Clear(); 2747 } 2748 } 2749 } 2750 2751 CallStaticJavaNode* ShenandoahWriteBarrierNode::pin_and_expand_null_check(PhaseIterGVN& igvn) { 2752 Node* val = in(ValueIn); 2753 2754 #ifdef ASSERT 2755 const Type* val_t = igvn.type(val); 2756 assert(val_t->meet(TypePtr::NULL_PTR) != val_t, "should be not null"); 2757 #endif 2758 2759 if (val->Opcode() == Op_CastPP && 2760 val->in(0) != NULL && 2761 val->in(0)->Opcode() == Op_IfTrue && 2762 val->in(0)->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none) && 2763 val->in(0)->in(0)->is_If() && 2764 val->in(0)->in(0)->in(1)->Opcode() == Op_Bool && 2765 val->in(0)->in(0)->in(1)->as_Bool()->_test._test == BoolTest::ne && 2766 val->in(0)->in(0)->in(1)->in(1)->Opcode() == Op_CmpP && 2767 val->in(0)->in(0)->in(1)->in(1)->in(1) == val->in(1) && 2768 val->in(0)->in(0)->in(1)->in(1)->in(2)->bottom_type() == TypePtr::NULL_PTR) { 2769 assert(val->in(0)->in(0)->in(1)->in(1)->in(1) == val->in(1), ""); 2770 CallStaticJavaNode* unc = val->in(0)->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none); 2771 return unc; 2772 } 2773 return NULL; 2774 } 2775 2776 void ShenandoahWriteBarrierNode::pin_and_expand_move_barrier(PhaseIdealLoop* phase) { 2777 Node* unc = pin_and_expand_null_check(phase->igvn()); 2778 Node* val = in(ValueIn); 2779 2780 if (unc != NULL) { 2781 Node* ctrl = phase->get_ctrl(this); 2782 Node* unc_ctrl = val->in(0); 2783 2784 // Don't move write barrier in a loop 2785 IdealLoopTree* loop = phase->get_loop(ctrl); 2786 IdealLoopTree* unc_loop = phase->get_loop(unc_ctrl); 2787 2788 if (!unc_loop->is_member(loop)) { 2789 return; 2790 } 2791 2792 Node* branch = no_branches(ctrl, unc_ctrl, false, phase); 2793 assert(branch == NULL || branch == NodeSentinel, "was not looking for a branch"); 2794 if (branch == NodeSentinel) { 2795 return; 2796 } 2797 2798 Node* mem = in(Memory); 2799 Node* old_mem = mem; 2800 2801 Node* mem_ctrl = NULL; 2802 int alias = phase->C->get_alias_index(adr_type()); 2803 mem = dom_mem(mem, mem_ctrl, this, unc_ctrl, alias, phase); 2804 if (mem == NULL) { 2805 return; 2806 } 2807 2808 Node* proj = find_out_with(Op_ShenandoahWBMemProj); 2809 if (mem != old_mem && !fix_mem_phis(mem, mem_ctrl, unc_ctrl, alias, phase)) { 2810 return; 2811 } 2812 2813 assert(mem == old_mem || memory_dominates_all_paths(mem, unc_ctrl, alias, phase), "can't fix the memory graph"); 2814 phase->set_ctrl_and_loop(this, unc_ctrl); 2815 if (in(Control) != NULL) { 2816 phase->igvn().replace_input_of(this, Control, unc_ctrl); 2817 } 2818 disconnect_barrier_mem(this, phase->igvn()); 2819 fix_memory_uses(mem, this, proj, unc_ctrl, phase->C->get_alias_index(adr_type()), phase); 2820 assert(proj->outcnt() > 0, "disconnected write barrier"); 2821 phase->igvn().replace_input_of(this, Memory, mem); 2822 phase->set_ctrl_and_loop(proj, unc_ctrl); 2823 } 2824 } 2825 2826 void ShenandoahWriteBarrierNode::pin_and_expand_helper(PhaseIdealLoop* phase) { 2827 Node* val = in(ValueIn); 2828 Node* ctrl = phase->get_ctrl(this); 2829 // Replace all uses of barrier's input that are dominated by ctrl 2830 // with the value returned by the barrier: no need to keep both 2831 // live. 2832 for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) { 2833 Node* u = val->fast_out(i); 2834 if (u != this) { 2835 if (u->is_Phi()) { 2836 int nb = 0; 2837 for (uint j = 1; j < u->req(); j++) { 2838 if (u->in(j) == val) { 2839 Node* c = u->in(0)->in(j); 2840 if (phase->is_dominator(ctrl, c)) { 2841 phase->igvn().replace_input_of(u, j, this); 2842 nb++; 2843 } 2844 } 2845 } 2846 if (nb > 0) { 2847 imax -= nb; 2848 --i; 2849 } 2850 } else { 2851 Node* c = phase->ctrl_or_self(u); 2852 if (is_dominator(ctrl, c, this, u, phase)) { 2853 phase->igvn().rehash_node_delayed(u); 2854 int nb = u->replace_edge(val, this); 2855 assert(nb > 0, "no update?"); 2856 --i, imax -= nb; 2857 } 2858 } 2859 } 2860 } 2861 } 2862 2863 Node* ShenandoahWriteBarrierNode::pick_phi(Node* phi1, Node* phi2, Node_Stack& phis, VectorSet& visited, PhaseIdealLoop* phase) { 2864 assert(phis.size() == 0, "stack needs to be empty"); 2865 uint i = 1; 2866 int phi_dominates = -1; 2867 for (;;) { 2868 assert(phi1->req() == phi2->req(), "strange pair of phis"); 2869 assert(phis.size() % 2 == 0, ""); 2870 Node* in1 = phi1->in(i); 2871 Node* in2 = phi2->in(i); 2872 2873 if (in1->is_MergeMem()) { 2874 in1 = in1->as_MergeMem()->base_memory(); 2875 } 2876 if (in2->is_MergeMem()) { 2877 in2 = in2->as_MergeMem()->base_memory(); 2878 } 2879 2880 if (in1 == in2) { 2881 //continue 2882 } else if (in1->is_Phi() && in2->is_Phi() && in1->in(0) == in2->in(0)) { 2883 assert(!visited.test_set(in1->_idx), "no loop"); 2884 assert(!visited.test_set(in2->_idx), "no loop"); 2885 phis.push(phi1, i+1); 2886 phis.push(phi2, i+1); 2887 phi1 = in1; 2888 phi2 = in2; 2889 i = 1; 2890 } else { 2891 Node* in1_c = phase->get_ctrl(in1); 2892 Node* in2_c = phase->get_ctrl(in2); 2893 if (is_dominator(in1_c, in2_c, in1, in2, phase)) { 2894 assert(!is_dominator(in2_c, in1_c, in2, in1, phase), "one has to dominate the other"); 2895 assert(phi_dominates == -1 || phi_dominates == 1, "all inputs must dominate"); 2896 phi_dominates = 1; 2897 } else { 2898 assert(is_dominator(in2_c, in1_c, in2, in1, phase), "one must dominate the other"); 2899 assert(!is_dominator(in1_c, in2_c, in1, in2, phase), "one has to dominate the other"); 2900 assert(phi_dominates == -1 || phi_dominates == 2, "all inputs must dominate"); 2901 phi_dominates = 2; 2902 } 2903 } 2904 i++; 2905 2906 while (i >= phi1->req() && phis.size() > 0) { 2907 i = phis.index(); 2908 phi2 = phis.node(); 2909 phis.pop(); 2910 phi1 = phis.node(); 2911 phis.pop(); 2912 } 2913 2914 if (i >= phi1->req() && phis.size() == 0) { 2915 Node* phi = NULL; 2916 if (phi_dominates == 1) { 2917 return phi2; 2918 } else if (phi_dominates == 2) { 2919 return phi1; 2920 } else { 2921 return phi1; 2922 } 2923 } 2924 } 2925 return NULL; 2926 } 2927 2928 bool ShenandoahWriteBarrierNode::mem_is_valid(Node* m, Node* c, PhaseIdealLoop* phase) { 2929 return m != NULL && get_ctrl(m, phase) == c; 2930 } 2931 2932 Node* ShenandoahWriteBarrierNode::find_raw_mem(Node* ctrl, Node* n, const Node_List& memory_nodes, PhaseIdealLoop* phase) { 2933 assert(n == NULL || phase->ctrl_or_self(n) == ctrl, ""); 2934 Node* raw_mem = memory_nodes[ctrl->_idx]; 2935 Node* c = ctrl; 2936 while (!mem_is_valid(raw_mem, c, phase) && 2937 (!c->is_CatchProj() || raw_mem == NULL || c->in(0)->in(0)->in(0) != get_ctrl(raw_mem, phase))) { 2938 c = phase->idom(c); 2939 raw_mem = memory_nodes[c->_idx]; 2940 } 2941 if (n != NULL && mem_is_valid(raw_mem, c, phase)) { 2942 while (!is_dominator_same_ctrl(c, raw_mem, n, phase) && phase->ctrl_or_self(raw_mem) == ctrl) { 2943 raw_mem = next_mem(raw_mem, Compile::AliasIdxRaw); 2944 } 2945 if (raw_mem->is_MergeMem()) { 2946 raw_mem = raw_mem->as_MergeMem()->memory_at(Compile::AliasIdxRaw); 2947 } 2948 if (!mem_is_valid(raw_mem, c, phase)) { 2949 do { 2950 c = phase->idom(c); 2951 raw_mem = memory_nodes[c->_idx]; 2952 } while (!mem_is_valid(raw_mem, c, phase) && 2953 (!c->is_CatchProj() || raw_mem == NULL || c->in(0)->in(0)->in(0) != get_ctrl(raw_mem, phase))); 2954 } 2955 } 2956 assert(raw_mem->bottom_type() == Type::MEMORY, ""); 2957 return raw_mem; 2958 } 2959 2960 Node* ShenandoahWriteBarrierNode::find_bottom_mem(Node* ctrl, PhaseIdealLoop* phase) { 2961 Node* mem = NULL; 2962 Node* c = ctrl; 2963 do { 2964 if (c->is_Region()) { 2965 Node* phi_bottom = NULL; 2966 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) { 2967 Node* u = c->fast_out(i); 2968 if (u->is_Phi() && u->bottom_type() == Type::MEMORY) { 2969 if (u->adr_type() == TypePtr::BOTTOM) { 2970 if (phi_bottom != NULL) { 2971 phi_bottom = NodeSentinel; 2972 } else { 2973 phi_bottom = u; 2974 } 2975 } 2976 } 2977 } 2978 if (phi_bottom != NULL) { 2979 if (phi_bottom != NodeSentinel) { 2980 mem = phi_bottom; 2981 } else { 2982 Node* phi = NULL; 2983 ResourceMark rm; 2984 Node_Stack phis(0); 2985 VectorSet visited(Thread::current()->resource_area()); 2986 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) { 2987 Node* u = c->fast_out(i); 2988 if (u->is_Phi() && u->bottom_type() == Type::MEMORY && u->adr_type() == TypePtr::BOTTOM) { 2989 if (phi == NULL) { 2990 phi = u; 2991 } else { 2992 phi = pick_phi(phi, u, phis, visited, phase); 2993 } 2994 } 2995 } 2996 mem = phi; 2997 } 2998 } 2999 } else { 3000 if (c->is_Call() && c->as_Call()->adr_type() != NULL) { 3001 CallProjections projs; 3002 c->as_Call()->extract_projections(&projs, true, false); 3003 if (projs.fallthrough_memproj != NULL) { 3004 if (projs.fallthrough_memproj->adr_type() == TypePtr::BOTTOM) { 3005 if (projs.catchall_memproj == NULL) { 3006 mem = projs.fallthrough_memproj; 3007 } else { 3008 if (phase->is_dominator(projs.fallthrough_catchproj, ctrl)) { 3009 mem = projs.fallthrough_memproj; 3010 } else { 3011 assert(phase->is_dominator(projs.catchall_catchproj, ctrl), "one proj must dominate barrier"); 3012 mem = projs.catchall_memproj; 3013 } 3014 } 3015 } 3016 } else { 3017 Node* proj = c->as_Call()->proj_out(TypeFunc::Memory); 3018 if (proj != NULL && 3019 proj->adr_type() == TypePtr::BOTTOM) { 3020 mem = proj; 3021 } 3022 } 3023 } else { 3024 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) { 3025 Node* u = c->fast_out(i); 3026 if (u->is_Proj() && 3027 u->bottom_type() == Type::MEMORY && 3028 u->adr_type() == TypePtr::BOTTOM) { 3029 assert(c->is_SafePoint() || c->is_MemBar() || c->is_Start(), ""); 3030 assert(mem == NULL, "only one proj"); 3031 mem = u; 3032 } 3033 } 3034 assert(!c->is_Call() || c->as_Call()->adr_type() != NULL || mem == NULL, "no mem projection expected"); 3035 } 3036 } 3037 c = phase->idom(c); 3038 } while (mem == NULL); 3039 return mem; 3040 } 3041 3042 void ShenandoahWriteBarrierNode::follow_barrier_uses(Node* n, Node* ctrl, Unique_Node_List& uses, PhaseIdealLoop* phase) { 3043 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { 3044 Node* u = n->fast_out(i); 3045 if (!u->is_CFG() && phase->get_ctrl(u) == ctrl && (!u->is_Phi() || !u->in(0)->is_Loop() || u->in(LoopNode::LoopBackControl) != n)) { 3046 uses.push(u); 3047 } 3048 } 3049 } 3050 3051 Node* ShenandoahWriteBarrierNode::get_ctrl(Node* n, PhaseIdealLoop* phase) { 3052 Node* c = phase->get_ctrl(n); 3053 if (n->is_Proj() && n->in(0)->is_Call()) { 3054 assert(c == n->in(0), ""); 3055 CallNode* call = c->as_Call(); 3056 CallProjections projs; 3057 call->extract_projections(&projs, true, false); 3058 if (projs.catchall_memproj != NULL) { 3059 if (projs.fallthrough_memproj == n) { 3060 c = projs.fallthrough_catchproj; 3061 } else { 3062 assert(projs.catchall_memproj == n, ""); 3063 c = projs.catchall_catchproj; 3064 } 3065 } 3066 } 3067 return c; 3068 } 3069 3070 Node* ShenandoahWriteBarrierNode::ctrl_or_self(Node* n, PhaseIdealLoop* phase) { 3071 if (phase->has_ctrl(n)) 3072 return get_ctrl(n, phase); 3073 else { 3074 assert (n->is_CFG(), "must be a CFG node"); 3075 return n; 3076 } 3077 } 3078 3079 #ifdef ASSERT 3080 static bool has_never_branch(Node* root) { 3081 for (uint i = 1; i < root->req(); i++) { 3082 Node* in = root->in(i); 3083 if (in != NULL && in->Opcode() == Op_Halt && in->in(0)->is_Proj() && in->in(0)->in(0)->Opcode() == Op_NeverBranch) { 3084 return true; 3085 } 3086 } 3087 return false; 3088 } 3089 #endif 3090 3091 void ShenandoahWriteBarrierNode::collect_memory_nodes(int alias, Node_List& memory_nodes, PhaseIdealLoop* phase) { 3092 Node_Stack stack(0); 3093 VectorSet visited(Thread::current()->resource_area()); 3094 Node_List regions; 3095 3096 // Walk the raw memory graph and create a mapping from CFG node to 3097 // memory node. Exclude phis for now. 3098 stack.push(phase->C->root(), 1); 3099 do { 3100 Node* n = stack.node(); 3101 int opc = n->Opcode(); 3102 uint i = stack.index(); 3103 if (i < n->req()) { 3104 Node* mem = NULL; 3105 if (opc == Op_Root) { 3106 Node* in = n->in(i); 3107 int in_opc = in->Opcode(); 3108 if (in_opc == Op_Return || in_opc == Op_Rethrow) { 3109 mem = in->in(TypeFunc::Memory); 3110 } else if (in_opc == Op_Halt) { 3111 if (!in->in(0)->is_Region()) { 3112 Node* proj = in->in(0); 3113 assert(proj->is_Proj(), ""); 3114 Node* in = proj->in(0); 3115 assert(in->is_CallStaticJava() || in->Opcode() == Op_NeverBranch || in->Opcode() == Op_Catch || proj->is_IfProj(), ""); 3116 if (in->is_CallStaticJava()) { 3117 mem = in->in(TypeFunc::Memory); 3118 } else if (in->Opcode() == Op_Catch) { 3119 Node* call = in->in(0)->in(0); 3120 assert(call->is_Call(), ""); 3121 mem = call->in(TypeFunc::Memory); 3122 } 3123 } 3124 } else { 3125 #ifdef ASSERT 3126 n->dump(); 3127 in->dump(); 3128 #endif 3129 ShouldNotReachHere(); 3130 } 3131 } else { 3132 assert(n->is_Phi() && n->bottom_type() == Type::MEMORY, ""); 3133 assert(n->adr_type() == TypePtr::BOTTOM || phase->C->get_alias_index(n->adr_type()) == alias, ""); 3134 mem = n->in(i); 3135 } 3136 i++; 3137 stack.set_index(i); 3138 if (mem == NULL) { 3139 continue; 3140 } 3141 for (;;) { 3142 if (visited.test_set(mem->_idx) || mem->is_Start()) { 3143 break; 3144 } 3145 if (mem->is_Phi()) { 3146 stack.push(mem, 2); 3147 mem = mem->in(1); 3148 } else if (mem->is_Proj()) { 3149 stack.push(mem, mem->req()); 3150 mem = mem->in(0); 3151 } else if (mem->is_SafePoint() || mem->is_MemBar()) { 3152 mem = mem->in(TypeFunc::Memory); 3153 } else if (mem->is_MergeMem()) { 3154 mem = mem->as_MergeMem()->memory_at(alias); 3155 } else if (mem->is_Store() || mem->is_LoadStore() || mem->is_ClearArray()) { 3156 stack.push(mem, mem->req()); 3157 mem = mem->in(MemNode::Memory); 3158 } else { 3159 #ifdef ASSERT 3160 mem->dump(); 3161 #endif 3162 ShouldNotReachHere(); 3163 } 3164 } 3165 } else { 3166 if (n->is_Phi()) { 3167 // Nothing 3168 } else if (!n->is_Root()) { 3169 Node* c = get_ctrl(n, phase); 3170 memory_nodes.map(c->_idx, n); 3171 } 3172 stack.pop(); 3173 } 3174 } while(stack.is_nonempty()); 3175 3176 // Iterate over CFG nodes in rpo and propagate memory state to 3177 // compute memory state at regions, creating new phis if needed. 3178 Node_List rpo_list; 3179 visited.Clear(); 3180 phase->rpo(phase->C->root(), stack, visited, rpo_list); 3181 Node* root = rpo_list.pop(); 3182 assert(root == phase->C->root(), ""); 3183 3184 const bool trace = false; 3185 #ifdef ASSERT 3186 if (trace) { 3187 for (int i = rpo_list.size() - 1; i >= 0; i--) { 3188 Node* c = rpo_list.at(i); 3189 if (memory_nodes[c->_idx] != NULL) { 3190 tty->print("X %d", c->_idx); memory_nodes[c->_idx]->dump(); 3191 } 3192 } 3193 } 3194 #endif 3195 uint last = phase->C->unique(); 3196 3197 #ifdef ASSERT 3198 uint8_t max_depth = 0; 3199 for (LoopTreeIterator iter(phase->ltree_root()); !iter.done(); iter.next()) { 3200 IdealLoopTree* lpt = iter.current(); 3201 max_depth = MAX2(max_depth, lpt->_nest); 3202 } 3203 #endif 3204 3205 bool progress = true; 3206 int iteration = 0; 3207 Node_List dead_phis; 3208 while (progress) { 3209 progress = false; 3210 iteration++; 3211 assert(iteration <= 2+max_depth || phase->C->has_irreducible_loop(), ""); 3212 if (trace) { tty->print_cr("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"); } 3213 IdealLoopTree* last_updated_ilt = NULL; 3214 for (int i = rpo_list.size() - 1; i >= 0; i--) { 3215 Node* c = rpo_list.at(i); 3216 3217 Node* prev_mem = memory_nodes[c->_idx]; 3218 if (c->is_Region()) { 3219 Node* prev_region = regions[c->_idx]; 3220 Node* unique = NULL; 3221 for (uint j = 1; j < c->req() && unique != NodeSentinel; j++) { 3222 Node* m = memory_nodes[c->in(j)->_idx]; 3223 assert(m != NULL || (c->is_Loop() && j == LoopNode::LoopBackControl && iteration == 1) || phase->C->has_irreducible_loop() || has_never_branch(phase->C->root()), "expect memory state"); 3224 if (m != NULL) { 3225 if (m == prev_region && ((c->is_Loop() && j == LoopNode::LoopBackControl) || (prev_region->is_Phi() && prev_region->in(0) == c))) { 3226 assert(c->is_Loop() && j == LoopNode::LoopBackControl || phase->C->has_irreducible_loop(), ""); 3227 // continue 3228 } else if (unique == NULL) { 3229 unique = m; 3230 } else if (m == unique) { 3231 // continue 3232 } else { 3233 unique = NodeSentinel; 3234 } 3235 } 3236 } 3237 assert(unique != NULL, "empty phi???"); 3238 if (unique != NodeSentinel) { 3239 if (prev_region != NULL && prev_region->is_Phi() && prev_region->in(0) == c) { 3240 dead_phis.push(prev_region); 3241 } 3242 regions.map(c->_idx, unique); 3243 } else { 3244 Node* phi = NULL; 3245 if (prev_region != NULL && prev_region->is_Phi() && prev_region->in(0) == c && prev_region->_idx >= last) { 3246 phi = prev_region; 3247 for (uint k = 1; k < c->req(); k++) { 3248 Node* m = memory_nodes[c->in(k)->_idx]; 3249 assert(m != NULL, "expect memory state"); 3250 phi->set_req(k, m); 3251 } 3252 } else { 3253 for (DUIterator_Fast jmax, j = c->fast_outs(jmax); j < jmax && phi == NULL; j++) { 3254 Node* u = c->fast_out(j); 3255 if (u->is_Phi() && u->bottom_type() == Type::MEMORY && 3256 (u->adr_type() == TypePtr::BOTTOM || phase->C->get_alias_index(u->adr_type()) == alias)) { 3257 phi = u; 3258 for (uint k = 1; k < c->req() && phi != NULL; k++) { 3259 Node* m = memory_nodes[c->in(k)->_idx]; 3260 assert(m != NULL, "expect memory state"); 3261 if (u->in(k) != m) { 3262 phi = NULL; 3263 } 3264 } 3265 } 3266 } 3267 if (phi == NULL) { 3268 phi = new PhiNode(c, Type::MEMORY, phase->C->get_adr_type(alias)); 3269 for (uint k = 1; k < c->req(); k++) { 3270 Node* m = memory_nodes[c->in(k)->_idx]; 3271 assert(m != NULL, "expect memory state"); 3272 phi->init_req(k, m); 3273 } 3274 } 3275 } 3276 assert(phi != NULL, ""); 3277 regions.map(c->_idx, phi); 3278 } 3279 Node* current_region = regions[c->_idx]; 3280 if (current_region != prev_region) { 3281 progress = true; 3282 if (prev_region == prev_mem) { 3283 memory_nodes.map(c->_idx, current_region); 3284 } 3285 } 3286 } else if (prev_mem == NULL || prev_mem->is_Phi() || ctrl_or_self(prev_mem, phase) != c) { 3287 Node* m = memory_nodes[phase->idom(c)->_idx]; 3288 assert(m != NULL, "expect memory state"); 3289 if (m != prev_mem) { 3290 memory_nodes.map(c->_idx, m); 3291 progress = true; 3292 } 3293 } 3294 #ifdef ASSERT 3295 if (trace) { tty->print("X %d", c->_idx); memory_nodes[c->_idx]->dump(); } 3296 #endif 3297 } 3298 } 3299 3300 // Replace existing phi with computed memory state for that region 3301 // if different (could be a new phi or a dominating memory node if 3302 // that phi was found to be useless). 3303 while (dead_phis.size() > 0) { 3304 Node* n = dead_phis.pop(); 3305 n->replace_by(phase->C->top()); 3306 n->destruct(); 3307 } 3308 for (int i = rpo_list.size() - 1; i >= 0; i--) { 3309 Node* c = rpo_list.at(i); 3310 if (c->is_Region()) { 3311 Node* n = regions[c->_idx]; 3312 if (n->is_Phi() && n->_idx >= last && n->in(0) == c) { 3313 phase->register_new_node(n, c); 3314 } 3315 } 3316 } 3317 for (int i = rpo_list.size() - 1; i >= 0; i--) { 3318 Node* c = rpo_list.at(i); 3319 if (c->is_Region()) { 3320 Node* n = regions[c->_idx]; 3321 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) { 3322 Node* u = c->fast_out(i); 3323 if (u->is_Phi() && u->bottom_type() == Type::MEMORY && 3324 u != n) { 3325 if (u->adr_type() == TypePtr::BOTTOM) { 3326 fix_memory_uses(u, n, n, c, alias, phase); 3327 } else if (phase->C->get_alias_index(u->adr_type()) == alias) { 3328 phase->lazy_replace(u, n); 3329 --i; --imax; 3330 } 3331 } 3332 } 3333 } 3334 } 3335 } 3336 3337 void ShenandoahWriteBarrierNode::fix_raw_mem(Node* ctrl, Node* region, Node* raw_mem, Node* raw_mem_for_ctrl, Node* raw_mem_phi, 3338 Node_List& memory_nodes, Unique_Node_List& uses, PhaseIdealLoop* phase) { 3339 const bool trace = false; 3340 DEBUG_ONLY(if (trace) { tty->print("ZZZ control is"); ctrl->dump(); }); 3341 DEBUG_ONLY(if (trace) { tty->print("ZZZ mem is"); raw_mem->dump(); }); 3342 GrowableArray<Node*> phis; 3343 if (raw_mem_for_ctrl != raw_mem) { 3344 Node* old = raw_mem_for_ctrl; 3345 Node* prev = NULL; 3346 while (old != raw_mem) { 3347 assert(old->is_Store() || old->is_LoadStore() || old->is_ClearArray(), ""); 3348 prev = old; 3349 old = old->in(MemNode::Memory); 3350 } 3351 assert(prev != NULL, ""); 3352 memory_nodes.map(ctrl->_idx, raw_mem); 3353 memory_nodes.map(region->_idx, raw_mem_for_ctrl); 3354 phase->igvn().replace_input_of(prev, MemNode::Memory, raw_mem_phi); 3355 } else { 3356 memory_nodes.map(region->_idx, raw_mem_phi); 3357 uses.clear(); 3358 uses.push(region); 3359 for(uint next = 0; next < uses.size(); next++ ) { 3360 Node *n = uses.at(next); 3361 assert(n->is_CFG(), ""); 3362 DEBUG_ONLY(if (trace) { tty->print("ZZZ ctrl"); n->dump(); }); 3363 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { 3364 Node* u = n->fast_out(i); 3365 if (!u->is_Root() && u->is_CFG() && u != n) { 3366 Node* m = memory_nodes[u->_idx]; 3367 if (u->is_Region() && !has_mem_phi(phase->C, u, Compile::AliasIdxRaw) && 3368 u->unique_ctrl_out()->Opcode() != Op_Halt) { 3369 DEBUG_ONLY(if (trace) { tty->print("ZZZ region"); u->dump(); }); 3370 DEBUG_ONLY(if (trace && m != NULL) { tty->print("ZZZ mem"); m->dump(); }); 3371 3372 if (!mem_is_valid(m, u, phase) || !m->is_Phi()) { 3373 bool push = true; 3374 bool create_phi = true; 3375 if (phase->is_dominator(region, u)) { 3376 create_phi = false; 3377 } else if (!phase->C->has_irreducible_loop()) { 3378 IdealLoopTree* loop = phase->get_loop(ctrl); 3379 bool do_check = true; 3380 IdealLoopTree* l = loop; 3381 create_phi = false; 3382 while (l != phase->ltree_root()) { 3383 if (phase->is_dominator(l->_head, u) && phase->is_dominator(phase->idom(u), l->_head)) { 3384 create_phi = true; 3385 do_check = false; 3386 break; 3387 } 3388 l = l->_parent; 3389 } 3390 3391 if (do_check) { 3392 assert(!create_phi, ""); 3393 IdealLoopTree* u_loop = phase->get_loop(u); 3394 if (u_loop != phase->ltree_root() && u_loop->is_member(loop)) { 3395 Node* c = ctrl; 3396 while (!phase->is_dominator(c, u_loop->tail())) { 3397 c = phase->idom(c); 3398 } 3399 if (!phase->is_dominator(c, u)) { 3400 do_check = false; 3401 } 3402 } 3403 } 3404 3405 if (do_check && phase->is_dominator(phase->idom(u), region)) { 3406 create_phi = true; 3407 } 3408 } 3409 if (create_phi) { 3410 Node* phi = new PhiNode(u, Type::MEMORY, TypeRawPtr::BOTTOM); 3411 phase->register_new_node(phi, u); 3412 phis.push(phi); 3413 DEBUG_ONLY(if (trace) { tty->print("ZZZ new phi"); phi->dump(); }); 3414 if (!mem_is_valid(m, u, phase)) { 3415 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting mem"); phi->dump(); }); 3416 memory_nodes.map(u->_idx, phi); 3417 } else { 3418 DEBUG_ONLY(if (trace) { tty->print("ZZZ NOT setting mem"); m->dump(); }); 3419 for (;;) { 3420 assert(m->is_Mem() || m->is_LoadStore() || m->is_Proj() /*|| m->is_MergeMem()*/, ""); 3421 Node* next = NULL; 3422 if (m->is_Proj()) { 3423 next = m->in(0); 3424 } else { 3425 next = m->in(MemNode::Memory); 3426 } 3427 if (phase->get_ctrl(next) != u) { 3428 break; 3429 } 3430 if (next->is_MergeMem()) { 3431 assert(phase->get_ctrl(next->as_MergeMem()->memory_at(Compile::AliasIdxRaw)) != u, ""); 3432 break; 3433 } 3434 if (next->is_Phi()) { 3435 assert(next->adr_type() == TypePtr::BOTTOM && next->in(0) == u, ""); 3436 break; 3437 } 3438 m = next; 3439 } 3440 3441 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting to phi"); m->dump(); }); 3442 assert(m->is_Mem() || m->is_LoadStore(), ""); 3443 phase->igvn().replace_input_of(m, MemNode::Memory, phi); 3444 push = false; 3445 } 3446 } else { 3447 DEBUG_ONLY(if (trace) { tty->print("ZZZ skipping region"); u->dump(); }); 3448 } 3449 if (push) { 3450 uses.push(u); 3451 } 3452 } 3453 } else if (!mem_is_valid(m, u, phase) && 3454 !(u->Opcode() == Op_CProj && u->in(0)->Opcode() == Op_NeverBranch && u->as_Proj()->_con == 1)) { 3455 uses.push(u); 3456 } 3457 } 3458 } 3459 } 3460 for (int i = 0; i < phis.length(); i++) { 3461 Node* n = phis.at(i); 3462 Node* r = n->in(0); 3463 DEBUG_ONLY(if (trace) { tty->print("ZZZ fixing new phi"); n->dump(); }); 3464 for (uint j = 1; j < n->req(); j++) { 3465 Node* m = find_raw_mem(r->in(j), NULL, memory_nodes, phase); 3466 phase->igvn().replace_input_of(n, j, m); 3467 DEBUG_ONLY(if (trace) { tty->print("ZZZ fixing new phi: %d", j); m->dump(); }); 3468 } 3469 } 3470 } 3471 uint last = phase->C->unique(); 3472 MergeMemNode* mm = NULL; 3473 int alias = Compile::AliasIdxRaw; 3474 DEBUG_ONLY(if (trace) { tty->print("ZZZ raw mem is"); raw_mem->dump(); }); 3475 for (DUIterator i = raw_mem->outs(); raw_mem->has_out(i); i++) { 3476 Node* u = raw_mem->out(i); 3477 if (u->_idx < last) { 3478 if (u->is_Mem()) { 3479 if (phase->C->get_alias_index(u->adr_type()) == alias) { 3480 Node* m = find_raw_mem(phase->get_ctrl(u), u, memory_nodes, phase); 3481 if (m != raw_mem) { 3482 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting memory of use"); u->dump(); }); 3483 phase->igvn().replace_input_of(u, MemNode::Memory, m); 3484 --i; 3485 } 3486 } 3487 } else if (u->is_MergeMem()) { 3488 MergeMemNode* u_mm = u->as_MergeMem(); 3489 if (u_mm->memory_at(alias) == raw_mem) { 3490 MergeMemNode* newmm = NULL; 3491 for (DUIterator_Fast jmax, j = u->fast_outs(jmax); j < jmax; j++) { 3492 Node* uu = u->fast_out(j); 3493 assert(!uu->is_MergeMem(), "chain of MergeMems?"); 3494 if (uu->is_Phi()) { 3495 assert(uu->adr_type() == TypePtr::BOTTOM, ""); 3496 Node* region = uu->in(0); 3497 int nb = 0; 3498 for (uint k = 1; k < uu->req(); k++) { 3499 if (uu->in(k) == u) { 3500 Node* m = find_raw_mem(region->in(k), NULL, memory_nodes, phase); 3501 if (m != raw_mem) { 3502 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting memory of phi %d", k); uu->dump(); }); 3503 if (newmm == NULL || 1) { 3504 newmm = clone_merge_mem(u, raw_mem, alias, m, phase->ctrl_or_self(m), i, phase); 3505 } 3506 if (newmm != u) { 3507 phase->igvn().replace_input_of(uu, k, newmm); 3508 nb++; 3509 --jmax; 3510 } 3511 } 3512 } 3513 } 3514 if (nb > 0) { 3515 --j; 3516 } 3517 } else { 3518 Node* m = find_raw_mem(phase->ctrl_or_self(uu), uu, memory_nodes, phase); 3519 if (m != raw_mem) { 3520 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting memory of use"); uu->dump(); }); 3521 if (newmm == NULL || 1) { 3522 newmm = clone_merge_mem(u, raw_mem, alias, m, phase->ctrl_or_self(m), i, phase); 3523 } 3524 if (newmm != u) { 3525 phase->igvn().replace_input_of(uu, uu->find_edge(u), newmm); 3526 --j, --jmax; 3527 } 3528 } 3529 } 3530 } 3531 } 3532 } else if (u->is_Phi()) { 3533 assert(u->bottom_type() == Type::MEMORY, "what else?"); 3534 if (u->adr_type() == TypeRawPtr::BOTTOM || u->adr_type() == TypePtr::BOTTOM) { 3535 Node* region = u->in(0); 3536 bool replaced = false; 3537 for (uint j = 1; j < u->req(); j++) { 3538 if (u->in(j) == raw_mem) { 3539 Node* m = find_raw_mem(region->in(j), NULL, memory_nodes, phase); 3540 Node* nnew = m; 3541 if (m != raw_mem) { 3542 if (u->adr_type() == TypePtr::BOTTOM) { 3543 if (mm == NULL || 1) { 3544 mm = allocate_merge_mem(raw_mem, alias, m, phase->ctrl_or_self(m), phase); 3545 } 3546 nnew = mm; 3547 } 3548 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting memory of phi %d", j); u->dump(); }); 3549 phase->igvn().replace_input_of(u, j, nnew); 3550 replaced = true; 3551 } 3552 } 3553 } 3554 if (replaced) { 3555 --i; 3556 } 3557 } 3558 } else if ((u->adr_type() == TypePtr::BOTTOM && u->Opcode() != Op_StrInflatedCopy) || 3559 u->adr_type() == NULL) { 3560 assert(u->adr_type() != NULL || 3561 u->Opcode() == Op_Rethrow || 3562 u->Opcode() == Op_Return || 3563 u->Opcode() == Op_SafePoint || 3564 (u->is_CallStaticJava() && u->as_CallStaticJava()->uncommon_trap_request() != 0) || 3565 (u->is_CallStaticJava() && u->as_CallStaticJava()->_entry_point == OptoRuntime::rethrow_stub()) || 3566 u->Opcode() == Op_CallLeaf, ""); 3567 Node* m = find_raw_mem(phase->ctrl_or_self(u), u, memory_nodes, phase); 3568 if (m != raw_mem) { 3569 if (mm == NULL || 1) { 3570 mm = allocate_merge_mem(raw_mem, alias, m, phase->get_ctrl(m), phase); 3571 } 3572 phase->igvn().replace_input_of(u, u->find_edge(raw_mem), mm); 3573 --i; 3574 } 3575 } else if (phase->C->get_alias_index(u->adr_type()) == alias) { 3576 Node* m = find_raw_mem(phase->ctrl_or_self(u), u, memory_nodes, phase); 3577 if (m != raw_mem) { 3578 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting memory of use"); u->dump(); }); 3579 phase->igvn().replace_input_of(u, u->find_edge(raw_mem), m); 3580 --i; 3581 } 3582 } 3583 } 3584 } 3585 #ifdef ASSERT 3586 assert(raw_mem_phi->outcnt() > 0, ""); 3587 for (int i = 0; i < phis.length(); i++) { 3588 Node* n = phis.at(i); 3589 assert(n->outcnt() > 0, "new phi must have uses now"); 3590 } 3591 #endif 3592 } 3593 3594 void ShenandoahWriteBarrierNode::test_evacuation_in_progress(Node* ctrl, int alias, Node*& raw_mem, Node*& wb_mem, 3595 IfNode*& evacuation_iff, Node*& evac_in_progress, 3596 Node*& evac_not_in_progress, PhaseIdealLoop* phase) { 3597 IdealLoopTree *loop = phase->get_loop(ctrl); 3598 Node* thread = new ThreadLocalNode(); 3599 phase->register_new_node(thread, ctrl); 3600 Node* offset = phase->igvn().MakeConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset())); 3601 phase->set_ctrl(offset, phase->C->root()); 3602 Node* gc_state_addr = new AddPNode(phase->C->top(), thread, offset); 3603 phase->register_new_node(gc_state_addr, ctrl); 3604 uint gc_state_idx = Compile::AliasIdxRaw; 3605 const TypePtr* gc_state_adr_type = NULL; // debug-mode-only argument 3606 debug_only(gc_state_adr_type = phase->C->get_adr_type(gc_state_idx)); 3607 3608 Node* gc_state = new LoadUBNode(ctrl, raw_mem, gc_state_addr, gc_state_adr_type, TypeInt::BYTE, MemNode::unordered); 3609 phase->register_new_node(gc_state, ctrl); 3610 3611 Node* evacuation_in_progress = new AndINode(gc_state, phase->igvn().intcon(ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL)); 3612 phase->register_new_node(evacuation_in_progress, ctrl); 3613 Node* evacuation_in_progress_cmp = new CmpINode(evacuation_in_progress, phase->igvn().zerocon(T_INT)); 3614 phase->register_new_node(evacuation_in_progress_cmp, ctrl); 3615 Node* evacuation_in_progress_test = new BoolNode(evacuation_in_progress_cmp, BoolTest::ne); 3616 phase->register_new_node(evacuation_in_progress_test, ctrl); 3617 evacuation_iff = new IfNode(ctrl, evacuation_in_progress_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN); 3618 phase->register_control(evacuation_iff, loop, ctrl); 3619 3620 assert(is_evacuation_in_progress_test(evacuation_iff), "Should match the shape"); 3621 assert(is_gc_state_load(gc_state), "Should match the shape"); 3622 3623 evac_not_in_progress = new IfFalseNode(evacuation_iff); 3624 phase->register_control(evac_not_in_progress, loop, evacuation_iff); 3625 evac_in_progress = new IfTrueNode(evacuation_iff); 3626 phase->register_control(evac_in_progress, loop, evacuation_iff); 3627 } 3628 3629 void ShenandoahWriteBarrierNode::evacuation_not_in_progress_null_check(Node*& c, Node*& val, Node* unc_ctrl, Node*& unc_region, PhaseIdealLoop* phase) { 3630 if (unc_ctrl != NULL) { 3631 // Clone the null check in this branch to allow implicit null check 3632 IdealLoopTree *loop = phase->get_loop(c); 3633 Node* iff = unc_ctrl->in(0); 3634 assert(iff->is_If(), "broken"); 3635 Node* new_iff = iff->clone(); 3636 new_iff->set_req(0, c); 3637 phase->register_control(new_iff, loop, c); 3638 Node* iffalse = new IfFalseNode(new_iff->as_If()); 3639 phase->register_control(iffalse, loop, new_iff); 3640 Node* iftrue = new IfTrueNode(new_iff->as_If()); 3641 phase->register_control(iftrue, loop, new_iff); 3642 c = iftrue; 3643 unc_region = new RegionNode(3); 3644 unc_region->init_req(1, iffalse); 3645 const Type *t = phase->igvn().type(val); 3646 assert(val->Opcode() == Op_CastPP, "expect cast to non null here"); 3647 Node* uncasted_val = val->in(1); 3648 val = new CastPPNode(uncasted_val, t); 3649 val->init_req(0, c); 3650 phase->register_new_node(val, c); 3651 } 3652 } 3653 3654 void ShenandoahWriteBarrierNode::evacuation_not_in_progress(Node* c, Node* val, Node* unc_ctrl, Node* raw_mem, Node* wb_mem, Node* region, 3655 Node* val_phi, Node* mem_phi, Node* raw_mem_phi, Node*& unc_region, PhaseIdealLoop* phase) { 3656 evacuation_not_in_progress_null_check(c, val, unc_ctrl, unc_region, phase); 3657 region->init_req(1, c); 3658 if (ShenandoahWriteBarrierRB) { 3659 Node* rbfalse = new ShenandoahReadBarrierNode(c, wb_mem, val); 3660 phase->register_new_node(rbfalse, c); 3661 val_phi->init_req(1, rbfalse); 3662 } else { 3663 val_phi->init_req(1, val); 3664 } 3665 mem_phi->init_req(1, wb_mem); 3666 raw_mem_phi->init_req(1, raw_mem); 3667 } 3668 3669 void ShenandoahWriteBarrierNode::evacuation_in_progress_null_check(Node*& c, Node*& val, Node* evacuation_iff, Node* unc, Node* unc_ctrl, 3670 Node* unc_region, Unique_Node_List& uses, PhaseIdealLoop* phase) { 3671 if (unc != NULL) { 3672 // Clone the null check in this branch to allow implicit null check 3673 IdealLoopTree *loop = phase->get_loop(c); 3674 Node* iff = unc_ctrl->in(0); 3675 assert(iff->is_If(), "broken"); 3676 Node* new_iff = iff->clone(); 3677 new_iff->set_req(0, c); 3678 phase->register_control(new_iff, loop, c); 3679 Node* iffalse = new IfFalseNode(new_iff->as_If()); 3680 phase->register_control(iffalse, loop, new_iff); 3681 Node* iftrue = new IfTrueNode(new_iff->as_If()); 3682 phase->register_control(iftrue, loop, new_iff); 3683 c = iftrue; 3684 unc_region->init_req(2, iffalse); 3685 3686 Node* proj = iff->as_If()->proj_out(0); 3687 assert(proj != unc_ctrl, "bad projection"); 3688 Node* use = proj->unique_ctrl_out(); 3689 3690 assert(use == unc || use->is_Region(), "what else?"); 3691 3692 uses.clear(); 3693 if (use == unc) { 3694 phase->set_idom(use, unc_region, phase->dom_depth(unc_region)+1); 3695 for (uint i = 1; i < unc->req(); i++) { 3696 Node* n = unc->in(i); 3697 if (phase->has_ctrl(n) && phase->get_ctrl(n) == proj) { 3698 uses.push(n); 3699 } 3700 } 3701 } else { 3702 assert(use->is_Region(), "what else?"); 3703 uint idx = 1; 3704 for (; use->in(idx) != proj; idx++); 3705 for (DUIterator_Fast imax, i = use->fast_outs(imax); i < imax; i++) { 3706 Node* u = use->fast_out(i); 3707 if (u->is_Phi() && phase->get_ctrl(u->in(idx)) == proj) { 3708 uses.push(u->in(idx)); 3709 } 3710 } 3711 } 3712 for(uint next = 0; next < uses.size(); next++ ) { 3713 Node *n = uses.at(next); 3714 assert(phase->get_ctrl(n) == proj, "bad control"); 3715 phase->set_ctrl_and_loop(n, unc_region); 3716 if (n->in(0) == proj) { 3717 phase->igvn().replace_input_of(n, 0, unc_region); 3718 } 3719 for (uint i = 0; i < n->req(); i++) { 3720 Node* m = n->in(i); 3721 if (m != NULL && phase->has_ctrl(m) && phase->get_ctrl(m) == proj) { 3722 uses.push(m); 3723 } 3724 } 3725 } 3726 3727 phase->igvn().rehash_node_delayed(use); 3728 int nb = use->replace_edge(proj, unc_region); 3729 assert(nb == 1, "only use expected"); 3730 phase->register_control(unc_region, phase->ltree_root(), evacuation_iff); 3731 3732 phase->igvn().replace_input_of(iff, 1, phase->igvn().intcon(1)); 3733 const Type *t = phase->igvn().type(val); 3734 assert(val->Opcode() == Op_CastPP, "expect cast to non null here"); 3735 Node* uncasted_val = val->in(1); 3736 val = new CastPPNode(uncasted_val, t); 3737 val->init_req(0, c); 3738 phase->register_new_node(val, c); 3739 } 3740 } 3741 3742 void ShenandoahWriteBarrierNode::in_cset_fast_test(Node*& c, Node* rbtrue, Node* raw_mem, Node* wb_mem, Node* region, Node* val_phi, Node* mem_phi, 3743 Node* raw_mem_phi, PhaseIdealLoop* phase) { 3744 if (ShenandoahWriteBarrierCsetTestInIR) { 3745 IdealLoopTree *loop = phase->get_loop(c); 3746 Node* raw_rbtrue = new CastP2XNode(c, rbtrue); 3747 phase->register_new_node(raw_rbtrue, c); 3748 Node* cset_offset = new URShiftXNode(raw_rbtrue, phase->igvn().intcon(ShenandoahHeapRegion::region_size_bytes_shift_jint())); 3749 phase->register_new_node(cset_offset, c); 3750 Node* in_cset_fast_test_base_addr = phase->igvn().makecon(TypeRawPtr::make(ShenandoahHeap::in_cset_fast_test_addr())); 3751 phase->set_ctrl(in_cset_fast_test_base_addr, phase->C->root()); 3752 Node* in_cset_fast_test_adr = new AddPNode(phase->C->top(), in_cset_fast_test_base_addr, cset_offset); 3753 phase->register_new_node(in_cset_fast_test_adr, c); 3754 uint in_cset_fast_test_idx = Compile::AliasIdxRaw; 3755 const TypePtr* in_cset_fast_test_adr_type = NULL; // debug-mode-only argument 3756 debug_only(in_cset_fast_test_adr_type = phase->C->get_adr_type(in_cset_fast_test_idx)); 3757 Node* in_cset_fast_test_load = new LoadUBNode(c, raw_mem, in_cset_fast_test_adr, in_cset_fast_test_adr_type, TypeInt::BOOL, MemNode::unordered); 3758 phase->register_new_node(in_cset_fast_test_load, c); 3759 Node* in_cset_fast_test_cmp = new CmpINode(in_cset_fast_test_load, phase->igvn().zerocon(T_INT)); 3760 phase->register_new_node(in_cset_fast_test_cmp, c); 3761 Node* in_cset_fast_test_test = new BoolNode(in_cset_fast_test_cmp, BoolTest::ne); 3762 phase->register_new_node(in_cset_fast_test_test, c); 3763 IfNode* in_cset_fast_test_iff = new IfNode(c, in_cset_fast_test_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN); 3764 phase->register_control(in_cset_fast_test_iff, loop, c); 3765 3766 Node* in_cset_fast_test_success = new IfFalseNode(in_cset_fast_test_iff); 3767 phase->register_control(in_cset_fast_test_success, loop, in_cset_fast_test_iff); 3768 3769 region->init_req(3, in_cset_fast_test_success); 3770 val_phi->init_req(3, rbtrue); 3771 mem_phi->init_req(3, wb_mem); 3772 raw_mem_phi->init_req(3, raw_mem); 3773 3774 Node* in_cset_fast_test_failure = new IfTrueNode(in_cset_fast_test_iff); 3775 phase->register_control(in_cset_fast_test_failure, loop, in_cset_fast_test_iff); 3776 3777 c = in_cset_fast_test_failure; 3778 } 3779 } 3780 3781 void ShenandoahWriteBarrierNode::evacuation_in_progress(Node* c, Node* val, Node* evacuation_iff, Node* unc, Node* unc_ctrl, 3782 Node* raw_mem, Node* wb_mem, Node* region, Node* val_phi, Node* mem_phi, 3783 Node* raw_mem_phi, Node* unc_region, int alias, Unique_Node_List& uses, 3784 PhaseIdealLoop* phase) { 3785 evacuation_in_progress_null_check(c, val, evacuation_iff, unc, unc_ctrl, unc_region, uses, phase); 3786 3787 IdealLoopTree *loop = phase->get_loop(c); 3788 Node* rbtrue = new ShenandoahReadBarrierNode(c, wb_mem, val); 3789 phase->register_new_node(rbtrue, c); 3790 3791 Node* in_cset_fast_test_failure = NULL; 3792 in_cset_fast_test(c, rbtrue, raw_mem, wb_mem, region, val_phi, mem_phi, raw_mem_phi, phase); 3793 3794 // The slow path stub consumes and produces raw memory in addition 3795 // to the existing memory edges 3796 Node* base = find_bottom_mem(c, phase); 3797 3798 MergeMemNode* mm = MergeMemNode::make(base); 3799 mm->set_memory_at(alias, wb_mem); 3800 mm->set_memory_at(Compile::AliasIdxRaw, raw_mem); 3801 phase->register_new_node(mm, c); 3802 3803 Node* call = new CallLeafNoFPNode(OptoRuntime::shenandoah_write_barrier_Type(), StubRoutines::shenandoah_wb_C(), "shenandoah_write_barrier", TypeRawPtr::BOTTOM); 3804 call->init_req(TypeFunc::Control, c); 3805 call->init_req(TypeFunc::I_O, phase->C->top()); 3806 call->init_req(TypeFunc::Memory, mm); 3807 call->init_req(TypeFunc::FramePtr, phase->C->top()); 3808 call->init_req(TypeFunc::ReturnAdr, phase->C->top()); 3809 call->init_req(TypeFunc::Parms, rbtrue); 3810 phase->register_control(call, loop, c); 3811 Node* ctrl_proj = new ProjNode(call, TypeFunc::Control); 3812 phase->register_control(ctrl_proj, loop, call); 3813 Node* mem_proj = new ProjNode(call, TypeFunc::Memory); 3814 phase->register_new_node(mem_proj, call); 3815 Node* res_proj = new ProjNode(call, TypeFunc::Parms); 3816 phase->register_new_node(res_proj, call); 3817 Node* res = new CheckCastPPNode(ctrl_proj, res_proj, phase->igvn().type(val)->is_oopptr()->cast_to_nonconst()); 3818 phase->register_new_node(res, ctrl_proj); 3819 region->init_req(2, ctrl_proj); 3820 val_phi->init_req(2, res); 3821 mem_phi->init_req(2, mem_proj); 3822 raw_mem_phi->init_req(2, mem_proj); 3823 phase->register_control(region, loop, evacuation_iff); 3824 3825 } 3826 3827 void ShenandoahWriteBarrierNode::pin_and_expand(PhaseIdealLoop* phase) { 3828 const bool trace = false; 3829 3830 // Collect raw memory state at CFG points in the entire graph and 3831 // record it in memory_nodes. Optimize the raw memory graph in the 3832 // process. Optimizing the memory graph also makes the memory graph 3833 // simpler. 3834 Node_List memory_nodes; 3835 collect_memory_nodes(Compile::AliasIdxRaw, memory_nodes, phase); 3836 3837 // Let's try to common write barriers again 3838 for (;;) { 3839 bool progress = false; 3840 for (int i = phase->C->shenandoah_barriers_count(); i > 0; i--) { 3841 ShenandoahBarrierNode* wb = phase->C->shenandoah_barrier(i-1); 3842 Node* ctrl = phase->get_ctrl(wb); 3843 if (wb->try_common(ctrl, phase) != NULL) { 3844 progress = true; 3845 } 3846 } 3847 if (!progress) { 3848 break; 3849 } 3850 } 3851 3852 for (int i = 0; i < phase->C->shenandoah_barriers_count(); i++) { 3853 ShenandoahWriteBarrierNode* wb = phase->C->shenandoah_barrier(i); 3854 Node* ctrl = phase->get_ctrl(wb); 3855 3856 Node* val = wb->in(ValueIn); 3857 if (ctrl->is_Proj() && ctrl->in(0)->is_CallJava()) { 3858 assert(is_dominator(phase->get_ctrl(val), ctrl->in(0)->in(0), val, ctrl->in(0), phase), "can't move"); 3859 phase->set_ctrl(wb, ctrl->in(0)->in(0)); 3860 } else if (ctrl->is_CallRuntime()) { 3861 assert(is_dominator(phase->get_ctrl(val), ctrl->in(0), val, ctrl, phase), "can't move"); 3862 phase->set_ctrl(wb, ctrl->in(0)); 3863 } 3864 3865 assert(wb->Opcode() == Op_ShenandoahWriteBarrier, "only for write barriers"); 3866 // Look for a null check that dominates this barrier and move the 3867 // barrier right after the null check to enable implicit null 3868 // checks 3869 wb->pin_and_expand_move_barrier(phase); 3870 3871 ctrl = phase->get_ctrl(wb); 3872 wb->pin_and_expand_helper(phase); 3873 } 3874 3875 Unique_Node_List uses; 3876 Unique_Node_List uses_to_ignore; 3877 for (int i = phase->C->shenandoah_barriers_count(); i > 0; i--) { 3878 int cnt = phase->C->shenandoah_barriers_count(); 3879 ShenandoahWriteBarrierNode* wb = phase->C->shenandoah_barrier(i-1); 3880 3881 uint last = phase->C->unique(); 3882 Node* ctrl = phase->get_ctrl(wb); 3883 3884 Node* raw_mem = find_raw_mem(ctrl, wb, memory_nodes, phase); 3885 Node* init_raw_mem = raw_mem; 3886 Node* raw_mem_for_ctrl = find_raw_mem(ctrl, NULL, memory_nodes, phase); 3887 int alias = phase->C->get_alias_index(wb->adr_type()); 3888 Node* wb_mem = wb->in(Memory); 3889 3890 Node* val = wb->in(ValueIn); 3891 Node* wbproj = wb->find_out_with(Op_ShenandoahWBMemProj); 3892 IdealLoopTree *loop = phase->get_loop(ctrl); 3893 3894 assert(val->Opcode() != Op_ShenandoahWriteBarrier || phase->C->has_irreducible_loop(), "No chain of write barriers"); 3895 3896 CallStaticJavaNode* unc = wb->pin_and_expand_null_check(phase->igvn()); 3897 Node* unc_ctrl = NULL; 3898 if (unc != NULL) { 3899 if (val->in(0) != ctrl) { 3900 unc = NULL; 3901 } else { 3902 unc_ctrl = val->in(0); 3903 } 3904 } 3905 3906 Node* uncasted_val = val; 3907 if (unc != NULL) { 3908 uncasted_val = val->in(1); 3909 } 3910 3911 Node* evac_in_progress = NULL; 3912 Node* evac_not_in_progress = NULL; 3913 IfNode* evacuation_iff = NULL; 3914 test_evacuation_in_progress(ctrl, alias, raw_mem, wb_mem, evacuation_iff, evac_in_progress, evac_not_in_progress, phase); 3915 3916 Node* region = new RegionNode(4); 3917 Node* val_phi = new PhiNode(region, val->bottom_type()->is_oopptr()->cast_to_nonconst()); 3918 Node* mem_phi = PhiNode::make(region, wb_mem, Type::MEMORY, phase->C->alias_type(wb->adr_type())->adr_type()); 3919 Node* raw_mem_phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM); 3920 3921 Node* unc_region = NULL; 3922 evacuation_not_in_progress(evac_not_in_progress, val, unc_ctrl, raw_mem, wb_mem, 3923 region, val_phi, mem_phi, raw_mem_phi, unc_region, 3924 phase); 3925 3926 evacuation_in_progress(evac_in_progress, val, evacuation_iff, unc, unc_ctrl, 3927 raw_mem, wb_mem, region, val_phi, mem_phi, raw_mem_phi, 3928 unc_region, alias, uses, 3929 phase); 3930 Node* out_val = val_phi; 3931 phase->register_new_node(val_phi, region); 3932 phase->register_new_node(mem_phi, region); 3933 phase->register_new_node(raw_mem_phi, region); 3934 3935 // Update the control of all nodes that should be after the 3936 // barrier control flow 3937 uses.clear(); 3938 // Every node that is control dependent on the barrier's input 3939 // control will be after the expanded barrier. The raw memory (if 3940 // its memory is control dependent on the barrier's input control) 3941 // must stay above the barrier. 3942 uses_to_ignore.clear(); 3943 if (phase->has_ctrl(init_raw_mem) && phase->get_ctrl(init_raw_mem) == ctrl && !init_raw_mem->is_Phi()) { 3944 uses_to_ignore.push(init_raw_mem); 3945 } 3946 for (uint next = 0; next < uses_to_ignore.size(); next++) { 3947 Node *n = uses_to_ignore.at(next); 3948 for (uint i = 0; i < n->req(); i++) { 3949 Node* in = n->in(i); 3950 if (in != NULL && phase->has_ctrl(in) && phase->get_ctrl(in) == ctrl) { 3951 uses_to_ignore.push(in); 3952 } 3953 } 3954 } 3955 for (DUIterator_Fast imax, i = ctrl->fast_outs(imax); i < imax; i++) { 3956 Node* u = ctrl->fast_out(i); 3957 if (u->_idx < last && 3958 u != wb && 3959 !uses_to_ignore.member(u) && 3960 (u->in(0) != ctrl || (!u->is_Region() && !u->is_Phi())) && 3961 (ctrl->Opcode() != Op_CatchProj || u->Opcode() != Op_CreateEx)) { 3962 Node* old_c = phase->ctrl_or_self(u); 3963 Node* c = old_c; 3964 if (c != ctrl || 3965 is_dominator_same_ctrl(old_c, wb, u, phase) || 3966 u->is_g1_marking_load()) { 3967 phase->igvn().rehash_node_delayed(u); 3968 int nb = u->replace_edge(ctrl, region); 3969 if (u->is_CFG()) { 3970 if (phase->idom(u) == ctrl) { 3971 phase->set_idom(u, region, phase->dom_depth(region)); 3972 } 3973 } else if (phase->get_ctrl(u) == ctrl) { 3974 assert(u != init_raw_mem, "should leave input raw mem above the barrier"); 3975 uses.push(u); 3976 } 3977 assert(nb == 1, "more than 1 ctrl input?"); 3978 --i, imax -= nb; 3979 } 3980 } 3981 } 3982 3983 if (wbproj != NULL) { 3984 phase->igvn().replace_input_of(wbproj, 0, phase->C->top()); 3985 phase->lazy_replace(wbproj, mem_phi); 3986 } 3987 if (unc != NULL) { 3988 for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) { 3989 Node* u = val->fast_out(i); 3990 Node* c = phase->ctrl_or_self(u); 3991 if (u != wb && (c != ctrl || is_dominator_same_ctrl(c, wb, u, phase))) { 3992 phase->igvn().rehash_node_delayed(u); 3993 int nb = u->replace_edge(val, out_val); 3994 --i, imax -= nb; 3995 } 3996 } 3997 if (val->outcnt() == 0) { 3998 phase->lazy_update(val, out_val); 3999 phase->igvn()._worklist.push(val); 4000 } 4001 } 4002 phase->lazy_replace(wb, out_val); 4003 4004 follow_barrier_uses(mem_phi, ctrl, uses, phase); 4005 follow_barrier_uses(out_val, ctrl, uses, phase); 4006 4007 for(uint next = 0; next < uses.size(); next++ ) { 4008 Node *n = uses.at(next); 4009 assert(phase->get_ctrl(n) == ctrl, "bad control"); 4010 assert(n != init_raw_mem, "should leave input raw mem above the barrier"); 4011 phase->set_ctrl(n, region); 4012 follow_barrier_uses(n, ctrl, uses, phase); 4013 } 4014 4015 // The slow path call produces memory: hook the raw memory phi 4016 // from the expanded write barrier with the rest of the graph 4017 // which may require adding memory phis at every post dominated 4018 // region and at enclosing loop heads. Use the memory state 4019 // collected in memory_nodes to fix the memory graph. Update that 4020 // memory state as we go. 4021 fix_raw_mem(ctrl,region, init_raw_mem, raw_mem_for_ctrl, raw_mem_phi, memory_nodes, uses, phase); 4022 assert(phase->C->shenandoah_barriers_count() == cnt - 1, "not replaced"); 4023 } 4024 4025 assert(phase->C->shenandoah_barriers_count() == 0, "all write barrier nodes should have been replaced"); 4026 } 4027 4028 void ShenandoahWriteBarrierNode::move_evacuation_test_out_of_loop(IfNode* iff, PhaseIdealLoop* phase) { 4029 // move test and its mem barriers out of the loop 4030 assert(is_evacuation_in_progress_test(iff), "inconsistent"); 4031 4032 IdealLoopTree *loop = phase->get_loop(iff); 4033 Node* loop_head = loop->_head; 4034 Node* entry_c = loop_head->in(LoopNode::EntryControl); 4035 4036 Node* load = iff->in(1)->in(1)->in(1)->in(1); 4037 assert(load->Opcode() == Op_LoadUB, "inconsistent"); 4038 Node* mem_ctrl = NULL; 4039 Node* mem = dom_mem(load->in(MemNode::Memory), loop_head, Compile::AliasIdxRaw, mem_ctrl, phase); 4040 phase->igvn().replace_input_of(load, MemNode::Memory, mem); 4041 phase->igvn().replace_input_of(load, 0, entry_c); 4042 phase->set_ctrl_and_loop(load, entry_c); 4043 } 4044 4045 void ShenandoahWriteBarrierNode::merge_back_to_back_evacuation_tests(Node* n, PhaseIdealLoop* phase) { 4046 if (phase->identical_backtoback_ifs(n)) { 4047 Node* n_ctrl = ShenandoahWriteBarrierNode::evacuation_in_progress_test_ctrl(n); 4048 if (phase->can_split_if(n_ctrl)) { 4049 IfNode* dom_if = phase->idom(n_ctrl)->as_If(); 4050 PhiNode* bolphi = PhiNode::make_blank(n_ctrl, n->in(1)); 4051 Node* proj_true = dom_if->proj_out(1); 4052 Node* proj_false = dom_if->proj_out(0); 4053 Node* con_true = phase->igvn().makecon(TypeInt::ONE); 4054 Node* con_false = phase->igvn().makecon(TypeInt::ZERO); 4055 4056 for (uint i = 1; i < n_ctrl->req(); i++) { 4057 if (phase->is_dominator(proj_true, n_ctrl->in(i))) { 4058 bolphi->init_req(i, con_true); 4059 } else { 4060 assert(phase->is_dominator(proj_false, n_ctrl->in(i)), "bad if"); 4061 bolphi->init_req(i, con_false); 4062 } 4063 } 4064 phase->register_new_node(bolphi, n_ctrl); 4065 phase->igvn().replace_input_of(n, 1, bolphi); 4066 phase->do_split_if(n); 4067 } 4068 } 4069 } 4070 4071 void ShenandoahWriteBarrierNode::optimize_after_expansion(const Node_List& evacuation_tests, const Node_List& gc_state_loads, Node_List &old_new, PhaseIdealLoop* phase) { 4072 bool progress; 4073 do { 4074 progress = false; 4075 for (uint i = 0; i < gc_state_loads.size(); i++) { 4076 Node* n = gc_state_loads.at(i); 4077 if (n->outcnt() != 0) { 4078 progress |= ShenandoahWriteBarrierNode::try_common_gc_state_load(n, phase); 4079 } 4080 } 4081 } while(progress); 4082 4083 for (uint i = 0; i < evacuation_tests.size(); i++) { 4084 Node* n = evacuation_tests.at(i); 4085 assert(is_evacuation_in_progress_test(n), "only evacuation test"); 4086 merge_back_to_back_evacuation_tests(n, phase); 4087 } 4088 if (!phase->C->major_progress()) { 4089 VectorSet seen(Thread::current()->resource_area()); 4090 for (uint i = 0; i < evacuation_tests.size(); i++) { 4091 Node* n = evacuation_tests.at(i); 4092 IdealLoopTree* loop = phase->get_loop(n); 4093 if (loop != phase->ltree_root() && 4094 loop->_child == NULL && 4095 !loop->_irreducible) { 4096 LoopNode* head = loop->_head->as_Loop(); 4097 if ((!head->is_CountedLoop() || head->as_CountedLoop()->is_main_loop() || head->as_CountedLoop()->is_normal_loop()) && 4098 !seen.test_set(head->_idx) && 4099 loop->policy_unswitching(phase)) { 4100 IfNode* iff = phase->find_unswitching_candidate(loop); 4101 if (iff != NULL && is_evacuation_in_progress_test(iff)) { 4102 if (head->is_strip_mined()) { 4103 head->verify_strip_mined(0); 4104 OuterStripMinedLoopNode* outer = head->as_CountedLoop()->outer_loop(); 4105 OuterStripMinedLoopEndNode* le = head->outer_loop_end(); 4106 Node* new_outer = new LoopNode(outer->in(LoopNode::EntryControl), outer->in(LoopNode::LoopBackControl)); 4107 phase->register_control(new_outer, phase->get_loop(outer), outer->in(LoopNode::EntryControl)); 4108 Node* new_le = new IfNode(le->in(0), le->in(1), le->_prob, le->_fcnt); 4109 phase->register_control(new_le, phase->get_loop(le), le->in(0)); 4110 phase->lazy_replace(outer, new_outer); 4111 phase->lazy_replace(le, new_le); 4112 head->clear_strip_mined(); 4113 } 4114 phase->do_unswitching(loop, old_new); 4115 } 4116 } 4117 } 4118 } 4119 } 4120 } 4121 4122 #ifdef ASSERT 4123 void ShenandoahBarrierNode::verify_raw_mem(RootNode* root) { 4124 const bool trace = false; 4125 ResourceMark rm; 4126 Unique_Node_List nodes; 4127 Unique_Node_List controls; 4128 Unique_Node_List memories; 4129 4130 nodes.push(root); 4131 for (uint next = 0; next < nodes.size(); next++) { 4132 Node *n = nodes.at(next); 4133 if (n->Opcode() == Op_CallLeafNoFP && n->as_Call()->_entry_point == StubRoutines::shenandoah_wb_C()) { 4134 controls.push(n); 4135 if (trace) { tty->print("XXXXXX verifying"); n->dump(); } 4136 for (uint next2 = 0; next2 < controls.size(); next2++) { 4137 Node *m = controls.at(next2); 4138 if (!m->is_Loop() || controls.member(m->in(LoopNode::EntryControl)) || 1) { 4139 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) { 4140 Node* u = m->fast_out(i); 4141 if (u->is_CFG() && !u->is_Root() && 4142 !(u->Opcode() == Op_CProj && u->in(0)->Opcode() == Op_NeverBranch && u->as_Proj()->_con == 1) && 4143 !(u->is_Region() && u->unique_ctrl_out()->Opcode() == Op_Halt)) { 4144 if (trace) { tty->print("XXXXXX pushing control"); u->dump(); } 4145 controls.push(u); 4146 } 4147 } 4148 } 4149 } 4150 memories.push(n->as_Call()->proj_out(TypeFunc::Memory)); 4151 for (uint next2 = 0; next2 < memories.size(); next2++) { 4152 Node *m = memories.at(next2); 4153 assert(m->bottom_type() == Type::MEMORY, ""); 4154 if (!m->is_Phi() || !m->in(0)->is_Loop() || controls.member(m->in(0)->in(LoopNode::EntryControl)) || 1) { 4155 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) { 4156 Node* u = m->fast_out(i); 4157 if (u->bottom_type() == Type::MEMORY && (u->is_Mem() || u->is_ClearArray())) { 4158 if (trace) { tty->print("XXXXXX pushing memory"); u->dump(); } 4159 memories.push(u); 4160 } else if (u->is_LoadStore()) { 4161 if (trace) { tty->print("XXXXXX pushing memory"); u->find_out_with(Op_SCMemProj)->dump(); } 4162 memories.push(u->find_out_with(Op_SCMemProj)); 4163 } else if (u->is_MergeMem() && u->as_MergeMem()->memory_at(Compile::AliasIdxRaw) == m) { 4164 if (trace) { tty->print("XXXXXX pushing memory"); u->dump(); } 4165 memories.push(u); 4166 } else if (u->is_Phi()) { 4167 assert(u->bottom_type() == Type::MEMORY, ""); 4168 if (u->adr_type() == TypeRawPtr::BOTTOM || u->adr_type() == TypePtr::BOTTOM) { 4169 assert(controls.member(u->in(0)), ""); 4170 if (trace) { tty->print("XXXXXX pushing memory"); u->dump(); } 4171 memories.push(u); 4172 } 4173 } else if (u->is_SafePoint() || u->is_MemBar()) { 4174 for (DUIterator_Fast jmax, j = u->fast_outs(jmax); j < jmax; j++) { 4175 Node* uu = u->fast_out(j); 4176 if (uu->bottom_type() == Type::MEMORY) { 4177 if (trace) { tty->print("XXXXXX pushing memory"); uu->dump(); } 4178 memories.push(uu); 4179 } 4180 } 4181 } 4182 } 4183 } 4184 } 4185 for (uint next2 = 0; next2 < controls.size(); next2++) { 4186 Node *m = controls.at(next2); 4187 if (m->is_Region()) { 4188 bool all_in = true; 4189 for (uint i = 1; i < m->req(); i++) { 4190 if (!controls.member(m->in(i))) { 4191 all_in = false; 4192 break; 4193 } 4194 } 4195 if (trace) { tty->print("XXX verifying %s", all_in ? "all in" : ""); m->dump(); } 4196 bool found_phi = false; 4197 for (DUIterator_Fast jmax, j = m->fast_outs(jmax); j < jmax && !found_phi; j++) { 4198 Node* u = m->fast_out(j); 4199 if (u->is_Phi() && memories.member(u)) { 4200 found_phi = true; 4201 for (uint i = 1; i < u->req() && found_phi; i++) { 4202 Node* k = u->in(i); 4203 if (memories.member(k) != controls.member(m->in(i))) { 4204 found_phi = false; 4205 } 4206 } 4207 } 4208 } 4209 assert(found_phi || all_in, ""); 4210 } 4211 } 4212 controls.clear(); 4213 memories.clear(); 4214 } 4215 for( uint i = 0; i < n->len(); ++i ) { 4216 Node *m = n->in(i); 4217 if (m != NULL) { 4218 nodes.push(m); 4219 } 4220 } 4221 } 4222 } 4223 #endif 4224 4225 static bool is_on_null_check_path(Block* b, Block* null_check_block) { 4226 if (null_check_block == NULL) { 4227 return false; 4228 } 4229 do { 4230 assert(null_check_block->_num_succs == 1, "only one succ on the path to unc"); 4231 if (b == null_check_block) { 4232 return true; 4233 } 4234 null_check_block = null_check_block->_succs[0]; 4235 } while(!null_check_block->head()->is_Root()); 4236 4237 return false; 4238 } 4239 4240 int PhaseCFG::replace_uses_with_shenandoah_barrier_helper(Node* n, Node* use, Node* val, Block* block, Block* null_check_block) { 4241 int nb = 0; 4242 Block* buse = get_block_for_node(use); 4243 if (is_on_null_check_path(buse, null_check_block)) { 4244 return 0; 4245 } 4246 if (use->is_Phi()) { 4247 for (uint j = 1; j < use->req(); j++) { 4248 if (use->in(j) == val) { 4249 Block* b = get_block_for_node(use->in(0)->in(j)); 4250 if ((block != b && block->dom_lca(b) == block) || 4251 block == b) { 4252 use->set_req(j, n); 4253 nb++; 4254 } 4255 } 4256 } 4257 } else { 4258 if ((block != buse && block->dom_lca(buse) == block) || 4259 (block == buse && !use->is_scheduled())) { 4260 // Let precedence edges alone (can confuse anti-dependence verification code) 4261 for (uint i = 0; i < use->req(); i++) { 4262 if (use->in(i) == val) { 4263 use->set_req(i, n); 4264 nb++; 4265 } 4266 } 4267 assert(nb > 0 || use->find_prec_edge(val) != -1, "no replacement?"); 4268 } 4269 } 4270 4271 return nb; 4272 } 4273 4274 void PhaseCFG::replace_uses_with_shenandoah_barrier(Node* n, Block* block, Node_List& worklist, GrowableArray<int>& ready_cnt, uint max_idx, uint& phi_cnt) { 4275 // Replace all uses of barrier's input that are dominated by the 4276 // barrier with the value returned by the barrier: no need to keep 4277 // both live. 4278 if (n->is_Mach() && n->as_Mach()->ideal_Opcode() == Op_ShenandoahReadBarrier) { 4279 MachNullCheckNode* null_check = NULL; 4280 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax && null_check == NULL; i++) { 4281 Node* use = n->fast_out(i); 4282 if (use->is_MachNullCheck()) { 4283 null_check = use->as_MachNullCheck(); 4284 } 4285 } 4286 Block* null_check_block = NULL; 4287 if (null_check != NULL) { 4288 Node* proj = null_check->find_out_with(Op_IfTrue); 4289 Node* head = proj->unique_out(); 4290 null_check_block = get_block_for_node(head); 4291 } 4292 4293 Node* val = n->in(ShenandoahBarrierNode::ValueIn); 4294 if (!val->bottom_type()->isa_narrowoop()) { 4295 for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) { 4296 Node* use = val->fast_out(i); 4297 if (use != n) { 4298 int nb = replace_uses_with_shenandoah_barrier_helper(n, use, val, block, null_check_block); 4299 if (nb > 0) { 4300 --i; imax -= nb; 4301 } 4302 } 4303 } 4304 } else { 4305 for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) { 4306 Node* u = val->fast_out(i); 4307 if (u->is_Mach() && u->as_Mach()->ideal_Opcode() == Op_DecodeN) { 4308 int projs = 0; 4309 for (DUIterator_Fast jmax, j = u->fast_outs(jmax); j < jmax; j++) { 4310 Node* uu = u->fast_out(j); 4311 assert(!uu->is_MachTemp(), ""); 4312 if (uu->is_MachProj() && uu->outcnt() == 0) { 4313 projs++; 4314 } else { 4315 int nb = replace_uses_with_shenandoah_barrier_helper(n, uu, u, block, null_check_block); 4316 if (nb > 0) { 4317 if (!u->is_scheduled()) { 4318 push_ready_nodes(n, uu, block, ready_cnt, worklist, max_idx, nb); 4319 } 4320 --j; jmax -= nb; 4321 } 4322 } 4323 } 4324 // The DecodeN may have gone dead 4325 if (u->outcnt() - projs == 0) { 4326 u->disconnect_inputs(NULL, C); 4327 Block* bu = get_block_for_node(u); 4328 unmap_node_from_block(u); 4329 if (bu == block) { 4330 if (u->is_scheduled()) { 4331 block->find_remove(u); 4332 phi_cnt--; 4333 } else { 4334 worklist.yank(u); 4335 block->remove_node(block->end_idx()-1); 4336 } 4337 } else { 4338 bu->find_remove(u); 4339 } 4340 for (DUIterator_Fast jmax, j = u->fast_outs(jmax); j < jmax; j++) { 4341 Node* uu = u->fast_out(j); 4342 assert(uu->is_MachProj() && uu->outcnt() == 0, ""); 4343 assert(bu == get_block_for_node(uu), ""); 4344 uu->disconnect_inputs(NULL, C); 4345 --j; --jmax; 4346 unmap_node_from_block(uu); 4347 if (bu == block) { 4348 if (u->is_scheduled()) { 4349 block->find_remove(uu); 4350 phi_cnt--; 4351 } else { 4352 worklist.yank(uu); 4353 block->remove_node(block->end_idx()-1); 4354 } 4355 } else { 4356 bu->find_remove(uu); 4357 } 4358 assert(uu->is_scheduled() == u->is_scheduled(), ""); 4359 } 4360 --i; --imax; 4361 } 4362 } 4363 } 4364 } 4365 } 4366 }