1 /* 2 * Copyright (c) 2015, 2019, Red Hat, Inc. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 27 #include "gc/shenandoah/c2/shenandoahSupport.hpp" 28 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp" 29 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp" 30 #include "gc/shenandoah/shenandoahForwarding.hpp" 31 #include "gc/shenandoah/shenandoahHeap.hpp" 32 #include "gc/shenandoah/shenandoahHeapRegion.hpp" 33 #include "gc/shenandoah/shenandoahRuntime.hpp" 34 #include "gc/shenandoah/shenandoahThreadLocalData.hpp" 35 #include "opto/arraycopynode.hpp" 36 #include "opto/block.hpp" 37 #include "opto/callnode.hpp" 38 #include "opto/castnode.hpp" 39 #include "opto/movenode.hpp" 40 #include "opto/phaseX.hpp" 41 #include "opto/rootnode.hpp" 42 #include "opto/runtime.hpp" 43 #include "opto/subnode.hpp" 44 45 bool ShenandoahBarrierC2Support::expand(Compile* C, PhaseIterGVN& igvn) { 46 ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state(); 47 if ((state->enqueue_barriers_count() + 48 state->load_reference_barriers_count()) > 0) { 49 bool attempt_more_loopopts = ShenandoahLoopOptsAfterExpansion; 50 C->clear_major_progress(); 51 PhaseIdealLoop ideal_loop(igvn, LoopOptsShenandoahExpand); 52 if (C->failing()) return false; 53 PhaseIdealLoop::verify(igvn); 54 DEBUG_ONLY(verify_raw_mem(C->root());) 55 if (attempt_more_loopopts) { 56 C->set_major_progress(); 57 if (!C->optimize_loops(igvn, LoopOptsShenandoahPostExpand)) { 58 return false; 59 } 60 C->clear_major_progress(); 61 if (C->range_check_cast_count() > 0) { 62 // No more loop optimizations. Remove all range check dependent CastIINodes. 63 C->remove_range_check_casts(igvn); 64 igvn.optimize(); 65 } 66 } 67 } 68 return true; 69 } 70 71 bool ShenandoahBarrierC2Support::is_gc_state_test(Node* iff, int mask) { 72 if (!UseShenandoahGC) { 73 return false; 74 } 75 assert(iff->is_If(), "bad input"); 76 if (iff->Opcode() != Op_If) { 77 return false; 78 } 79 Node* bol = iff->in(1); 80 if (!bol->is_Bool() || bol->as_Bool()->_test._test != BoolTest::ne) { 81 return false; 82 } 83 Node* cmp = bol->in(1); 84 if (cmp->Opcode() != Op_CmpI) { 85 return false; 86 } 87 Node* in1 = cmp->in(1); 88 Node* in2 = cmp->in(2); 89 if (in2->find_int_con(-1) != 0) { 90 return false; 91 } 92 if (in1->Opcode() != Op_AndI) { 93 return false; 94 } 95 in2 = in1->in(2); 96 if (in2->find_int_con(-1) != mask) { 97 return false; 98 } 99 in1 = in1->in(1); 100 101 return is_gc_state_load(in1); 102 } 103 104 bool ShenandoahBarrierC2Support::is_heap_stable_test(Node* iff) { 105 return is_gc_state_test(iff, ShenandoahHeap::HAS_FORWARDED); 106 } 107 108 bool ShenandoahBarrierC2Support::is_gc_state_load(Node *n) { 109 if (!UseShenandoahGC) { 110 return false; 111 } 112 if (n->Opcode() != Op_LoadB && n->Opcode() != Op_LoadUB) { 113 return false; 114 } 115 Node* addp = n->in(MemNode::Address); 116 if (!addp->is_AddP()) { 117 return false; 118 } 119 Node* base = addp->in(AddPNode::Address); 120 Node* off = addp->in(AddPNode::Offset); 121 if (base->Opcode() != Op_ThreadLocal) { 122 return false; 123 } 124 if (off->find_intptr_t_con(-1) != in_bytes(ShenandoahThreadLocalData::gc_state_offset())) { 125 return false; 126 } 127 return true; 128 } 129 130 bool ShenandoahBarrierC2Support::has_safepoint_between(Node* start, Node* stop, PhaseIdealLoop *phase) { 131 assert(phase->is_dominator(stop, start), "bad inputs"); 132 ResourceMark rm; 133 Unique_Node_List wq; 134 wq.push(start); 135 for (uint next = 0; next < wq.size(); next++) { 136 Node *m = wq.at(next); 137 if (m == stop) { 138 continue; 139 } 140 if (m->is_SafePoint() && !m->is_CallLeaf()) { 141 return true; 142 } 143 if (m->is_Region()) { 144 for (uint i = 1; i < m->req(); i++) { 145 wq.push(m->in(i)); 146 } 147 } else { 148 wq.push(m->in(0)); 149 } 150 } 151 return false; 152 } 153 154 #ifdef ASSERT 155 bool ShenandoahBarrierC2Support::verify_helper(Node* in, Node_Stack& phis, VectorSet& visited, verify_type t, bool trace, Unique_Node_List& barriers_used) { 156 assert(phis.size() == 0, ""); 157 158 while (true) { 159 if (in->bottom_type() == TypePtr::NULL_PTR) { 160 if (trace) {tty->print_cr("NULL");} 161 } else if (!in->bottom_type()->make_ptr()->make_oopptr()) { 162 if (trace) {tty->print_cr("Non oop");} 163 } else { 164 if (in->is_ConstraintCast()) { 165 in = in->in(1); 166 continue; 167 } else if (in->is_AddP()) { 168 assert(!in->in(AddPNode::Address)->is_top(), "no raw memory access"); 169 in = in->in(AddPNode::Address); 170 continue; 171 } else if (in->is_Con()) { 172 if (trace) { 173 tty->print("Found constant"); 174 in->dump(); 175 } 176 } else if (in->Opcode() == Op_Parm) { 177 if (trace) { 178 tty->print("Found argument"); 179 } 180 } else if (in->Opcode() == Op_CreateEx) { 181 if (trace) { 182 tty->print("Found create-exception"); 183 } 184 } else if (in->Opcode() == Op_LoadP && in->adr_type() == TypeRawPtr::BOTTOM) { 185 if (trace) { 186 tty->print("Found raw LoadP (OSR argument?)"); 187 } 188 } else if (in->Opcode() == Op_ShenandoahLoadReferenceBarrier) { 189 if (t == ShenandoahOopStore) { 190 uint i = 0; 191 for (; i < phis.size(); i++) { 192 Node* n = phis.node_at(i); 193 if (n->Opcode() == Op_ShenandoahEnqueueBarrier) { 194 break; 195 } 196 } 197 if (i == phis.size()) { 198 return false; 199 } 200 } 201 barriers_used.push(in); 202 if (trace) {tty->print("Found barrier"); in->dump();} 203 } else if (in->Opcode() == Op_ShenandoahEnqueueBarrier) { 204 if (t != ShenandoahOopStore) { 205 in = in->in(1); 206 continue; 207 } 208 if (trace) {tty->print("Found enqueue barrier"); in->dump();} 209 phis.push(in, in->req()); 210 in = in->in(1); 211 continue; 212 } else if (in->is_Proj() && in->in(0)->is_Allocate()) { 213 if (trace) { 214 tty->print("Found alloc"); 215 in->in(0)->dump(); 216 } 217 } else if (in->is_Proj() && (in->in(0)->Opcode() == Op_CallStaticJava || in->in(0)->Opcode() == Op_CallDynamicJava)) { 218 if (trace) { 219 tty->print("Found Java call"); 220 } 221 } else if (in->is_Phi()) { 222 if (!visited.test_set(in->_idx)) { 223 if (trace) {tty->print("Pushed phi:"); in->dump();} 224 phis.push(in, 2); 225 in = in->in(1); 226 continue; 227 } 228 if (trace) {tty->print("Already seen phi:"); in->dump();} 229 } else if (in->Opcode() == Op_CMoveP || in->Opcode() == Op_CMoveN) { 230 if (!visited.test_set(in->_idx)) { 231 if (trace) {tty->print("Pushed cmovep:"); in->dump();} 232 phis.push(in, CMoveNode::IfTrue); 233 in = in->in(CMoveNode::IfFalse); 234 continue; 235 } 236 if (trace) {tty->print("Already seen cmovep:"); in->dump();} 237 } else if (in->Opcode() == Op_EncodeP || in->Opcode() == Op_DecodeN) { 238 in = in->in(1); 239 continue; 240 } else { 241 return false; 242 } 243 } 244 bool cont = false; 245 while (phis.is_nonempty()) { 246 uint idx = phis.index(); 247 Node* phi = phis.node(); 248 if (idx >= phi->req()) { 249 if (trace) {tty->print("Popped phi:"); phi->dump();} 250 phis.pop(); 251 continue; 252 } 253 if (trace) {tty->print("Next entry(%d) for phi:", idx); phi->dump();} 254 in = phi->in(idx); 255 phis.set_index(idx+1); 256 cont = true; 257 break; 258 } 259 if (!cont) { 260 break; 261 } 262 } 263 return true; 264 } 265 266 void ShenandoahBarrierC2Support::report_verify_failure(const char* msg, Node* n1, Node* n2) { 267 if (n1 != NULL) { 268 n1->dump(+10); 269 } 270 if (n2 != NULL) { 271 n2->dump(+10); 272 } 273 fatal("%s", msg); 274 } 275 276 void ShenandoahBarrierC2Support::verify(RootNode* root) { 277 ResourceMark rm; 278 Unique_Node_List wq; 279 GrowableArray<Node*> barriers; 280 Unique_Node_List barriers_used; 281 Node_Stack phis(0); 282 VectorSet visited(Thread::current()->resource_area()); 283 const bool trace = false; 284 const bool verify_no_useless_barrier = false; 285 286 wq.push(root); 287 for (uint next = 0; next < wq.size(); next++) { 288 Node *n = wq.at(next); 289 if (n->is_Load()) { 290 const bool trace = false; 291 if (trace) {tty->print("Verifying"); n->dump();} 292 if (n->Opcode() == Op_LoadRange || n->Opcode() == Op_LoadKlass || n->Opcode() == Op_LoadNKlass) { 293 if (trace) {tty->print_cr("Load range/klass");} 294 } else { 295 const TypePtr* adr_type = n->as_Load()->adr_type(); 296 297 if (adr_type->isa_oopptr() && adr_type->is_oopptr()->offset() == oopDesc::mark_offset_in_bytes()) { 298 if (trace) {tty->print_cr("Mark load");} 299 } else if (adr_type->isa_instptr() && 300 adr_type->is_instptr()->klass()->is_subtype_of(Compile::current()->env()->Reference_klass()) && 301 adr_type->is_instptr()->offset() == java_lang_ref_Reference::referent_offset) { 302 if (trace) {tty->print_cr("Reference.get()");} 303 } else if (!verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahLoad, trace, barriers_used)) { 304 report_verify_failure("Shenandoah verification: Load should have barriers", n); 305 } 306 } 307 } else if (n->is_Store()) { 308 const bool trace = false; 309 310 if (trace) {tty->print("Verifying"); n->dump();} 311 if (n->in(MemNode::ValueIn)->bottom_type()->make_oopptr()) { 312 Node* adr = n->in(MemNode::Address); 313 bool verify = true; 314 315 if (adr->is_AddP() && adr->in(AddPNode::Base)->is_top()) { 316 adr = adr->in(AddPNode::Address); 317 if (adr->is_AddP()) { 318 assert(adr->in(AddPNode::Base)->is_top(), ""); 319 adr = adr->in(AddPNode::Address); 320 if (adr->Opcode() == Op_LoadP && 321 adr->in(MemNode::Address)->in(AddPNode::Base)->is_top() && 322 adr->in(MemNode::Address)->in(AddPNode::Address)->Opcode() == Op_ThreadLocal && 323 adr->in(MemNode::Address)->in(AddPNode::Offset)->find_intptr_t_con(-1) == in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset())) { 324 if (trace) {tty->print_cr("SATB prebarrier");} 325 verify = false; 326 } 327 } 328 } 329 330 if (verify && !verify_helper(n->in(MemNode::ValueIn), phis, visited, ShenandoahStoreValEnqueueBarrier ? ShenandoahOopStore : ShenandoahValue, trace, barriers_used)) { 331 report_verify_failure("Shenandoah verification: Store should have barriers", n); 332 } 333 } 334 if (!verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahStore, trace, barriers_used)) { 335 report_verify_failure("Shenandoah verification: Store (address) should have barriers", n); 336 } 337 } else if (n->Opcode() == Op_CmpP) { 338 const bool trace = false; 339 340 Node* in1 = n->in(1); 341 Node* in2 = n->in(2); 342 if (in1->bottom_type()->isa_oopptr()) { 343 if (trace) {tty->print("Verifying"); n->dump();} 344 345 bool mark_inputs = false; 346 if (in1->bottom_type() == TypePtr::NULL_PTR || in2->bottom_type() == TypePtr::NULL_PTR || 347 (in1->is_Con() || in2->is_Con())) { 348 if (trace) {tty->print_cr("Comparison against a constant");} 349 mark_inputs = true; 350 } else if ((in1->is_CheckCastPP() && in1->in(1)->is_Proj() && in1->in(1)->in(0)->is_Allocate()) || 351 (in2->is_CheckCastPP() && in2->in(1)->is_Proj() && in2->in(1)->in(0)->is_Allocate())) { 352 if (trace) {tty->print_cr("Comparison with newly alloc'ed object");} 353 mark_inputs = true; 354 } else { 355 assert(in2->bottom_type()->isa_oopptr(), ""); 356 357 if (!verify_helper(in1, phis, visited, ShenandoahStore, trace, barriers_used) || 358 !verify_helper(in2, phis, visited, ShenandoahStore, trace, barriers_used)) { 359 report_verify_failure("Shenandoah verification: Cmp should have barriers", n); 360 } 361 } 362 if (verify_no_useless_barrier && 363 mark_inputs && 364 (!verify_helper(in1, phis, visited, ShenandoahValue, trace, barriers_used) || 365 !verify_helper(in2, phis, visited, ShenandoahValue, trace, barriers_used))) { 366 phis.clear(); 367 visited.reset(); 368 } 369 } 370 } else if (n->is_LoadStore()) { 371 if (n->in(MemNode::ValueIn)->bottom_type()->make_ptr() && 372 !verify_helper(n->in(MemNode::ValueIn), phis, visited, ShenandoahStoreValEnqueueBarrier ? ShenandoahOopStore : ShenandoahValue, trace, barriers_used)) { 373 report_verify_failure("Shenandoah verification: LoadStore (value) should have barriers", n); 374 } 375 376 if (n->in(MemNode::Address)->bottom_type()->make_oopptr() && !verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahStore, trace, barriers_used)) { 377 report_verify_failure("Shenandoah verification: LoadStore (address) should have barriers", n); 378 } 379 } else if (n->Opcode() == Op_CallLeafNoFP || n->Opcode() == Op_CallLeaf) { 380 CallNode* call = n->as_Call(); 381 382 static struct { 383 const char* name; 384 struct { 385 int pos; 386 verify_type t; 387 } args[6]; 388 } calls[] = { 389 "aescrypt_encryptBlock", 390 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad }, 391 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 392 "aescrypt_decryptBlock", 393 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad }, 394 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 395 "multiplyToLen", 396 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad }, { TypeFunc::Parms+4, ShenandoahStore }, 397 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 398 "squareToLen", 399 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad }, { -1, ShenandoahNone}, 400 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 401 "montgomery_multiply", 402 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad }, 403 { TypeFunc::Parms+6, ShenandoahStore }, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 404 "montgomery_square", 405 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+5, ShenandoahStore }, 406 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 407 "mulAdd", 408 { { TypeFunc::Parms, ShenandoahStore }, { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone}, 409 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 410 "vectorizedMismatch", 411 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone}, 412 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 413 "updateBytesCRC32", 414 { { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone}, 415 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 416 "updateBytesAdler32", 417 { { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone}, 418 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 419 "updateBytesCRC32C", 420 { { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahLoad}, { -1, ShenandoahNone}, 421 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 422 "counterMode_AESCrypt", 423 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad }, 424 { TypeFunc::Parms+3, ShenandoahStore }, { TypeFunc::Parms+5, ShenandoahStore }, { TypeFunc::Parms+6, ShenandoahStore } }, 425 "cipherBlockChaining_encryptAESCrypt", 426 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad }, 427 { TypeFunc::Parms+3, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 428 "cipherBlockChaining_decryptAESCrypt", 429 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad }, 430 { TypeFunc::Parms+3, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 431 "shenandoah_clone_barrier", 432 { { TypeFunc::Parms, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone}, 433 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 434 "ghash_processBlocks", 435 { { TypeFunc::Parms, ShenandoahStore }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad }, 436 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 437 "sha1_implCompress", 438 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone }, 439 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 440 "sha256_implCompress", 441 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone }, 442 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 443 "sha512_implCompress", 444 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone }, 445 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 446 "sha1_implCompressMB", 447 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone }, 448 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 449 "sha256_implCompressMB", 450 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone }, 451 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 452 "sha512_implCompressMB", 453 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone }, 454 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 455 "encodeBlock", 456 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahStore }, { -1, ShenandoahNone }, 457 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 458 }; 459 460 if (call->is_call_to_arraycopystub()) { 461 Node* dest = NULL; 462 const TypeTuple* args = n->as_Call()->_tf->domain(); 463 for (uint i = TypeFunc::Parms, j = 0; i < args->cnt(); i++) { 464 if (args->field_at(i)->isa_ptr()) { 465 j++; 466 if (j == 2) { 467 dest = n->in(i); 468 break; 469 } 470 } 471 } 472 if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahLoad, trace, barriers_used) || 473 !verify_helper(dest, phis, visited, ShenandoahStore, trace, barriers_used)) { 474 report_verify_failure("Shenandoah verification: ArrayCopy should have barriers", n); 475 } 476 } else if (strlen(call->_name) > 5 && 477 !strcmp(call->_name + strlen(call->_name) - 5, "_fill")) { 478 if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahStore, trace, barriers_used)) { 479 report_verify_failure("Shenandoah verification: _fill should have barriers", n); 480 } 481 } else if (!strcmp(call->_name, "shenandoah_wb_pre")) { 482 // skip 483 } else { 484 const int calls_len = sizeof(calls) / sizeof(calls[0]); 485 int i = 0; 486 for (; i < calls_len; i++) { 487 if (!strcmp(calls[i].name, call->_name)) { 488 break; 489 } 490 } 491 if (i != calls_len) { 492 const uint args_len = sizeof(calls[0].args) / sizeof(calls[0].args[0]); 493 for (uint j = 0; j < args_len; j++) { 494 int pos = calls[i].args[j].pos; 495 if (pos == -1) { 496 break; 497 } 498 if (!verify_helper(call->in(pos), phis, visited, calls[i].args[j].t, trace, barriers_used)) { 499 report_verify_failure("Shenandoah verification: intrinsic calls should have barriers", n); 500 } 501 } 502 for (uint j = TypeFunc::Parms; j < call->req(); j++) { 503 if (call->in(j)->bottom_type()->make_ptr() && 504 call->in(j)->bottom_type()->make_ptr()->isa_oopptr()) { 505 uint k = 0; 506 for (; k < args_len && calls[i].args[k].pos != (int)j; k++); 507 if (k == args_len) { 508 fatal("arg %d for call %s not covered", j, call->_name); 509 } 510 } 511 } 512 } else { 513 for (uint j = TypeFunc::Parms; j < call->req(); j++) { 514 if (call->in(j)->bottom_type()->make_ptr() && 515 call->in(j)->bottom_type()->make_ptr()->isa_oopptr()) { 516 fatal("%s not covered", call->_name); 517 } 518 } 519 } 520 } 521 } else if (n->Opcode() == Op_ShenandoahEnqueueBarrier || n->Opcode() == Op_ShenandoahLoadReferenceBarrier) { 522 // skip 523 } else if (n->is_AddP() 524 || n->is_Phi() 525 || n->is_ConstraintCast() 526 || n->Opcode() == Op_Return 527 || n->Opcode() == Op_CMoveP 528 || n->Opcode() == Op_CMoveN 529 || n->Opcode() == Op_Rethrow 530 || n->is_MemBar() 531 || n->Opcode() == Op_Conv2B 532 || n->Opcode() == Op_SafePoint 533 || n->is_CallJava() 534 || n->Opcode() == Op_Unlock 535 || n->Opcode() == Op_EncodeP 536 || n->Opcode() == Op_DecodeN) { 537 // nothing to do 538 } else { 539 static struct { 540 int opcode; 541 struct { 542 int pos; 543 verify_type t; 544 } inputs[2]; 545 } others[] = { 546 Op_FastLock, 547 { { 1, ShenandoahLoad }, { -1, ShenandoahNone} }, 548 Op_Lock, 549 { { TypeFunc::Parms, ShenandoahLoad }, { -1, ShenandoahNone} }, 550 Op_ArrayCopy, 551 { { ArrayCopyNode::Src, ShenandoahLoad }, { ArrayCopyNode::Dest, ShenandoahStore } }, 552 Op_StrCompressedCopy, 553 { { 2, ShenandoahLoad }, { 3, ShenandoahStore } }, 554 Op_StrInflatedCopy, 555 { { 2, ShenandoahLoad }, { 3, ShenandoahStore } }, 556 Op_AryEq, 557 { { 2, ShenandoahLoad }, { 3, ShenandoahLoad } }, 558 Op_StrIndexOf, 559 { { 2, ShenandoahLoad }, { 4, ShenandoahLoad } }, 560 Op_StrComp, 561 { { 2, ShenandoahLoad }, { 4, ShenandoahLoad } }, 562 Op_StrEquals, 563 { { 2, ShenandoahLoad }, { 3, ShenandoahLoad } }, 564 Op_EncodeISOArray, 565 { { 2, ShenandoahLoad }, { 3, ShenandoahStore } }, 566 Op_HasNegatives, 567 { { 2, ShenandoahLoad }, { -1, ShenandoahNone} }, 568 Op_CastP2X, 569 { { 1, ShenandoahLoad }, { -1, ShenandoahNone} }, 570 Op_StrIndexOfChar, 571 { { 2, ShenandoahLoad }, { -1, ShenandoahNone } }, 572 }; 573 574 const int others_len = sizeof(others) / sizeof(others[0]); 575 int i = 0; 576 for (; i < others_len; i++) { 577 if (others[i].opcode == n->Opcode()) { 578 break; 579 } 580 } 581 uint stop = n->is_Call() ? n->as_Call()->tf()->domain()->cnt() : n->req(); 582 if (i != others_len) { 583 const uint inputs_len = sizeof(others[0].inputs) / sizeof(others[0].inputs[0]); 584 for (uint j = 0; j < inputs_len; j++) { 585 int pos = others[i].inputs[j].pos; 586 if (pos == -1) { 587 break; 588 } 589 if (!verify_helper(n->in(pos), phis, visited, others[i].inputs[j].t, trace, barriers_used)) { 590 report_verify_failure("Shenandoah verification: intrinsic calls should have barriers", n); 591 } 592 } 593 for (uint j = 1; j < stop; j++) { 594 if (n->in(j) != NULL && n->in(j)->bottom_type()->make_ptr() && 595 n->in(j)->bottom_type()->make_ptr()->make_oopptr()) { 596 uint k = 0; 597 for (; k < inputs_len && others[i].inputs[k].pos != (int)j; k++); 598 if (k == inputs_len) { 599 fatal("arg %d for node %s not covered", j, n->Name()); 600 } 601 } 602 } 603 } else { 604 for (uint j = 1; j < stop; j++) { 605 if (n->in(j) != NULL && n->in(j)->bottom_type()->make_ptr() && 606 n->in(j)->bottom_type()->make_ptr()->make_oopptr()) { 607 fatal("%s not covered", n->Name()); 608 } 609 } 610 } 611 } 612 613 if (n->is_SafePoint()) { 614 SafePointNode* sfpt = n->as_SafePoint(); 615 if (verify_no_useless_barrier && sfpt->jvms() != NULL) { 616 for (uint i = sfpt->jvms()->scloff(); i < sfpt->jvms()->endoff(); i++) { 617 if (!verify_helper(sfpt->in(i), phis, visited, ShenandoahLoad, trace, barriers_used)) { 618 phis.clear(); 619 visited.reset(); 620 } 621 } 622 } 623 } 624 } 625 626 if (verify_no_useless_barrier) { 627 for (int i = 0; i < barriers.length(); i++) { 628 Node* n = barriers.at(i); 629 if (!barriers_used.member(n)) { 630 tty->print("XXX useless barrier"); n->dump(-2); 631 ShouldNotReachHere(); 632 } 633 } 634 } 635 } 636 #endif 637 638 bool ShenandoahBarrierC2Support::is_dominator_same_ctrl(Node* c, Node* d, Node* n, PhaseIdealLoop* phase) { 639 // That both nodes have the same control is not sufficient to prove 640 // domination, verify that there's no path from d to n 641 ResourceMark rm; 642 Unique_Node_List wq; 643 wq.push(d); 644 for (uint next = 0; next < wq.size(); next++) { 645 Node *m = wq.at(next); 646 if (m == n) { 647 return false; 648 } 649 if (m->is_Phi() && m->in(0)->is_Loop()) { 650 assert(phase->ctrl_or_self(m->in(LoopNode::EntryControl)) != c, "following loop entry should lead to new control"); 651 } else { 652 if (m->is_Store() || m->is_LoadStore()) { 653 // Take anti-dependencies into account 654 Node* mem = m->in(MemNode::Memory); 655 for (DUIterator_Fast imax, i = mem->fast_outs(imax); i < imax; i++) { 656 Node* u = mem->fast_out(i); 657 if (u->is_Load() && phase->C->can_alias(m->adr_type(), phase->C->get_alias_index(u->adr_type())) && 658 phase->ctrl_or_self(u) == c) { 659 wq.push(u); 660 } 661 } 662 } 663 for (uint i = 0; i < m->req(); i++) { 664 if (m->in(i) != NULL && phase->ctrl_or_self(m->in(i)) == c) { 665 wq.push(m->in(i)); 666 } 667 } 668 } 669 } 670 return true; 671 } 672 673 bool ShenandoahBarrierC2Support::is_dominator(Node* d_c, Node* n_c, Node* d, Node* n, PhaseIdealLoop* phase) { 674 if (d_c != n_c) { 675 return phase->is_dominator(d_c, n_c); 676 } 677 return is_dominator_same_ctrl(d_c, d, n, phase); 678 } 679 680 Node* next_mem(Node* mem, int alias) { 681 Node* res = NULL; 682 if (mem->is_Proj()) { 683 res = mem->in(0); 684 } else if (mem->is_SafePoint() || mem->is_MemBar()) { 685 res = mem->in(TypeFunc::Memory); 686 } else if (mem->is_Phi()) { 687 res = mem->in(1); 688 } else if (mem->is_MergeMem()) { 689 res = mem->as_MergeMem()->memory_at(alias); 690 } else if (mem->is_Store() || mem->is_LoadStore() || mem->is_ClearArray()) { 691 assert(alias = Compile::AliasIdxRaw, "following raw memory can't lead to a barrier"); 692 res = mem->in(MemNode::Memory); 693 } else { 694 #ifdef ASSERT 695 mem->dump(); 696 #endif 697 ShouldNotReachHere(); 698 } 699 return res; 700 } 701 702 Node* ShenandoahBarrierC2Support::no_branches(Node* c, Node* dom, bool allow_one_proj, PhaseIdealLoop* phase) { 703 Node* iffproj = NULL; 704 while (c != dom) { 705 Node* next = phase->idom(c); 706 assert(next->unique_ctrl_out() == c || c->is_Proj() || c->is_Region(), "multiple control flow out but no proj or region?"); 707 if (c->is_Region()) { 708 ResourceMark rm; 709 Unique_Node_List wq; 710 wq.push(c); 711 for (uint i = 0; i < wq.size(); i++) { 712 Node *n = wq.at(i); 713 if (n == next) { 714 continue; 715 } 716 if (n->is_Region()) { 717 for (uint j = 1; j < n->req(); j++) { 718 wq.push(n->in(j)); 719 } 720 } else { 721 wq.push(n->in(0)); 722 } 723 } 724 for (uint i = 0; i < wq.size(); i++) { 725 Node *n = wq.at(i); 726 assert(n->is_CFG(), ""); 727 if (n->is_Multi()) { 728 for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) { 729 Node* u = n->fast_out(j); 730 if (u->is_CFG()) { 731 if (!wq.member(u) && !u->as_Proj()->is_uncommon_trap_proj(Deoptimization::Reason_none)) { 732 return NodeSentinel; 733 } 734 } 735 } 736 } 737 } 738 } else if (c->is_Proj()) { 739 if (c->is_IfProj()) { 740 if (c->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none) != NULL) { 741 // continue; 742 } else { 743 if (!allow_one_proj) { 744 return NodeSentinel; 745 } 746 if (iffproj == NULL) { 747 iffproj = c; 748 } else { 749 return NodeSentinel; 750 } 751 } 752 } else if (c->Opcode() == Op_JumpProj) { 753 return NodeSentinel; // unsupported 754 } else if (c->Opcode() == Op_CatchProj) { 755 return NodeSentinel; // unsupported 756 } else if (c->Opcode() == Op_CProj && next->Opcode() == Op_NeverBranch) { 757 return NodeSentinel; // unsupported 758 } else { 759 assert(next->unique_ctrl_out() == c, "unsupported branch pattern"); 760 } 761 } 762 c = next; 763 } 764 return iffproj; 765 } 766 767 Node* ShenandoahBarrierC2Support::dom_mem(Node* mem, Node* ctrl, int alias, Node*& mem_ctrl, PhaseIdealLoop* phase) { 768 ResourceMark rm; 769 VectorSet wq(Thread::current()->resource_area()); 770 wq.set(mem->_idx); 771 mem_ctrl = phase->ctrl_or_self(mem); 772 while (!phase->is_dominator(mem_ctrl, ctrl) || mem_ctrl == ctrl) { 773 mem = next_mem(mem, alias); 774 if (wq.test_set(mem->_idx)) { 775 return NULL; 776 } 777 mem_ctrl = phase->ctrl_or_self(mem); 778 } 779 if (mem->is_MergeMem()) { 780 mem = mem->as_MergeMem()->memory_at(alias); 781 mem_ctrl = phase->ctrl_or_self(mem); 782 } 783 return mem; 784 } 785 786 Node* ShenandoahBarrierC2Support::find_bottom_mem(Node* ctrl, PhaseIdealLoop* phase) { 787 Node* mem = NULL; 788 Node* c = ctrl; 789 do { 790 if (c->is_Region()) { 791 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax && mem == NULL; i++) { 792 Node* u = c->fast_out(i); 793 if (u->is_Phi() && u->bottom_type() == Type::MEMORY) { 794 if (u->adr_type() == TypePtr::BOTTOM) { 795 mem = u; 796 } 797 } 798 } 799 } else { 800 if (c->is_Call() && c->as_Call()->adr_type() != NULL) { 801 CallProjections projs; 802 c->as_Call()->extract_projections(&projs, true, false); 803 if (projs.fallthrough_memproj != NULL) { 804 if (projs.fallthrough_memproj->adr_type() == TypePtr::BOTTOM) { 805 if (projs.catchall_memproj == NULL) { 806 mem = projs.fallthrough_memproj; 807 } else { 808 if (phase->is_dominator(projs.fallthrough_catchproj, ctrl)) { 809 mem = projs.fallthrough_memproj; 810 } else { 811 assert(phase->is_dominator(projs.catchall_catchproj, ctrl), "one proj must dominate barrier"); 812 mem = projs.catchall_memproj; 813 } 814 } 815 } 816 } else { 817 Node* proj = c->as_Call()->proj_out(TypeFunc::Memory); 818 if (proj != NULL && 819 proj->adr_type() == TypePtr::BOTTOM) { 820 mem = proj; 821 } 822 } 823 } else { 824 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) { 825 Node* u = c->fast_out(i); 826 if (u->is_Proj() && 827 u->bottom_type() == Type::MEMORY && 828 u->adr_type() == TypePtr::BOTTOM) { 829 assert(c->is_SafePoint() || c->is_MemBar() || c->is_Start(), ""); 830 assert(mem == NULL, "only one proj"); 831 mem = u; 832 } 833 } 834 assert(!c->is_Call() || c->as_Call()->adr_type() != NULL || mem == NULL, "no mem projection expected"); 835 } 836 } 837 c = phase->idom(c); 838 } while (mem == NULL); 839 return mem; 840 } 841 842 void ShenandoahBarrierC2Support::follow_barrier_uses(Node* n, Node* ctrl, Unique_Node_List& uses, PhaseIdealLoop* phase) { 843 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { 844 Node* u = n->fast_out(i); 845 if (!u->is_CFG() && phase->get_ctrl(u) == ctrl && (!u->is_Phi() || !u->in(0)->is_Loop() || u->in(LoopNode::LoopBackControl) != n)) { 846 uses.push(u); 847 } 848 } 849 } 850 851 static void hide_strip_mined_loop(OuterStripMinedLoopNode* outer, CountedLoopNode* inner, PhaseIdealLoop* phase) { 852 OuterStripMinedLoopEndNode* le = inner->outer_loop_end(); 853 Node* new_outer = new LoopNode(outer->in(LoopNode::EntryControl), outer->in(LoopNode::LoopBackControl)); 854 phase->register_control(new_outer, phase->get_loop(outer), outer->in(LoopNode::EntryControl)); 855 Node* new_le = new IfNode(le->in(0), le->in(1), le->_prob, le->_fcnt); 856 phase->register_control(new_le, phase->get_loop(le), le->in(0)); 857 phase->lazy_replace(outer, new_outer); 858 phase->lazy_replace(le, new_le); 859 inner->clear_strip_mined(); 860 } 861 862 void ShenandoahBarrierC2Support::test_gc_state(Node*& ctrl, Node* raw_mem, Node*& test_fail_ctrl, 863 PhaseIdealLoop* phase, int flags) { 864 PhaseIterGVN& igvn = phase->igvn(); 865 Node* old_ctrl = ctrl; 866 867 Node* thread = new ThreadLocalNode(); 868 Node* gc_state_offset = igvn.MakeConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset())); 869 Node* gc_state_addr = new AddPNode(phase->C->top(), thread, gc_state_offset); 870 Node* gc_state = new LoadBNode(old_ctrl, raw_mem, gc_state_addr, 871 DEBUG_ONLY(phase->C->get_adr_type(Compile::AliasIdxRaw)) NOT_DEBUG(NULL), 872 TypeInt::BYTE, MemNode::unordered); 873 Node* gc_state_and = new AndINode(gc_state, igvn.intcon(flags)); 874 Node* gc_state_cmp = new CmpINode(gc_state_and, igvn.zerocon(T_INT)); 875 Node* gc_state_bool = new BoolNode(gc_state_cmp, BoolTest::ne); 876 877 IfNode* gc_state_iff = new IfNode(old_ctrl, gc_state_bool, PROB_UNLIKELY(0.999), COUNT_UNKNOWN); 878 ctrl = new IfTrueNode(gc_state_iff); 879 test_fail_ctrl = new IfFalseNode(gc_state_iff); 880 881 IdealLoopTree* loop = phase->get_loop(old_ctrl); 882 phase->register_control(gc_state_iff, loop, old_ctrl); 883 phase->register_control(ctrl, loop, gc_state_iff); 884 phase->register_control(test_fail_ctrl, loop, gc_state_iff); 885 886 phase->register_new_node(thread, old_ctrl); 887 phase->register_new_node(gc_state_addr, old_ctrl); 888 phase->register_new_node(gc_state, old_ctrl); 889 phase->register_new_node(gc_state_and, old_ctrl); 890 phase->register_new_node(gc_state_cmp, old_ctrl); 891 phase->register_new_node(gc_state_bool, old_ctrl); 892 893 phase->set_ctrl(gc_state_offset, phase->C->root()); 894 895 assert(is_gc_state_test(gc_state_iff, flags), "Should match the shape"); 896 } 897 898 void ShenandoahBarrierC2Support::test_null(Node*& ctrl, Node* val, Node*& null_ctrl, PhaseIdealLoop* phase) { 899 Node* old_ctrl = ctrl; 900 PhaseIterGVN& igvn = phase->igvn(); 901 902 const Type* val_t = igvn.type(val); 903 if (val_t->meet(TypePtr::NULL_PTR) == val_t) { 904 Node* null_cmp = new CmpPNode(val, igvn.zerocon(T_OBJECT)); 905 Node* null_test = new BoolNode(null_cmp, BoolTest::ne); 906 907 IfNode* null_iff = new IfNode(old_ctrl, null_test, PROB_LIKELY(0.999), COUNT_UNKNOWN); 908 ctrl = new IfTrueNode(null_iff); 909 null_ctrl = new IfFalseNode(null_iff); 910 911 IdealLoopTree* loop = phase->get_loop(old_ctrl); 912 phase->register_control(null_iff, loop, old_ctrl); 913 phase->register_control(ctrl, loop, null_iff); 914 phase->register_control(null_ctrl, loop, null_iff); 915 916 phase->register_new_node(null_cmp, old_ctrl); 917 phase->register_new_node(null_test, old_ctrl); 918 } 919 } 920 921 void ShenandoahBarrierC2Support::test_in_cset(Node*& ctrl, Node*& not_cset_ctrl, Node* val, Node* raw_mem, PhaseIdealLoop* phase) { 922 Node* old_ctrl = ctrl; 923 PhaseIterGVN& igvn = phase->igvn(); 924 925 Node* raw_val = new CastP2XNode(old_ctrl, val); 926 Node* cset_idx = new URShiftXNode(raw_val, igvn.intcon(ShenandoahHeapRegion::region_size_bytes_shift_jint())); 927 928 // Figure out the target cset address with raw pointer math. 929 // This avoids matching AddP+LoadB that would emit inefficient code. 930 // See JDK-8245465. 931 Node* cset_addr_ptr = igvn.makecon(TypeRawPtr::make(ShenandoahHeap::in_cset_fast_test_addr())); 932 Node* cset_addr = new CastP2XNode(old_ctrl, cset_addr_ptr); 933 Node* cset_load_addr = new AddXNode(cset_addr, cset_idx); 934 Node* cset_load_ptr = new CastX2PNode(cset_load_addr); 935 936 Node* cset_load = new LoadBNode(old_ctrl, raw_mem, cset_load_ptr, 937 DEBUG_ONLY(phase->C->get_adr_type(Compile::AliasIdxRaw)) NOT_DEBUG(NULL), 938 TypeInt::BYTE, MemNode::unordered); 939 Node* cset_cmp = new CmpINode(cset_load, igvn.zerocon(T_INT)); 940 Node* cset_bool = new BoolNode(cset_cmp, BoolTest::ne); 941 942 IfNode* cset_iff = new IfNode(old_ctrl, cset_bool, PROB_UNLIKELY(0.999), COUNT_UNKNOWN); 943 ctrl = new IfTrueNode(cset_iff); 944 not_cset_ctrl = new IfFalseNode(cset_iff); 945 946 IdealLoopTree *loop = phase->get_loop(old_ctrl); 947 phase->register_control(cset_iff, loop, old_ctrl); 948 phase->register_control(ctrl, loop, cset_iff); 949 phase->register_control(not_cset_ctrl, loop, cset_iff); 950 951 phase->set_ctrl(cset_addr_ptr, phase->C->root()); 952 953 phase->register_new_node(raw_val, old_ctrl); 954 phase->register_new_node(cset_idx, old_ctrl); 955 phase->register_new_node(cset_addr, old_ctrl); 956 phase->register_new_node(cset_load_addr, old_ctrl); 957 phase->register_new_node(cset_load_ptr, old_ctrl); 958 phase->register_new_node(cset_load, old_ctrl); 959 phase->register_new_node(cset_cmp, old_ctrl); 960 phase->register_new_node(cset_bool, old_ctrl); 961 } 962 963 void ShenandoahBarrierC2Support::call_lrb_stub(Node*& ctrl, Node*& val, Node* load_addr, Node*& result_mem, Node* raw_mem, bool is_native, PhaseIdealLoop* phase) { 964 IdealLoopTree*loop = phase->get_loop(ctrl); 965 const TypePtr* obj_type = phase->igvn().type(val)->is_oopptr(); 966 967 // The slow path stub consumes and produces raw memory in addition 968 // to the existing memory edges 969 Node* base = find_bottom_mem(ctrl, phase); 970 MergeMemNode* mm = MergeMemNode::make(base); 971 mm->set_memory_at(Compile::AliasIdxRaw, raw_mem); 972 phase->register_new_node(mm, ctrl); 973 974 address target = LP64_ONLY(UseCompressedOops) NOT_LP64(false) ? 975 CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_narrow) : 976 CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier); 977 978 address calladdr = is_native ? CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_native) 979 : target; 980 const char* name = is_native ? "load_reference_barrier_native" : "load_reference_barrier"; 981 Node* call = new CallLeafNode(ShenandoahBarrierSetC2::shenandoah_load_reference_barrier_Type(), calladdr, name, TypeRawPtr::BOTTOM); 982 983 call->init_req(TypeFunc::Control, ctrl); 984 call->init_req(TypeFunc::I_O, phase->C->top()); 985 call->init_req(TypeFunc::Memory, mm); 986 call->init_req(TypeFunc::FramePtr, phase->C->top()); 987 call->init_req(TypeFunc::ReturnAdr, phase->C->top()); 988 call->init_req(TypeFunc::Parms, val); 989 call->init_req(TypeFunc::Parms+1, load_addr); 990 phase->register_control(call, loop, ctrl); 991 ctrl = new ProjNode(call, TypeFunc::Control); 992 phase->register_control(ctrl, loop, call); 993 result_mem = new ProjNode(call, TypeFunc::Memory); 994 phase->register_new_node(result_mem, call); 995 val = new ProjNode(call, TypeFunc::Parms); 996 phase->register_new_node(val, call); 997 val = new CheckCastPPNode(ctrl, val, obj_type); 998 phase->register_new_node(val, ctrl); 999 } 1000 1001 void ShenandoahBarrierC2Support::fix_ctrl(Node* barrier, Node* region, const MemoryGraphFixer& fixer, Unique_Node_List& uses, Unique_Node_List& uses_to_ignore, uint last, PhaseIdealLoop* phase) { 1002 Node* ctrl = phase->get_ctrl(barrier); 1003 Node* init_raw_mem = fixer.find_mem(ctrl, barrier); 1004 1005 // Update the control of all nodes that should be after the 1006 // barrier control flow 1007 uses.clear(); 1008 // Every node that is control dependent on the barrier's input 1009 // control will be after the expanded barrier. The raw memory (if 1010 // its memory is control dependent on the barrier's input control) 1011 // must stay above the barrier. 1012 uses_to_ignore.clear(); 1013 if (phase->has_ctrl(init_raw_mem) && phase->get_ctrl(init_raw_mem) == ctrl && !init_raw_mem->is_Phi()) { 1014 uses_to_ignore.push(init_raw_mem); 1015 } 1016 for (uint next = 0; next < uses_to_ignore.size(); next++) { 1017 Node *n = uses_to_ignore.at(next); 1018 for (uint i = 0; i < n->req(); i++) { 1019 Node* in = n->in(i); 1020 if (in != NULL && phase->has_ctrl(in) && phase->get_ctrl(in) == ctrl) { 1021 uses_to_ignore.push(in); 1022 } 1023 } 1024 } 1025 for (DUIterator_Fast imax, i = ctrl->fast_outs(imax); i < imax; i++) { 1026 Node* u = ctrl->fast_out(i); 1027 if (u->_idx < last && 1028 u != barrier && 1029 !uses_to_ignore.member(u) && 1030 (u->in(0) != ctrl || (!u->is_Region() && !u->is_Phi())) && 1031 (ctrl->Opcode() != Op_CatchProj || u->Opcode() != Op_CreateEx)) { 1032 Node* old_c = phase->ctrl_or_self(u); 1033 Node* c = old_c; 1034 if (c != ctrl || 1035 is_dominator_same_ctrl(old_c, barrier, u, phase) || 1036 ShenandoahBarrierSetC2::is_shenandoah_state_load(u)) { 1037 phase->igvn().rehash_node_delayed(u); 1038 int nb = u->replace_edge(ctrl, region); 1039 if (u->is_CFG()) { 1040 if (phase->idom(u) == ctrl) { 1041 phase->set_idom(u, region, phase->dom_depth(region)); 1042 } 1043 } else if (phase->get_ctrl(u) == ctrl) { 1044 assert(u != init_raw_mem, "should leave input raw mem above the barrier"); 1045 uses.push(u); 1046 } 1047 assert(nb == 1, "more than 1 ctrl input?"); 1048 --i, imax -= nb; 1049 } 1050 } 1051 } 1052 } 1053 1054 static Node* create_phis_on_call_return(Node* ctrl, Node* c, Node* n, Node* n_clone, const CallProjections& projs, PhaseIdealLoop* phase) { 1055 Node* region = NULL; 1056 while (c != ctrl) { 1057 if (c->is_Region()) { 1058 region = c; 1059 } 1060 c = phase->idom(c); 1061 } 1062 assert(region != NULL, ""); 1063 Node* phi = new PhiNode(region, n->bottom_type()); 1064 for (uint j = 1; j < region->req(); j++) { 1065 Node* in = region->in(j); 1066 if (phase->is_dominator(projs.fallthrough_catchproj, in)) { 1067 phi->init_req(j, n); 1068 } else if (phase->is_dominator(projs.catchall_catchproj, in)) { 1069 phi->init_req(j, n_clone); 1070 } else { 1071 phi->init_req(j, create_phis_on_call_return(ctrl, in, n, n_clone, projs, phase)); 1072 } 1073 } 1074 phase->register_new_node(phi, region); 1075 return phi; 1076 } 1077 1078 void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) { 1079 ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state(); 1080 1081 Unique_Node_List uses; 1082 for (int i = 0; i < state->enqueue_barriers_count(); i++) { 1083 Node* barrier = state->enqueue_barrier(i); 1084 Node* ctrl = phase->get_ctrl(barrier); 1085 IdealLoopTree* loop = phase->get_loop(ctrl); 1086 if (loop->_head->is_OuterStripMinedLoop()) { 1087 // Expanding a barrier here will break loop strip mining 1088 // verification. Transform the loop so the loop nest doesn't 1089 // appear as strip mined. 1090 OuterStripMinedLoopNode* outer = loop->_head->as_OuterStripMinedLoop(); 1091 hide_strip_mined_loop(outer, outer->unique_ctrl_out()->as_CountedLoop(), phase); 1092 } 1093 } 1094 1095 Node_Stack stack(0); 1096 Node_List clones; 1097 for (int i = state->load_reference_barriers_count() - 1; i >= 0; i--) { 1098 ShenandoahLoadReferenceBarrierNode* lrb = state->load_reference_barrier(i); 1099 1100 Node* ctrl = phase->get_ctrl(lrb); 1101 Node* val = lrb->in(ShenandoahLoadReferenceBarrierNode::ValueIn); 1102 1103 CallStaticJavaNode* unc = NULL; 1104 Node* unc_ctrl = NULL; 1105 Node* uncasted_val = val; 1106 1107 for (DUIterator_Fast imax, i = lrb->fast_outs(imax); i < imax; i++) { 1108 Node* u = lrb->fast_out(i); 1109 if (u->Opcode() == Op_CastPP && 1110 u->in(0) != NULL && 1111 phase->is_dominator(u->in(0), ctrl)) { 1112 const Type* u_t = phase->igvn().type(u); 1113 1114 if (u_t->meet(TypePtr::NULL_PTR) != u_t && 1115 u->in(0)->Opcode() == Op_IfTrue && 1116 u->in(0)->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none) && 1117 u->in(0)->in(0)->is_If() && 1118 u->in(0)->in(0)->in(1)->Opcode() == Op_Bool && 1119 u->in(0)->in(0)->in(1)->as_Bool()->_test._test == BoolTest::ne && 1120 u->in(0)->in(0)->in(1)->in(1)->Opcode() == Op_CmpP && 1121 u->in(0)->in(0)->in(1)->in(1)->in(1) == val && 1122 u->in(0)->in(0)->in(1)->in(1)->in(2)->bottom_type() == TypePtr::NULL_PTR) { 1123 IdealLoopTree* loop = phase->get_loop(ctrl); 1124 IdealLoopTree* unc_loop = phase->get_loop(u->in(0)); 1125 1126 if (!unc_loop->is_member(loop)) { 1127 continue; 1128 } 1129 1130 Node* branch = no_branches(ctrl, u->in(0), false, phase); 1131 assert(branch == NULL || branch == NodeSentinel, "was not looking for a branch"); 1132 if (branch == NodeSentinel) { 1133 continue; 1134 } 1135 1136 Node* iff = u->in(0)->in(0); 1137 Node* bol = iff->in(1)->clone(); 1138 Node* cmp = bol->in(1)->clone(); 1139 cmp->set_req(1, lrb); 1140 bol->set_req(1, cmp); 1141 phase->igvn().replace_input_of(iff, 1, bol); 1142 phase->set_ctrl(lrb, iff->in(0)); 1143 phase->register_new_node(cmp, iff->in(0)); 1144 phase->register_new_node(bol, iff->in(0)); 1145 break; 1146 } 1147 } 1148 } 1149 if ((ctrl->is_Proj() && ctrl->in(0)->is_CallJava()) || ctrl->is_CallJava()) { 1150 CallNode* call = ctrl->is_Proj() ? ctrl->in(0)->as_CallJava() : ctrl->as_CallJava(); 1151 if (call->entry_point() == OptoRuntime::rethrow_stub()) { 1152 // The rethrow call may have too many projections to be 1153 // properly handled here. Given there's no reason for a 1154 // barrier to depend on the call, move it above the call 1155 stack.push(lrb, 0); 1156 do { 1157 Node* n = stack.node(); 1158 uint idx = stack.index(); 1159 if (idx < n->req()) { 1160 Node* in = n->in(idx); 1161 stack.set_index(idx+1); 1162 if (in != NULL) { 1163 if (phase->has_ctrl(in)) { 1164 if (phase->is_dominator(call, phase->get_ctrl(in))) { 1165 #ifdef ASSERT 1166 for (uint i = 0; i < stack.size(); i++) { 1167 assert(stack.node_at(i) != in, "node shouldn't have been seen yet"); 1168 } 1169 #endif 1170 stack.push(in, 0); 1171 } 1172 } else { 1173 assert(phase->is_dominator(in, call->in(0)), "no dependency on the call"); 1174 } 1175 } 1176 } else { 1177 phase->set_ctrl(n, call->in(0)); 1178 stack.pop(); 1179 } 1180 } while(stack.size() > 0); 1181 continue; 1182 } 1183 CallProjections projs; 1184 call->extract_projections(&projs, false, false); 1185 1186 #ifdef ASSERT 1187 VectorSet cloned(Thread::current()->resource_area()); 1188 #endif 1189 Node* lrb_clone = lrb->clone(); 1190 phase->register_new_node(lrb_clone, projs.catchall_catchproj); 1191 phase->set_ctrl(lrb, projs.fallthrough_catchproj); 1192 1193 stack.push(lrb, 0); 1194 clones.push(lrb_clone); 1195 1196 do { 1197 assert(stack.size() == clones.size(), ""); 1198 Node* n = stack.node(); 1199 #ifdef ASSERT 1200 if (n->is_Load()) { 1201 Node* mem = n->in(MemNode::Memory); 1202 for (DUIterator_Fast jmax, j = mem->fast_outs(jmax); j < jmax; j++) { 1203 Node* u = mem->fast_out(j); 1204 assert(!u->is_Store() || !u->is_LoadStore() || phase->get_ctrl(u) != ctrl, "anti dependent store?"); 1205 } 1206 } 1207 #endif 1208 uint idx = stack.index(); 1209 Node* n_clone = clones.at(clones.size()-1); 1210 if (idx < n->outcnt()) { 1211 Node* u = n->raw_out(idx); 1212 Node* c = phase->ctrl_or_self(u); 1213 if (phase->is_dominator(call, c) && phase->is_dominator(c, projs.fallthrough_proj)) { 1214 stack.set_index(idx+1); 1215 assert(!u->is_CFG(), ""); 1216 stack.push(u, 0); 1217 assert(!cloned.test_set(u->_idx), "only one clone"); 1218 Node* u_clone = u->clone(); 1219 int nb = u_clone->replace_edge(n, n_clone); 1220 assert(nb > 0, "should have replaced some uses"); 1221 phase->register_new_node(u_clone, projs.catchall_catchproj); 1222 clones.push(u_clone); 1223 phase->set_ctrl(u, projs.fallthrough_catchproj); 1224 } else { 1225 bool replaced = false; 1226 if (u->is_Phi()) { 1227 for (uint k = 1; k < u->req(); k++) { 1228 if (u->in(k) == n) { 1229 if (phase->is_dominator(projs.catchall_catchproj, u->in(0)->in(k))) { 1230 phase->igvn().replace_input_of(u, k, n_clone); 1231 replaced = true; 1232 } else if (!phase->is_dominator(projs.fallthrough_catchproj, u->in(0)->in(k))) { 1233 phase->igvn().replace_input_of(u, k, create_phis_on_call_return(ctrl, u->in(0)->in(k), n, n_clone, projs, phase)); 1234 replaced = true; 1235 } 1236 } 1237 } 1238 } else { 1239 if (phase->is_dominator(projs.catchall_catchproj, c)) { 1240 phase->igvn().rehash_node_delayed(u); 1241 int nb = u->replace_edge(n, n_clone); 1242 assert(nb > 0, "should have replaced some uses"); 1243 replaced = true; 1244 } else if (!phase->is_dominator(projs.fallthrough_catchproj, c)) { 1245 if (u->is_If()) { 1246 // Can't break If/Bool/Cmp chain 1247 assert(n->is_Bool(), "unexpected If shape"); 1248 assert(stack.node_at(stack.size()-2)->is_Cmp(), "unexpected If shape"); 1249 assert(n_clone->is_Bool(), "unexpected clone"); 1250 assert(clones.at(clones.size()-2)->is_Cmp(), "unexpected clone"); 1251 Node* bol_clone = n->clone(); 1252 Node* cmp_clone = stack.node_at(stack.size()-2)->clone(); 1253 bol_clone->set_req(1, cmp_clone); 1254 1255 Node* nn = stack.node_at(stack.size()-3); 1256 Node* nn_clone = clones.at(clones.size()-3); 1257 assert(nn->Opcode() == nn_clone->Opcode(), "mismatch"); 1258 1259 int nb = cmp_clone->replace_edge(nn, create_phis_on_call_return(ctrl, c, nn, nn_clone, projs, phase)); 1260 assert(nb > 0, "should have replaced some uses"); 1261 1262 phase->register_new_node(bol_clone, u->in(0)); 1263 phase->register_new_node(cmp_clone, u->in(0)); 1264 1265 phase->igvn().replace_input_of(u, 1, bol_clone); 1266 1267 } else { 1268 phase->igvn().rehash_node_delayed(u); 1269 int nb = u->replace_edge(n, create_phis_on_call_return(ctrl, c, n, n_clone, projs, phase)); 1270 assert(nb > 0, "should have replaced some uses"); 1271 } 1272 replaced = true; 1273 } 1274 } 1275 if (!replaced) { 1276 stack.set_index(idx+1); 1277 } 1278 } 1279 } else { 1280 stack.pop(); 1281 clones.pop(); 1282 } 1283 } while (stack.size() > 0); 1284 assert(stack.size() == 0 && clones.size() == 0, ""); 1285 } 1286 } 1287 1288 for (int i = 0; i < state->load_reference_barriers_count(); i++) { 1289 ShenandoahLoadReferenceBarrierNode* lrb = state->load_reference_barrier(i); 1290 Node* ctrl = phase->get_ctrl(lrb); 1291 IdealLoopTree* loop = phase->get_loop(ctrl); 1292 if (loop->_head->is_OuterStripMinedLoop()) { 1293 // Expanding a barrier here will break loop strip mining 1294 // verification. Transform the loop so the loop nest doesn't 1295 // appear as strip mined. 1296 OuterStripMinedLoopNode* outer = loop->_head->as_OuterStripMinedLoop(); 1297 hide_strip_mined_loop(outer, outer->unique_ctrl_out()->as_CountedLoop(), phase); 1298 } 1299 } 1300 1301 // Expand load-reference-barriers 1302 MemoryGraphFixer fixer(Compile::AliasIdxRaw, true, phase); 1303 Unique_Node_List uses_to_ignore; 1304 for (int i = state->load_reference_barriers_count() - 1; i >= 0; i--) { 1305 ShenandoahLoadReferenceBarrierNode* lrb = state->load_reference_barrier(i); 1306 uint last = phase->C->unique(); 1307 Node* ctrl = phase->get_ctrl(lrb); 1308 Node* val = lrb->in(ShenandoahLoadReferenceBarrierNode::ValueIn); 1309 1310 1311 Node* orig_ctrl = ctrl; 1312 1313 Node* raw_mem = fixer.find_mem(ctrl, lrb); 1314 Node* init_raw_mem = raw_mem; 1315 Node* raw_mem_for_ctrl = fixer.find_mem(ctrl, NULL); 1316 1317 IdealLoopTree *loop = phase->get_loop(ctrl); 1318 1319 Node* heap_stable_ctrl = NULL; 1320 Node* null_ctrl = NULL; 1321 1322 assert(val->bottom_type()->make_oopptr(), "need oop"); 1323 assert(val->bottom_type()->make_oopptr()->const_oop() == NULL, "expect non-constant"); 1324 1325 enum { _heap_stable = 1, _not_cset, _evac_path, PATH_LIMIT }; 1326 Node* region = new RegionNode(PATH_LIMIT); 1327 Node* val_phi = new PhiNode(region, val->bottom_type()->is_oopptr()); 1328 Node* raw_mem_phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM); 1329 1330 // Stable path. 1331 test_gc_state(ctrl, raw_mem, heap_stable_ctrl, phase, ShenandoahHeap::HAS_FORWARDED); 1332 IfNode* heap_stable_iff = heap_stable_ctrl->in(0)->as_If(); 1333 1334 // Heap stable case 1335 region->init_req(_heap_stable, heap_stable_ctrl); 1336 val_phi->init_req(_heap_stable, val); 1337 raw_mem_phi->init_req(_heap_stable, raw_mem); 1338 1339 // Test for in-cset. 1340 // Wires !in_cset(obj) to slot 2 of region and phis 1341 Node* not_cset_ctrl = NULL; 1342 test_in_cset(ctrl, not_cset_ctrl, val, raw_mem, phase); 1343 if (not_cset_ctrl != NULL) { 1344 region->init_req(_not_cset, not_cset_ctrl); 1345 val_phi->init_req(_not_cset, val); 1346 raw_mem_phi->init_req(_not_cset, raw_mem); 1347 } 1348 1349 // Resolve object when orig-value is in cset. 1350 // Make the unconditional resolve for fwdptr. 1351 1352 // Call lrb-stub and wire up that path in slots 4 1353 Node* result_mem = NULL; 1354 1355 Node* addr; 1356 if (ShenandoahSelfFixing) { 1357 VectorSet visited(Thread::current()->resource_area()); 1358 addr = get_load_addr(phase, visited, lrb); 1359 } else { 1360 addr = phase->igvn().zerocon(T_OBJECT); 1361 } 1362 if (addr->Opcode() == Op_AddP) { 1363 Node* orig_base = addr->in(AddPNode::Base); 1364 Node* base = new CheckCastPPNode(ctrl, orig_base, orig_base->bottom_type(), true); 1365 phase->register_new_node(base, ctrl); 1366 if (addr->in(AddPNode::Base) == addr->in((AddPNode::Address))) { 1367 // Field access 1368 addr = addr->clone(); 1369 addr->set_req(AddPNode::Base, base); 1370 addr->set_req(AddPNode::Address, base); 1371 phase->register_new_node(addr, ctrl); 1372 } else { 1373 Node* addr2 = addr->in(AddPNode::Address); 1374 if (addr2->Opcode() == Op_AddP && addr2->in(AddPNode::Base) == addr2->in(AddPNode::Address) && 1375 addr2->in(AddPNode::Base) == orig_base) { 1376 addr2 = addr2->clone(); 1377 addr2->set_req(AddPNode::Base, base); 1378 addr2->set_req(AddPNode::Address, base); 1379 phase->register_new_node(addr2, ctrl); 1380 addr = addr->clone(); 1381 addr->set_req(AddPNode::Base, base); 1382 addr->set_req(AddPNode::Address, addr2); 1383 phase->register_new_node(addr, ctrl); 1384 } 1385 } 1386 } 1387 call_lrb_stub(ctrl, val, addr, result_mem, raw_mem, lrb->is_native(), phase); 1388 region->init_req(_evac_path, ctrl); 1389 val_phi->init_req(_evac_path, val); 1390 raw_mem_phi->init_req(_evac_path, result_mem); 1391 1392 phase->register_control(region, loop, heap_stable_iff); 1393 Node* out_val = val_phi; 1394 phase->register_new_node(val_phi, region); 1395 phase->register_new_node(raw_mem_phi, region); 1396 1397 fix_ctrl(lrb, region, fixer, uses, uses_to_ignore, last, phase); 1398 1399 ctrl = orig_ctrl; 1400 1401 phase->igvn().replace_node(lrb, out_val); 1402 1403 follow_barrier_uses(out_val, ctrl, uses, phase); 1404 1405 for(uint next = 0; next < uses.size(); next++ ) { 1406 Node *n = uses.at(next); 1407 assert(phase->get_ctrl(n) == ctrl, "bad control"); 1408 assert(n != init_raw_mem, "should leave input raw mem above the barrier"); 1409 phase->set_ctrl(n, region); 1410 follow_barrier_uses(n, ctrl, uses, phase); 1411 } 1412 1413 // The slow path call produces memory: hook the raw memory phi 1414 // from the expanded load reference barrier with the rest of the graph 1415 // which may require adding memory phis at every post dominated 1416 // region and at enclosing loop heads. Use the memory state 1417 // collected in memory_nodes to fix the memory graph. Update that 1418 // memory state as we go. 1419 fixer.fix_mem(ctrl, region, init_raw_mem, raw_mem_for_ctrl, raw_mem_phi, uses); 1420 } 1421 // Done expanding load-reference-barriers. 1422 assert(ShenandoahBarrierSetC2::bsc2()->state()->load_reference_barriers_count() == 0, "all load reference barrier nodes should have been replaced"); 1423 1424 for (int i = state->enqueue_barriers_count() - 1; i >= 0; i--) { 1425 Node* barrier = state->enqueue_barrier(i); 1426 Node* pre_val = barrier->in(1); 1427 1428 if (phase->igvn().type(pre_val)->higher_equal(TypePtr::NULL_PTR)) { 1429 ShouldNotReachHere(); 1430 continue; 1431 } 1432 1433 Node* ctrl = phase->get_ctrl(barrier); 1434 1435 if (ctrl->is_Proj() && ctrl->in(0)->is_CallJava()) { 1436 assert(is_dominator(phase->get_ctrl(pre_val), ctrl->in(0)->in(0), pre_val, ctrl->in(0), phase), "can't move"); 1437 ctrl = ctrl->in(0)->in(0); 1438 phase->set_ctrl(barrier, ctrl); 1439 } else if (ctrl->is_CallRuntime()) { 1440 assert(is_dominator(phase->get_ctrl(pre_val), ctrl->in(0), pre_val, ctrl, phase), "can't move"); 1441 ctrl = ctrl->in(0); 1442 phase->set_ctrl(barrier, ctrl); 1443 } 1444 1445 Node* init_ctrl = ctrl; 1446 IdealLoopTree* loop = phase->get_loop(ctrl); 1447 Node* raw_mem = fixer.find_mem(ctrl, barrier); 1448 Node* init_raw_mem = raw_mem; 1449 Node* raw_mem_for_ctrl = fixer.find_mem(ctrl, NULL); 1450 Node* heap_stable_ctrl = NULL; 1451 Node* null_ctrl = NULL; 1452 uint last = phase->C->unique(); 1453 1454 enum { _heap_stable = 1, _heap_unstable, PATH_LIMIT }; 1455 Node* region = new RegionNode(PATH_LIMIT); 1456 Node* phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM); 1457 1458 enum { _fast_path = 1, _slow_path, _null_path, PATH_LIMIT2 }; 1459 Node* region2 = new RegionNode(PATH_LIMIT2); 1460 Node* phi2 = PhiNode::make(region2, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM); 1461 1462 // Stable path. 1463 test_gc_state(ctrl, raw_mem, heap_stable_ctrl, phase, ShenandoahHeap::MARKING); 1464 region->init_req(_heap_stable, heap_stable_ctrl); 1465 phi->init_req(_heap_stable, raw_mem); 1466 1467 // Null path 1468 Node* reg2_ctrl = NULL; 1469 test_null(ctrl, pre_val, null_ctrl, phase); 1470 if (null_ctrl != NULL) { 1471 reg2_ctrl = null_ctrl->in(0); 1472 region2->init_req(_null_path, null_ctrl); 1473 phi2->init_req(_null_path, raw_mem); 1474 } else { 1475 region2->del_req(_null_path); 1476 phi2->del_req(_null_path); 1477 } 1478 1479 const int index_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()); 1480 const int buffer_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()); 1481 Node* thread = new ThreadLocalNode(); 1482 phase->register_new_node(thread, ctrl); 1483 Node* buffer_adr = new AddPNode(phase->C->top(), thread, phase->igvn().MakeConX(buffer_offset)); 1484 phase->register_new_node(buffer_adr, ctrl); 1485 Node* index_adr = new AddPNode(phase->C->top(), thread, phase->igvn().MakeConX(index_offset)); 1486 phase->register_new_node(index_adr, ctrl); 1487 1488 BasicType index_bt = TypeX_X->basic_type(); 1489 assert(sizeof(size_t) == type2aelembytes(index_bt), "Loading G1 SATBMarkQueue::_index with wrong size."); 1490 const TypePtr* adr_type = TypeRawPtr::BOTTOM; 1491 Node* index = new LoadXNode(ctrl, raw_mem, index_adr, adr_type, TypeX_X, MemNode::unordered); 1492 phase->register_new_node(index, ctrl); 1493 Node* index_cmp = new CmpXNode(index, phase->igvn().MakeConX(0)); 1494 phase->register_new_node(index_cmp, ctrl); 1495 Node* index_test = new BoolNode(index_cmp, BoolTest::ne); 1496 phase->register_new_node(index_test, ctrl); 1497 IfNode* queue_full_iff = new IfNode(ctrl, index_test, PROB_LIKELY(0.999), COUNT_UNKNOWN); 1498 if (reg2_ctrl == NULL) reg2_ctrl = queue_full_iff; 1499 phase->register_control(queue_full_iff, loop, ctrl); 1500 Node* not_full = new IfTrueNode(queue_full_iff); 1501 phase->register_control(not_full, loop, queue_full_iff); 1502 Node* full = new IfFalseNode(queue_full_iff); 1503 phase->register_control(full, loop, queue_full_iff); 1504 1505 ctrl = not_full; 1506 1507 Node* next_index = new SubXNode(index, phase->igvn().MakeConX(sizeof(intptr_t))); 1508 phase->register_new_node(next_index, ctrl); 1509 1510 Node* buffer = new LoadPNode(ctrl, raw_mem, buffer_adr, adr_type, TypeRawPtr::NOTNULL, MemNode::unordered); 1511 phase->register_new_node(buffer, ctrl); 1512 Node *log_addr = new AddPNode(phase->C->top(), buffer, next_index); 1513 phase->register_new_node(log_addr, ctrl); 1514 Node* log_store = new StorePNode(ctrl, raw_mem, log_addr, adr_type, pre_val, MemNode::unordered); 1515 phase->register_new_node(log_store, ctrl); 1516 // update the index 1517 Node* index_update = new StoreXNode(ctrl, log_store, index_adr, adr_type, next_index, MemNode::unordered); 1518 phase->register_new_node(index_update, ctrl); 1519 1520 // Fast-path case 1521 region2->init_req(_fast_path, ctrl); 1522 phi2->init_req(_fast_path, index_update); 1523 1524 ctrl = full; 1525 1526 Node* base = find_bottom_mem(ctrl, phase); 1527 1528 MergeMemNode* mm = MergeMemNode::make(base); 1529 mm->set_memory_at(Compile::AliasIdxRaw, raw_mem); 1530 phase->register_new_node(mm, ctrl); 1531 1532 Node* call = new CallLeafNode(ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type(), CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), "shenandoah_wb_pre", TypeRawPtr::BOTTOM); 1533 call->init_req(TypeFunc::Control, ctrl); 1534 call->init_req(TypeFunc::I_O, phase->C->top()); 1535 call->init_req(TypeFunc::Memory, mm); 1536 call->init_req(TypeFunc::FramePtr, phase->C->top()); 1537 call->init_req(TypeFunc::ReturnAdr, phase->C->top()); 1538 call->init_req(TypeFunc::Parms, pre_val); 1539 call->init_req(TypeFunc::Parms+1, thread); 1540 phase->register_control(call, loop, ctrl); 1541 1542 Node* ctrl_proj = new ProjNode(call, TypeFunc::Control); 1543 phase->register_control(ctrl_proj, loop, call); 1544 Node* mem_proj = new ProjNode(call, TypeFunc::Memory); 1545 phase->register_new_node(mem_proj, call); 1546 1547 // Slow-path case 1548 region2->init_req(_slow_path, ctrl_proj); 1549 phi2->init_req(_slow_path, mem_proj); 1550 1551 phase->register_control(region2, loop, reg2_ctrl); 1552 phase->register_new_node(phi2, region2); 1553 1554 region->init_req(_heap_unstable, region2); 1555 phi->init_req(_heap_unstable, phi2); 1556 1557 phase->register_control(region, loop, heap_stable_ctrl->in(0)); 1558 phase->register_new_node(phi, region); 1559 1560 fix_ctrl(barrier, region, fixer, uses, uses_to_ignore, last, phase); 1561 for(uint next = 0; next < uses.size(); next++ ) { 1562 Node *n = uses.at(next); 1563 assert(phase->get_ctrl(n) == init_ctrl, "bad control"); 1564 assert(n != init_raw_mem, "should leave input raw mem above the barrier"); 1565 phase->set_ctrl(n, region); 1566 follow_barrier_uses(n, init_ctrl, uses, phase); 1567 } 1568 fixer.fix_mem(init_ctrl, region, init_raw_mem, raw_mem_for_ctrl, phi, uses); 1569 1570 phase->igvn().replace_node(barrier, pre_val); 1571 } 1572 assert(state->enqueue_barriers_count() == 0, "all enqueue barrier nodes should have been replaced"); 1573 1574 } 1575 1576 Node* ShenandoahBarrierC2Support::get_load_addr(PhaseIdealLoop* phase, VectorSet& visited, Node* in) { 1577 if (visited.test_set(in->_idx)) { 1578 return NULL; 1579 } 1580 switch (in->Opcode()) { 1581 case Op_Proj: 1582 return get_load_addr(phase, visited, in->in(0)); 1583 case Op_CastPP: 1584 case Op_CheckCastPP: 1585 case Op_DecodeN: 1586 case Op_EncodeP: 1587 return get_load_addr(phase, visited, in->in(1)); 1588 case Op_LoadN: 1589 case Op_LoadP: 1590 return in->in(MemNode::Address); 1591 case Op_CompareAndExchangeN: 1592 case Op_CompareAndExchangeP: 1593 case Op_GetAndSetN: 1594 case Op_GetAndSetP: 1595 case Op_ShenandoahCompareAndExchangeP: 1596 case Op_ShenandoahCompareAndExchangeN: 1597 // Those instructions would just have stored a different 1598 // value into the field. No use to attempt to fix it at this point. 1599 return phase->igvn().zerocon(T_OBJECT); 1600 case Op_CMoveP: 1601 case Op_CMoveN: { 1602 Node* t = get_load_addr(phase, visited, in->in(CMoveNode::IfTrue)); 1603 Node* f = get_load_addr(phase, visited, in->in(CMoveNode::IfFalse)); 1604 // Handle unambiguous cases: single address reported on both branches. 1605 if (t != NULL && f == NULL) return t; 1606 if (t == NULL && f != NULL) return f; 1607 if (t != NULL && t == f) return t; 1608 // Ambiguity. 1609 return phase->igvn().zerocon(T_OBJECT); 1610 } 1611 case Op_Phi: { 1612 Node* addr = NULL; 1613 for (uint i = 1; i < in->req(); i++) { 1614 Node* addr1 = get_load_addr(phase, visited, in->in(i)); 1615 if (addr == NULL) { 1616 addr = addr1; 1617 } 1618 if (addr != addr1) { 1619 return phase->igvn().zerocon(T_OBJECT); 1620 } 1621 } 1622 return addr; 1623 } 1624 case Op_ShenandoahLoadReferenceBarrier: 1625 return get_load_addr(phase, visited, in->in(ShenandoahLoadReferenceBarrierNode::ValueIn)); 1626 case Op_ShenandoahEnqueueBarrier: 1627 return get_load_addr(phase, visited, in->in(1)); 1628 case Op_CallDynamicJava: 1629 case Op_CallLeaf: 1630 case Op_CallStaticJava: 1631 case Op_ConN: 1632 case Op_ConP: 1633 case Op_Parm: 1634 case Op_CreateEx: 1635 return phase->igvn().zerocon(T_OBJECT); 1636 default: 1637 #ifdef ASSERT 1638 fatal("Unknown node in get_load_addr: %s", NodeClassNames[in->Opcode()]); 1639 #endif 1640 return phase->igvn().zerocon(T_OBJECT); 1641 } 1642 1643 } 1644 1645 void ShenandoahBarrierC2Support::move_gc_state_test_out_of_loop(IfNode* iff, PhaseIdealLoop* phase) { 1646 IdealLoopTree *loop = phase->get_loop(iff); 1647 Node* loop_head = loop->_head; 1648 Node* entry_c = loop_head->in(LoopNode::EntryControl); 1649 1650 Node* bol = iff->in(1); 1651 Node* cmp = bol->in(1); 1652 Node* andi = cmp->in(1); 1653 Node* load = andi->in(1); 1654 1655 assert(is_gc_state_load(load), "broken"); 1656 if (!phase->is_dominator(load->in(0), entry_c)) { 1657 Node* mem_ctrl = NULL; 1658 Node* mem = dom_mem(load->in(MemNode::Memory), loop_head, Compile::AliasIdxRaw, mem_ctrl, phase); 1659 load = load->clone(); 1660 load->set_req(MemNode::Memory, mem); 1661 load->set_req(0, entry_c); 1662 phase->register_new_node(load, entry_c); 1663 andi = andi->clone(); 1664 andi->set_req(1, load); 1665 phase->register_new_node(andi, entry_c); 1666 cmp = cmp->clone(); 1667 cmp->set_req(1, andi); 1668 phase->register_new_node(cmp, entry_c); 1669 bol = bol->clone(); 1670 bol->set_req(1, cmp); 1671 phase->register_new_node(bol, entry_c); 1672 1673 phase->igvn().replace_input_of(iff, 1, bol); 1674 } 1675 } 1676 1677 bool ShenandoahBarrierC2Support::identical_backtoback_ifs(Node* n, PhaseIdealLoop* phase) { 1678 if (!n->is_If() || n->is_CountedLoopEnd()) { 1679 return false; 1680 } 1681 Node* region = n->in(0); 1682 1683 if (!region->is_Region()) { 1684 return false; 1685 } 1686 Node* dom = phase->idom(region); 1687 if (!dom->is_If()) { 1688 return false; 1689 } 1690 1691 if (!is_heap_stable_test(n) || !is_heap_stable_test(dom)) { 1692 return false; 1693 } 1694 1695 IfNode* dom_if = dom->as_If(); 1696 Node* proj_true = dom_if->proj_out(1); 1697 Node* proj_false = dom_if->proj_out(0); 1698 1699 for (uint i = 1; i < region->req(); i++) { 1700 if (phase->is_dominator(proj_true, region->in(i))) { 1701 continue; 1702 } 1703 if (phase->is_dominator(proj_false, region->in(i))) { 1704 continue; 1705 } 1706 return false; 1707 } 1708 1709 return true; 1710 } 1711 1712 void ShenandoahBarrierC2Support::merge_back_to_back_tests(Node* n, PhaseIdealLoop* phase) { 1713 assert(is_heap_stable_test(n), "no other tests"); 1714 if (identical_backtoback_ifs(n, phase)) { 1715 Node* n_ctrl = n->in(0); 1716 if (phase->can_split_if(n_ctrl)) { 1717 IfNode* dom_if = phase->idom(n_ctrl)->as_If(); 1718 if (is_heap_stable_test(n)) { 1719 Node* gc_state_load = n->in(1)->in(1)->in(1)->in(1); 1720 assert(is_gc_state_load(gc_state_load), "broken"); 1721 Node* dom_gc_state_load = dom_if->in(1)->in(1)->in(1)->in(1); 1722 assert(is_gc_state_load(dom_gc_state_load), "broken"); 1723 if (gc_state_load != dom_gc_state_load) { 1724 phase->igvn().replace_node(gc_state_load, dom_gc_state_load); 1725 } 1726 } 1727 PhiNode* bolphi = PhiNode::make_blank(n_ctrl, n->in(1)); 1728 Node* proj_true = dom_if->proj_out(1); 1729 Node* proj_false = dom_if->proj_out(0); 1730 Node* con_true = phase->igvn().makecon(TypeInt::ONE); 1731 Node* con_false = phase->igvn().makecon(TypeInt::ZERO); 1732 1733 for (uint i = 1; i < n_ctrl->req(); i++) { 1734 if (phase->is_dominator(proj_true, n_ctrl->in(i))) { 1735 bolphi->init_req(i, con_true); 1736 } else { 1737 assert(phase->is_dominator(proj_false, n_ctrl->in(i)), "bad if"); 1738 bolphi->init_req(i, con_false); 1739 } 1740 } 1741 phase->register_new_node(bolphi, n_ctrl); 1742 phase->igvn().replace_input_of(n, 1, bolphi); 1743 phase->do_split_if(n); 1744 } 1745 } 1746 } 1747 1748 IfNode* ShenandoahBarrierC2Support::find_unswitching_candidate(const IdealLoopTree* loop, PhaseIdealLoop* phase) { 1749 // Find first invariant test that doesn't exit the loop 1750 LoopNode *head = loop->_head->as_Loop(); 1751 IfNode* unswitch_iff = NULL; 1752 Node* n = head->in(LoopNode::LoopBackControl); 1753 int loop_has_sfpts = -1; 1754 while (n != head) { 1755 Node* n_dom = phase->idom(n); 1756 if (n->is_Region()) { 1757 if (n_dom->is_If()) { 1758 IfNode* iff = n_dom->as_If(); 1759 if (iff->in(1)->is_Bool()) { 1760 BoolNode* bol = iff->in(1)->as_Bool(); 1761 if (bol->in(1)->is_Cmp()) { 1762 // If condition is invariant and not a loop exit, 1763 // then found reason to unswitch. 1764 if (is_heap_stable_test(iff) && 1765 (loop_has_sfpts == -1 || loop_has_sfpts == 0)) { 1766 assert(!loop->is_loop_exit(iff), "both branches should be in the loop"); 1767 if (loop_has_sfpts == -1) { 1768 for(uint i = 0; i < loop->_body.size(); i++) { 1769 Node *m = loop->_body[i]; 1770 if (m->is_SafePoint() && !m->is_CallLeaf()) { 1771 loop_has_sfpts = 1; 1772 break; 1773 } 1774 } 1775 if (loop_has_sfpts == -1) { 1776 loop_has_sfpts = 0; 1777 } 1778 } 1779 if (!loop_has_sfpts) { 1780 unswitch_iff = iff; 1781 } 1782 } 1783 } 1784 } 1785 } 1786 } 1787 n = n_dom; 1788 } 1789 return unswitch_iff; 1790 } 1791 1792 1793 void ShenandoahBarrierC2Support::optimize_after_expansion(VectorSet &visited, Node_Stack &stack, Node_List &old_new, PhaseIdealLoop* phase) { 1794 Node_List heap_stable_tests; 1795 stack.push(phase->C->start(), 0); 1796 do { 1797 Node* n = stack.node(); 1798 uint i = stack.index(); 1799 1800 if (i < n->outcnt()) { 1801 Node* u = n->raw_out(i); 1802 stack.set_index(i+1); 1803 if (!visited.test_set(u->_idx)) { 1804 stack.push(u, 0); 1805 } 1806 } else { 1807 stack.pop(); 1808 if (n->is_If() && is_heap_stable_test(n)) { 1809 heap_stable_tests.push(n); 1810 } 1811 } 1812 } while (stack.size() > 0); 1813 1814 for (uint i = 0; i < heap_stable_tests.size(); i++) { 1815 Node* n = heap_stable_tests.at(i); 1816 assert(is_heap_stable_test(n), "only evacuation test"); 1817 merge_back_to_back_tests(n, phase); 1818 } 1819 1820 if (!phase->C->major_progress()) { 1821 VectorSet seen(Thread::current()->resource_area()); 1822 for (uint i = 0; i < heap_stable_tests.size(); i++) { 1823 Node* n = heap_stable_tests.at(i); 1824 IdealLoopTree* loop = phase->get_loop(n); 1825 if (loop != phase->ltree_root() && 1826 loop->_child == NULL && 1827 !loop->_irreducible) { 1828 Node* head = loop->_head; 1829 if (head->is_Loop() && 1830 (!head->is_CountedLoop() || head->as_CountedLoop()->is_main_loop() || head->as_CountedLoop()->is_normal_loop()) && 1831 !seen.test_set(head->_idx)) { 1832 IfNode* iff = find_unswitching_candidate(loop, phase); 1833 if (iff != NULL) { 1834 Node* bol = iff->in(1); 1835 if (head->as_Loop()->is_strip_mined()) { 1836 head->as_Loop()->verify_strip_mined(0); 1837 } 1838 move_gc_state_test_out_of_loop(iff, phase); 1839 1840 AutoNodeBudget node_budget(phase); 1841 1842 if (loop->policy_unswitching(phase)) { 1843 if (head->as_Loop()->is_strip_mined()) { 1844 OuterStripMinedLoopNode* outer = head->as_CountedLoop()->outer_loop(); 1845 hide_strip_mined_loop(outer, head->as_CountedLoop(), phase); 1846 } 1847 phase->do_unswitching(loop, old_new); 1848 } else { 1849 // Not proceeding with unswitching. Move load back in 1850 // the loop. 1851 phase->igvn().replace_input_of(iff, 1, bol); 1852 } 1853 } 1854 } 1855 } 1856 } 1857 } 1858 } 1859 1860 #ifdef ASSERT 1861 void ShenandoahBarrierC2Support::verify_raw_mem(RootNode* root) { 1862 const bool trace = false; 1863 ResourceMark rm; 1864 Unique_Node_List nodes; 1865 Unique_Node_List controls; 1866 Unique_Node_List memories; 1867 1868 nodes.push(root); 1869 for (uint next = 0; next < nodes.size(); next++) { 1870 Node *n = nodes.at(next); 1871 if (ShenandoahBarrierSetC2::is_shenandoah_lrb_call(n)) { 1872 controls.push(n); 1873 if (trace) { tty->print("XXXXXX verifying"); n->dump(); } 1874 for (uint next2 = 0; next2 < controls.size(); next2++) { 1875 Node *m = controls.at(next2); 1876 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) { 1877 Node* u = m->fast_out(i); 1878 if (u->is_CFG() && !u->is_Root() && 1879 !(u->Opcode() == Op_CProj && u->in(0)->Opcode() == Op_NeverBranch && u->as_Proj()->_con == 1) && 1880 !(u->is_Region() && u->unique_ctrl_out()->Opcode() == Op_Halt)) { 1881 if (trace) { tty->print("XXXXXX pushing control"); u->dump(); } 1882 controls.push(u); 1883 } 1884 } 1885 } 1886 memories.push(n->as_Call()->proj_out(TypeFunc::Memory)); 1887 for (uint next2 = 0; next2 < memories.size(); next2++) { 1888 Node *m = memories.at(next2); 1889 assert(m->bottom_type() == Type::MEMORY, ""); 1890 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) { 1891 Node* u = m->fast_out(i); 1892 if (u->bottom_type() == Type::MEMORY && (u->is_Mem() || u->is_ClearArray())) { 1893 if (trace) { tty->print("XXXXXX pushing memory"); u->dump(); } 1894 memories.push(u); 1895 } else if (u->is_LoadStore()) { 1896 if (trace) { tty->print("XXXXXX pushing memory"); u->find_out_with(Op_SCMemProj)->dump(); } 1897 memories.push(u->find_out_with(Op_SCMemProj)); 1898 } else if (u->is_MergeMem() && u->as_MergeMem()->memory_at(Compile::AliasIdxRaw) == m) { 1899 if (trace) { tty->print("XXXXXX pushing memory"); u->dump(); } 1900 memories.push(u); 1901 } else if (u->is_Phi()) { 1902 assert(u->bottom_type() == Type::MEMORY, ""); 1903 if (u->adr_type() == TypeRawPtr::BOTTOM || u->adr_type() == TypePtr::BOTTOM) { 1904 assert(controls.member(u->in(0)), ""); 1905 if (trace) { tty->print("XXXXXX pushing memory"); u->dump(); } 1906 memories.push(u); 1907 } 1908 } else if (u->is_SafePoint() || u->is_MemBar()) { 1909 for (DUIterator_Fast jmax, j = u->fast_outs(jmax); j < jmax; j++) { 1910 Node* uu = u->fast_out(j); 1911 if (uu->bottom_type() == Type::MEMORY) { 1912 if (trace) { tty->print("XXXXXX pushing memory"); uu->dump(); } 1913 memories.push(uu); 1914 } 1915 } 1916 } 1917 } 1918 } 1919 for (uint next2 = 0; next2 < controls.size(); next2++) { 1920 Node *m = controls.at(next2); 1921 if (m->is_Region()) { 1922 bool all_in = true; 1923 for (uint i = 1; i < m->req(); i++) { 1924 if (!controls.member(m->in(i))) { 1925 all_in = false; 1926 break; 1927 } 1928 } 1929 if (trace) { tty->print("XXX verifying %s", all_in ? "all in" : ""); m->dump(); } 1930 bool found_phi = false; 1931 for (DUIterator_Fast jmax, j = m->fast_outs(jmax); j < jmax && !found_phi; j++) { 1932 Node* u = m->fast_out(j); 1933 if (u->is_Phi() && memories.member(u)) { 1934 found_phi = true; 1935 for (uint i = 1; i < u->req() && found_phi; i++) { 1936 Node* k = u->in(i); 1937 if (memories.member(k) != controls.member(m->in(i))) { 1938 found_phi = false; 1939 } 1940 } 1941 } 1942 } 1943 assert(found_phi || all_in, ""); 1944 } 1945 } 1946 controls.clear(); 1947 memories.clear(); 1948 } 1949 for( uint i = 0; i < n->len(); ++i ) { 1950 Node *m = n->in(i); 1951 if (m != NULL) { 1952 nodes.push(m); 1953 } 1954 } 1955 } 1956 } 1957 #endif 1958 1959 ShenandoahEnqueueBarrierNode::ShenandoahEnqueueBarrierNode(Node* val) : Node(NULL, val) { 1960 ShenandoahBarrierSetC2::bsc2()->state()->add_enqueue_barrier(this); 1961 } 1962 1963 const Type* ShenandoahEnqueueBarrierNode::bottom_type() const { 1964 if (in(1) == NULL || in(1)->is_top()) { 1965 return Type::TOP; 1966 } 1967 const Type* t = in(1)->bottom_type(); 1968 if (t == TypePtr::NULL_PTR) { 1969 return t; 1970 } 1971 return t->is_oopptr(); 1972 } 1973 1974 const Type* ShenandoahEnqueueBarrierNode::Value(PhaseGVN* phase) const { 1975 if (in(1) == NULL) { 1976 return Type::TOP; 1977 } 1978 const Type* t = phase->type(in(1)); 1979 if (t == Type::TOP) { 1980 return Type::TOP; 1981 } 1982 if (t == TypePtr::NULL_PTR) { 1983 return t; 1984 } 1985 return t->is_oopptr(); 1986 } 1987 1988 int ShenandoahEnqueueBarrierNode::needed(Node* n) { 1989 if (n == NULL || 1990 n->is_Allocate() || 1991 n->Opcode() == Op_ShenandoahEnqueueBarrier || 1992 n->bottom_type() == TypePtr::NULL_PTR || 1993 (n->bottom_type()->make_oopptr() != NULL && n->bottom_type()->make_oopptr()->const_oop() != NULL)) { 1994 return NotNeeded; 1995 } 1996 if (n->is_Phi() || 1997 n->is_CMove()) { 1998 return MaybeNeeded; 1999 } 2000 return Needed; 2001 } 2002 2003 Node* ShenandoahEnqueueBarrierNode::next(Node* n) { 2004 for (;;) { 2005 if (n == NULL) { 2006 return n; 2007 } else if (n->bottom_type() == TypePtr::NULL_PTR) { 2008 return n; 2009 } else if (n->bottom_type()->make_oopptr() != NULL && n->bottom_type()->make_oopptr()->const_oop() != NULL) { 2010 return n; 2011 } else if (n->is_ConstraintCast() || 2012 n->Opcode() == Op_DecodeN || 2013 n->Opcode() == Op_EncodeP) { 2014 n = n->in(1); 2015 } else if (n->is_Proj()) { 2016 n = n->in(0); 2017 } else { 2018 return n; 2019 } 2020 } 2021 ShouldNotReachHere(); 2022 return NULL; 2023 } 2024 2025 Node* ShenandoahEnqueueBarrierNode::Identity(PhaseGVN* phase) { 2026 PhaseIterGVN* igvn = phase->is_IterGVN(); 2027 2028 Node* n = next(in(1)); 2029 2030 int cont = needed(n); 2031 2032 if (cont == NotNeeded) { 2033 return in(1); 2034 } else if (cont == MaybeNeeded) { 2035 if (igvn == NULL) { 2036 phase->record_for_igvn(this); 2037 return this; 2038 } else { 2039 ResourceMark rm; 2040 Unique_Node_List wq; 2041 uint wq_i = 0; 2042 2043 for (;;) { 2044 if (n->is_Phi()) { 2045 for (uint i = 1; i < n->req(); i++) { 2046 Node* m = n->in(i); 2047 if (m != NULL) { 2048 wq.push(m); 2049 } 2050 } 2051 } else { 2052 assert(n->is_CMove(), "nothing else here"); 2053 Node* m = n->in(CMoveNode::IfFalse); 2054 wq.push(m); 2055 m = n->in(CMoveNode::IfTrue); 2056 wq.push(m); 2057 } 2058 Node* orig_n = NULL; 2059 do { 2060 if (wq_i >= wq.size()) { 2061 return in(1); 2062 } 2063 n = wq.at(wq_i); 2064 wq_i++; 2065 orig_n = n; 2066 n = next(n); 2067 cont = needed(n); 2068 if (cont == Needed) { 2069 return this; 2070 } 2071 } while (cont != MaybeNeeded || (orig_n != n && wq.member(n))); 2072 } 2073 } 2074 } 2075 2076 return this; 2077 } 2078 2079 #ifdef ASSERT 2080 static bool has_never_branch(Node* root) { 2081 for (uint i = 1; i < root->req(); i++) { 2082 Node* in = root->in(i); 2083 if (in != NULL && in->Opcode() == Op_Halt && in->in(0)->is_Proj() && in->in(0)->in(0)->Opcode() == Op_NeverBranch) { 2084 return true; 2085 } 2086 } 2087 return false; 2088 } 2089 #endif 2090 2091 void MemoryGraphFixer::collect_memory_nodes() { 2092 Node_Stack stack(0); 2093 VectorSet visited(Thread::current()->resource_area()); 2094 Node_List regions; 2095 2096 // Walk the raw memory graph and create a mapping from CFG node to 2097 // memory node. Exclude phis for now. 2098 stack.push(_phase->C->root(), 1); 2099 do { 2100 Node* n = stack.node(); 2101 int opc = n->Opcode(); 2102 uint i = stack.index(); 2103 if (i < n->req()) { 2104 Node* mem = NULL; 2105 if (opc == Op_Root) { 2106 Node* in = n->in(i); 2107 int in_opc = in->Opcode(); 2108 if (in_opc == Op_Return || in_opc == Op_Rethrow) { 2109 mem = in->in(TypeFunc::Memory); 2110 } else if (in_opc == Op_Halt) { 2111 if (in->in(0)->is_Region()) { 2112 Node* r = in->in(0); 2113 for (uint j = 1; j < r->req(); j++) { 2114 assert(r->in(j)->Opcode() != Op_NeverBranch, ""); 2115 } 2116 } else { 2117 Node* proj = in->in(0); 2118 assert(proj->is_Proj(), ""); 2119 Node* in = proj->in(0); 2120 assert(in->is_CallStaticJava() || in->Opcode() == Op_NeverBranch || in->Opcode() == Op_Catch || proj->is_IfProj(), ""); 2121 if (in->is_CallStaticJava()) { 2122 mem = in->in(TypeFunc::Memory); 2123 } else if (in->Opcode() == Op_Catch) { 2124 Node* call = in->in(0)->in(0); 2125 assert(call->is_Call(), ""); 2126 mem = call->in(TypeFunc::Memory); 2127 } else if (in->Opcode() == Op_NeverBranch) { 2128 Node* head = in->in(0); 2129 assert(head->is_Region(), "unexpected infinite loop graph shape"); 2130 2131 Node* phi_mem = NULL; 2132 for (DUIterator_Fast jmax, j = head->fast_outs(jmax); j < jmax; j++) { 2133 Node* u = head->fast_out(j); 2134 if (u->is_Phi() && u->bottom_type() == Type::MEMORY) { 2135 if (_phase->C->get_alias_index(u->adr_type()) == _alias) { 2136 assert(phi_mem == NULL || phi_mem->adr_type() == TypePtr::BOTTOM, ""); 2137 phi_mem = u; 2138 } else if (u->adr_type() == TypePtr::BOTTOM) { 2139 assert(phi_mem == NULL || _phase->C->get_alias_index(phi_mem->adr_type()) == _alias, ""); 2140 if (phi_mem == NULL) { 2141 phi_mem = u; 2142 } 2143 } 2144 } 2145 } 2146 if (phi_mem == NULL) { 2147 for (uint j = 1; j < head->req(); j++) { 2148 Node* tail = head->in(j); 2149 if (!_phase->is_dominator(head, tail)) { 2150 continue; 2151 } 2152 Node* c = tail; 2153 while (c != head) { 2154 if (c->is_SafePoint() && !c->is_CallLeaf()) { 2155 Node* m =c->in(TypeFunc::Memory); 2156 if (m->is_MergeMem()) { 2157 m = m->as_MergeMem()->memory_at(_alias); 2158 } 2159 assert(mem == NULL || mem == m, "several memory states"); 2160 mem = m; 2161 } 2162 c = _phase->idom(c); 2163 } 2164 assert(mem != NULL, "should have found safepoint"); 2165 } 2166 assert(mem != NULL, "should have found safepoint"); 2167 } else { 2168 mem = phi_mem; 2169 } 2170 } 2171 } 2172 } else { 2173 #ifdef ASSERT 2174 n->dump(); 2175 in->dump(); 2176 #endif 2177 ShouldNotReachHere(); 2178 } 2179 } else { 2180 assert(n->is_Phi() && n->bottom_type() == Type::MEMORY, ""); 2181 assert(n->adr_type() == TypePtr::BOTTOM || _phase->C->get_alias_index(n->adr_type()) == _alias, ""); 2182 mem = n->in(i); 2183 } 2184 i++; 2185 stack.set_index(i); 2186 if (mem == NULL) { 2187 continue; 2188 } 2189 for (;;) { 2190 if (visited.test_set(mem->_idx) || mem->is_Start()) { 2191 break; 2192 } 2193 if (mem->is_Phi()) { 2194 stack.push(mem, 2); 2195 mem = mem->in(1); 2196 } else if (mem->is_Proj()) { 2197 stack.push(mem, mem->req()); 2198 mem = mem->in(0); 2199 } else if (mem->is_SafePoint() || mem->is_MemBar()) { 2200 mem = mem->in(TypeFunc::Memory); 2201 } else if (mem->is_MergeMem()) { 2202 MergeMemNode* mm = mem->as_MergeMem(); 2203 mem = mm->memory_at(_alias); 2204 } else if (mem->is_Store() || mem->is_LoadStore() || mem->is_ClearArray()) { 2205 assert(_alias == Compile::AliasIdxRaw, ""); 2206 stack.push(mem, mem->req()); 2207 mem = mem->in(MemNode::Memory); 2208 } else { 2209 #ifdef ASSERT 2210 mem->dump(); 2211 #endif 2212 ShouldNotReachHere(); 2213 } 2214 } 2215 } else { 2216 if (n->is_Phi()) { 2217 // Nothing 2218 } else if (!n->is_Root()) { 2219 Node* c = get_ctrl(n); 2220 _memory_nodes.map(c->_idx, n); 2221 } 2222 stack.pop(); 2223 } 2224 } while(stack.is_nonempty()); 2225 2226 // Iterate over CFG nodes in rpo and propagate memory state to 2227 // compute memory state at regions, creating new phis if needed. 2228 Node_List rpo_list; 2229 visited.clear(); 2230 _phase->rpo(_phase->C->root(), stack, visited, rpo_list); 2231 Node* root = rpo_list.pop(); 2232 assert(root == _phase->C->root(), ""); 2233 2234 const bool trace = false; 2235 #ifdef ASSERT 2236 if (trace) { 2237 for (int i = rpo_list.size() - 1; i >= 0; i--) { 2238 Node* c = rpo_list.at(i); 2239 if (_memory_nodes[c->_idx] != NULL) { 2240 tty->print("X %d", c->_idx); _memory_nodes[c->_idx]->dump(); 2241 } 2242 } 2243 } 2244 #endif 2245 uint last = _phase->C->unique(); 2246 2247 #ifdef ASSERT 2248 uint8_t max_depth = 0; 2249 for (LoopTreeIterator iter(_phase->ltree_root()); !iter.done(); iter.next()) { 2250 IdealLoopTree* lpt = iter.current(); 2251 max_depth = MAX2(max_depth, lpt->_nest); 2252 } 2253 #endif 2254 2255 bool progress = true; 2256 int iteration = 0; 2257 Node_List dead_phis; 2258 while (progress) { 2259 progress = false; 2260 iteration++; 2261 assert(iteration <= 2+max_depth || _phase->C->has_irreducible_loop() || has_never_branch(_phase->C->root()), ""); 2262 if (trace) { tty->print_cr("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"); } 2263 2264 for (int i = rpo_list.size() - 1; i >= 0; i--) { 2265 Node* c = rpo_list.at(i); 2266 2267 Node* prev_mem = _memory_nodes[c->_idx]; 2268 if (c->is_Region() && (_include_lsm || !c->is_OuterStripMinedLoop())) { 2269 Node* prev_region = regions[c->_idx]; 2270 Node* unique = NULL; 2271 for (uint j = 1; j < c->req() && unique != NodeSentinel; j++) { 2272 Node* m = _memory_nodes[c->in(j)->_idx]; 2273 assert(m != NULL || (c->is_Loop() && j == LoopNode::LoopBackControl && iteration == 1) || _phase->C->has_irreducible_loop() || has_never_branch(_phase->C->root()), "expect memory state"); 2274 if (m != NULL) { 2275 if (m == prev_region && ((c->is_Loop() && j == LoopNode::LoopBackControl) || (prev_region->is_Phi() && prev_region->in(0) == c))) { 2276 assert(c->is_Loop() && j == LoopNode::LoopBackControl || _phase->C->has_irreducible_loop() || has_never_branch(_phase->C->root()), ""); 2277 // continue 2278 } else if (unique == NULL) { 2279 unique = m; 2280 } else if (m == unique) { 2281 // continue 2282 } else { 2283 unique = NodeSentinel; 2284 } 2285 } 2286 } 2287 assert(unique != NULL, "empty phi???"); 2288 if (unique != NodeSentinel) { 2289 if (prev_region != NULL && prev_region->is_Phi() && prev_region->in(0) == c) { 2290 dead_phis.push(prev_region); 2291 } 2292 regions.map(c->_idx, unique); 2293 } else { 2294 Node* phi = NULL; 2295 if (prev_region != NULL && prev_region->is_Phi() && prev_region->in(0) == c && prev_region->_idx >= last) { 2296 phi = prev_region; 2297 for (uint k = 1; k < c->req(); k++) { 2298 Node* m = _memory_nodes[c->in(k)->_idx]; 2299 assert(m != NULL, "expect memory state"); 2300 phi->set_req(k, m); 2301 } 2302 } else { 2303 for (DUIterator_Fast jmax, j = c->fast_outs(jmax); j < jmax && phi == NULL; j++) { 2304 Node* u = c->fast_out(j); 2305 if (u->is_Phi() && u->bottom_type() == Type::MEMORY && 2306 (u->adr_type() == TypePtr::BOTTOM || _phase->C->get_alias_index(u->adr_type()) == _alias)) { 2307 phi = u; 2308 for (uint k = 1; k < c->req() && phi != NULL; k++) { 2309 Node* m = _memory_nodes[c->in(k)->_idx]; 2310 assert(m != NULL, "expect memory state"); 2311 if (u->in(k) != m) { 2312 phi = NULL; 2313 } 2314 } 2315 } 2316 } 2317 if (phi == NULL) { 2318 phi = new PhiNode(c, Type::MEMORY, _phase->C->get_adr_type(_alias)); 2319 for (uint k = 1; k < c->req(); k++) { 2320 Node* m = _memory_nodes[c->in(k)->_idx]; 2321 assert(m != NULL, "expect memory state"); 2322 phi->init_req(k, m); 2323 } 2324 } 2325 } 2326 assert(phi != NULL, ""); 2327 regions.map(c->_idx, phi); 2328 } 2329 Node* current_region = regions[c->_idx]; 2330 if (current_region != prev_region) { 2331 progress = true; 2332 if (prev_region == prev_mem) { 2333 _memory_nodes.map(c->_idx, current_region); 2334 } 2335 } 2336 } else if (prev_mem == NULL || prev_mem->is_Phi() || ctrl_or_self(prev_mem) != c) { 2337 Node* m = _memory_nodes[_phase->idom(c)->_idx]; 2338 assert(m != NULL, "expect memory state"); 2339 if (m != prev_mem) { 2340 _memory_nodes.map(c->_idx, m); 2341 progress = true; 2342 } 2343 } 2344 #ifdef ASSERT 2345 if (trace) { tty->print("X %d", c->_idx); _memory_nodes[c->_idx]->dump(); } 2346 #endif 2347 } 2348 } 2349 2350 // Replace existing phi with computed memory state for that region 2351 // if different (could be a new phi or a dominating memory node if 2352 // that phi was found to be useless). 2353 while (dead_phis.size() > 0) { 2354 Node* n = dead_phis.pop(); 2355 n->replace_by(_phase->C->top()); 2356 n->destruct(); 2357 } 2358 for (int i = rpo_list.size() - 1; i >= 0; i--) { 2359 Node* c = rpo_list.at(i); 2360 if (c->is_Region() && (_include_lsm || !c->is_OuterStripMinedLoop())) { 2361 Node* n = regions[c->_idx]; 2362 if (n->is_Phi() && n->_idx >= last && n->in(0) == c) { 2363 _phase->register_new_node(n, c); 2364 } 2365 } 2366 } 2367 for (int i = rpo_list.size() - 1; i >= 0; i--) { 2368 Node* c = rpo_list.at(i); 2369 if (c->is_Region() && (_include_lsm || !c->is_OuterStripMinedLoop())) { 2370 Node* n = regions[c->_idx]; 2371 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) { 2372 Node* u = c->fast_out(i); 2373 if (u->is_Phi() && u->bottom_type() == Type::MEMORY && 2374 u != n) { 2375 if (u->adr_type() == TypePtr::BOTTOM) { 2376 fix_memory_uses(u, n, n, c); 2377 } else if (_phase->C->get_alias_index(u->adr_type()) == _alias) { 2378 _phase->lazy_replace(u, n); 2379 --i; --imax; 2380 } 2381 } 2382 } 2383 } 2384 } 2385 } 2386 2387 Node* MemoryGraphFixer::get_ctrl(Node* n) const { 2388 Node* c = _phase->get_ctrl(n); 2389 if (n->is_Proj() && n->in(0) != NULL && n->in(0)->is_Call()) { 2390 assert(c == n->in(0), ""); 2391 CallNode* call = c->as_Call(); 2392 CallProjections projs; 2393 call->extract_projections(&projs, true, false); 2394 if (projs.catchall_memproj != NULL) { 2395 if (projs.fallthrough_memproj == n) { 2396 c = projs.fallthrough_catchproj; 2397 } else { 2398 assert(projs.catchall_memproj == n, ""); 2399 c = projs.catchall_catchproj; 2400 } 2401 } 2402 } 2403 return c; 2404 } 2405 2406 Node* MemoryGraphFixer::ctrl_or_self(Node* n) const { 2407 if (_phase->has_ctrl(n)) 2408 return get_ctrl(n); 2409 else { 2410 assert (n->is_CFG(), "must be a CFG node"); 2411 return n; 2412 } 2413 } 2414 2415 bool MemoryGraphFixer::mem_is_valid(Node* m, Node* c) const { 2416 return m != NULL && get_ctrl(m) == c; 2417 } 2418 2419 Node* MemoryGraphFixer::find_mem(Node* ctrl, Node* n) const { 2420 assert(n == NULL || _phase->ctrl_or_self(n) == ctrl, ""); 2421 Node* mem = _memory_nodes[ctrl->_idx]; 2422 Node* c = ctrl; 2423 while (!mem_is_valid(mem, c) && 2424 (!c->is_CatchProj() || mem == NULL || c->in(0)->in(0)->in(0) != get_ctrl(mem))) { 2425 c = _phase->idom(c); 2426 mem = _memory_nodes[c->_idx]; 2427 } 2428 if (n != NULL && mem_is_valid(mem, c)) { 2429 while (!ShenandoahBarrierC2Support::is_dominator_same_ctrl(c, mem, n, _phase) && _phase->ctrl_or_self(mem) == ctrl) { 2430 mem = next_mem(mem, _alias); 2431 } 2432 if (mem->is_MergeMem()) { 2433 mem = mem->as_MergeMem()->memory_at(_alias); 2434 } 2435 if (!mem_is_valid(mem, c)) { 2436 do { 2437 c = _phase->idom(c); 2438 mem = _memory_nodes[c->_idx]; 2439 } while (!mem_is_valid(mem, c) && 2440 (!c->is_CatchProj() || mem == NULL || c->in(0)->in(0)->in(0) != get_ctrl(mem))); 2441 } 2442 } 2443 assert(mem->bottom_type() == Type::MEMORY, ""); 2444 return mem; 2445 } 2446 2447 bool MemoryGraphFixer::has_mem_phi(Node* region) const { 2448 for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) { 2449 Node* use = region->fast_out(i); 2450 if (use->is_Phi() && use->bottom_type() == Type::MEMORY && 2451 (_phase->C->get_alias_index(use->adr_type()) == _alias)) { 2452 return true; 2453 } 2454 } 2455 return false; 2456 } 2457 2458 void MemoryGraphFixer::fix_mem(Node* ctrl, Node* new_ctrl, Node* mem, Node* mem_for_ctrl, Node* new_mem, Unique_Node_List& uses) { 2459 assert(_phase->ctrl_or_self(new_mem) == new_ctrl, ""); 2460 const bool trace = false; 2461 DEBUG_ONLY(if (trace) { tty->print("ZZZ control is"); ctrl->dump(); }); 2462 DEBUG_ONLY(if (trace) { tty->print("ZZZ mem is"); mem->dump(); }); 2463 GrowableArray<Node*> phis; 2464 if (mem_for_ctrl != mem) { 2465 Node* old = mem_for_ctrl; 2466 Node* prev = NULL; 2467 while (old != mem) { 2468 prev = old; 2469 if (old->is_Store() || old->is_ClearArray() || old->is_LoadStore()) { 2470 assert(_alias == Compile::AliasIdxRaw, ""); 2471 old = old->in(MemNode::Memory); 2472 } else if (old->Opcode() == Op_SCMemProj) { 2473 assert(_alias == Compile::AliasIdxRaw, ""); 2474 old = old->in(0); 2475 } else { 2476 ShouldNotReachHere(); 2477 } 2478 } 2479 assert(prev != NULL, ""); 2480 if (new_ctrl != ctrl) { 2481 _memory_nodes.map(ctrl->_idx, mem); 2482 _memory_nodes.map(new_ctrl->_idx, mem_for_ctrl); 2483 } 2484 uint input = (uint)MemNode::Memory; 2485 _phase->igvn().replace_input_of(prev, input, new_mem); 2486 } else { 2487 uses.clear(); 2488 _memory_nodes.map(new_ctrl->_idx, new_mem); 2489 uses.push(new_ctrl); 2490 for(uint next = 0; next < uses.size(); next++ ) { 2491 Node *n = uses.at(next); 2492 assert(n->is_CFG(), ""); 2493 DEBUG_ONLY(if (trace) { tty->print("ZZZ ctrl"); n->dump(); }); 2494 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { 2495 Node* u = n->fast_out(i); 2496 if (!u->is_Root() && u->is_CFG() && u != n) { 2497 Node* m = _memory_nodes[u->_idx]; 2498 if (u->is_Region() && (!u->is_OuterStripMinedLoop() || _include_lsm) && 2499 !has_mem_phi(u) && 2500 u->unique_ctrl_out()->Opcode() != Op_Halt) { 2501 DEBUG_ONLY(if (trace) { tty->print("ZZZ region"); u->dump(); }); 2502 DEBUG_ONLY(if (trace && m != NULL) { tty->print("ZZZ mem"); m->dump(); }); 2503 2504 if (!mem_is_valid(m, u) || !m->is_Phi()) { 2505 bool push = true; 2506 bool create_phi = true; 2507 if (_phase->is_dominator(new_ctrl, u)) { 2508 create_phi = false; 2509 } 2510 if (create_phi) { 2511 Node* phi = new PhiNode(u, Type::MEMORY, _phase->C->get_adr_type(_alias)); 2512 _phase->register_new_node(phi, u); 2513 phis.push(phi); 2514 DEBUG_ONLY(if (trace) { tty->print("ZZZ new phi"); phi->dump(); }); 2515 if (!mem_is_valid(m, u)) { 2516 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting mem"); phi->dump(); }); 2517 _memory_nodes.map(u->_idx, phi); 2518 } else { 2519 DEBUG_ONLY(if (trace) { tty->print("ZZZ NOT setting mem"); m->dump(); }); 2520 for (;;) { 2521 assert(m->is_Mem() || m->is_LoadStore() || m->is_Proj(), ""); 2522 Node* next = NULL; 2523 if (m->is_Proj()) { 2524 next = m->in(0); 2525 } else { 2526 assert(m->is_Mem() || m->is_LoadStore(), ""); 2527 assert(_alias == Compile::AliasIdxRaw, ""); 2528 next = m->in(MemNode::Memory); 2529 } 2530 if (_phase->get_ctrl(next) != u) { 2531 break; 2532 } 2533 if (next->is_MergeMem()) { 2534 assert(_phase->get_ctrl(next->as_MergeMem()->memory_at(_alias)) != u, ""); 2535 break; 2536 } 2537 if (next->is_Phi()) { 2538 assert(next->adr_type() == TypePtr::BOTTOM && next->in(0) == u, ""); 2539 break; 2540 } 2541 m = next; 2542 } 2543 2544 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting to phi"); m->dump(); }); 2545 assert(m->is_Mem() || m->is_LoadStore(), ""); 2546 uint input = (uint)MemNode::Memory; 2547 _phase->igvn().replace_input_of(m, input, phi); 2548 push = false; 2549 } 2550 } else { 2551 DEBUG_ONLY(if (trace) { tty->print("ZZZ skipping region"); u->dump(); }); 2552 } 2553 if (push) { 2554 uses.push(u); 2555 } 2556 } 2557 } else if (!mem_is_valid(m, u) && 2558 !(u->Opcode() == Op_CProj && u->in(0)->Opcode() == Op_NeverBranch && u->as_Proj()->_con == 1)) { 2559 uses.push(u); 2560 } 2561 } 2562 } 2563 } 2564 for (int i = 0; i < phis.length(); i++) { 2565 Node* n = phis.at(i); 2566 Node* r = n->in(0); 2567 DEBUG_ONLY(if (trace) { tty->print("ZZZ fixing new phi"); n->dump(); }); 2568 for (uint j = 1; j < n->req(); j++) { 2569 Node* m = find_mem(r->in(j), NULL); 2570 _phase->igvn().replace_input_of(n, j, m); 2571 DEBUG_ONLY(if (trace) { tty->print("ZZZ fixing new phi: %d", j); m->dump(); }); 2572 } 2573 } 2574 } 2575 uint last = _phase->C->unique(); 2576 MergeMemNode* mm = NULL; 2577 int alias = _alias; 2578 DEBUG_ONLY(if (trace) { tty->print("ZZZ raw mem is"); mem->dump(); }); 2579 // Process loads first to not miss an anti-dependency: if the memory 2580 // edge of a store is updated before a load is processed then an 2581 // anti-dependency may be missed. 2582 for (DUIterator i = mem->outs(); mem->has_out(i); i++) { 2583 Node* u = mem->out(i); 2584 if (u->_idx < last && u->is_Load() && _phase->C->get_alias_index(u->adr_type()) == alias) { 2585 Node* m = find_mem(_phase->get_ctrl(u), u); 2586 if (m != mem) { 2587 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting memory of use"); u->dump(); }); 2588 _phase->igvn().replace_input_of(u, MemNode::Memory, m); 2589 --i; 2590 } 2591 } 2592 } 2593 for (DUIterator i = mem->outs(); mem->has_out(i); i++) { 2594 Node* u = mem->out(i); 2595 if (u->_idx < last) { 2596 if (u->is_Mem()) { 2597 if (_phase->C->get_alias_index(u->adr_type()) == alias) { 2598 Node* m = find_mem(_phase->get_ctrl(u), u); 2599 if (m != mem) { 2600 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting memory of use"); u->dump(); }); 2601 _phase->igvn().replace_input_of(u, MemNode::Memory, m); 2602 --i; 2603 } 2604 } 2605 } else if (u->is_MergeMem()) { 2606 MergeMemNode* u_mm = u->as_MergeMem(); 2607 if (u_mm->memory_at(alias) == mem) { 2608 MergeMemNode* newmm = NULL; 2609 for (DUIterator_Fast jmax, j = u->fast_outs(jmax); j < jmax; j++) { 2610 Node* uu = u->fast_out(j); 2611 assert(!uu->is_MergeMem(), "chain of MergeMems?"); 2612 if (uu->is_Phi()) { 2613 assert(uu->adr_type() == TypePtr::BOTTOM, ""); 2614 Node* region = uu->in(0); 2615 int nb = 0; 2616 for (uint k = 1; k < uu->req(); k++) { 2617 if (uu->in(k) == u) { 2618 Node* m = find_mem(region->in(k), NULL); 2619 if (m != mem) { 2620 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting memory of phi %d", k); uu->dump(); }); 2621 newmm = clone_merge_mem(u, mem, m, _phase->ctrl_or_self(m), i); 2622 if (newmm != u) { 2623 _phase->igvn().replace_input_of(uu, k, newmm); 2624 nb++; 2625 --jmax; 2626 } 2627 } 2628 } 2629 } 2630 if (nb > 0) { 2631 --j; 2632 } 2633 } else { 2634 Node* m = find_mem(_phase->ctrl_or_self(uu), uu); 2635 if (m != mem) { 2636 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting memory of use"); uu->dump(); }); 2637 newmm = clone_merge_mem(u, mem, m, _phase->ctrl_or_self(m), i); 2638 if (newmm != u) { 2639 _phase->igvn().replace_input_of(uu, uu->find_edge(u), newmm); 2640 --j, --jmax; 2641 } 2642 } 2643 } 2644 } 2645 } 2646 } else if (u->is_Phi()) { 2647 assert(u->bottom_type() == Type::MEMORY, "what else?"); 2648 if (_phase->C->get_alias_index(u->adr_type()) == alias || u->adr_type() == TypePtr::BOTTOM) { 2649 Node* region = u->in(0); 2650 bool replaced = false; 2651 for (uint j = 1; j < u->req(); j++) { 2652 if (u->in(j) == mem) { 2653 Node* m = find_mem(region->in(j), NULL); 2654 Node* nnew = m; 2655 if (m != mem) { 2656 if (u->adr_type() == TypePtr::BOTTOM) { 2657 mm = allocate_merge_mem(mem, m, _phase->ctrl_or_self(m)); 2658 nnew = mm; 2659 } 2660 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting memory of phi %d", j); u->dump(); }); 2661 _phase->igvn().replace_input_of(u, j, nnew); 2662 replaced = true; 2663 } 2664 } 2665 } 2666 if (replaced) { 2667 --i; 2668 } 2669 } 2670 } else if ((u->adr_type() == TypePtr::BOTTOM && u->Opcode() != Op_StrInflatedCopy) || 2671 u->adr_type() == NULL) { 2672 assert(u->adr_type() != NULL || 2673 u->Opcode() == Op_Rethrow || 2674 u->Opcode() == Op_Return || 2675 u->Opcode() == Op_SafePoint || 2676 (u->is_CallStaticJava() && u->as_CallStaticJava()->uncommon_trap_request() != 0) || 2677 (u->is_CallStaticJava() && u->as_CallStaticJava()->_entry_point == OptoRuntime::rethrow_stub()) || 2678 u->Opcode() == Op_CallLeaf, ""); 2679 Node* m = find_mem(_phase->ctrl_or_self(u), u); 2680 if (m != mem) { 2681 mm = allocate_merge_mem(mem, m, _phase->get_ctrl(m)); 2682 _phase->igvn().replace_input_of(u, u->find_edge(mem), mm); 2683 --i; 2684 } 2685 } else if (_phase->C->get_alias_index(u->adr_type()) == alias) { 2686 Node* m = find_mem(_phase->ctrl_or_self(u), u); 2687 if (m != mem) { 2688 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting memory of use"); u->dump(); }); 2689 _phase->igvn().replace_input_of(u, u->find_edge(mem), m); 2690 --i; 2691 } 2692 } else if (u->adr_type() != TypePtr::BOTTOM && 2693 _memory_nodes[_phase->ctrl_or_self(u)->_idx] == u) { 2694 Node* m = find_mem(_phase->ctrl_or_self(u), u); 2695 assert(m != mem, ""); 2696 // u is on the wrong slice... 2697 assert(u->is_ClearArray(), ""); 2698 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting memory of use"); u->dump(); }); 2699 _phase->igvn().replace_input_of(u, u->find_edge(mem), m); 2700 --i; 2701 } 2702 } 2703 } 2704 #ifdef ASSERT 2705 assert(new_mem->outcnt() > 0, ""); 2706 for (int i = 0; i < phis.length(); i++) { 2707 Node* n = phis.at(i); 2708 assert(n->outcnt() > 0, "new phi must have uses now"); 2709 } 2710 #endif 2711 } 2712 2713 MergeMemNode* MemoryGraphFixer::allocate_merge_mem(Node* mem, Node* rep_proj, Node* rep_ctrl) const { 2714 MergeMemNode* mm = MergeMemNode::make(mem); 2715 mm->set_memory_at(_alias, rep_proj); 2716 _phase->register_new_node(mm, rep_ctrl); 2717 return mm; 2718 } 2719 2720 MergeMemNode* MemoryGraphFixer::clone_merge_mem(Node* u, Node* mem, Node* rep_proj, Node* rep_ctrl, DUIterator& i) const { 2721 MergeMemNode* newmm = NULL; 2722 MergeMemNode* u_mm = u->as_MergeMem(); 2723 Node* c = _phase->get_ctrl(u); 2724 if (_phase->is_dominator(c, rep_ctrl)) { 2725 c = rep_ctrl; 2726 } else { 2727 assert(_phase->is_dominator(rep_ctrl, c), "one must dominate the other"); 2728 } 2729 if (u->outcnt() == 1) { 2730 if (u->req() > (uint)_alias && u->in(_alias) == mem) { 2731 _phase->igvn().replace_input_of(u, _alias, rep_proj); 2732 --i; 2733 } else { 2734 _phase->igvn().rehash_node_delayed(u); 2735 u_mm->set_memory_at(_alias, rep_proj); 2736 } 2737 newmm = u_mm; 2738 _phase->set_ctrl_and_loop(u, c); 2739 } else { 2740 // can't simply clone u and then change one of its input because 2741 // it adds and then removes an edge which messes with the 2742 // DUIterator 2743 newmm = MergeMemNode::make(u_mm->base_memory()); 2744 for (uint j = 0; j < u->req(); j++) { 2745 if (j < newmm->req()) { 2746 if (j == (uint)_alias) { 2747 newmm->set_req(j, rep_proj); 2748 } else if (newmm->in(j) != u->in(j)) { 2749 newmm->set_req(j, u->in(j)); 2750 } 2751 } else if (j == (uint)_alias) { 2752 newmm->add_req(rep_proj); 2753 } else { 2754 newmm->add_req(u->in(j)); 2755 } 2756 } 2757 if ((uint)_alias >= u->req()) { 2758 newmm->set_memory_at(_alias, rep_proj); 2759 } 2760 _phase->register_new_node(newmm, c); 2761 } 2762 return newmm; 2763 } 2764 2765 bool MemoryGraphFixer::should_process_phi(Node* phi) const { 2766 if (phi->adr_type() == TypePtr::BOTTOM) { 2767 Node* region = phi->in(0); 2768 for (DUIterator_Fast jmax, j = region->fast_outs(jmax); j < jmax; j++) { 2769 Node* uu = region->fast_out(j); 2770 if (uu->is_Phi() && uu != phi && uu->bottom_type() == Type::MEMORY && _phase->C->get_alias_index(uu->adr_type()) == _alias) { 2771 return false; 2772 } 2773 } 2774 return true; 2775 } 2776 return _phase->C->get_alias_index(phi->adr_type()) == _alias; 2777 } 2778 2779 void MemoryGraphFixer::fix_memory_uses(Node* mem, Node* replacement, Node* rep_proj, Node* rep_ctrl) const { 2780 uint last = _phase-> C->unique(); 2781 MergeMemNode* mm = NULL; 2782 assert(mem->bottom_type() == Type::MEMORY, ""); 2783 for (DUIterator i = mem->outs(); mem->has_out(i); i++) { 2784 Node* u = mem->out(i); 2785 if (u != replacement && u->_idx < last) { 2786 if (u->is_MergeMem()) { 2787 MergeMemNode* u_mm = u->as_MergeMem(); 2788 if (u_mm->memory_at(_alias) == mem) { 2789 MergeMemNode* newmm = NULL; 2790 for (DUIterator_Fast jmax, j = u->fast_outs(jmax); j < jmax; j++) { 2791 Node* uu = u->fast_out(j); 2792 assert(!uu->is_MergeMem(), "chain of MergeMems?"); 2793 if (uu->is_Phi()) { 2794 if (should_process_phi(uu)) { 2795 Node* region = uu->in(0); 2796 int nb = 0; 2797 for (uint k = 1; k < uu->req(); k++) { 2798 if (uu->in(k) == u && _phase->is_dominator(rep_ctrl, region->in(k))) { 2799 if (newmm == NULL) { 2800 newmm = clone_merge_mem(u, mem, rep_proj, rep_ctrl, i); 2801 } 2802 if (newmm != u) { 2803 _phase->igvn().replace_input_of(uu, k, newmm); 2804 nb++; 2805 --jmax; 2806 } 2807 } 2808 } 2809 if (nb > 0) { 2810 --j; 2811 } 2812 } 2813 } else { 2814 if (rep_ctrl != uu && ShenandoahBarrierC2Support::is_dominator(rep_ctrl, _phase->ctrl_or_self(uu), replacement, uu, _phase)) { 2815 if (newmm == NULL) { 2816 newmm = clone_merge_mem(u, mem, rep_proj, rep_ctrl, i); 2817 } 2818 if (newmm != u) { 2819 _phase->igvn().replace_input_of(uu, uu->find_edge(u), newmm); 2820 --j, --jmax; 2821 } 2822 } 2823 } 2824 } 2825 } 2826 } else if (u->is_Phi()) { 2827 assert(u->bottom_type() == Type::MEMORY, "what else?"); 2828 Node* region = u->in(0); 2829 if (should_process_phi(u)) { 2830 bool replaced = false; 2831 for (uint j = 1; j < u->req(); j++) { 2832 if (u->in(j) == mem && _phase->is_dominator(rep_ctrl, region->in(j))) { 2833 Node* nnew = rep_proj; 2834 if (u->adr_type() == TypePtr::BOTTOM) { 2835 if (mm == NULL) { 2836 mm = allocate_merge_mem(mem, rep_proj, rep_ctrl); 2837 } 2838 nnew = mm; 2839 } 2840 _phase->igvn().replace_input_of(u, j, nnew); 2841 replaced = true; 2842 } 2843 } 2844 if (replaced) { 2845 --i; 2846 } 2847 2848 } 2849 } else if ((u->adr_type() == TypePtr::BOTTOM && u->Opcode() != Op_StrInflatedCopy) || 2850 u->adr_type() == NULL) { 2851 assert(u->adr_type() != NULL || 2852 u->Opcode() == Op_Rethrow || 2853 u->Opcode() == Op_Return || 2854 u->Opcode() == Op_SafePoint || 2855 u->Opcode() == Op_StoreIConditional || 2856 u->Opcode() == Op_StoreLConditional || 2857 (u->is_CallStaticJava() && u->as_CallStaticJava()->uncommon_trap_request() != 0) || 2858 (u->is_CallStaticJava() && u->as_CallStaticJava()->_entry_point == OptoRuntime::rethrow_stub()) || 2859 u->Opcode() == Op_CallLeaf, "%s", u->Name()); 2860 if (ShenandoahBarrierC2Support::is_dominator(rep_ctrl, _phase->ctrl_or_self(u), replacement, u, _phase)) { 2861 if (mm == NULL) { 2862 mm = allocate_merge_mem(mem, rep_proj, rep_ctrl); 2863 } 2864 _phase->igvn().replace_input_of(u, u->find_edge(mem), mm); 2865 --i; 2866 } 2867 } else if (_phase->C->get_alias_index(u->adr_type()) == _alias) { 2868 if (ShenandoahBarrierC2Support::is_dominator(rep_ctrl, _phase->ctrl_or_self(u), replacement, u, _phase)) { 2869 _phase->igvn().replace_input_of(u, u->find_edge(mem), rep_proj); 2870 --i; 2871 } 2872 } 2873 } 2874 } 2875 } 2876 2877 ShenandoahLoadReferenceBarrierNode::ShenandoahLoadReferenceBarrierNode(Node* ctrl, Node* obj, bool native) 2878 : Node(ctrl, obj), _native(native) { 2879 ShenandoahBarrierSetC2::bsc2()->state()->add_load_reference_barrier(this); 2880 } 2881 2882 bool ShenandoahLoadReferenceBarrierNode::is_native() const { 2883 return _native; 2884 } 2885 2886 uint ShenandoahLoadReferenceBarrierNode::size_of() const { 2887 return sizeof(*this); 2888 } 2889 2890 uint ShenandoahLoadReferenceBarrierNode::hash() const { 2891 return Node::hash() + (_native ? 1 : 0); 2892 } 2893 2894 bool ShenandoahLoadReferenceBarrierNode::cmp( const Node &n ) const { 2895 return Node::cmp(n) && n.Opcode() == Op_ShenandoahLoadReferenceBarrier && 2896 _native == ((const ShenandoahLoadReferenceBarrierNode&)n)._native; 2897 } 2898 2899 const Type* ShenandoahLoadReferenceBarrierNode::bottom_type() const { 2900 if (in(ValueIn) == NULL || in(ValueIn)->is_top()) { 2901 return Type::TOP; 2902 } 2903 const Type* t = in(ValueIn)->bottom_type(); 2904 if (t == TypePtr::NULL_PTR) { 2905 return t; 2906 } 2907 return t->is_oopptr(); 2908 } 2909 2910 const Type* ShenandoahLoadReferenceBarrierNode::Value(PhaseGVN* phase) const { 2911 // Either input is TOP ==> the result is TOP 2912 const Type *t2 = phase->type(in(ValueIn)); 2913 if( t2 == Type::TOP ) return Type::TOP; 2914 2915 if (t2 == TypePtr::NULL_PTR) { 2916 return t2; 2917 } 2918 2919 const Type* type = t2->is_oopptr(); 2920 return type; 2921 } 2922 2923 Node* ShenandoahLoadReferenceBarrierNode::Identity(PhaseGVN* phase) { 2924 Node* value = in(ValueIn); 2925 if (!needs_barrier(phase, value)) { 2926 return value; 2927 } 2928 return this; 2929 } 2930 2931 bool ShenandoahLoadReferenceBarrierNode::needs_barrier(PhaseGVN* phase, Node* n) { 2932 Unique_Node_List visited; 2933 return needs_barrier_impl(phase, n, visited); 2934 } 2935 2936 bool ShenandoahLoadReferenceBarrierNode::needs_barrier_impl(PhaseGVN* phase, Node* n, Unique_Node_List &visited) { 2937 if (n == NULL) return false; 2938 if (visited.member(n)) { 2939 return false; // Been there. 2940 } 2941 visited.push(n); 2942 2943 if (n->is_Allocate()) { 2944 // tty->print_cr("optimize barrier on alloc"); 2945 return false; 2946 } 2947 if (n->is_Call()) { 2948 // tty->print_cr("optimize barrier on call"); 2949 return false; 2950 } 2951 2952 const Type* type = phase->type(n); 2953 if (type == Type::TOP) { 2954 return false; 2955 } 2956 if (type->make_ptr()->higher_equal(TypePtr::NULL_PTR)) { 2957 // tty->print_cr("optimize barrier on null"); 2958 return false; 2959 } 2960 if (type->make_oopptr() && type->make_oopptr()->const_oop() != NULL) { 2961 // tty->print_cr("optimize barrier on constant"); 2962 return false; 2963 } 2964 2965 switch (n->Opcode()) { 2966 case Op_AddP: 2967 return true; // TODO: Can refine? 2968 case Op_LoadP: 2969 case Op_ShenandoahCompareAndExchangeN: 2970 case Op_ShenandoahCompareAndExchangeP: 2971 case Op_CompareAndExchangeN: 2972 case Op_CompareAndExchangeP: 2973 case Op_GetAndSetN: 2974 case Op_GetAndSetP: 2975 return true; 2976 case Op_Phi: { 2977 for (uint i = 1; i < n->req(); i++) { 2978 if (needs_barrier_impl(phase, n->in(i), visited)) return true; 2979 } 2980 return false; 2981 } 2982 case Op_CheckCastPP: 2983 case Op_CastPP: 2984 return needs_barrier_impl(phase, n->in(1), visited); 2985 case Op_Proj: 2986 return needs_barrier_impl(phase, n->in(0), visited); 2987 case Op_ShenandoahLoadReferenceBarrier: 2988 // tty->print_cr("optimize barrier on barrier"); 2989 return false; 2990 case Op_Parm: 2991 // tty->print_cr("optimize barrier on input arg"); 2992 return false; 2993 case Op_DecodeN: 2994 case Op_EncodeP: 2995 return needs_barrier_impl(phase, n->in(1), visited); 2996 case Op_LoadN: 2997 return true; 2998 case Op_CMoveN: 2999 case Op_CMoveP: 3000 return needs_barrier_impl(phase, n->in(2), visited) || 3001 needs_barrier_impl(phase, n->in(3), visited); 3002 case Op_ShenandoahEnqueueBarrier: 3003 return needs_barrier_impl(phase, n->in(1), visited); 3004 case Op_CreateEx: 3005 return false; 3006 default: 3007 break; 3008 } 3009 #ifdef ASSERT 3010 tty->print("need barrier on?: "); 3011 tty->print_cr("ins:"); 3012 n->dump(2); 3013 tty->print_cr("outs:"); 3014 n->dump(-2); 3015 ShouldNotReachHere(); 3016 #endif 3017 return true; 3018 }