1 /* 2 * Copyright (c) 2015, 2019, Red Hat, Inc. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 27 #include "classfile/javaClasses.hpp" 28 #include "gc/shenandoah/c2/shenandoahSupport.hpp" 29 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp" 30 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp" 31 #include "gc/shenandoah/shenandoahForwarding.hpp" 32 #include "gc/shenandoah/shenandoahHeap.hpp" 33 #include "gc/shenandoah/shenandoahHeapRegion.hpp" 34 #include "gc/shenandoah/shenandoahRuntime.hpp" 35 #include "gc/shenandoah/shenandoahThreadLocalData.hpp" 36 #include "opto/arraycopynode.hpp" 37 #include "opto/block.hpp" 38 #include "opto/callnode.hpp" 39 #include "opto/castnode.hpp" 40 #include "opto/movenode.hpp" 41 #include "opto/phaseX.hpp" 42 #include "opto/rootnode.hpp" 43 #include "opto/runtime.hpp" 44 #include "opto/subnode.hpp" 45 46 bool ShenandoahBarrierC2Support::expand(Compile* C, PhaseIterGVN& igvn) { 47 ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state(); 48 if ((state->enqueue_barriers_count() + 49 state->load_reference_barriers_count()) > 0) { 50 bool attempt_more_loopopts = ShenandoahLoopOptsAfterExpansion; 51 C->clear_major_progress(); 52 PhaseIdealLoop ideal_loop(igvn, LoopOptsShenandoahExpand); 53 if (C->failing()) return false; 54 PhaseIdealLoop::verify(igvn); 55 DEBUG_ONLY(verify_raw_mem(C->root());) 56 if (attempt_more_loopopts) { 57 C->set_major_progress(); 58 if (!C->optimize_loops(igvn, LoopOptsShenandoahPostExpand)) { 59 return false; 60 } 61 C->clear_major_progress(); 62 if (C->range_check_cast_count() > 0) { 63 // No more loop optimizations. Remove all range check dependent CastIINodes. 64 C->remove_range_check_casts(igvn); 65 igvn.optimize(); 66 } 67 } 68 } 69 return true; 70 } 71 72 bool ShenandoahBarrierC2Support::is_gc_state_test(Node* iff, int mask) { 73 if (!UseShenandoahGC) { 74 return false; 75 } 76 assert(iff->is_If(), "bad input"); 77 if (iff->Opcode() != Op_If) { 78 return false; 79 } 80 Node* bol = iff->in(1); 81 if (!bol->is_Bool() || bol->as_Bool()->_test._test != BoolTest::ne) { 82 return false; 83 } 84 Node* cmp = bol->in(1); 85 if (cmp->Opcode() != Op_CmpI) { 86 return false; 87 } 88 Node* in1 = cmp->in(1); 89 Node* in2 = cmp->in(2); 90 if (in2->find_int_con(-1) != 0) { 91 return false; 92 } 93 if (in1->Opcode() != Op_AndI) { 94 return false; 95 } 96 in2 = in1->in(2); 97 if (in2->find_int_con(-1) != mask) { 98 return false; 99 } 100 in1 = in1->in(1); 101 102 return is_gc_state_load(in1); 103 } 104 105 bool ShenandoahBarrierC2Support::is_heap_stable_test(Node* iff) { 106 return is_gc_state_test(iff, ShenandoahHeap::HAS_FORWARDED); 107 } 108 109 bool ShenandoahBarrierC2Support::is_gc_state_load(Node *n) { 110 if (!UseShenandoahGC) { 111 return false; 112 } 113 if (n->Opcode() != Op_LoadB && n->Opcode() != Op_LoadUB) { 114 return false; 115 } 116 Node* addp = n->in(MemNode::Address); 117 if (!addp->is_AddP()) { 118 return false; 119 } 120 Node* base = addp->in(AddPNode::Address); 121 Node* off = addp->in(AddPNode::Offset); 122 if (base->Opcode() != Op_ThreadLocal) { 123 return false; 124 } 125 if (off->find_intptr_t_con(-1) != in_bytes(ShenandoahThreadLocalData::gc_state_offset())) { 126 return false; 127 } 128 return true; 129 } 130 131 bool ShenandoahBarrierC2Support::has_safepoint_between(Node* start, Node* stop, PhaseIdealLoop *phase) { 132 assert(phase->is_dominator(stop, start), "bad inputs"); 133 ResourceMark rm; 134 Unique_Node_List wq; 135 wq.push(start); 136 for (uint next = 0; next < wq.size(); next++) { 137 Node *m = wq.at(next); 138 if (m == stop) { 139 continue; 140 } 141 if (m->is_SafePoint() && !m->is_CallLeaf()) { 142 return true; 143 } 144 if (m->is_Region()) { 145 for (uint i = 1; i < m->req(); i++) { 146 wq.push(m->in(i)); 147 } 148 } else { 149 wq.push(m->in(0)); 150 } 151 } 152 return false; 153 } 154 155 #ifdef ASSERT 156 bool ShenandoahBarrierC2Support::verify_helper(Node* in, Node_Stack& phis, VectorSet& visited, verify_type t, bool trace, Unique_Node_List& barriers_used) { 157 assert(phis.size() == 0, ""); 158 159 while (true) { 160 if (in->bottom_type() == TypePtr::NULL_PTR) { 161 if (trace) {tty->print_cr("NULL");} 162 } else if (!in->bottom_type()->make_ptr()->make_oopptr()) { 163 if (trace) {tty->print_cr("Non oop");} 164 } else { 165 if (in->is_ConstraintCast()) { 166 in = in->in(1); 167 continue; 168 } else if (in->is_AddP()) { 169 assert(!in->in(AddPNode::Address)->is_top(), "no raw memory access"); 170 in = in->in(AddPNode::Address); 171 continue; 172 } else if (in->is_Con()) { 173 if (trace) { 174 tty->print("Found constant"); 175 in->dump(); 176 } 177 } else if (in->Opcode() == Op_Parm) { 178 if (trace) { 179 tty->print("Found argument"); 180 } 181 } else if (in->Opcode() == Op_CreateEx) { 182 if (trace) { 183 tty->print("Found create-exception"); 184 } 185 } else if (in->Opcode() == Op_LoadP && in->adr_type() == TypeRawPtr::BOTTOM) { 186 if (trace) { 187 tty->print("Found raw LoadP (OSR argument?)"); 188 } 189 } else if (in->Opcode() == Op_ShenandoahLoadReferenceBarrier) { 190 if (t == ShenandoahOopStore) { 191 uint i = 0; 192 for (; i < phis.size(); i++) { 193 Node* n = phis.node_at(i); 194 if (n->Opcode() == Op_ShenandoahEnqueueBarrier) { 195 break; 196 } 197 } 198 if (i == phis.size()) { 199 return false; 200 } 201 } 202 barriers_used.push(in); 203 if (trace) {tty->print("Found barrier"); in->dump();} 204 } else if (in->Opcode() == Op_ShenandoahEnqueueBarrier) { 205 if (t != ShenandoahOopStore) { 206 in = in->in(1); 207 continue; 208 } 209 if (trace) {tty->print("Found enqueue barrier"); in->dump();} 210 phis.push(in, in->req()); 211 in = in->in(1); 212 continue; 213 } else if (in->is_Proj() && in->in(0)->is_Allocate()) { 214 if (trace) { 215 tty->print("Found alloc"); 216 in->in(0)->dump(); 217 } 218 } else if (in->is_Proj() && (in->in(0)->Opcode() == Op_CallStaticJava || in->in(0)->Opcode() == Op_CallDynamicJava)) { 219 if (trace) { 220 tty->print("Found Java call"); 221 } 222 } else if (in->is_Phi()) { 223 if (!visited.test_set(in->_idx)) { 224 if (trace) {tty->print("Pushed phi:"); in->dump();} 225 phis.push(in, 2); 226 in = in->in(1); 227 continue; 228 } 229 if (trace) {tty->print("Already seen phi:"); in->dump();} 230 } else if (in->Opcode() == Op_CMoveP || in->Opcode() == Op_CMoveN) { 231 if (!visited.test_set(in->_idx)) { 232 if (trace) {tty->print("Pushed cmovep:"); in->dump();} 233 phis.push(in, CMoveNode::IfTrue); 234 in = in->in(CMoveNode::IfFalse); 235 continue; 236 } 237 if (trace) {tty->print("Already seen cmovep:"); in->dump();} 238 } else if (in->Opcode() == Op_EncodeP || in->Opcode() == Op_DecodeN) { 239 in = in->in(1); 240 continue; 241 } else { 242 return false; 243 } 244 } 245 bool cont = false; 246 while (phis.is_nonempty()) { 247 uint idx = phis.index(); 248 Node* phi = phis.node(); 249 if (idx >= phi->req()) { 250 if (trace) {tty->print("Popped phi:"); phi->dump();} 251 phis.pop(); 252 continue; 253 } 254 if (trace) {tty->print("Next entry(%d) for phi:", idx); phi->dump();} 255 in = phi->in(idx); 256 phis.set_index(idx+1); 257 cont = true; 258 break; 259 } 260 if (!cont) { 261 break; 262 } 263 } 264 return true; 265 } 266 267 void ShenandoahBarrierC2Support::report_verify_failure(const char* msg, Node* n1, Node* n2) { 268 if (n1 != NULL) { 269 n1->dump(+10); 270 } 271 if (n2 != NULL) { 272 n2->dump(+10); 273 } 274 fatal("%s", msg); 275 } 276 277 void ShenandoahBarrierC2Support::verify(RootNode* root) { 278 ResourceMark rm; 279 Unique_Node_List wq; 280 GrowableArray<Node*> barriers; 281 Unique_Node_List barriers_used; 282 Node_Stack phis(0); 283 VectorSet visited; 284 const bool trace = false; 285 const bool verify_no_useless_barrier = false; 286 287 wq.push(root); 288 for (uint next = 0; next < wq.size(); next++) { 289 Node *n = wq.at(next); 290 if (n->is_Load()) { 291 const bool trace = false; 292 if (trace) {tty->print("Verifying"); n->dump();} 293 if (n->Opcode() == Op_LoadRange || n->Opcode() == Op_LoadKlass || n->Opcode() == Op_LoadNKlass) { 294 if (trace) {tty->print_cr("Load range/klass");} 295 } else { 296 const TypePtr* adr_type = n->as_Load()->adr_type(); 297 298 if (adr_type->isa_oopptr() && adr_type->is_oopptr()->offset() == oopDesc::mark_offset_in_bytes()) { 299 if (trace) {tty->print_cr("Mark load");} 300 } else if (adr_type->isa_instptr() && 301 adr_type->is_instptr()->klass()->is_subtype_of(Compile::current()->env()->Reference_klass()) && 302 adr_type->is_instptr()->offset() == java_lang_ref_Reference::referent_offset()) { 303 if (trace) {tty->print_cr("Reference.get()");} 304 } else if (!verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahLoad, trace, barriers_used)) { 305 report_verify_failure("Shenandoah verification: Load should have barriers", n); 306 } 307 } 308 } else if (n->is_Store()) { 309 const bool trace = false; 310 311 if (trace) {tty->print("Verifying"); n->dump();} 312 if (n->in(MemNode::ValueIn)->bottom_type()->make_oopptr()) { 313 Node* adr = n->in(MemNode::Address); 314 bool verify = true; 315 316 if (adr->is_AddP() && adr->in(AddPNode::Base)->is_top()) { 317 adr = adr->in(AddPNode::Address); 318 if (adr->is_AddP()) { 319 assert(adr->in(AddPNode::Base)->is_top(), ""); 320 adr = adr->in(AddPNode::Address); 321 if (adr->Opcode() == Op_LoadP && 322 adr->in(MemNode::Address)->in(AddPNode::Base)->is_top() && 323 adr->in(MemNode::Address)->in(AddPNode::Address)->Opcode() == Op_ThreadLocal && 324 adr->in(MemNode::Address)->in(AddPNode::Offset)->find_intptr_t_con(-1) == in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset())) { 325 if (trace) {tty->print_cr("SATB prebarrier");} 326 verify = false; 327 } 328 } 329 } 330 331 if (verify && !verify_helper(n->in(MemNode::ValueIn), phis, visited, ShenandoahStoreValEnqueueBarrier ? ShenandoahOopStore : ShenandoahValue, trace, barriers_used)) { 332 report_verify_failure("Shenandoah verification: Store should have barriers", n); 333 } 334 } 335 if (!verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahStore, trace, barriers_used)) { 336 report_verify_failure("Shenandoah verification: Store (address) should have barriers", n); 337 } 338 } else if (n->Opcode() == Op_CmpP) { 339 const bool trace = false; 340 341 Node* in1 = n->in(1); 342 Node* in2 = n->in(2); 343 if (in1->bottom_type()->isa_oopptr()) { 344 if (trace) {tty->print("Verifying"); n->dump();} 345 346 bool mark_inputs = false; 347 if (in1->bottom_type() == TypePtr::NULL_PTR || in2->bottom_type() == TypePtr::NULL_PTR || 348 (in1->is_Con() || in2->is_Con())) { 349 if (trace) {tty->print_cr("Comparison against a constant");} 350 mark_inputs = true; 351 } else if ((in1->is_CheckCastPP() && in1->in(1)->is_Proj() && in1->in(1)->in(0)->is_Allocate()) || 352 (in2->is_CheckCastPP() && in2->in(1)->is_Proj() && in2->in(1)->in(0)->is_Allocate())) { 353 if (trace) {tty->print_cr("Comparison with newly alloc'ed object");} 354 mark_inputs = true; 355 } else { 356 assert(in2->bottom_type()->isa_oopptr(), ""); 357 358 if (!verify_helper(in1, phis, visited, ShenandoahStore, trace, barriers_used) || 359 !verify_helper(in2, phis, visited, ShenandoahStore, trace, barriers_used)) { 360 report_verify_failure("Shenandoah verification: Cmp should have barriers", n); 361 } 362 } 363 if (verify_no_useless_barrier && 364 mark_inputs && 365 (!verify_helper(in1, phis, visited, ShenandoahValue, trace, barriers_used) || 366 !verify_helper(in2, phis, visited, ShenandoahValue, trace, barriers_used))) { 367 phis.clear(); 368 visited.reset(); 369 } 370 } 371 } else if (n->is_LoadStore()) { 372 if (n->in(MemNode::ValueIn)->bottom_type()->make_ptr() && 373 !verify_helper(n->in(MemNode::ValueIn), phis, visited, ShenandoahStoreValEnqueueBarrier ? ShenandoahOopStore : ShenandoahValue, trace, barriers_used)) { 374 report_verify_failure("Shenandoah verification: LoadStore (value) should have barriers", n); 375 } 376 377 if (n->in(MemNode::Address)->bottom_type()->make_oopptr() && !verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahStore, trace, barriers_used)) { 378 report_verify_failure("Shenandoah verification: LoadStore (address) should have barriers", n); 379 } 380 } else if (n->Opcode() == Op_CallLeafNoFP || n->Opcode() == Op_CallLeaf) { 381 CallNode* call = n->as_Call(); 382 383 static struct { 384 const char* name; 385 struct { 386 int pos; 387 verify_type t; 388 } args[6]; 389 } calls[] = { 390 "aescrypt_encryptBlock", 391 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad }, 392 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 393 "aescrypt_decryptBlock", 394 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad }, 395 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 396 "multiplyToLen", 397 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad }, { TypeFunc::Parms+4, ShenandoahStore }, 398 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 399 "squareToLen", 400 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad }, { -1, ShenandoahNone}, 401 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 402 "montgomery_multiply", 403 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad }, 404 { TypeFunc::Parms+6, ShenandoahStore }, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 405 "montgomery_square", 406 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+5, ShenandoahStore }, 407 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 408 "mulAdd", 409 { { TypeFunc::Parms, ShenandoahStore }, { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone}, 410 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 411 "vectorizedMismatch", 412 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone}, 413 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 414 "updateBytesCRC32", 415 { { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone}, 416 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 417 "updateBytesAdler32", 418 { { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone}, 419 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 420 "updateBytesCRC32C", 421 { { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahLoad}, { -1, ShenandoahNone}, 422 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 423 "counterMode_AESCrypt", 424 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad }, 425 { TypeFunc::Parms+3, ShenandoahStore }, { TypeFunc::Parms+5, ShenandoahStore }, { TypeFunc::Parms+6, ShenandoahStore } }, 426 "cipherBlockChaining_encryptAESCrypt", 427 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad }, 428 { TypeFunc::Parms+3, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 429 "cipherBlockChaining_decryptAESCrypt", 430 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad }, 431 { TypeFunc::Parms+3, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 432 "shenandoah_clone_barrier", 433 { { TypeFunc::Parms, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone}, 434 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 435 "ghash_processBlocks", 436 { { TypeFunc::Parms, ShenandoahStore }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad }, 437 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 438 "sha1_implCompress", 439 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone }, 440 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 441 "sha256_implCompress", 442 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone }, 443 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 444 "sha512_implCompress", 445 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone }, 446 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 447 "sha1_implCompressMB", 448 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone }, 449 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 450 "sha256_implCompressMB", 451 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone }, 452 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 453 "sha512_implCompressMB", 454 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone }, 455 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 456 "encodeBlock", 457 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahStore }, { -1, ShenandoahNone }, 458 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} }, 459 }; 460 461 if (call->is_call_to_arraycopystub()) { 462 Node* dest = NULL; 463 const TypeTuple* args = n->as_Call()->_tf->domain(); 464 for (uint i = TypeFunc::Parms, j = 0; i < args->cnt(); i++) { 465 if (args->field_at(i)->isa_ptr()) { 466 j++; 467 if (j == 2) { 468 dest = n->in(i); 469 break; 470 } 471 } 472 } 473 if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahLoad, trace, barriers_used) || 474 !verify_helper(dest, phis, visited, ShenandoahStore, trace, barriers_used)) { 475 report_verify_failure("Shenandoah verification: ArrayCopy should have barriers", n); 476 } 477 } else if (strlen(call->_name) > 5 && 478 !strcmp(call->_name + strlen(call->_name) - 5, "_fill")) { 479 if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahStore, trace, barriers_used)) { 480 report_verify_failure("Shenandoah verification: _fill should have barriers", n); 481 } 482 } else if (!strcmp(call->_name, "shenandoah_wb_pre")) { 483 // skip 484 } else { 485 const int calls_len = sizeof(calls) / sizeof(calls[0]); 486 int i = 0; 487 for (; i < calls_len; i++) { 488 if (!strcmp(calls[i].name, call->_name)) { 489 break; 490 } 491 } 492 if (i != calls_len) { 493 const uint args_len = sizeof(calls[0].args) / sizeof(calls[0].args[0]); 494 for (uint j = 0; j < args_len; j++) { 495 int pos = calls[i].args[j].pos; 496 if (pos == -1) { 497 break; 498 } 499 if (!verify_helper(call->in(pos), phis, visited, calls[i].args[j].t, trace, barriers_used)) { 500 report_verify_failure("Shenandoah verification: intrinsic calls should have barriers", n); 501 } 502 } 503 for (uint j = TypeFunc::Parms; j < call->req(); j++) { 504 if (call->in(j)->bottom_type()->make_ptr() && 505 call->in(j)->bottom_type()->make_ptr()->isa_oopptr()) { 506 uint k = 0; 507 for (; k < args_len && calls[i].args[k].pos != (int)j; k++); 508 if (k == args_len) { 509 fatal("arg %d for call %s not covered", j, call->_name); 510 } 511 } 512 } 513 } else { 514 for (uint j = TypeFunc::Parms; j < call->req(); j++) { 515 if (call->in(j)->bottom_type()->make_ptr() && 516 call->in(j)->bottom_type()->make_ptr()->isa_oopptr()) { 517 fatal("%s not covered", call->_name); 518 } 519 } 520 } 521 } 522 } else if (n->Opcode() == Op_ShenandoahEnqueueBarrier || n->Opcode() == Op_ShenandoahLoadReferenceBarrier) { 523 // skip 524 } else if (n->is_AddP() 525 || n->is_Phi() 526 || n->is_ConstraintCast() 527 || n->Opcode() == Op_Return 528 || n->Opcode() == Op_CMoveP 529 || n->Opcode() == Op_CMoveN 530 || n->Opcode() == Op_Rethrow 531 || n->is_MemBar() 532 || n->Opcode() == Op_Conv2B 533 || n->Opcode() == Op_SafePoint 534 || n->is_CallJava() 535 || n->Opcode() == Op_Unlock 536 || n->Opcode() == Op_EncodeP 537 || n->Opcode() == Op_DecodeN) { 538 // nothing to do 539 } else { 540 static struct { 541 int opcode; 542 struct { 543 int pos; 544 verify_type t; 545 } inputs[2]; 546 } others[] = { 547 Op_FastLock, 548 { { 1, ShenandoahLoad }, { -1, ShenandoahNone} }, 549 Op_Lock, 550 { { TypeFunc::Parms, ShenandoahLoad }, { -1, ShenandoahNone} }, 551 Op_ArrayCopy, 552 { { ArrayCopyNode::Src, ShenandoahLoad }, { ArrayCopyNode::Dest, ShenandoahStore } }, 553 Op_StrCompressedCopy, 554 { { 2, ShenandoahLoad }, { 3, ShenandoahStore } }, 555 Op_StrInflatedCopy, 556 { { 2, ShenandoahLoad }, { 3, ShenandoahStore } }, 557 Op_AryEq, 558 { { 2, ShenandoahLoad }, { 3, ShenandoahLoad } }, 559 Op_StrIndexOf, 560 { { 2, ShenandoahLoad }, { 4, ShenandoahLoad } }, 561 Op_StrComp, 562 { { 2, ShenandoahLoad }, { 4, ShenandoahLoad } }, 563 Op_StrEquals, 564 { { 2, ShenandoahLoad }, { 3, ShenandoahLoad } }, 565 Op_EncodeISOArray, 566 { { 2, ShenandoahLoad }, { 3, ShenandoahStore } }, 567 Op_HasNegatives, 568 { { 2, ShenandoahLoad }, { -1, ShenandoahNone} }, 569 Op_CastP2X, 570 { { 1, ShenandoahLoad }, { -1, ShenandoahNone} }, 571 Op_StrIndexOfChar, 572 { { 2, ShenandoahLoad }, { -1, ShenandoahNone } }, 573 }; 574 575 const int others_len = sizeof(others) / sizeof(others[0]); 576 int i = 0; 577 for (; i < others_len; i++) { 578 if (others[i].opcode == n->Opcode()) { 579 break; 580 } 581 } 582 uint stop = n->is_Call() ? n->as_Call()->tf()->domain()->cnt() : n->req(); 583 if (i != others_len) { 584 const uint inputs_len = sizeof(others[0].inputs) / sizeof(others[0].inputs[0]); 585 for (uint j = 0; j < inputs_len; j++) { 586 int pos = others[i].inputs[j].pos; 587 if (pos == -1) { 588 break; 589 } 590 if (!verify_helper(n->in(pos), phis, visited, others[i].inputs[j].t, trace, barriers_used)) { 591 report_verify_failure("Shenandoah verification: intrinsic calls should have barriers", n); 592 } 593 } 594 for (uint j = 1; j < stop; j++) { 595 if (n->in(j) != NULL && n->in(j)->bottom_type()->make_ptr() && 596 n->in(j)->bottom_type()->make_ptr()->make_oopptr()) { 597 uint k = 0; 598 for (; k < inputs_len && others[i].inputs[k].pos != (int)j; k++); 599 if (k == inputs_len) { 600 fatal("arg %d for node %s not covered", j, n->Name()); 601 } 602 } 603 } 604 } else { 605 for (uint j = 1; j < stop; j++) { 606 if (n->in(j) != NULL && n->in(j)->bottom_type()->make_ptr() && 607 n->in(j)->bottom_type()->make_ptr()->make_oopptr()) { 608 fatal("%s not covered", n->Name()); 609 } 610 } 611 } 612 } 613 614 if (n->is_SafePoint()) { 615 SafePointNode* sfpt = n->as_SafePoint(); 616 if (verify_no_useless_barrier && sfpt->jvms() != NULL) { 617 for (uint i = sfpt->jvms()->scloff(); i < sfpt->jvms()->endoff(); i++) { 618 if (!verify_helper(sfpt->in(i), phis, visited, ShenandoahLoad, trace, barriers_used)) { 619 phis.clear(); 620 visited.reset(); 621 } 622 } 623 } 624 } 625 } 626 627 if (verify_no_useless_barrier) { 628 for (int i = 0; i < barriers.length(); i++) { 629 Node* n = barriers.at(i); 630 if (!barriers_used.member(n)) { 631 tty->print("XXX useless barrier"); n->dump(-2); 632 ShouldNotReachHere(); 633 } 634 } 635 } 636 } 637 #endif 638 639 bool ShenandoahBarrierC2Support::is_dominator_same_ctrl(Node* c, Node* d, Node* n, PhaseIdealLoop* phase) { 640 // That both nodes have the same control is not sufficient to prove 641 // domination, verify that there's no path from d to n 642 ResourceMark rm; 643 Unique_Node_List wq; 644 wq.push(d); 645 for (uint next = 0; next < wq.size(); next++) { 646 Node *m = wq.at(next); 647 if (m == n) { 648 return false; 649 } 650 if (m->is_Phi() && m->in(0)->is_Loop()) { 651 assert(phase->ctrl_or_self(m->in(LoopNode::EntryControl)) != c, "following loop entry should lead to new control"); 652 } else { 653 if (m->is_Store() || m->is_LoadStore()) { 654 // Take anti-dependencies into account 655 Node* mem = m->in(MemNode::Memory); 656 for (DUIterator_Fast imax, i = mem->fast_outs(imax); i < imax; i++) { 657 Node* u = mem->fast_out(i); 658 if (u->is_Load() && phase->C->can_alias(m->adr_type(), phase->C->get_alias_index(u->adr_type())) && 659 phase->ctrl_or_self(u) == c) { 660 wq.push(u); 661 } 662 } 663 } 664 for (uint i = 0; i < m->req(); i++) { 665 if (m->in(i) != NULL && phase->ctrl_or_self(m->in(i)) == c) { 666 wq.push(m->in(i)); 667 } 668 } 669 } 670 } 671 return true; 672 } 673 674 bool ShenandoahBarrierC2Support::is_dominator(Node* d_c, Node* n_c, Node* d, Node* n, PhaseIdealLoop* phase) { 675 if (d_c != n_c) { 676 return phase->is_dominator(d_c, n_c); 677 } 678 return is_dominator_same_ctrl(d_c, d, n, phase); 679 } 680 681 Node* next_mem(Node* mem, int alias) { 682 Node* res = NULL; 683 if (mem->is_Proj()) { 684 res = mem->in(0); 685 } else if (mem->is_SafePoint() || mem->is_MemBar()) { 686 res = mem->in(TypeFunc::Memory); 687 } else if (mem->is_Phi()) { 688 res = mem->in(1); 689 } else if (mem->is_MergeMem()) { 690 res = mem->as_MergeMem()->memory_at(alias); 691 } else if (mem->is_Store() || mem->is_LoadStore() || mem->is_ClearArray()) { 692 assert(alias = Compile::AliasIdxRaw, "following raw memory can't lead to a barrier"); 693 res = mem->in(MemNode::Memory); 694 } else { 695 #ifdef ASSERT 696 mem->dump(); 697 #endif 698 ShouldNotReachHere(); 699 } 700 return res; 701 } 702 703 Node* ShenandoahBarrierC2Support::no_branches(Node* c, Node* dom, bool allow_one_proj, PhaseIdealLoop* phase) { 704 Node* iffproj = NULL; 705 while (c != dom) { 706 Node* next = phase->idom(c); 707 assert(next->unique_ctrl_out() == c || c->is_Proj() || c->is_Region(), "multiple control flow out but no proj or region?"); 708 if (c->is_Region()) { 709 ResourceMark rm; 710 Unique_Node_List wq; 711 wq.push(c); 712 for (uint i = 0; i < wq.size(); i++) { 713 Node *n = wq.at(i); 714 if (n == next) { 715 continue; 716 } 717 if (n->is_Region()) { 718 for (uint j = 1; j < n->req(); j++) { 719 wq.push(n->in(j)); 720 } 721 } else { 722 wq.push(n->in(0)); 723 } 724 } 725 for (uint i = 0; i < wq.size(); i++) { 726 Node *n = wq.at(i); 727 assert(n->is_CFG(), ""); 728 if (n->is_Multi()) { 729 for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) { 730 Node* u = n->fast_out(j); 731 if (u->is_CFG()) { 732 if (!wq.member(u) && !u->as_Proj()->is_uncommon_trap_proj(Deoptimization::Reason_none)) { 733 return NodeSentinel; 734 } 735 } 736 } 737 } 738 } 739 } else if (c->is_Proj()) { 740 if (c->is_IfProj()) { 741 if (c->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none) != NULL) { 742 // continue; 743 } else { 744 if (!allow_one_proj) { 745 return NodeSentinel; 746 } 747 if (iffproj == NULL) { 748 iffproj = c; 749 } else { 750 return NodeSentinel; 751 } 752 } 753 } else if (c->Opcode() == Op_JumpProj) { 754 return NodeSentinel; // unsupported 755 } else if (c->Opcode() == Op_CatchProj) { 756 return NodeSentinel; // unsupported 757 } else if (c->Opcode() == Op_CProj && next->Opcode() == Op_NeverBranch) { 758 return NodeSentinel; // unsupported 759 } else { 760 assert(next->unique_ctrl_out() == c, "unsupported branch pattern"); 761 } 762 } 763 c = next; 764 } 765 return iffproj; 766 } 767 768 Node* ShenandoahBarrierC2Support::dom_mem(Node* mem, Node* ctrl, int alias, Node*& mem_ctrl, PhaseIdealLoop* phase) { 769 ResourceMark rm; 770 VectorSet wq; 771 wq.set(mem->_idx); 772 mem_ctrl = phase->ctrl_or_self(mem); 773 while (!phase->is_dominator(mem_ctrl, ctrl) || mem_ctrl == ctrl) { 774 mem = next_mem(mem, alias); 775 if (wq.test_set(mem->_idx)) { 776 return NULL; 777 } 778 mem_ctrl = phase->ctrl_or_self(mem); 779 } 780 if (mem->is_MergeMem()) { 781 mem = mem->as_MergeMem()->memory_at(alias); 782 mem_ctrl = phase->ctrl_or_self(mem); 783 } 784 return mem; 785 } 786 787 Node* ShenandoahBarrierC2Support::find_bottom_mem(Node* ctrl, PhaseIdealLoop* phase) { 788 Node* mem = NULL; 789 Node* c = ctrl; 790 do { 791 if (c->is_Region()) { 792 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax && mem == NULL; i++) { 793 Node* u = c->fast_out(i); 794 if (u->is_Phi() && u->bottom_type() == Type::MEMORY) { 795 if (u->adr_type() == TypePtr::BOTTOM) { 796 mem = u; 797 } 798 } 799 } 800 } else { 801 if (c->is_Call() && c->as_Call()->adr_type() != NULL) { 802 CallProjections projs; 803 c->as_Call()->extract_projections(&projs, true, false); 804 if (projs.fallthrough_memproj != NULL) { 805 if (projs.fallthrough_memproj->adr_type() == TypePtr::BOTTOM) { 806 if (projs.catchall_memproj == NULL) { 807 mem = projs.fallthrough_memproj; 808 } else { 809 if (phase->is_dominator(projs.fallthrough_catchproj, ctrl)) { 810 mem = projs.fallthrough_memproj; 811 } else { 812 assert(phase->is_dominator(projs.catchall_catchproj, ctrl), "one proj must dominate barrier"); 813 mem = projs.catchall_memproj; 814 } 815 } 816 } 817 } else { 818 Node* proj = c->as_Call()->proj_out(TypeFunc::Memory); 819 if (proj != NULL && 820 proj->adr_type() == TypePtr::BOTTOM) { 821 mem = proj; 822 } 823 } 824 } else { 825 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) { 826 Node* u = c->fast_out(i); 827 if (u->is_Proj() && 828 u->bottom_type() == Type::MEMORY && 829 u->adr_type() == TypePtr::BOTTOM) { 830 assert(c->is_SafePoint() || c->is_MemBar() || c->is_Start(), ""); 831 assert(mem == NULL, "only one proj"); 832 mem = u; 833 } 834 } 835 assert(!c->is_Call() || c->as_Call()->adr_type() != NULL || mem == NULL, "no mem projection expected"); 836 } 837 } 838 c = phase->idom(c); 839 } while (mem == NULL); 840 return mem; 841 } 842 843 void ShenandoahBarrierC2Support::follow_barrier_uses(Node* n, Node* ctrl, Unique_Node_List& uses, PhaseIdealLoop* phase) { 844 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { 845 Node* u = n->fast_out(i); 846 if (!u->is_CFG() && phase->get_ctrl(u) == ctrl && (!u->is_Phi() || !u->in(0)->is_Loop() || u->in(LoopNode::LoopBackControl) != n)) { 847 uses.push(u); 848 } 849 } 850 } 851 852 static void hide_strip_mined_loop(OuterStripMinedLoopNode* outer, CountedLoopNode* inner, PhaseIdealLoop* phase) { 853 OuterStripMinedLoopEndNode* le = inner->outer_loop_end(); 854 Node* new_outer = new LoopNode(outer->in(LoopNode::EntryControl), outer->in(LoopNode::LoopBackControl)); 855 phase->register_control(new_outer, phase->get_loop(outer), outer->in(LoopNode::EntryControl)); 856 Node* new_le = new IfNode(le->in(0), le->in(1), le->_prob, le->_fcnt); 857 phase->register_control(new_le, phase->get_loop(le), le->in(0)); 858 phase->lazy_replace(outer, new_outer); 859 phase->lazy_replace(le, new_le); 860 inner->clear_strip_mined(); 861 } 862 863 void ShenandoahBarrierC2Support::test_gc_state(Node*& ctrl, Node* raw_mem, Node*& test_fail_ctrl, 864 PhaseIdealLoop* phase, int flags) { 865 PhaseIterGVN& igvn = phase->igvn(); 866 Node* old_ctrl = ctrl; 867 868 Node* thread = new ThreadLocalNode(); 869 Node* gc_state_offset = igvn.MakeConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset())); 870 Node* gc_state_addr = new AddPNode(phase->C->top(), thread, gc_state_offset); 871 Node* gc_state = new LoadBNode(old_ctrl, raw_mem, gc_state_addr, 872 DEBUG_ONLY(phase->C->get_adr_type(Compile::AliasIdxRaw)) NOT_DEBUG(NULL), 873 TypeInt::BYTE, MemNode::unordered); 874 Node* gc_state_and = new AndINode(gc_state, igvn.intcon(flags)); 875 Node* gc_state_cmp = new CmpINode(gc_state_and, igvn.zerocon(T_INT)); 876 Node* gc_state_bool = new BoolNode(gc_state_cmp, BoolTest::ne); 877 878 IfNode* gc_state_iff = new IfNode(old_ctrl, gc_state_bool, PROB_UNLIKELY(0.999), COUNT_UNKNOWN); 879 ctrl = new IfTrueNode(gc_state_iff); 880 test_fail_ctrl = new IfFalseNode(gc_state_iff); 881 882 IdealLoopTree* loop = phase->get_loop(old_ctrl); 883 phase->register_control(gc_state_iff, loop, old_ctrl); 884 phase->register_control(ctrl, loop, gc_state_iff); 885 phase->register_control(test_fail_ctrl, loop, gc_state_iff); 886 887 phase->register_new_node(thread, old_ctrl); 888 phase->register_new_node(gc_state_addr, old_ctrl); 889 phase->register_new_node(gc_state, old_ctrl); 890 phase->register_new_node(gc_state_and, old_ctrl); 891 phase->register_new_node(gc_state_cmp, old_ctrl); 892 phase->register_new_node(gc_state_bool, old_ctrl); 893 894 phase->set_ctrl(gc_state_offset, phase->C->root()); 895 896 assert(is_gc_state_test(gc_state_iff, flags), "Should match the shape"); 897 } 898 899 void ShenandoahBarrierC2Support::test_null(Node*& ctrl, Node* val, Node*& null_ctrl, PhaseIdealLoop* phase) { 900 Node* old_ctrl = ctrl; 901 PhaseIterGVN& igvn = phase->igvn(); 902 903 const Type* val_t = igvn.type(val); 904 if (val_t->meet(TypePtr::NULL_PTR) == val_t) { 905 Node* null_cmp = new CmpPNode(val, igvn.zerocon(T_OBJECT)); 906 Node* null_test = new BoolNode(null_cmp, BoolTest::ne); 907 908 IfNode* null_iff = new IfNode(old_ctrl, null_test, PROB_LIKELY(0.999), COUNT_UNKNOWN); 909 ctrl = new IfTrueNode(null_iff); 910 null_ctrl = new IfFalseNode(null_iff); 911 912 IdealLoopTree* loop = phase->get_loop(old_ctrl); 913 phase->register_control(null_iff, loop, old_ctrl); 914 phase->register_control(ctrl, loop, null_iff); 915 phase->register_control(null_ctrl, loop, null_iff); 916 917 phase->register_new_node(null_cmp, old_ctrl); 918 phase->register_new_node(null_test, old_ctrl); 919 } 920 } 921 922 void ShenandoahBarrierC2Support::test_in_cset(Node*& ctrl, Node*& not_cset_ctrl, Node* val, Node* raw_mem, PhaseIdealLoop* phase) { 923 Node* old_ctrl = ctrl; 924 PhaseIterGVN& igvn = phase->igvn(); 925 926 Node* raw_val = new CastP2XNode(old_ctrl, val); 927 Node* cset_idx = new URShiftXNode(raw_val, igvn.intcon(ShenandoahHeapRegion::region_size_bytes_shift_jint())); 928 929 // Figure out the target cset address with raw pointer math. 930 // This avoids matching AddP+LoadB that would emit inefficient code. 931 // See JDK-8245465. 932 Node* cset_addr_ptr = igvn.makecon(TypeRawPtr::make(ShenandoahHeap::in_cset_fast_test_addr())); 933 Node* cset_addr = new CastP2XNode(old_ctrl, cset_addr_ptr); 934 Node* cset_load_addr = new AddXNode(cset_addr, cset_idx); 935 Node* cset_load_ptr = new CastX2PNode(cset_load_addr); 936 937 Node* cset_load = new LoadBNode(old_ctrl, raw_mem, cset_load_ptr, 938 DEBUG_ONLY(phase->C->get_adr_type(Compile::AliasIdxRaw)) NOT_DEBUG(NULL), 939 TypeInt::BYTE, MemNode::unordered); 940 Node* cset_cmp = new CmpINode(cset_load, igvn.zerocon(T_INT)); 941 Node* cset_bool = new BoolNode(cset_cmp, BoolTest::ne); 942 943 IfNode* cset_iff = new IfNode(old_ctrl, cset_bool, PROB_UNLIKELY(0.999), COUNT_UNKNOWN); 944 ctrl = new IfTrueNode(cset_iff); 945 not_cset_ctrl = new IfFalseNode(cset_iff); 946 947 IdealLoopTree *loop = phase->get_loop(old_ctrl); 948 phase->register_control(cset_iff, loop, old_ctrl); 949 phase->register_control(ctrl, loop, cset_iff); 950 phase->register_control(not_cset_ctrl, loop, cset_iff); 951 952 phase->set_ctrl(cset_addr_ptr, phase->C->root()); 953 954 phase->register_new_node(raw_val, old_ctrl); 955 phase->register_new_node(cset_idx, old_ctrl); 956 phase->register_new_node(cset_addr, old_ctrl); 957 phase->register_new_node(cset_load_addr, old_ctrl); 958 phase->register_new_node(cset_load_ptr, old_ctrl); 959 phase->register_new_node(cset_load, old_ctrl); 960 phase->register_new_node(cset_cmp, old_ctrl); 961 phase->register_new_node(cset_bool, old_ctrl); 962 } 963 964 void ShenandoahBarrierC2Support::call_lrb_stub(Node*& ctrl, Node*& val, Node* load_addr, Node*& result_mem, Node* raw_mem, bool is_native, PhaseIdealLoop* phase) { 965 IdealLoopTree*loop = phase->get_loop(ctrl); 966 const TypePtr* obj_type = phase->igvn().type(val)->is_oopptr(); 967 968 // The slow path stub consumes and produces raw memory in addition 969 // to the existing memory edges 970 Node* base = find_bottom_mem(ctrl, phase); 971 MergeMemNode* mm = MergeMemNode::make(base); 972 mm->set_memory_at(Compile::AliasIdxRaw, raw_mem); 973 phase->register_new_node(mm, ctrl); 974 975 address target = LP64_ONLY(UseCompressedOops) NOT_LP64(false) ? 976 CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_narrow) : 977 CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier); 978 979 address calladdr = is_native ? CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_native) 980 : target; 981 const char* name = is_native ? "load_reference_barrier_native" : "load_reference_barrier"; 982 Node* call = new CallLeafNode(ShenandoahBarrierSetC2::shenandoah_load_reference_barrier_Type(), calladdr, name, TypeRawPtr::BOTTOM); 983 984 call->init_req(TypeFunc::Control, ctrl); 985 call->init_req(TypeFunc::I_O, phase->C->top()); 986 call->init_req(TypeFunc::Memory, mm); 987 call->init_req(TypeFunc::FramePtr, phase->C->top()); 988 call->init_req(TypeFunc::ReturnAdr, phase->C->top()); 989 call->init_req(TypeFunc::Parms, val); 990 call->init_req(TypeFunc::Parms+1, load_addr); 991 phase->register_control(call, loop, ctrl); 992 ctrl = new ProjNode(call, TypeFunc::Control); 993 phase->register_control(ctrl, loop, call); 994 result_mem = new ProjNode(call, TypeFunc::Memory); 995 phase->register_new_node(result_mem, call); 996 val = new ProjNode(call, TypeFunc::Parms); 997 phase->register_new_node(val, call); 998 val = new CheckCastPPNode(ctrl, val, obj_type); 999 phase->register_new_node(val, ctrl); 1000 } 1001 1002 void ShenandoahBarrierC2Support::fix_ctrl(Node* barrier, Node* region, const MemoryGraphFixer& fixer, Unique_Node_List& uses, Unique_Node_List& uses_to_ignore, uint last, PhaseIdealLoop* phase) { 1003 Node* ctrl = phase->get_ctrl(barrier); 1004 Node* init_raw_mem = fixer.find_mem(ctrl, barrier); 1005 1006 // Update the control of all nodes that should be after the 1007 // barrier control flow 1008 uses.clear(); 1009 // Every node that is control dependent on the barrier's input 1010 // control will be after the expanded barrier. The raw memory (if 1011 // its memory is control dependent on the barrier's input control) 1012 // must stay above the barrier. 1013 uses_to_ignore.clear(); 1014 if (phase->has_ctrl(init_raw_mem) && phase->get_ctrl(init_raw_mem) == ctrl && !init_raw_mem->is_Phi()) { 1015 uses_to_ignore.push(init_raw_mem); 1016 } 1017 for (uint next = 0; next < uses_to_ignore.size(); next++) { 1018 Node *n = uses_to_ignore.at(next); 1019 for (uint i = 0; i < n->req(); i++) { 1020 Node* in = n->in(i); 1021 if (in != NULL && phase->has_ctrl(in) && phase->get_ctrl(in) == ctrl) { 1022 uses_to_ignore.push(in); 1023 } 1024 } 1025 } 1026 for (DUIterator_Fast imax, i = ctrl->fast_outs(imax); i < imax; i++) { 1027 Node* u = ctrl->fast_out(i); 1028 if (u->_idx < last && 1029 u != barrier && 1030 !uses_to_ignore.member(u) && 1031 (u->in(0) != ctrl || (!u->is_Region() && !u->is_Phi())) && 1032 (ctrl->Opcode() != Op_CatchProj || u->Opcode() != Op_CreateEx)) { 1033 Node* old_c = phase->ctrl_or_self(u); 1034 Node* c = old_c; 1035 if (c != ctrl || 1036 is_dominator_same_ctrl(old_c, barrier, u, phase) || 1037 ShenandoahBarrierSetC2::is_shenandoah_state_load(u)) { 1038 phase->igvn().rehash_node_delayed(u); 1039 int nb = u->replace_edge(ctrl, region); 1040 if (u->is_CFG()) { 1041 if (phase->idom(u) == ctrl) { 1042 phase->set_idom(u, region, phase->dom_depth(region)); 1043 } 1044 } else if (phase->get_ctrl(u) == ctrl) { 1045 assert(u != init_raw_mem, "should leave input raw mem above the barrier"); 1046 uses.push(u); 1047 } 1048 assert(nb == 1, "more than 1 ctrl input?"); 1049 --i, imax -= nb; 1050 } 1051 } 1052 } 1053 } 1054 1055 static Node* create_phis_on_call_return(Node* ctrl, Node* c, Node* n, Node* n_clone, const CallProjections& projs, PhaseIdealLoop* phase) { 1056 Node* region = NULL; 1057 while (c != ctrl) { 1058 if (c->is_Region()) { 1059 region = c; 1060 } 1061 c = phase->idom(c); 1062 } 1063 assert(region != NULL, ""); 1064 Node* phi = new PhiNode(region, n->bottom_type()); 1065 for (uint j = 1; j < region->req(); j++) { 1066 Node* in = region->in(j); 1067 if (phase->is_dominator(projs.fallthrough_catchproj, in)) { 1068 phi->init_req(j, n); 1069 } else if (phase->is_dominator(projs.catchall_catchproj, in)) { 1070 phi->init_req(j, n_clone); 1071 } else { 1072 phi->init_req(j, create_phis_on_call_return(ctrl, in, n, n_clone, projs, phase)); 1073 } 1074 } 1075 phase->register_new_node(phi, region); 1076 return phi; 1077 } 1078 1079 void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) { 1080 ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state(); 1081 1082 Unique_Node_List uses; 1083 for (int i = 0; i < state->enqueue_barriers_count(); i++) { 1084 Node* barrier = state->enqueue_barrier(i); 1085 Node* ctrl = phase->get_ctrl(barrier); 1086 IdealLoopTree* loop = phase->get_loop(ctrl); 1087 Node* head = loop->head(); 1088 if (head->is_OuterStripMinedLoop()) { 1089 // Expanding a barrier here will break loop strip mining 1090 // verification. Transform the loop so the loop nest doesn't 1091 // appear as strip mined. 1092 OuterStripMinedLoopNode* outer = head->as_OuterStripMinedLoop(); 1093 hide_strip_mined_loop(outer, outer->unique_ctrl_out()->as_CountedLoop(), phase); 1094 } 1095 } 1096 1097 Node_Stack stack(0); 1098 Node_List clones; 1099 for (int i = state->load_reference_barriers_count() - 1; i >= 0; i--) { 1100 ShenandoahLoadReferenceBarrierNode* lrb = state->load_reference_barrier(i); 1101 1102 Node* ctrl = phase->get_ctrl(lrb); 1103 Node* val = lrb->in(ShenandoahLoadReferenceBarrierNode::ValueIn); 1104 1105 CallStaticJavaNode* unc = NULL; 1106 Node* unc_ctrl = NULL; 1107 Node* uncasted_val = val; 1108 1109 for (DUIterator_Fast imax, i = lrb->fast_outs(imax); i < imax; i++) { 1110 Node* u = lrb->fast_out(i); 1111 if (u->Opcode() == Op_CastPP && 1112 u->in(0) != NULL && 1113 phase->is_dominator(u->in(0), ctrl)) { 1114 const Type* u_t = phase->igvn().type(u); 1115 1116 if (u_t->meet(TypePtr::NULL_PTR) != u_t && 1117 u->in(0)->Opcode() == Op_IfTrue && 1118 u->in(0)->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none) && 1119 u->in(0)->in(0)->is_If() && 1120 u->in(0)->in(0)->in(1)->Opcode() == Op_Bool && 1121 u->in(0)->in(0)->in(1)->as_Bool()->_test._test == BoolTest::ne && 1122 u->in(0)->in(0)->in(1)->in(1)->Opcode() == Op_CmpP && 1123 u->in(0)->in(0)->in(1)->in(1)->in(1) == val && 1124 u->in(0)->in(0)->in(1)->in(1)->in(2)->bottom_type() == TypePtr::NULL_PTR) { 1125 IdealLoopTree* loop = phase->get_loop(ctrl); 1126 IdealLoopTree* unc_loop = phase->get_loop(u->in(0)); 1127 1128 if (!unc_loop->is_member(loop)) { 1129 continue; 1130 } 1131 1132 Node* branch = no_branches(ctrl, u->in(0), false, phase); 1133 assert(branch == NULL || branch == NodeSentinel, "was not looking for a branch"); 1134 if (branch == NodeSentinel) { 1135 continue; 1136 } 1137 1138 Node* iff = u->in(0)->in(0); 1139 Node* bol = iff->in(1)->clone(); 1140 Node* cmp = bol->in(1)->clone(); 1141 cmp->set_req(1, lrb); 1142 bol->set_req(1, cmp); 1143 phase->igvn().replace_input_of(iff, 1, bol); 1144 phase->set_ctrl(lrb, iff->in(0)); 1145 phase->register_new_node(cmp, iff->in(0)); 1146 phase->register_new_node(bol, iff->in(0)); 1147 break; 1148 } 1149 } 1150 } 1151 if ((ctrl->is_Proj() && ctrl->in(0)->is_CallJava()) || ctrl->is_CallJava()) { 1152 CallNode* call = ctrl->is_Proj() ? ctrl->in(0)->as_CallJava() : ctrl->as_CallJava(); 1153 if (call->entry_point() == OptoRuntime::rethrow_stub()) { 1154 // The rethrow call may have too many projections to be 1155 // properly handled here. Given there's no reason for a 1156 // barrier to depend on the call, move it above the call 1157 stack.push(lrb, 0); 1158 do { 1159 Node* n = stack.node(); 1160 uint idx = stack.index(); 1161 if (idx < n->req()) { 1162 Node* in = n->in(idx); 1163 stack.set_index(idx+1); 1164 if (in != NULL) { 1165 if (phase->has_ctrl(in)) { 1166 if (phase->is_dominator(call, phase->get_ctrl(in))) { 1167 #ifdef ASSERT 1168 for (uint i = 0; i < stack.size(); i++) { 1169 assert(stack.node_at(i) != in, "node shouldn't have been seen yet"); 1170 } 1171 #endif 1172 stack.push(in, 0); 1173 } 1174 } else { 1175 assert(phase->is_dominator(in, call->in(0)), "no dependency on the call"); 1176 } 1177 } 1178 } else { 1179 phase->set_ctrl(n, call->in(0)); 1180 stack.pop(); 1181 } 1182 } while(stack.size() > 0); 1183 continue; 1184 } 1185 CallProjections projs; 1186 call->extract_projections(&projs, false, false); 1187 1188 #ifdef ASSERT 1189 VectorSet cloned; 1190 #endif 1191 Node* lrb_clone = lrb->clone(); 1192 phase->register_new_node(lrb_clone, projs.catchall_catchproj); 1193 phase->set_ctrl(lrb, projs.fallthrough_catchproj); 1194 1195 stack.push(lrb, 0); 1196 clones.push(lrb_clone); 1197 1198 do { 1199 assert(stack.size() == clones.size(), ""); 1200 Node* n = stack.node(); 1201 #ifdef ASSERT 1202 if (n->is_Load()) { 1203 Node* mem = n->in(MemNode::Memory); 1204 for (DUIterator_Fast jmax, j = mem->fast_outs(jmax); j < jmax; j++) { 1205 Node* u = mem->fast_out(j); 1206 assert(!u->is_Store() || !u->is_LoadStore() || phase->get_ctrl(u) != ctrl, "anti dependent store?"); 1207 } 1208 } 1209 #endif 1210 uint idx = stack.index(); 1211 Node* n_clone = clones.at(clones.size()-1); 1212 if (idx < n->outcnt()) { 1213 Node* u = n->raw_out(idx); 1214 Node* c = phase->ctrl_or_self(u); 1215 if (phase->is_dominator(call, c) && phase->is_dominator(c, projs.fallthrough_proj)) { 1216 stack.set_index(idx+1); 1217 assert(!u->is_CFG(), ""); 1218 stack.push(u, 0); 1219 assert(!cloned.test_set(u->_idx), "only one clone"); 1220 Node* u_clone = u->clone(); 1221 int nb = u_clone->replace_edge(n, n_clone); 1222 assert(nb > 0, "should have replaced some uses"); 1223 phase->register_new_node(u_clone, projs.catchall_catchproj); 1224 clones.push(u_clone); 1225 phase->set_ctrl(u, projs.fallthrough_catchproj); 1226 } else { 1227 bool replaced = false; 1228 if (u->is_Phi()) { 1229 for (uint k = 1; k < u->req(); k++) { 1230 if (u->in(k) == n) { 1231 if (phase->is_dominator(projs.catchall_catchproj, u->in(0)->in(k))) { 1232 phase->igvn().replace_input_of(u, k, n_clone); 1233 replaced = true; 1234 } else if (!phase->is_dominator(projs.fallthrough_catchproj, u->in(0)->in(k))) { 1235 phase->igvn().replace_input_of(u, k, create_phis_on_call_return(ctrl, u->in(0)->in(k), n, n_clone, projs, phase)); 1236 replaced = true; 1237 } 1238 } 1239 } 1240 } else { 1241 if (phase->is_dominator(projs.catchall_catchproj, c)) { 1242 phase->igvn().rehash_node_delayed(u); 1243 int nb = u->replace_edge(n, n_clone); 1244 assert(nb > 0, "should have replaced some uses"); 1245 replaced = true; 1246 } else if (!phase->is_dominator(projs.fallthrough_catchproj, c)) { 1247 if (u->is_If()) { 1248 // Can't break If/Bool/Cmp chain 1249 assert(n->is_Bool(), "unexpected If shape"); 1250 assert(stack.node_at(stack.size()-2)->is_Cmp(), "unexpected If shape"); 1251 assert(n_clone->is_Bool(), "unexpected clone"); 1252 assert(clones.at(clones.size()-2)->is_Cmp(), "unexpected clone"); 1253 Node* bol_clone = n->clone(); 1254 Node* cmp_clone = stack.node_at(stack.size()-2)->clone(); 1255 bol_clone->set_req(1, cmp_clone); 1256 1257 Node* nn = stack.node_at(stack.size()-3); 1258 Node* nn_clone = clones.at(clones.size()-3); 1259 assert(nn->Opcode() == nn_clone->Opcode(), "mismatch"); 1260 1261 int nb = cmp_clone->replace_edge(nn, create_phis_on_call_return(ctrl, c, nn, nn_clone, projs, phase)); 1262 assert(nb > 0, "should have replaced some uses"); 1263 1264 phase->register_new_node(bol_clone, u->in(0)); 1265 phase->register_new_node(cmp_clone, u->in(0)); 1266 1267 phase->igvn().replace_input_of(u, 1, bol_clone); 1268 1269 } else { 1270 phase->igvn().rehash_node_delayed(u); 1271 int nb = u->replace_edge(n, create_phis_on_call_return(ctrl, c, n, n_clone, projs, phase)); 1272 assert(nb > 0, "should have replaced some uses"); 1273 } 1274 replaced = true; 1275 } 1276 } 1277 if (!replaced) { 1278 stack.set_index(idx+1); 1279 } 1280 } 1281 } else { 1282 stack.pop(); 1283 clones.pop(); 1284 } 1285 } while (stack.size() > 0); 1286 assert(stack.size() == 0 && clones.size() == 0, ""); 1287 } 1288 } 1289 1290 for (int i = 0; i < state->load_reference_barriers_count(); i++) { 1291 ShenandoahLoadReferenceBarrierNode* lrb = state->load_reference_barrier(i); 1292 Node* ctrl = phase->get_ctrl(lrb); 1293 IdealLoopTree* loop = phase->get_loop(ctrl); 1294 Node* head = loop->head(); 1295 if (head->is_OuterStripMinedLoop()) { 1296 // Expanding a barrier here will break loop strip mining 1297 // verification. Transform the loop so the loop nest doesn't 1298 // appear as strip mined. 1299 OuterStripMinedLoopNode* outer = head->as_OuterStripMinedLoop(); 1300 hide_strip_mined_loop(outer, outer->unique_ctrl_out()->as_CountedLoop(), phase); 1301 } 1302 } 1303 1304 // Expand load-reference-barriers 1305 MemoryGraphFixer fixer(Compile::AliasIdxRaw, true, phase); 1306 Unique_Node_List uses_to_ignore; 1307 for (int i = state->load_reference_barriers_count() - 1; i >= 0; i--) { 1308 ShenandoahLoadReferenceBarrierNode* lrb = state->load_reference_barrier(i); 1309 uint last = phase->C->unique(); 1310 Node* ctrl = phase->get_ctrl(lrb); 1311 Node* val = lrb->in(ShenandoahLoadReferenceBarrierNode::ValueIn); 1312 1313 1314 Node* orig_ctrl = ctrl; 1315 1316 Node* raw_mem = fixer.find_mem(ctrl, lrb); 1317 Node* init_raw_mem = raw_mem; 1318 Node* raw_mem_for_ctrl = fixer.find_mem(ctrl, NULL); 1319 1320 IdealLoopTree *loop = phase->get_loop(ctrl); 1321 1322 Node* heap_stable_ctrl = NULL; 1323 Node* null_ctrl = NULL; 1324 1325 assert(val->bottom_type()->make_oopptr(), "need oop"); 1326 assert(val->bottom_type()->make_oopptr()->const_oop() == NULL, "expect non-constant"); 1327 1328 enum { _heap_stable = 1, _not_cset, _evac_path, PATH_LIMIT }; 1329 Node* region = new RegionNode(PATH_LIMIT); 1330 Node* val_phi = new PhiNode(region, val->bottom_type()->is_oopptr()); 1331 Node* raw_mem_phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM); 1332 1333 // Stable path. 1334 test_gc_state(ctrl, raw_mem, heap_stable_ctrl, phase, ShenandoahHeap::HAS_FORWARDED); 1335 IfNode* heap_stable_iff = heap_stable_ctrl->in(0)->as_If(); 1336 1337 // Heap stable case 1338 region->init_req(_heap_stable, heap_stable_ctrl); 1339 val_phi->init_req(_heap_stable, val); 1340 raw_mem_phi->init_req(_heap_stable, raw_mem); 1341 1342 // Test for in-cset. 1343 // Wires !in_cset(obj) to slot 2 of region and phis 1344 Node* not_cset_ctrl = NULL; 1345 test_in_cset(ctrl, not_cset_ctrl, val, raw_mem, phase); 1346 if (not_cset_ctrl != NULL) { 1347 region->init_req(_not_cset, not_cset_ctrl); 1348 val_phi->init_req(_not_cset, val); 1349 raw_mem_phi->init_req(_not_cset, raw_mem); 1350 } 1351 1352 // Resolve object when orig-value is in cset. 1353 // Make the unconditional resolve for fwdptr. 1354 1355 // Call lrb-stub and wire up that path in slots 4 1356 Node* result_mem = NULL; 1357 1358 Node* addr; 1359 if (ShenandoahSelfFixing) { 1360 VectorSet visited; 1361 addr = get_load_addr(phase, visited, lrb); 1362 } else { 1363 addr = phase->igvn().zerocon(T_OBJECT); 1364 } 1365 if (addr->Opcode() == Op_AddP) { 1366 Node* orig_base = addr->in(AddPNode::Base); 1367 Node* base = new CheckCastPPNode(ctrl, orig_base, orig_base->bottom_type(), true); 1368 phase->register_new_node(base, ctrl); 1369 if (addr->in(AddPNode::Base) == addr->in((AddPNode::Address))) { 1370 // Field access 1371 addr = addr->clone(); 1372 addr->set_req(AddPNode::Base, base); 1373 addr->set_req(AddPNode::Address, base); 1374 phase->register_new_node(addr, ctrl); 1375 } else { 1376 Node* addr2 = addr->in(AddPNode::Address); 1377 if (addr2->Opcode() == Op_AddP && addr2->in(AddPNode::Base) == addr2->in(AddPNode::Address) && 1378 addr2->in(AddPNode::Base) == orig_base) { 1379 addr2 = addr2->clone(); 1380 addr2->set_req(AddPNode::Base, base); 1381 addr2->set_req(AddPNode::Address, base); 1382 phase->register_new_node(addr2, ctrl); 1383 addr = addr->clone(); 1384 addr->set_req(AddPNode::Base, base); 1385 addr->set_req(AddPNode::Address, addr2); 1386 phase->register_new_node(addr, ctrl); 1387 } 1388 } 1389 } 1390 call_lrb_stub(ctrl, val, addr, result_mem, raw_mem, lrb->is_native(), phase); 1391 region->init_req(_evac_path, ctrl); 1392 val_phi->init_req(_evac_path, val); 1393 raw_mem_phi->init_req(_evac_path, result_mem); 1394 1395 phase->register_control(region, loop, heap_stable_iff); 1396 Node* out_val = val_phi; 1397 phase->register_new_node(val_phi, region); 1398 phase->register_new_node(raw_mem_phi, region); 1399 1400 fix_ctrl(lrb, region, fixer, uses, uses_to_ignore, last, phase); 1401 1402 ctrl = orig_ctrl; 1403 1404 phase->igvn().replace_node(lrb, out_val); 1405 1406 follow_barrier_uses(out_val, ctrl, uses, phase); 1407 1408 for(uint next = 0; next < uses.size(); next++ ) { 1409 Node *n = uses.at(next); 1410 assert(phase->get_ctrl(n) == ctrl, "bad control"); 1411 assert(n != init_raw_mem, "should leave input raw mem above the barrier"); 1412 phase->set_ctrl(n, region); 1413 follow_barrier_uses(n, ctrl, uses, phase); 1414 } 1415 1416 // The slow path call produces memory: hook the raw memory phi 1417 // from the expanded load reference barrier with the rest of the graph 1418 // which may require adding memory phis at every post dominated 1419 // region and at enclosing loop heads. Use the memory state 1420 // collected in memory_nodes to fix the memory graph. Update that 1421 // memory state as we go. 1422 fixer.fix_mem(ctrl, region, init_raw_mem, raw_mem_for_ctrl, raw_mem_phi, uses); 1423 } 1424 // Done expanding load-reference-barriers. 1425 assert(ShenandoahBarrierSetC2::bsc2()->state()->load_reference_barriers_count() == 0, "all load reference barrier nodes should have been replaced"); 1426 1427 for (int i = state->enqueue_barriers_count() - 1; i >= 0; i--) { 1428 Node* barrier = state->enqueue_barrier(i); 1429 Node* pre_val = barrier->in(1); 1430 1431 if (phase->igvn().type(pre_val)->higher_equal(TypePtr::NULL_PTR)) { 1432 ShouldNotReachHere(); 1433 continue; 1434 } 1435 1436 Node* ctrl = phase->get_ctrl(barrier); 1437 1438 if (ctrl->is_Proj() && ctrl->in(0)->is_CallJava()) { 1439 assert(is_dominator(phase->get_ctrl(pre_val), ctrl->in(0)->in(0), pre_val, ctrl->in(0), phase), "can't move"); 1440 ctrl = ctrl->in(0)->in(0); 1441 phase->set_ctrl(barrier, ctrl); 1442 } else if (ctrl->is_CallRuntime()) { 1443 assert(is_dominator(phase->get_ctrl(pre_val), ctrl->in(0), pre_val, ctrl, phase), "can't move"); 1444 ctrl = ctrl->in(0); 1445 phase->set_ctrl(barrier, ctrl); 1446 } 1447 1448 Node* init_ctrl = ctrl; 1449 IdealLoopTree* loop = phase->get_loop(ctrl); 1450 Node* raw_mem = fixer.find_mem(ctrl, barrier); 1451 Node* init_raw_mem = raw_mem; 1452 Node* raw_mem_for_ctrl = fixer.find_mem(ctrl, NULL); 1453 Node* heap_stable_ctrl = NULL; 1454 Node* null_ctrl = NULL; 1455 uint last = phase->C->unique(); 1456 1457 enum { _heap_stable = 1, _heap_unstable, PATH_LIMIT }; 1458 Node* region = new RegionNode(PATH_LIMIT); 1459 Node* phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM); 1460 1461 enum { _fast_path = 1, _slow_path, _null_path, PATH_LIMIT2 }; 1462 Node* region2 = new RegionNode(PATH_LIMIT2); 1463 Node* phi2 = PhiNode::make(region2, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM); 1464 1465 // Stable path. 1466 test_gc_state(ctrl, raw_mem, heap_stable_ctrl, phase, ShenandoahHeap::MARKING); 1467 region->init_req(_heap_stable, heap_stable_ctrl); 1468 phi->init_req(_heap_stable, raw_mem); 1469 1470 // Null path 1471 Node* reg2_ctrl = NULL; 1472 test_null(ctrl, pre_val, null_ctrl, phase); 1473 if (null_ctrl != NULL) { 1474 reg2_ctrl = null_ctrl->in(0); 1475 region2->init_req(_null_path, null_ctrl); 1476 phi2->init_req(_null_path, raw_mem); 1477 } else { 1478 region2->del_req(_null_path); 1479 phi2->del_req(_null_path); 1480 } 1481 1482 const int index_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()); 1483 const int buffer_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()); 1484 Node* thread = new ThreadLocalNode(); 1485 phase->register_new_node(thread, ctrl); 1486 Node* buffer_adr = new AddPNode(phase->C->top(), thread, phase->igvn().MakeConX(buffer_offset)); 1487 phase->register_new_node(buffer_adr, ctrl); 1488 Node* index_adr = new AddPNode(phase->C->top(), thread, phase->igvn().MakeConX(index_offset)); 1489 phase->register_new_node(index_adr, ctrl); 1490 1491 BasicType index_bt = TypeX_X->basic_type(); 1492 assert(sizeof(size_t) == type2aelembytes(index_bt), "Loading G1 SATBMarkQueue::_index with wrong size."); 1493 const TypePtr* adr_type = TypeRawPtr::BOTTOM; 1494 Node* index = new LoadXNode(ctrl, raw_mem, index_adr, adr_type, TypeX_X, MemNode::unordered); 1495 phase->register_new_node(index, ctrl); 1496 Node* index_cmp = new CmpXNode(index, phase->igvn().MakeConX(0)); 1497 phase->register_new_node(index_cmp, ctrl); 1498 Node* index_test = new BoolNode(index_cmp, BoolTest::ne); 1499 phase->register_new_node(index_test, ctrl); 1500 IfNode* queue_full_iff = new IfNode(ctrl, index_test, PROB_LIKELY(0.999), COUNT_UNKNOWN); 1501 if (reg2_ctrl == NULL) reg2_ctrl = queue_full_iff; 1502 phase->register_control(queue_full_iff, loop, ctrl); 1503 Node* not_full = new IfTrueNode(queue_full_iff); 1504 phase->register_control(not_full, loop, queue_full_iff); 1505 Node* full = new IfFalseNode(queue_full_iff); 1506 phase->register_control(full, loop, queue_full_iff); 1507 1508 ctrl = not_full; 1509 1510 Node* next_index = new SubXNode(index, phase->igvn().MakeConX(sizeof(intptr_t))); 1511 phase->register_new_node(next_index, ctrl); 1512 1513 Node* buffer = new LoadPNode(ctrl, raw_mem, buffer_adr, adr_type, TypeRawPtr::NOTNULL, MemNode::unordered); 1514 phase->register_new_node(buffer, ctrl); 1515 Node *log_addr = new AddPNode(phase->C->top(), buffer, next_index); 1516 phase->register_new_node(log_addr, ctrl); 1517 Node* log_store = new StorePNode(ctrl, raw_mem, log_addr, adr_type, pre_val, MemNode::unordered); 1518 phase->register_new_node(log_store, ctrl); 1519 // update the index 1520 Node* index_update = new StoreXNode(ctrl, log_store, index_adr, adr_type, next_index, MemNode::unordered); 1521 phase->register_new_node(index_update, ctrl); 1522 1523 // Fast-path case 1524 region2->init_req(_fast_path, ctrl); 1525 phi2->init_req(_fast_path, index_update); 1526 1527 ctrl = full; 1528 1529 Node* base = find_bottom_mem(ctrl, phase); 1530 1531 MergeMemNode* mm = MergeMemNode::make(base); 1532 mm->set_memory_at(Compile::AliasIdxRaw, raw_mem); 1533 phase->register_new_node(mm, ctrl); 1534 1535 Node* call = new CallLeafNode(ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type(), CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), "shenandoah_wb_pre", TypeRawPtr::BOTTOM); 1536 call->init_req(TypeFunc::Control, ctrl); 1537 call->init_req(TypeFunc::I_O, phase->C->top()); 1538 call->init_req(TypeFunc::Memory, mm); 1539 call->init_req(TypeFunc::FramePtr, phase->C->top()); 1540 call->init_req(TypeFunc::ReturnAdr, phase->C->top()); 1541 call->init_req(TypeFunc::Parms, pre_val); 1542 call->init_req(TypeFunc::Parms+1, thread); 1543 phase->register_control(call, loop, ctrl); 1544 1545 Node* ctrl_proj = new ProjNode(call, TypeFunc::Control); 1546 phase->register_control(ctrl_proj, loop, call); 1547 Node* mem_proj = new ProjNode(call, TypeFunc::Memory); 1548 phase->register_new_node(mem_proj, call); 1549 1550 // Slow-path case 1551 region2->init_req(_slow_path, ctrl_proj); 1552 phi2->init_req(_slow_path, mem_proj); 1553 1554 phase->register_control(region2, loop, reg2_ctrl); 1555 phase->register_new_node(phi2, region2); 1556 1557 region->init_req(_heap_unstable, region2); 1558 phi->init_req(_heap_unstable, phi2); 1559 1560 phase->register_control(region, loop, heap_stable_ctrl->in(0)); 1561 phase->register_new_node(phi, region); 1562 1563 fix_ctrl(barrier, region, fixer, uses, uses_to_ignore, last, phase); 1564 for(uint next = 0; next < uses.size(); next++ ) { 1565 Node *n = uses.at(next); 1566 assert(phase->get_ctrl(n) == init_ctrl, "bad control"); 1567 assert(n != init_raw_mem, "should leave input raw mem above the barrier"); 1568 phase->set_ctrl(n, region); 1569 follow_barrier_uses(n, init_ctrl, uses, phase); 1570 } 1571 fixer.fix_mem(init_ctrl, region, init_raw_mem, raw_mem_for_ctrl, phi, uses); 1572 1573 phase->igvn().replace_node(barrier, pre_val); 1574 } 1575 assert(state->enqueue_barriers_count() == 0, "all enqueue barrier nodes should have been replaced"); 1576 1577 } 1578 1579 Node* ShenandoahBarrierC2Support::get_load_addr(PhaseIdealLoop* phase, VectorSet& visited, Node* in) { 1580 if (visited.test_set(in->_idx)) { 1581 return NULL; 1582 } 1583 switch (in->Opcode()) { 1584 case Op_Proj: 1585 return get_load_addr(phase, visited, in->in(0)); 1586 case Op_CastPP: 1587 case Op_CheckCastPP: 1588 case Op_DecodeN: 1589 case Op_EncodeP: 1590 return get_load_addr(phase, visited, in->in(1)); 1591 case Op_LoadN: 1592 case Op_LoadP: 1593 return in->in(MemNode::Address); 1594 case Op_CompareAndExchangeN: 1595 case Op_CompareAndExchangeP: 1596 case Op_GetAndSetN: 1597 case Op_GetAndSetP: 1598 case Op_ShenandoahCompareAndExchangeP: 1599 case Op_ShenandoahCompareAndExchangeN: 1600 // Those instructions would just have stored a different 1601 // value into the field. No use to attempt to fix it at this point. 1602 return phase->igvn().zerocon(T_OBJECT); 1603 case Op_CMoveP: 1604 case Op_CMoveN: { 1605 Node* t = get_load_addr(phase, visited, in->in(CMoveNode::IfTrue)); 1606 Node* f = get_load_addr(phase, visited, in->in(CMoveNode::IfFalse)); 1607 // Handle unambiguous cases: single address reported on both branches. 1608 if (t != NULL && f == NULL) return t; 1609 if (t == NULL && f != NULL) return f; 1610 if (t != NULL && t == f) return t; 1611 // Ambiguity. 1612 return phase->igvn().zerocon(T_OBJECT); 1613 } 1614 case Op_Phi: { 1615 Node* addr = NULL; 1616 for (uint i = 1; i < in->req(); i++) { 1617 Node* addr1 = get_load_addr(phase, visited, in->in(i)); 1618 if (addr == NULL) { 1619 addr = addr1; 1620 } 1621 if (addr != addr1) { 1622 return phase->igvn().zerocon(T_OBJECT); 1623 } 1624 } 1625 return addr; 1626 } 1627 case Op_ShenandoahLoadReferenceBarrier: 1628 return get_load_addr(phase, visited, in->in(ShenandoahLoadReferenceBarrierNode::ValueIn)); 1629 case Op_ShenandoahEnqueueBarrier: 1630 return get_load_addr(phase, visited, in->in(1)); 1631 case Op_CallDynamicJava: 1632 case Op_CallLeaf: 1633 case Op_CallStaticJava: 1634 case Op_ConN: 1635 case Op_ConP: 1636 case Op_Parm: 1637 case Op_CreateEx: 1638 return phase->igvn().zerocon(T_OBJECT); 1639 default: 1640 #ifdef ASSERT 1641 fatal("Unknown node in get_load_addr: %s", NodeClassNames[in->Opcode()]); 1642 #endif 1643 return phase->igvn().zerocon(T_OBJECT); 1644 } 1645 1646 } 1647 1648 void ShenandoahBarrierC2Support::move_gc_state_test_out_of_loop(IfNode* iff, PhaseIdealLoop* phase) { 1649 IdealLoopTree *loop = phase->get_loop(iff); 1650 Node* loop_head = loop->_head; 1651 Node* entry_c = loop_head->in(LoopNode::EntryControl); 1652 1653 Node* bol = iff->in(1); 1654 Node* cmp = bol->in(1); 1655 Node* andi = cmp->in(1); 1656 Node* load = andi->in(1); 1657 1658 assert(is_gc_state_load(load), "broken"); 1659 if (!phase->is_dominator(load->in(0), entry_c)) { 1660 Node* mem_ctrl = NULL; 1661 Node* mem = dom_mem(load->in(MemNode::Memory), loop_head, Compile::AliasIdxRaw, mem_ctrl, phase); 1662 load = load->clone(); 1663 load->set_req(MemNode::Memory, mem); 1664 load->set_req(0, entry_c); 1665 phase->register_new_node(load, entry_c); 1666 andi = andi->clone(); 1667 andi->set_req(1, load); 1668 phase->register_new_node(andi, entry_c); 1669 cmp = cmp->clone(); 1670 cmp->set_req(1, andi); 1671 phase->register_new_node(cmp, entry_c); 1672 bol = bol->clone(); 1673 bol->set_req(1, cmp); 1674 phase->register_new_node(bol, entry_c); 1675 1676 phase->igvn().replace_input_of(iff, 1, bol); 1677 } 1678 } 1679 1680 bool ShenandoahBarrierC2Support::identical_backtoback_ifs(Node* n, PhaseIdealLoop* phase) { 1681 if (!n->is_If() || n->is_CountedLoopEnd()) { 1682 return false; 1683 } 1684 Node* region = n->in(0); 1685 1686 if (!region->is_Region()) { 1687 return false; 1688 } 1689 Node* dom = phase->idom(region); 1690 if (!dom->is_If()) { 1691 return false; 1692 } 1693 1694 if (!is_heap_stable_test(n) || !is_heap_stable_test(dom)) { 1695 return false; 1696 } 1697 1698 IfNode* dom_if = dom->as_If(); 1699 Node* proj_true = dom_if->proj_out(1); 1700 Node* proj_false = dom_if->proj_out(0); 1701 1702 for (uint i = 1; i < region->req(); i++) { 1703 if (phase->is_dominator(proj_true, region->in(i))) { 1704 continue; 1705 } 1706 if (phase->is_dominator(proj_false, region->in(i))) { 1707 continue; 1708 } 1709 return false; 1710 } 1711 1712 return true; 1713 } 1714 1715 void ShenandoahBarrierC2Support::merge_back_to_back_tests(Node* n, PhaseIdealLoop* phase) { 1716 assert(is_heap_stable_test(n), "no other tests"); 1717 if (identical_backtoback_ifs(n, phase)) { 1718 Node* n_ctrl = n->in(0); 1719 if (phase->can_split_if(n_ctrl)) { 1720 IfNode* dom_if = phase->idom(n_ctrl)->as_If(); 1721 if (is_heap_stable_test(n)) { 1722 Node* gc_state_load = n->in(1)->in(1)->in(1)->in(1); 1723 assert(is_gc_state_load(gc_state_load), "broken"); 1724 Node* dom_gc_state_load = dom_if->in(1)->in(1)->in(1)->in(1); 1725 assert(is_gc_state_load(dom_gc_state_load), "broken"); 1726 if (gc_state_load != dom_gc_state_load) { 1727 phase->igvn().replace_node(gc_state_load, dom_gc_state_load); 1728 } 1729 } 1730 PhiNode* bolphi = PhiNode::make_blank(n_ctrl, n->in(1)); 1731 Node* proj_true = dom_if->proj_out(1); 1732 Node* proj_false = dom_if->proj_out(0); 1733 Node* con_true = phase->igvn().makecon(TypeInt::ONE); 1734 Node* con_false = phase->igvn().makecon(TypeInt::ZERO); 1735 1736 for (uint i = 1; i < n_ctrl->req(); i++) { 1737 if (phase->is_dominator(proj_true, n_ctrl->in(i))) { 1738 bolphi->init_req(i, con_true); 1739 } else { 1740 assert(phase->is_dominator(proj_false, n_ctrl->in(i)), "bad if"); 1741 bolphi->init_req(i, con_false); 1742 } 1743 } 1744 phase->register_new_node(bolphi, n_ctrl); 1745 phase->igvn().replace_input_of(n, 1, bolphi); 1746 phase->do_split_if(n); 1747 } 1748 } 1749 } 1750 1751 IfNode* ShenandoahBarrierC2Support::find_unswitching_candidate(const IdealLoopTree* loop, PhaseIdealLoop* phase) { 1752 // Find first invariant test that doesn't exit the loop 1753 LoopNode *head = loop->_head->as_Loop(); 1754 IfNode* unswitch_iff = NULL; 1755 Node* n = head->in(LoopNode::LoopBackControl); 1756 int loop_has_sfpts = -1; 1757 while (n != head) { 1758 Node* n_dom = phase->idom(n); 1759 if (n->is_Region()) { 1760 if (n_dom->is_If()) { 1761 IfNode* iff = n_dom->as_If(); 1762 if (iff->in(1)->is_Bool()) { 1763 BoolNode* bol = iff->in(1)->as_Bool(); 1764 if (bol->in(1)->is_Cmp()) { 1765 // If condition is invariant and not a loop exit, 1766 // then found reason to unswitch. 1767 if (is_heap_stable_test(iff) && 1768 (loop_has_sfpts == -1 || loop_has_sfpts == 0)) { 1769 assert(!loop->is_loop_exit(iff), "both branches should be in the loop"); 1770 if (loop_has_sfpts == -1) { 1771 for(uint i = 0; i < loop->_body.size(); i++) { 1772 Node *m = loop->_body[i]; 1773 if (m->is_SafePoint() && !m->is_CallLeaf()) { 1774 loop_has_sfpts = 1; 1775 break; 1776 } 1777 } 1778 if (loop_has_sfpts == -1) { 1779 loop_has_sfpts = 0; 1780 } 1781 } 1782 if (!loop_has_sfpts) { 1783 unswitch_iff = iff; 1784 } 1785 } 1786 } 1787 } 1788 } 1789 } 1790 n = n_dom; 1791 } 1792 return unswitch_iff; 1793 } 1794 1795 1796 void ShenandoahBarrierC2Support::optimize_after_expansion(VectorSet &visited, Node_Stack &stack, Node_List &old_new, PhaseIdealLoop* phase) { 1797 Node_List heap_stable_tests; 1798 stack.push(phase->C->start(), 0); 1799 do { 1800 Node* n = stack.node(); 1801 uint i = stack.index(); 1802 1803 if (i < n->outcnt()) { 1804 Node* u = n->raw_out(i); 1805 stack.set_index(i+1); 1806 if (!visited.test_set(u->_idx)) { 1807 stack.push(u, 0); 1808 } 1809 } else { 1810 stack.pop(); 1811 if (n->is_If() && is_heap_stable_test(n)) { 1812 heap_stable_tests.push(n); 1813 } 1814 } 1815 } while (stack.size() > 0); 1816 1817 for (uint i = 0; i < heap_stable_tests.size(); i++) { 1818 Node* n = heap_stable_tests.at(i); 1819 assert(is_heap_stable_test(n), "only evacuation test"); 1820 merge_back_to_back_tests(n, phase); 1821 } 1822 1823 if (!phase->C->major_progress()) { 1824 VectorSet seen; 1825 for (uint i = 0; i < heap_stable_tests.size(); i++) { 1826 Node* n = heap_stable_tests.at(i); 1827 IdealLoopTree* loop = phase->get_loop(n); 1828 if (loop != phase->ltree_root() && 1829 loop->_child == NULL && 1830 !loop->_irreducible) { 1831 Node* head = loop->_head; 1832 if (head->is_Loop() && 1833 (!head->is_CountedLoop() || head->as_CountedLoop()->is_main_loop() || head->as_CountedLoop()->is_normal_loop()) && 1834 !seen.test_set(head->_idx)) { 1835 IfNode* iff = find_unswitching_candidate(loop, phase); 1836 if (iff != NULL) { 1837 Node* bol = iff->in(1); 1838 if (head->as_Loop()->is_strip_mined()) { 1839 head->as_Loop()->verify_strip_mined(0); 1840 } 1841 move_gc_state_test_out_of_loop(iff, phase); 1842 1843 AutoNodeBudget node_budget(phase); 1844 1845 if (loop->policy_unswitching(phase)) { 1846 if (head->as_Loop()->is_strip_mined()) { 1847 OuterStripMinedLoopNode* outer = head->as_CountedLoop()->outer_loop(); 1848 hide_strip_mined_loop(outer, head->as_CountedLoop(), phase); 1849 } 1850 phase->do_unswitching(loop, old_new); 1851 } else { 1852 // Not proceeding with unswitching. Move load back in 1853 // the loop. 1854 phase->igvn().replace_input_of(iff, 1, bol); 1855 } 1856 } 1857 } 1858 } 1859 } 1860 } 1861 } 1862 1863 #ifdef ASSERT 1864 void ShenandoahBarrierC2Support::verify_raw_mem(RootNode* root) { 1865 const bool trace = false; 1866 ResourceMark rm; 1867 Unique_Node_List nodes; 1868 Unique_Node_List controls; 1869 Unique_Node_List memories; 1870 1871 nodes.push(root); 1872 for (uint next = 0; next < nodes.size(); next++) { 1873 Node *n = nodes.at(next); 1874 if (ShenandoahBarrierSetC2::is_shenandoah_lrb_call(n)) { 1875 controls.push(n); 1876 if (trace) { tty->print("XXXXXX verifying"); n->dump(); } 1877 for (uint next2 = 0; next2 < controls.size(); next2++) { 1878 Node *m = controls.at(next2); 1879 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) { 1880 Node* u = m->fast_out(i); 1881 if (u->is_CFG() && !u->is_Root() && 1882 !(u->Opcode() == Op_CProj && u->in(0)->Opcode() == Op_NeverBranch && u->as_Proj()->_con == 1) && 1883 !(u->is_Region() && u->unique_ctrl_out()->Opcode() == Op_Halt)) { 1884 if (trace) { tty->print("XXXXXX pushing control"); u->dump(); } 1885 controls.push(u); 1886 } 1887 } 1888 } 1889 memories.push(n->as_Call()->proj_out(TypeFunc::Memory)); 1890 for (uint next2 = 0; next2 < memories.size(); next2++) { 1891 Node *m = memories.at(next2); 1892 assert(m->bottom_type() == Type::MEMORY, ""); 1893 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) { 1894 Node* u = m->fast_out(i); 1895 if (u->bottom_type() == Type::MEMORY && (u->is_Mem() || u->is_ClearArray())) { 1896 if (trace) { tty->print("XXXXXX pushing memory"); u->dump(); } 1897 memories.push(u); 1898 } else if (u->is_LoadStore()) { 1899 if (trace) { tty->print("XXXXXX pushing memory"); u->find_out_with(Op_SCMemProj)->dump(); } 1900 memories.push(u->find_out_with(Op_SCMemProj)); 1901 } else if (u->is_MergeMem() && u->as_MergeMem()->memory_at(Compile::AliasIdxRaw) == m) { 1902 if (trace) { tty->print("XXXXXX pushing memory"); u->dump(); } 1903 memories.push(u); 1904 } else if (u->is_Phi()) { 1905 assert(u->bottom_type() == Type::MEMORY, ""); 1906 if (u->adr_type() == TypeRawPtr::BOTTOM || u->adr_type() == TypePtr::BOTTOM) { 1907 assert(controls.member(u->in(0)), ""); 1908 if (trace) { tty->print("XXXXXX pushing memory"); u->dump(); } 1909 memories.push(u); 1910 } 1911 } else if (u->is_SafePoint() || u->is_MemBar()) { 1912 for (DUIterator_Fast jmax, j = u->fast_outs(jmax); j < jmax; j++) { 1913 Node* uu = u->fast_out(j); 1914 if (uu->bottom_type() == Type::MEMORY) { 1915 if (trace) { tty->print("XXXXXX pushing memory"); uu->dump(); } 1916 memories.push(uu); 1917 } 1918 } 1919 } 1920 } 1921 } 1922 for (uint next2 = 0; next2 < controls.size(); next2++) { 1923 Node *m = controls.at(next2); 1924 if (m->is_Region()) { 1925 bool all_in = true; 1926 for (uint i = 1; i < m->req(); i++) { 1927 if (!controls.member(m->in(i))) { 1928 all_in = false; 1929 break; 1930 } 1931 } 1932 if (trace) { tty->print("XXX verifying %s", all_in ? "all in" : ""); m->dump(); } 1933 bool found_phi = false; 1934 for (DUIterator_Fast jmax, j = m->fast_outs(jmax); j < jmax && !found_phi; j++) { 1935 Node* u = m->fast_out(j); 1936 if (u->is_Phi() && memories.member(u)) { 1937 found_phi = true; 1938 for (uint i = 1; i < u->req() && found_phi; i++) { 1939 Node* k = u->in(i); 1940 if (memories.member(k) != controls.member(m->in(i))) { 1941 found_phi = false; 1942 } 1943 } 1944 } 1945 } 1946 assert(found_phi || all_in, ""); 1947 } 1948 } 1949 controls.clear(); 1950 memories.clear(); 1951 } 1952 for( uint i = 0; i < n->len(); ++i ) { 1953 Node *m = n->in(i); 1954 if (m != NULL) { 1955 nodes.push(m); 1956 } 1957 } 1958 } 1959 } 1960 #endif 1961 1962 ShenandoahEnqueueBarrierNode::ShenandoahEnqueueBarrierNode(Node* val) : Node(NULL, val) { 1963 ShenandoahBarrierSetC2::bsc2()->state()->add_enqueue_barrier(this); 1964 } 1965 1966 const Type* ShenandoahEnqueueBarrierNode::bottom_type() const { 1967 if (in(1) == NULL || in(1)->is_top()) { 1968 return Type::TOP; 1969 } 1970 const Type* t = in(1)->bottom_type(); 1971 if (t == TypePtr::NULL_PTR) { 1972 return t; 1973 } 1974 return t->is_oopptr(); 1975 } 1976 1977 const Type* ShenandoahEnqueueBarrierNode::Value(PhaseGVN* phase) const { 1978 if (in(1) == NULL) { 1979 return Type::TOP; 1980 } 1981 const Type* t = phase->type(in(1)); 1982 if (t == Type::TOP) { 1983 return Type::TOP; 1984 } 1985 if (t == TypePtr::NULL_PTR) { 1986 return t; 1987 } 1988 return t->is_oopptr(); 1989 } 1990 1991 int ShenandoahEnqueueBarrierNode::needed(Node* n) { 1992 if (n == NULL || 1993 n->is_Allocate() || 1994 n->Opcode() == Op_ShenandoahEnqueueBarrier || 1995 n->bottom_type() == TypePtr::NULL_PTR || 1996 (n->bottom_type()->make_oopptr() != NULL && n->bottom_type()->make_oopptr()->const_oop() != NULL)) { 1997 return NotNeeded; 1998 } 1999 if (n->is_Phi() || 2000 n->is_CMove()) { 2001 return MaybeNeeded; 2002 } 2003 return Needed; 2004 } 2005 2006 Node* ShenandoahEnqueueBarrierNode::next(Node* n) { 2007 for (;;) { 2008 if (n == NULL) { 2009 return n; 2010 } else if (n->bottom_type() == TypePtr::NULL_PTR) { 2011 return n; 2012 } else if (n->bottom_type()->make_oopptr() != NULL && n->bottom_type()->make_oopptr()->const_oop() != NULL) { 2013 return n; 2014 } else if (n->is_ConstraintCast() || 2015 n->Opcode() == Op_DecodeN || 2016 n->Opcode() == Op_EncodeP) { 2017 n = n->in(1); 2018 } else if (n->is_Proj()) { 2019 n = n->in(0); 2020 } else { 2021 return n; 2022 } 2023 } 2024 ShouldNotReachHere(); 2025 return NULL; 2026 } 2027 2028 Node* ShenandoahEnqueueBarrierNode::Identity(PhaseGVN* phase) { 2029 PhaseIterGVN* igvn = phase->is_IterGVN(); 2030 2031 Node* n = next(in(1)); 2032 2033 int cont = needed(n); 2034 2035 if (cont == NotNeeded) { 2036 return in(1); 2037 } else if (cont == MaybeNeeded) { 2038 if (igvn == NULL) { 2039 phase->record_for_igvn(this); 2040 return this; 2041 } else { 2042 ResourceMark rm; 2043 Unique_Node_List wq; 2044 uint wq_i = 0; 2045 2046 for (;;) { 2047 if (n->is_Phi()) { 2048 for (uint i = 1; i < n->req(); i++) { 2049 Node* m = n->in(i); 2050 if (m != NULL) { 2051 wq.push(m); 2052 } 2053 } 2054 } else { 2055 assert(n->is_CMove(), "nothing else here"); 2056 Node* m = n->in(CMoveNode::IfFalse); 2057 wq.push(m); 2058 m = n->in(CMoveNode::IfTrue); 2059 wq.push(m); 2060 } 2061 Node* orig_n = NULL; 2062 do { 2063 if (wq_i >= wq.size()) { 2064 return in(1); 2065 } 2066 n = wq.at(wq_i); 2067 wq_i++; 2068 orig_n = n; 2069 n = next(n); 2070 cont = needed(n); 2071 if (cont == Needed) { 2072 return this; 2073 } 2074 } while (cont != MaybeNeeded || (orig_n != n && wq.member(n))); 2075 } 2076 } 2077 } 2078 2079 return this; 2080 } 2081 2082 #ifdef ASSERT 2083 static bool has_never_branch(Node* root) { 2084 for (uint i = 1; i < root->req(); i++) { 2085 Node* in = root->in(i); 2086 if (in != NULL && in->Opcode() == Op_Halt && in->in(0)->is_Proj() && in->in(0)->in(0)->Opcode() == Op_NeverBranch) { 2087 return true; 2088 } 2089 } 2090 return false; 2091 } 2092 #endif 2093 2094 void MemoryGraphFixer::collect_memory_nodes() { 2095 Node_Stack stack(0); 2096 VectorSet visited; 2097 Node_List regions; 2098 2099 // Walk the raw memory graph and create a mapping from CFG node to 2100 // memory node. Exclude phis for now. 2101 stack.push(_phase->C->root(), 1); 2102 do { 2103 Node* n = stack.node(); 2104 int opc = n->Opcode(); 2105 uint i = stack.index(); 2106 if (i < n->req()) { 2107 Node* mem = NULL; 2108 if (opc == Op_Root) { 2109 Node* in = n->in(i); 2110 int in_opc = in->Opcode(); 2111 if (in_opc == Op_Return || in_opc == Op_Rethrow) { 2112 mem = in->in(TypeFunc::Memory); 2113 } else if (in_opc == Op_Halt) { 2114 if (in->in(0)->is_Region()) { 2115 Node* r = in->in(0); 2116 for (uint j = 1; j < r->req(); j++) { 2117 assert(r->in(j)->Opcode() != Op_NeverBranch, ""); 2118 } 2119 } else { 2120 Node* proj = in->in(0); 2121 assert(proj->is_Proj(), ""); 2122 Node* in = proj->in(0); 2123 assert(in->is_CallStaticJava() || in->Opcode() == Op_NeverBranch || in->Opcode() == Op_Catch || proj->is_IfProj(), ""); 2124 if (in->is_CallStaticJava()) { 2125 mem = in->in(TypeFunc::Memory); 2126 } else if (in->Opcode() == Op_Catch) { 2127 Node* call = in->in(0)->in(0); 2128 assert(call->is_Call(), ""); 2129 mem = call->in(TypeFunc::Memory); 2130 } else if (in->Opcode() == Op_NeverBranch) { 2131 Node* head = in->in(0); 2132 assert(head->is_Region(), "unexpected infinite loop graph shape"); 2133 2134 Node* phi_mem = NULL; 2135 for (DUIterator_Fast jmax, j = head->fast_outs(jmax); j < jmax; j++) { 2136 Node* u = head->fast_out(j); 2137 if (u->is_Phi() && u->bottom_type() == Type::MEMORY) { 2138 if (_phase->C->get_alias_index(u->adr_type()) == _alias) { 2139 assert(phi_mem == NULL || phi_mem->adr_type() == TypePtr::BOTTOM, ""); 2140 phi_mem = u; 2141 } else if (u->adr_type() == TypePtr::BOTTOM) { 2142 assert(phi_mem == NULL || _phase->C->get_alias_index(phi_mem->adr_type()) == _alias, ""); 2143 if (phi_mem == NULL) { 2144 phi_mem = u; 2145 } 2146 } 2147 } 2148 } 2149 if (phi_mem == NULL) { 2150 for (uint j = 1; j < head->req(); j++) { 2151 Node* tail = head->in(j); 2152 if (!_phase->is_dominator(head, tail)) { 2153 continue; 2154 } 2155 Node* c = tail; 2156 while (c != head) { 2157 if (c->is_SafePoint() && !c->is_CallLeaf()) { 2158 Node* m =c->in(TypeFunc::Memory); 2159 if (m->is_MergeMem()) { 2160 m = m->as_MergeMem()->memory_at(_alias); 2161 } 2162 assert(mem == NULL || mem == m, "several memory states"); 2163 mem = m; 2164 } 2165 c = _phase->idom(c); 2166 } 2167 assert(mem != NULL, "should have found safepoint"); 2168 } 2169 assert(mem != NULL, "should have found safepoint"); 2170 } else { 2171 mem = phi_mem; 2172 } 2173 } 2174 } 2175 } else { 2176 #ifdef ASSERT 2177 n->dump(); 2178 in->dump(); 2179 #endif 2180 ShouldNotReachHere(); 2181 } 2182 } else { 2183 assert(n->is_Phi() && n->bottom_type() == Type::MEMORY, ""); 2184 assert(n->adr_type() == TypePtr::BOTTOM || _phase->C->get_alias_index(n->adr_type()) == _alias, ""); 2185 mem = n->in(i); 2186 } 2187 i++; 2188 stack.set_index(i); 2189 if (mem == NULL) { 2190 continue; 2191 } 2192 for (;;) { 2193 if (visited.test_set(mem->_idx) || mem->is_Start()) { 2194 break; 2195 } 2196 if (mem->is_Phi()) { 2197 stack.push(mem, 2); 2198 mem = mem->in(1); 2199 } else if (mem->is_Proj()) { 2200 stack.push(mem, mem->req()); 2201 mem = mem->in(0); 2202 } else if (mem->is_SafePoint() || mem->is_MemBar()) { 2203 mem = mem->in(TypeFunc::Memory); 2204 } else if (mem->is_MergeMem()) { 2205 MergeMemNode* mm = mem->as_MergeMem(); 2206 mem = mm->memory_at(_alias); 2207 } else if (mem->is_Store() || mem->is_LoadStore() || mem->is_ClearArray()) { 2208 assert(_alias == Compile::AliasIdxRaw, ""); 2209 stack.push(mem, mem->req()); 2210 mem = mem->in(MemNode::Memory); 2211 } else { 2212 #ifdef ASSERT 2213 mem->dump(); 2214 #endif 2215 ShouldNotReachHere(); 2216 } 2217 } 2218 } else { 2219 if (n->is_Phi()) { 2220 // Nothing 2221 } else if (!n->is_Root()) { 2222 Node* c = get_ctrl(n); 2223 _memory_nodes.map(c->_idx, n); 2224 } 2225 stack.pop(); 2226 } 2227 } while(stack.is_nonempty()); 2228 2229 // Iterate over CFG nodes in rpo and propagate memory state to 2230 // compute memory state at regions, creating new phis if needed. 2231 Node_List rpo_list; 2232 visited.clear(); 2233 _phase->rpo(_phase->C->root(), stack, visited, rpo_list); 2234 Node* root = rpo_list.pop(); 2235 assert(root == _phase->C->root(), ""); 2236 2237 const bool trace = false; 2238 #ifdef ASSERT 2239 if (trace) { 2240 for (int i = rpo_list.size() - 1; i >= 0; i--) { 2241 Node* c = rpo_list.at(i); 2242 if (_memory_nodes[c->_idx] != NULL) { 2243 tty->print("X %d", c->_idx); _memory_nodes[c->_idx]->dump(); 2244 } 2245 } 2246 } 2247 #endif 2248 uint last = _phase->C->unique(); 2249 2250 #ifdef ASSERT 2251 uint8_t max_depth = 0; 2252 for (LoopTreeIterator iter(_phase->ltree_root()); !iter.done(); iter.next()) { 2253 IdealLoopTree* lpt = iter.current(); 2254 max_depth = MAX2(max_depth, lpt->_nest); 2255 } 2256 #endif 2257 2258 bool progress = true; 2259 int iteration = 0; 2260 Node_List dead_phis; 2261 while (progress) { 2262 progress = false; 2263 iteration++; 2264 assert(iteration <= 2+max_depth || _phase->C->has_irreducible_loop() || has_never_branch(_phase->C->root()), ""); 2265 if (trace) { tty->print_cr("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"); } 2266 2267 for (int i = rpo_list.size() - 1; i >= 0; i--) { 2268 Node* c = rpo_list.at(i); 2269 2270 Node* prev_mem = _memory_nodes[c->_idx]; 2271 if (c->is_Region() && (_include_lsm || !c->is_OuterStripMinedLoop())) { 2272 Node* prev_region = regions[c->_idx]; 2273 Node* unique = NULL; 2274 for (uint j = 1; j < c->req() && unique != NodeSentinel; j++) { 2275 Node* m = _memory_nodes[c->in(j)->_idx]; 2276 assert(m != NULL || (c->is_Loop() && j == LoopNode::LoopBackControl && iteration == 1) || _phase->C->has_irreducible_loop() || has_never_branch(_phase->C->root()), "expect memory state"); 2277 if (m != NULL) { 2278 if (m == prev_region && ((c->is_Loop() && j == LoopNode::LoopBackControl) || (prev_region->is_Phi() && prev_region->in(0) == c))) { 2279 assert(c->is_Loop() && j == LoopNode::LoopBackControl || _phase->C->has_irreducible_loop() || has_never_branch(_phase->C->root()), ""); 2280 // continue 2281 } else if (unique == NULL) { 2282 unique = m; 2283 } else if (m == unique) { 2284 // continue 2285 } else { 2286 unique = NodeSentinel; 2287 } 2288 } 2289 } 2290 assert(unique != NULL, "empty phi???"); 2291 if (unique != NodeSentinel) { 2292 if (prev_region != NULL && prev_region->is_Phi() && prev_region->in(0) == c) { 2293 dead_phis.push(prev_region); 2294 } 2295 regions.map(c->_idx, unique); 2296 } else { 2297 Node* phi = NULL; 2298 if (prev_region != NULL && prev_region->is_Phi() && prev_region->in(0) == c && prev_region->_idx >= last) { 2299 phi = prev_region; 2300 for (uint k = 1; k < c->req(); k++) { 2301 Node* m = _memory_nodes[c->in(k)->_idx]; 2302 assert(m != NULL, "expect memory state"); 2303 phi->set_req(k, m); 2304 } 2305 } else { 2306 for (DUIterator_Fast jmax, j = c->fast_outs(jmax); j < jmax && phi == NULL; j++) { 2307 Node* u = c->fast_out(j); 2308 if (u->is_Phi() && u->bottom_type() == Type::MEMORY && 2309 (u->adr_type() == TypePtr::BOTTOM || _phase->C->get_alias_index(u->adr_type()) == _alias)) { 2310 phi = u; 2311 for (uint k = 1; k < c->req() && phi != NULL; k++) { 2312 Node* m = _memory_nodes[c->in(k)->_idx]; 2313 assert(m != NULL, "expect memory state"); 2314 if (u->in(k) != m) { 2315 phi = NULL; 2316 } 2317 } 2318 } 2319 } 2320 if (phi == NULL) { 2321 phi = new PhiNode(c, Type::MEMORY, _phase->C->get_adr_type(_alias)); 2322 for (uint k = 1; k < c->req(); k++) { 2323 Node* m = _memory_nodes[c->in(k)->_idx]; 2324 assert(m != NULL, "expect memory state"); 2325 phi->init_req(k, m); 2326 } 2327 } 2328 } 2329 assert(phi != NULL, ""); 2330 regions.map(c->_idx, phi); 2331 } 2332 Node* current_region = regions[c->_idx]; 2333 if (current_region != prev_region) { 2334 progress = true; 2335 if (prev_region == prev_mem) { 2336 _memory_nodes.map(c->_idx, current_region); 2337 } 2338 } 2339 } else if (prev_mem == NULL || prev_mem->is_Phi() || ctrl_or_self(prev_mem) != c) { 2340 Node* m = _memory_nodes[_phase->idom(c)->_idx]; 2341 assert(m != NULL, "expect memory state"); 2342 if (m != prev_mem) { 2343 _memory_nodes.map(c->_idx, m); 2344 progress = true; 2345 } 2346 } 2347 #ifdef ASSERT 2348 if (trace) { tty->print("X %d", c->_idx); _memory_nodes[c->_idx]->dump(); } 2349 #endif 2350 } 2351 } 2352 2353 // Replace existing phi with computed memory state for that region 2354 // if different (could be a new phi or a dominating memory node if 2355 // that phi was found to be useless). 2356 while (dead_phis.size() > 0) { 2357 Node* n = dead_phis.pop(); 2358 n->replace_by(_phase->C->top()); 2359 n->destruct(); 2360 } 2361 for (int i = rpo_list.size() - 1; i >= 0; i--) { 2362 Node* c = rpo_list.at(i); 2363 if (c->is_Region() && (_include_lsm || !c->is_OuterStripMinedLoop())) { 2364 Node* n = regions[c->_idx]; 2365 if (n->is_Phi() && n->_idx >= last && n->in(0) == c) { 2366 _phase->register_new_node(n, c); 2367 } 2368 } 2369 } 2370 for (int i = rpo_list.size() - 1; i >= 0; i--) { 2371 Node* c = rpo_list.at(i); 2372 if (c->is_Region() && (_include_lsm || !c->is_OuterStripMinedLoop())) { 2373 Node* n = regions[c->_idx]; 2374 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) { 2375 Node* u = c->fast_out(i); 2376 if (u->is_Phi() && u->bottom_type() == Type::MEMORY && 2377 u != n) { 2378 if (u->adr_type() == TypePtr::BOTTOM) { 2379 fix_memory_uses(u, n, n, c); 2380 } else if (_phase->C->get_alias_index(u->adr_type()) == _alias) { 2381 _phase->lazy_replace(u, n); 2382 --i; --imax; 2383 } 2384 } 2385 } 2386 } 2387 } 2388 } 2389 2390 Node* MemoryGraphFixer::get_ctrl(Node* n) const { 2391 Node* c = _phase->get_ctrl(n); 2392 if (n->is_Proj() && n->in(0) != NULL && n->in(0)->is_Call()) { 2393 assert(c == n->in(0), ""); 2394 CallNode* call = c->as_Call(); 2395 CallProjections projs; 2396 call->extract_projections(&projs, true, false); 2397 if (projs.catchall_memproj != NULL) { 2398 if (projs.fallthrough_memproj == n) { 2399 c = projs.fallthrough_catchproj; 2400 } else { 2401 assert(projs.catchall_memproj == n, ""); 2402 c = projs.catchall_catchproj; 2403 } 2404 } 2405 } 2406 return c; 2407 } 2408 2409 Node* MemoryGraphFixer::ctrl_or_self(Node* n) const { 2410 if (_phase->has_ctrl(n)) 2411 return get_ctrl(n); 2412 else { 2413 assert (n->is_CFG(), "must be a CFG node"); 2414 return n; 2415 } 2416 } 2417 2418 bool MemoryGraphFixer::mem_is_valid(Node* m, Node* c) const { 2419 return m != NULL && get_ctrl(m) == c; 2420 } 2421 2422 Node* MemoryGraphFixer::find_mem(Node* ctrl, Node* n) const { 2423 assert(n == NULL || _phase->ctrl_or_self(n) == ctrl, ""); 2424 Node* mem = _memory_nodes[ctrl->_idx]; 2425 Node* c = ctrl; 2426 while (!mem_is_valid(mem, c) && 2427 (!c->is_CatchProj() || mem == NULL || c->in(0)->in(0)->in(0) != get_ctrl(mem))) { 2428 c = _phase->idom(c); 2429 mem = _memory_nodes[c->_idx]; 2430 } 2431 if (n != NULL && mem_is_valid(mem, c)) { 2432 while (!ShenandoahBarrierC2Support::is_dominator_same_ctrl(c, mem, n, _phase) && _phase->ctrl_or_self(mem) == ctrl) { 2433 mem = next_mem(mem, _alias); 2434 } 2435 if (mem->is_MergeMem()) { 2436 mem = mem->as_MergeMem()->memory_at(_alias); 2437 } 2438 if (!mem_is_valid(mem, c)) { 2439 do { 2440 c = _phase->idom(c); 2441 mem = _memory_nodes[c->_idx]; 2442 } while (!mem_is_valid(mem, c) && 2443 (!c->is_CatchProj() || mem == NULL || c->in(0)->in(0)->in(0) != get_ctrl(mem))); 2444 } 2445 } 2446 assert(mem->bottom_type() == Type::MEMORY, ""); 2447 return mem; 2448 } 2449 2450 bool MemoryGraphFixer::has_mem_phi(Node* region) const { 2451 for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) { 2452 Node* use = region->fast_out(i); 2453 if (use->is_Phi() && use->bottom_type() == Type::MEMORY && 2454 (_phase->C->get_alias_index(use->adr_type()) == _alias)) { 2455 return true; 2456 } 2457 } 2458 return false; 2459 } 2460 2461 void MemoryGraphFixer::fix_mem(Node* ctrl, Node* new_ctrl, Node* mem, Node* mem_for_ctrl, Node* new_mem, Unique_Node_List& uses) { 2462 assert(_phase->ctrl_or_self(new_mem) == new_ctrl, ""); 2463 const bool trace = false; 2464 DEBUG_ONLY(if (trace) { tty->print("ZZZ control is"); ctrl->dump(); }); 2465 DEBUG_ONLY(if (trace) { tty->print("ZZZ mem is"); mem->dump(); }); 2466 GrowableArray<Node*> phis; 2467 if (mem_for_ctrl != mem) { 2468 Node* old = mem_for_ctrl; 2469 Node* prev = NULL; 2470 while (old != mem) { 2471 prev = old; 2472 if (old->is_Store() || old->is_ClearArray() || old->is_LoadStore()) { 2473 assert(_alias == Compile::AliasIdxRaw, ""); 2474 old = old->in(MemNode::Memory); 2475 } else if (old->Opcode() == Op_SCMemProj) { 2476 assert(_alias == Compile::AliasIdxRaw, ""); 2477 old = old->in(0); 2478 } else { 2479 ShouldNotReachHere(); 2480 } 2481 } 2482 assert(prev != NULL, ""); 2483 if (new_ctrl != ctrl) { 2484 _memory_nodes.map(ctrl->_idx, mem); 2485 _memory_nodes.map(new_ctrl->_idx, mem_for_ctrl); 2486 } 2487 uint input = (uint)MemNode::Memory; 2488 _phase->igvn().replace_input_of(prev, input, new_mem); 2489 } else { 2490 uses.clear(); 2491 _memory_nodes.map(new_ctrl->_idx, new_mem); 2492 uses.push(new_ctrl); 2493 for(uint next = 0; next < uses.size(); next++ ) { 2494 Node *n = uses.at(next); 2495 assert(n->is_CFG(), ""); 2496 DEBUG_ONLY(if (trace) { tty->print("ZZZ ctrl"); n->dump(); }); 2497 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { 2498 Node* u = n->fast_out(i); 2499 if (!u->is_Root() && u->is_CFG() && u != n) { 2500 Node* m = _memory_nodes[u->_idx]; 2501 if (u->is_Region() && (!u->is_OuterStripMinedLoop() || _include_lsm) && 2502 !has_mem_phi(u) && 2503 u->unique_ctrl_out()->Opcode() != Op_Halt) { 2504 DEBUG_ONLY(if (trace) { tty->print("ZZZ region"); u->dump(); }); 2505 DEBUG_ONLY(if (trace && m != NULL) { tty->print("ZZZ mem"); m->dump(); }); 2506 2507 if (!mem_is_valid(m, u) || !m->is_Phi()) { 2508 bool push = true; 2509 bool create_phi = true; 2510 if (_phase->is_dominator(new_ctrl, u)) { 2511 create_phi = false; 2512 } 2513 if (create_phi) { 2514 Node* phi = new PhiNode(u, Type::MEMORY, _phase->C->get_adr_type(_alias)); 2515 _phase->register_new_node(phi, u); 2516 phis.push(phi); 2517 DEBUG_ONLY(if (trace) { tty->print("ZZZ new phi"); phi->dump(); }); 2518 if (!mem_is_valid(m, u)) { 2519 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting mem"); phi->dump(); }); 2520 _memory_nodes.map(u->_idx, phi); 2521 } else { 2522 DEBUG_ONLY(if (trace) { tty->print("ZZZ NOT setting mem"); m->dump(); }); 2523 for (;;) { 2524 assert(m->is_Mem() || m->is_LoadStore() || m->is_Proj(), ""); 2525 Node* next = NULL; 2526 if (m->is_Proj()) { 2527 next = m->in(0); 2528 } else { 2529 assert(m->is_Mem() || m->is_LoadStore(), ""); 2530 assert(_alias == Compile::AliasIdxRaw, ""); 2531 next = m->in(MemNode::Memory); 2532 } 2533 if (_phase->get_ctrl(next) != u) { 2534 break; 2535 } 2536 if (next->is_MergeMem()) { 2537 assert(_phase->get_ctrl(next->as_MergeMem()->memory_at(_alias)) != u, ""); 2538 break; 2539 } 2540 if (next->is_Phi()) { 2541 assert(next->adr_type() == TypePtr::BOTTOM && next->in(0) == u, ""); 2542 break; 2543 } 2544 m = next; 2545 } 2546 2547 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting to phi"); m->dump(); }); 2548 assert(m->is_Mem() || m->is_LoadStore(), ""); 2549 uint input = (uint)MemNode::Memory; 2550 _phase->igvn().replace_input_of(m, input, phi); 2551 push = false; 2552 } 2553 } else { 2554 DEBUG_ONLY(if (trace) { tty->print("ZZZ skipping region"); u->dump(); }); 2555 } 2556 if (push) { 2557 uses.push(u); 2558 } 2559 } 2560 } else if (!mem_is_valid(m, u) && 2561 !(u->Opcode() == Op_CProj && u->in(0)->Opcode() == Op_NeverBranch && u->as_Proj()->_con == 1)) { 2562 uses.push(u); 2563 } 2564 } 2565 } 2566 } 2567 for (int i = 0; i < phis.length(); i++) { 2568 Node* n = phis.at(i); 2569 Node* r = n->in(0); 2570 DEBUG_ONLY(if (trace) { tty->print("ZZZ fixing new phi"); n->dump(); }); 2571 for (uint j = 1; j < n->req(); j++) { 2572 Node* m = find_mem(r->in(j), NULL); 2573 _phase->igvn().replace_input_of(n, j, m); 2574 DEBUG_ONLY(if (trace) { tty->print("ZZZ fixing new phi: %d", j); m->dump(); }); 2575 } 2576 } 2577 } 2578 uint last = _phase->C->unique(); 2579 MergeMemNode* mm = NULL; 2580 int alias = _alias; 2581 DEBUG_ONLY(if (trace) { tty->print("ZZZ raw mem is"); mem->dump(); }); 2582 // Process loads first to not miss an anti-dependency: if the memory 2583 // edge of a store is updated before a load is processed then an 2584 // anti-dependency may be missed. 2585 for (DUIterator i = mem->outs(); mem->has_out(i); i++) { 2586 Node* u = mem->out(i); 2587 if (u->_idx < last && u->is_Load() && _phase->C->get_alias_index(u->adr_type()) == alias) { 2588 Node* m = find_mem(_phase->get_ctrl(u), u); 2589 if (m != mem) { 2590 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting memory of use"); u->dump(); }); 2591 _phase->igvn().replace_input_of(u, MemNode::Memory, m); 2592 --i; 2593 } 2594 } 2595 } 2596 for (DUIterator i = mem->outs(); mem->has_out(i); i++) { 2597 Node* u = mem->out(i); 2598 if (u->_idx < last) { 2599 if (u->is_Mem()) { 2600 if (_phase->C->get_alias_index(u->adr_type()) == alias) { 2601 Node* m = find_mem(_phase->get_ctrl(u), u); 2602 if (m != mem) { 2603 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting memory of use"); u->dump(); }); 2604 _phase->igvn().replace_input_of(u, MemNode::Memory, m); 2605 --i; 2606 } 2607 } 2608 } else if (u->is_MergeMem()) { 2609 MergeMemNode* u_mm = u->as_MergeMem(); 2610 if (u_mm->memory_at(alias) == mem) { 2611 MergeMemNode* newmm = NULL; 2612 for (DUIterator_Fast jmax, j = u->fast_outs(jmax); j < jmax; j++) { 2613 Node* uu = u->fast_out(j); 2614 assert(!uu->is_MergeMem(), "chain of MergeMems?"); 2615 if (uu->is_Phi()) { 2616 assert(uu->adr_type() == TypePtr::BOTTOM, ""); 2617 Node* region = uu->in(0); 2618 int nb = 0; 2619 for (uint k = 1; k < uu->req(); k++) { 2620 if (uu->in(k) == u) { 2621 Node* m = find_mem(region->in(k), NULL); 2622 if (m != mem) { 2623 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting memory of phi %d", k); uu->dump(); }); 2624 newmm = clone_merge_mem(u, mem, m, _phase->ctrl_or_self(m), i); 2625 if (newmm != u) { 2626 _phase->igvn().replace_input_of(uu, k, newmm); 2627 nb++; 2628 --jmax; 2629 } 2630 } 2631 } 2632 } 2633 if (nb > 0) { 2634 --j; 2635 } 2636 } else { 2637 Node* m = find_mem(_phase->ctrl_or_self(uu), uu); 2638 if (m != mem) { 2639 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting memory of use"); uu->dump(); }); 2640 newmm = clone_merge_mem(u, mem, m, _phase->ctrl_or_self(m), i); 2641 if (newmm != u) { 2642 _phase->igvn().replace_input_of(uu, uu->find_edge(u), newmm); 2643 --j, --jmax; 2644 } 2645 } 2646 } 2647 } 2648 } 2649 } else if (u->is_Phi()) { 2650 assert(u->bottom_type() == Type::MEMORY, "what else?"); 2651 if (_phase->C->get_alias_index(u->adr_type()) == alias || u->adr_type() == TypePtr::BOTTOM) { 2652 Node* region = u->in(0); 2653 bool replaced = false; 2654 for (uint j = 1; j < u->req(); j++) { 2655 if (u->in(j) == mem) { 2656 Node* m = find_mem(region->in(j), NULL); 2657 Node* nnew = m; 2658 if (m != mem) { 2659 if (u->adr_type() == TypePtr::BOTTOM) { 2660 mm = allocate_merge_mem(mem, m, _phase->ctrl_or_self(m)); 2661 nnew = mm; 2662 } 2663 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting memory of phi %d", j); u->dump(); }); 2664 _phase->igvn().replace_input_of(u, j, nnew); 2665 replaced = true; 2666 } 2667 } 2668 } 2669 if (replaced) { 2670 --i; 2671 } 2672 } 2673 } else if ((u->adr_type() == TypePtr::BOTTOM && u->Opcode() != Op_StrInflatedCopy) || 2674 u->adr_type() == NULL) { 2675 assert(u->adr_type() != NULL || 2676 u->Opcode() == Op_Rethrow || 2677 u->Opcode() == Op_Return || 2678 u->Opcode() == Op_SafePoint || 2679 (u->is_CallStaticJava() && u->as_CallStaticJava()->uncommon_trap_request() != 0) || 2680 (u->is_CallStaticJava() && u->as_CallStaticJava()->_entry_point == OptoRuntime::rethrow_stub()) || 2681 u->Opcode() == Op_CallLeaf, ""); 2682 Node* m = find_mem(_phase->ctrl_or_self(u), u); 2683 if (m != mem) { 2684 mm = allocate_merge_mem(mem, m, _phase->get_ctrl(m)); 2685 _phase->igvn().replace_input_of(u, u->find_edge(mem), mm); 2686 --i; 2687 } 2688 } else if (_phase->C->get_alias_index(u->adr_type()) == alias) { 2689 Node* m = find_mem(_phase->ctrl_or_self(u), u); 2690 if (m != mem) { 2691 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting memory of use"); u->dump(); }); 2692 _phase->igvn().replace_input_of(u, u->find_edge(mem), m); 2693 --i; 2694 } 2695 } else if (u->adr_type() != TypePtr::BOTTOM && 2696 _memory_nodes[_phase->ctrl_or_self(u)->_idx] == u) { 2697 Node* m = find_mem(_phase->ctrl_or_self(u), u); 2698 assert(m != mem, ""); 2699 // u is on the wrong slice... 2700 assert(u->is_ClearArray(), ""); 2701 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting memory of use"); u->dump(); }); 2702 _phase->igvn().replace_input_of(u, u->find_edge(mem), m); 2703 --i; 2704 } 2705 } 2706 } 2707 #ifdef ASSERT 2708 assert(new_mem->outcnt() > 0, ""); 2709 for (int i = 0; i < phis.length(); i++) { 2710 Node* n = phis.at(i); 2711 assert(n->outcnt() > 0, "new phi must have uses now"); 2712 } 2713 #endif 2714 } 2715 2716 MergeMemNode* MemoryGraphFixer::allocate_merge_mem(Node* mem, Node* rep_proj, Node* rep_ctrl) const { 2717 MergeMemNode* mm = MergeMemNode::make(mem); 2718 mm->set_memory_at(_alias, rep_proj); 2719 _phase->register_new_node(mm, rep_ctrl); 2720 return mm; 2721 } 2722 2723 MergeMemNode* MemoryGraphFixer::clone_merge_mem(Node* u, Node* mem, Node* rep_proj, Node* rep_ctrl, DUIterator& i) const { 2724 MergeMemNode* newmm = NULL; 2725 MergeMemNode* u_mm = u->as_MergeMem(); 2726 Node* c = _phase->get_ctrl(u); 2727 if (_phase->is_dominator(c, rep_ctrl)) { 2728 c = rep_ctrl; 2729 } else { 2730 assert(_phase->is_dominator(rep_ctrl, c), "one must dominate the other"); 2731 } 2732 if (u->outcnt() == 1) { 2733 if (u->req() > (uint)_alias && u->in(_alias) == mem) { 2734 _phase->igvn().replace_input_of(u, _alias, rep_proj); 2735 --i; 2736 } else { 2737 _phase->igvn().rehash_node_delayed(u); 2738 u_mm->set_memory_at(_alias, rep_proj); 2739 } 2740 newmm = u_mm; 2741 _phase->set_ctrl_and_loop(u, c); 2742 } else { 2743 // can't simply clone u and then change one of its input because 2744 // it adds and then removes an edge which messes with the 2745 // DUIterator 2746 newmm = MergeMemNode::make(u_mm->base_memory()); 2747 for (uint j = 0; j < u->req(); j++) { 2748 if (j < newmm->req()) { 2749 if (j == (uint)_alias) { 2750 newmm->set_req(j, rep_proj); 2751 } else if (newmm->in(j) != u->in(j)) { 2752 newmm->set_req(j, u->in(j)); 2753 } 2754 } else if (j == (uint)_alias) { 2755 newmm->add_req(rep_proj); 2756 } else { 2757 newmm->add_req(u->in(j)); 2758 } 2759 } 2760 if ((uint)_alias >= u->req()) { 2761 newmm->set_memory_at(_alias, rep_proj); 2762 } 2763 _phase->register_new_node(newmm, c); 2764 } 2765 return newmm; 2766 } 2767 2768 bool MemoryGraphFixer::should_process_phi(Node* phi) const { 2769 if (phi->adr_type() == TypePtr::BOTTOM) { 2770 Node* region = phi->in(0); 2771 for (DUIterator_Fast jmax, j = region->fast_outs(jmax); j < jmax; j++) { 2772 Node* uu = region->fast_out(j); 2773 if (uu->is_Phi() && uu != phi && uu->bottom_type() == Type::MEMORY && _phase->C->get_alias_index(uu->adr_type()) == _alias) { 2774 return false; 2775 } 2776 } 2777 return true; 2778 } 2779 return _phase->C->get_alias_index(phi->adr_type()) == _alias; 2780 } 2781 2782 void MemoryGraphFixer::fix_memory_uses(Node* mem, Node* replacement, Node* rep_proj, Node* rep_ctrl) const { 2783 uint last = _phase-> C->unique(); 2784 MergeMemNode* mm = NULL; 2785 assert(mem->bottom_type() == Type::MEMORY, ""); 2786 for (DUIterator i = mem->outs(); mem->has_out(i); i++) { 2787 Node* u = mem->out(i); 2788 if (u != replacement && u->_idx < last) { 2789 if (u->is_MergeMem()) { 2790 MergeMemNode* u_mm = u->as_MergeMem(); 2791 if (u_mm->memory_at(_alias) == mem) { 2792 MergeMemNode* newmm = NULL; 2793 for (DUIterator_Fast jmax, j = u->fast_outs(jmax); j < jmax; j++) { 2794 Node* uu = u->fast_out(j); 2795 assert(!uu->is_MergeMem(), "chain of MergeMems?"); 2796 if (uu->is_Phi()) { 2797 if (should_process_phi(uu)) { 2798 Node* region = uu->in(0); 2799 int nb = 0; 2800 for (uint k = 1; k < uu->req(); k++) { 2801 if (uu->in(k) == u && _phase->is_dominator(rep_ctrl, region->in(k))) { 2802 if (newmm == NULL) { 2803 newmm = clone_merge_mem(u, mem, rep_proj, rep_ctrl, i); 2804 } 2805 if (newmm != u) { 2806 _phase->igvn().replace_input_of(uu, k, newmm); 2807 nb++; 2808 --jmax; 2809 } 2810 } 2811 } 2812 if (nb > 0) { 2813 --j; 2814 } 2815 } 2816 } else { 2817 if (rep_ctrl != uu && ShenandoahBarrierC2Support::is_dominator(rep_ctrl, _phase->ctrl_or_self(uu), replacement, uu, _phase)) { 2818 if (newmm == NULL) { 2819 newmm = clone_merge_mem(u, mem, rep_proj, rep_ctrl, i); 2820 } 2821 if (newmm != u) { 2822 _phase->igvn().replace_input_of(uu, uu->find_edge(u), newmm); 2823 --j, --jmax; 2824 } 2825 } 2826 } 2827 } 2828 } 2829 } else if (u->is_Phi()) { 2830 assert(u->bottom_type() == Type::MEMORY, "what else?"); 2831 Node* region = u->in(0); 2832 if (should_process_phi(u)) { 2833 bool replaced = false; 2834 for (uint j = 1; j < u->req(); j++) { 2835 if (u->in(j) == mem && _phase->is_dominator(rep_ctrl, region->in(j))) { 2836 Node* nnew = rep_proj; 2837 if (u->adr_type() == TypePtr::BOTTOM) { 2838 if (mm == NULL) { 2839 mm = allocate_merge_mem(mem, rep_proj, rep_ctrl); 2840 } 2841 nnew = mm; 2842 } 2843 _phase->igvn().replace_input_of(u, j, nnew); 2844 replaced = true; 2845 } 2846 } 2847 if (replaced) { 2848 --i; 2849 } 2850 2851 } 2852 } else if ((u->adr_type() == TypePtr::BOTTOM && u->Opcode() != Op_StrInflatedCopy) || 2853 u->adr_type() == NULL) { 2854 assert(u->adr_type() != NULL || 2855 u->Opcode() == Op_Rethrow || 2856 u->Opcode() == Op_Return || 2857 u->Opcode() == Op_SafePoint || 2858 u->Opcode() == Op_StoreIConditional || 2859 u->Opcode() == Op_StoreLConditional || 2860 (u->is_CallStaticJava() && u->as_CallStaticJava()->uncommon_trap_request() != 0) || 2861 (u->is_CallStaticJava() && u->as_CallStaticJava()->_entry_point == OptoRuntime::rethrow_stub()) || 2862 u->Opcode() == Op_CallLeaf, "%s", u->Name()); 2863 if (ShenandoahBarrierC2Support::is_dominator(rep_ctrl, _phase->ctrl_or_self(u), replacement, u, _phase)) { 2864 if (mm == NULL) { 2865 mm = allocate_merge_mem(mem, rep_proj, rep_ctrl); 2866 } 2867 _phase->igvn().replace_input_of(u, u->find_edge(mem), mm); 2868 --i; 2869 } 2870 } else if (_phase->C->get_alias_index(u->adr_type()) == _alias) { 2871 if (ShenandoahBarrierC2Support::is_dominator(rep_ctrl, _phase->ctrl_or_self(u), replacement, u, _phase)) { 2872 _phase->igvn().replace_input_of(u, u->find_edge(mem), rep_proj); 2873 --i; 2874 } 2875 } 2876 } 2877 } 2878 } 2879 2880 ShenandoahLoadReferenceBarrierNode::ShenandoahLoadReferenceBarrierNode(Node* ctrl, Node* obj, bool native) 2881 : Node(ctrl, obj), _native(native) { 2882 ShenandoahBarrierSetC2::bsc2()->state()->add_load_reference_barrier(this); 2883 } 2884 2885 bool ShenandoahLoadReferenceBarrierNode::is_native() const { 2886 return _native; 2887 } 2888 2889 uint ShenandoahLoadReferenceBarrierNode::size_of() const { 2890 return sizeof(*this); 2891 } 2892 2893 uint ShenandoahLoadReferenceBarrierNode::hash() const { 2894 return Node::hash() + (_native ? 1 : 0); 2895 } 2896 2897 bool ShenandoahLoadReferenceBarrierNode::cmp( const Node &n ) const { 2898 return Node::cmp(n) && n.Opcode() == Op_ShenandoahLoadReferenceBarrier && 2899 _native == ((const ShenandoahLoadReferenceBarrierNode&)n)._native; 2900 } 2901 2902 const Type* ShenandoahLoadReferenceBarrierNode::bottom_type() const { 2903 if (in(ValueIn) == NULL || in(ValueIn)->is_top()) { 2904 return Type::TOP; 2905 } 2906 const Type* t = in(ValueIn)->bottom_type(); 2907 if (t == TypePtr::NULL_PTR) { 2908 return t; 2909 } 2910 return t->is_oopptr(); 2911 } 2912 2913 const Type* ShenandoahLoadReferenceBarrierNode::Value(PhaseGVN* phase) const { 2914 // Either input is TOP ==> the result is TOP 2915 const Type *t2 = phase->type(in(ValueIn)); 2916 if( t2 == Type::TOP ) return Type::TOP; 2917 2918 if (t2 == TypePtr::NULL_PTR) { 2919 return t2; 2920 } 2921 2922 const Type* type = t2->is_oopptr(); 2923 return type; 2924 } 2925 2926 Node* ShenandoahLoadReferenceBarrierNode::Identity(PhaseGVN* phase) { 2927 Node* value = in(ValueIn); 2928 if (!needs_barrier(phase, value)) { 2929 return value; 2930 } 2931 return this; 2932 } 2933 2934 bool ShenandoahLoadReferenceBarrierNode::needs_barrier(PhaseGVN* phase, Node* n) { 2935 Unique_Node_List visited; 2936 return needs_barrier_impl(phase, n, visited); 2937 } 2938 2939 bool ShenandoahLoadReferenceBarrierNode::needs_barrier_impl(PhaseGVN* phase, Node* n, Unique_Node_List &visited) { 2940 if (n == NULL) return false; 2941 if (visited.member(n)) { 2942 return false; // Been there. 2943 } 2944 visited.push(n); 2945 2946 if (n->is_Allocate()) { 2947 // tty->print_cr("optimize barrier on alloc"); 2948 return false; 2949 } 2950 if (n->is_Call()) { 2951 // tty->print_cr("optimize barrier on call"); 2952 return false; 2953 } 2954 2955 const Type* type = phase->type(n); 2956 if (type == Type::TOP) { 2957 return false; 2958 } 2959 if (type->make_ptr()->higher_equal(TypePtr::NULL_PTR)) { 2960 // tty->print_cr("optimize barrier on null"); 2961 return false; 2962 } 2963 if (type->make_oopptr() && type->make_oopptr()->const_oop() != NULL) { 2964 // tty->print_cr("optimize barrier on constant"); 2965 return false; 2966 } 2967 2968 switch (n->Opcode()) { 2969 case Op_AddP: 2970 return true; // TODO: Can refine? 2971 case Op_LoadP: 2972 case Op_ShenandoahCompareAndExchangeN: 2973 case Op_ShenandoahCompareAndExchangeP: 2974 case Op_CompareAndExchangeN: 2975 case Op_CompareAndExchangeP: 2976 case Op_GetAndSetN: 2977 case Op_GetAndSetP: 2978 return true; 2979 case Op_Phi: { 2980 for (uint i = 1; i < n->req(); i++) { 2981 if (needs_barrier_impl(phase, n->in(i), visited)) return true; 2982 } 2983 return false; 2984 } 2985 case Op_CheckCastPP: 2986 case Op_CastPP: 2987 return needs_barrier_impl(phase, n->in(1), visited); 2988 case Op_Proj: 2989 return needs_barrier_impl(phase, n->in(0), visited); 2990 case Op_ShenandoahLoadReferenceBarrier: 2991 // tty->print_cr("optimize barrier on barrier"); 2992 return false; 2993 case Op_Parm: 2994 // tty->print_cr("optimize barrier on input arg"); 2995 return false; 2996 case Op_DecodeN: 2997 case Op_EncodeP: 2998 return needs_barrier_impl(phase, n->in(1), visited); 2999 case Op_LoadN: 3000 return true; 3001 case Op_CMoveN: 3002 case Op_CMoveP: 3003 return needs_barrier_impl(phase, n->in(2), visited) || 3004 needs_barrier_impl(phase, n->in(3), visited); 3005 case Op_ShenandoahEnqueueBarrier: 3006 return needs_barrier_impl(phase, n->in(1), visited); 3007 case Op_CreateEx: 3008 return false; 3009 default: 3010 break; 3011 } 3012 #ifdef ASSERT 3013 tty->print("need barrier on?: "); 3014 tty->print_cr("ins:"); 3015 n->dump(2); 3016 tty->print_cr("outs:"); 3017 n->dump(-2); 3018 ShouldNotReachHere(); 3019 #endif 3020 return true; 3021 }