1 /* 2 * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 #include "gc/shared/c2/barrierSetC2.hpp" 27 #include "opto/arraycopynode.hpp" 28 #include "opto/convertnode.hpp" 29 #include "opto/graphKit.hpp" 30 #include "opto/idealKit.hpp" 31 #include "opto/macro.hpp" 32 #include "opto/narrowptrnode.hpp" 33 #include "utilities/macros.hpp" 34 35 // By default this is a no-op. 36 void BarrierSetC2::resolve_address(C2Access& access) const { } 37 38 void* C2ParseAccess::barrier_set_state() const { 39 return _kit->barrier_set_state(); 40 } 41 42 PhaseGVN& C2ParseAccess::gvn() const { return _kit->gvn(); } 43 44 bool C2Access::needs_cpu_membar() const { 45 bool mismatched = (_decorators & C2_MISMATCHED) != 0; 46 bool is_unordered = (_decorators & MO_UNORDERED) != 0; 47 bool anonymous = (_decorators & C2_UNSAFE_ACCESS) != 0; 48 bool in_heap = (_decorators & IN_HEAP) != 0; 49 50 bool is_write = (_decorators & C2_WRITE_ACCESS) != 0; 51 bool is_read = (_decorators & C2_READ_ACCESS) != 0; 52 bool is_atomic = is_read && is_write; 53 54 if (is_atomic) { 55 // Atomics always need to be wrapped in CPU membars 56 return true; 57 } 58 59 if (anonymous) { 60 // We will need memory barriers unless we can determine a unique 61 // alias category for this reference. (Note: If for some reason 62 // the barriers get omitted and the unsafe reference begins to "pollute" 63 // the alias analysis of the rest of the graph, either Compile::can_alias 64 // or Compile::must_alias will throw a diagnostic assert.) 65 if (!in_heap || !is_unordered || (mismatched && !_addr.type()->isa_aryptr())) { 66 return true; 67 } 68 } 69 70 return false; 71 } 72 73 Node* BarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& val) const { 74 DecoratorSet decorators = access.decorators(); 75 76 bool mismatched = (decorators & C2_MISMATCHED) != 0; 77 bool unaligned = (decorators & C2_UNALIGNED) != 0; 78 bool unsafe = (decorators & C2_UNSAFE_ACCESS) != 0; 79 bool requires_atomic_access = (decorators & MO_UNORDERED) == 0; 80 81 bool in_native = (decorators & IN_NATIVE) != 0; 82 assert(!in_native, "not supported yet"); 83 84 MemNode::MemOrd mo = access.mem_node_mo(); 85 86 Node* store; 87 if (access.is_parse_access()) { 88 C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access); 89 90 GraphKit* kit = parse_access.kit(); 91 if (access.type() == T_DOUBLE) { 92 Node* new_val = kit->dstore_rounding(val.node()); 93 val.set_node(new_val); 94 } 95 96 store = kit->store_to_memory(kit->control(), access.addr().node(), val.node(), access.type(), 97 access.addr().type(), mo, requires_atomic_access, unaligned, mismatched, unsafe); 98 access.set_raw_access(store); 99 } else { 100 assert(!requires_atomic_access, "not yet supported"); 101 assert(access.is_opt_access(), "either parse or opt access"); 102 C2OptAccess& opt_access = static_cast<C2OptAccess&>(access); 103 Node* ctl = opt_access.ctl(); 104 MergeMemNode* mm = opt_access.mem(); 105 PhaseGVN& gvn = opt_access.gvn(); 106 const TypePtr* adr_type = access.addr().type(); 107 int alias = gvn.C->get_alias_index(adr_type); 108 Node* mem = mm->memory_at(alias); 109 110 StoreNode* st = StoreNode::make(gvn, ctl, mem, access.addr().node(), adr_type, val.node(), access.type(), mo); 111 if (unaligned) { 112 st->set_unaligned_access(); 113 } 114 if (mismatched) { 115 st->set_mismatched_access(); 116 } 117 store = gvn.transform(st); 118 if (store == st) { 119 mm->set_memory_at(alias, st); 120 } 121 } 122 return store; 123 } 124 125 Node* BarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const { 126 DecoratorSet decorators = access.decorators(); 127 128 Node* adr = access.addr().node(); 129 const TypePtr* adr_type = access.addr().type(); 130 131 bool mismatched = (decorators & C2_MISMATCHED) != 0; 132 bool requires_atomic_access = (decorators & MO_UNORDERED) == 0; 133 bool unaligned = (decorators & C2_UNALIGNED) != 0; 134 bool control_dependent = (decorators & C2_CONTROL_DEPENDENT_LOAD) != 0; 135 bool pinned = (decorators & C2_PINNED_LOAD) != 0; 136 bool unsafe = (decorators & C2_UNSAFE_ACCESS) != 0; 137 138 bool in_native = (decorators & IN_NATIVE) != 0; 139 140 MemNode::MemOrd mo = access.mem_node_mo(); 141 LoadNode::ControlDependency dep = pinned ? LoadNode::Pinned : LoadNode::DependsOnlyOnTest; 142 143 Node* load; 144 if (access.is_parse_access()) { 145 C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access); 146 GraphKit* kit = parse_access.kit(); 147 Node* control = control_dependent ? kit->control() : NULL; 148 149 if (in_native) { 150 load = kit->make_load(control, adr, val_type, access.type(), mo); 151 } else { 152 load = kit->make_load(control, adr, val_type, access.type(), adr_type, mo, 153 dep, requires_atomic_access, unaligned, mismatched, unsafe); 154 } 155 access.set_raw_access(load); 156 } else { 157 assert(!requires_atomic_access, "not yet supported"); 158 assert(access.is_opt_access(), "either parse or opt access"); 159 C2OptAccess& opt_access = static_cast<C2OptAccess&>(access); 160 Node* control = control_dependent ? opt_access.ctl() : NULL; 161 MergeMemNode* mm = opt_access.mem(); 162 PhaseGVN& gvn = opt_access.gvn(); 163 Node* mem = mm->memory_at(gvn.C->get_alias_index(adr_type)); 164 load = LoadNode::make(gvn, control, mem, adr, adr_type, val_type, access.type(), mo, dep, unaligned, mismatched); 165 load = gvn.transform(load); 166 } 167 168 return load; 169 } 170 171 class C2AccessFence: public StackObj { 172 C2Access& _access; 173 Node* _leading_membar; 174 175 public: 176 C2AccessFence(C2Access& access) : 177 _access(access), _leading_membar(NULL) { 178 GraphKit* kit = NULL; 179 if (access.is_parse_access()) { 180 C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access); 181 kit = parse_access.kit(); 182 } 183 DecoratorSet decorators = access.decorators(); 184 185 bool is_write = (decorators & C2_WRITE_ACCESS) != 0; 186 bool is_read = (decorators & C2_READ_ACCESS) != 0; 187 bool is_atomic = is_read && is_write; 188 189 bool is_volatile = (decorators & MO_SEQ_CST) != 0; 190 bool is_release = (decorators & MO_RELEASE) != 0; 191 192 if (is_atomic) { 193 assert(kit != NULL, "unsupported at optimization time"); 194 // Memory-model-wise, a LoadStore acts like a little synchronized 195 // block, so needs barriers on each side. These don't translate 196 // into actual barriers on most machines, but we still need rest of 197 // compiler to respect ordering. 198 if (is_release) { 199 _leading_membar = kit->insert_mem_bar(Op_MemBarRelease); 200 } else if (is_volatile) { 201 if (support_IRIW_for_not_multiple_copy_atomic_cpu) { 202 _leading_membar = kit->insert_mem_bar(Op_MemBarVolatile); 203 } else { 204 _leading_membar = kit->insert_mem_bar(Op_MemBarRelease); 205 } 206 } 207 } else if (is_write) { 208 // If reference is volatile, prevent following memory ops from 209 // floating down past the volatile write. Also prevents commoning 210 // another volatile read. 211 if (is_volatile || is_release) { 212 assert(kit != NULL, "unsupported at optimization time"); 213 _leading_membar = kit->insert_mem_bar(Op_MemBarRelease); 214 } 215 } else { 216 // Memory barrier to prevent normal and 'unsafe' accesses from 217 // bypassing each other. Happens after null checks, so the 218 // exception paths do not take memory state from the memory barrier, 219 // so there's no problems making a strong assert about mixing users 220 // of safe & unsafe memory. 221 if (is_volatile && support_IRIW_for_not_multiple_copy_atomic_cpu) { 222 assert(kit != NULL, "unsupported at optimization time"); 223 _leading_membar = kit->insert_mem_bar(Op_MemBarVolatile); 224 } 225 } 226 227 if (access.needs_cpu_membar()) { 228 assert(kit != NULL, "unsupported at optimization time"); 229 kit->insert_mem_bar(Op_MemBarCPUOrder); 230 } 231 232 if (is_atomic) { 233 // 4984716: MemBars must be inserted before this 234 // memory node in order to avoid a false 235 // dependency which will confuse the scheduler. 236 access.set_memory(); 237 } 238 } 239 240 ~C2AccessFence() { 241 GraphKit* kit = NULL; 242 if (_access.is_parse_access()) { 243 C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(_access); 244 kit = parse_access.kit(); 245 } 246 DecoratorSet decorators = _access.decorators(); 247 248 bool is_write = (decorators & C2_WRITE_ACCESS) != 0; 249 bool is_read = (decorators & C2_READ_ACCESS) != 0; 250 bool is_atomic = is_read && is_write; 251 252 bool is_volatile = (decorators & MO_SEQ_CST) != 0; 253 bool is_acquire = (decorators & MO_ACQUIRE) != 0; 254 255 // If reference is volatile, prevent following volatiles ops from 256 // floating up before the volatile access. 257 if (_access.needs_cpu_membar()) { 258 kit->insert_mem_bar(Op_MemBarCPUOrder); 259 } 260 261 if (is_atomic) { 262 assert(kit != NULL, "unsupported at optimization time"); 263 if (is_acquire || is_volatile) { 264 Node* n = _access.raw_access(); 265 Node* mb = kit->insert_mem_bar(Op_MemBarAcquire, n); 266 if (_leading_membar != NULL) { 267 MemBarNode::set_load_store_pair(_leading_membar->as_MemBar(), mb->as_MemBar()); 268 } 269 } 270 } else if (is_write) { 271 // If not multiple copy atomic, we do the MemBarVolatile before the load. 272 if (is_volatile && !support_IRIW_for_not_multiple_copy_atomic_cpu) { 273 assert(kit != NULL, "unsupported at optimization time"); 274 Node* n = _access.raw_access(); 275 Node* mb = kit->insert_mem_bar(Op_MemBarVolatile, n); // Use fat membar 276 if (_leading_membar != NULL) { 277 MemBarNode::set_store_pair(_leading_membar->as_MemBar(), mb->as_MemBar()); 278 } 279 } 280 } else { 281 if (is_volatile || is_acquire) { 282 assert(kit != NULL, "unsupported at optimization time"); 283 Node* n = _access.raw_access(); 284 assert(_leading_membar == NULL || support_IRIW_for_not_multiple_copy_atomic_cpu, "no leading membar expected"); 285 Node* mb = kit->insert_mem_bar(Op_MemBarAcquire, n); 286 mb->as_MemBar()->set_trailing_load(); 287 } 288 } 289 } 290 }; 291 292 Node* BarrierSetC2::store_at(C2Access& access, C2AccessValue& val) const { 293 C2AccessFence fence(access); 294 resolve_address(access); 295 return store_at_resolved(access, val); 296 } 297 298 Node* BarrierSetC2::load_at(C2Access& access, const Type* val_type) const { 299 C2AccessFence fence(access); 300 resolve_address(access); 301 return load_at_resolved(access, val_type); 302 } 303 304 MemNode::MemOrd C2Access::mem_node_mo() const { 305 bool is_write = (_decorators & C2_WRITE_ACCESS) != 0; 306 bool is_read = (_decorators & C2_READ_ACCESS) != 0; 307 if ((_decorators & MO_SEQ_CST) != 0) { 308 if (is_write && is_read) { 309 // For atomic operations 310 return MemNode::seqcst; 311 } else if (is_write) { 312 return MemNode::release; 313 } else { 314 assert(is_read, "what else?"); 315 return MemNode::acquire; 316 } 317 } else if ((_decorators & MO_RELEASE) != 0) { 318 return MemNode::release; 319 } else if ((_decorators & MO_ACQUIRE) != 0) { 320 return MemNode::acquire; 321 } else if (is_write) { 322 // Volatile fields need releasing stores. 323 // Non-volatile fields also need releasing stores if they hold an 324 // object reference, because the object reference might point to 325 // a freshly created object. 326 // Conservatively release stores of object references. 327 return StoreNode::release_if_reference(_type); 328 } else { 329 return MemNode::unordered; 330 } 331 } 332 333 void C2Access::fixup_decorators() { 334 bool default_mo = (_decorators & MO_DECORATOR_MASK) == 0; 335 bool is_unordered = (_decorators & MO_UNORDERED) != 0 || default_mo; 336 bool anonymous = (_decorators & C2_UNSAFE_ACCESS) != 0; 337 338 bool is_read = (_decorators & C2_READ_ACCESS) != 0; 339 bool is_write = (_decorators & C2_WRITE_ACCESS) != 0; 340 341 if (AlwaysAtomicAccesses && is_unordered) { 342 _decorators &= ~MO_DECORATOR_MASK; // clear the MO bits 343 _decorators |= MO_RELAXED; // Force the MO_RELAXED decorator with AlwaysAtomicAccess 344 } 345 346 _decorators = AccessInternal::decorator_fixup(_decorators); 347 348 if (is_read && !is_write && anonymous) { 349 // To be valid, unsafe loads may depend on other conditions than 350 // the one that guards them: pin the Load node 351 _decorators |= C2_CONTROL_DEPENDENT_LOAD; 352 _decorators |= C2_PINNED_LOAD; 353 const TypePtr* adr_type = _addr.type(); 354 Node* adr = _addr.node(); 355 if (!needs_cpu_membar() && adr_type->isa_instptr()) { 356 assert(adr_type->meet(TypePtr::NULL_PTR) != adr_type->remove_speculative(), "should be not null"); 357 intptr_t offset = Type::OffsetBot; 358 AddPNode::Ideal_base_and_offset(adr, &gvn(), offset); 359 if (offset >= 0) { 360 int s = Klass::layout_helper_size_in_bytes(adr_type->isa_instptr()->klass()->layout_helper()); 361 if (offset < s) { 362 // Guaranteed to be a valid access, no need to pin it 363 _decorators ^= C2_CONTROL_DEPENDENT_LOAD; 364 _decorators ^= C2_PINNED_LOAD; 365 } 366 } 367 } 368 } 369 } 370 371 //--------------------------- atomic operations--------------------------------- 372 373 void BarrierSetC2::pin_atomic_op(C2AtomicParseAccess& access) const { 374 if (!access.needs_pinning()) { 375 return; 376 } 377 // SCMemProjNodes represent the memory state of a LoadStore. Their 378 // main role is to prevent LoadStore nodes from being optimized away 379 // when their results aren't used. 380 assert(access.is_parse_access(), "entry not supported at optimization time"); 381 C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access); 382 GraphKit* kit = parse_access.kit(); 383 Node* load_store = access.raw_access(); 384 assert(load_store != NULL, "must pin atomic op"); 385 Node* proj = kit->gvn().transform(new SCMemProjNode(load_store)); 386 kit->set_memory(proj, access.alias_idx()); 387 } 388 389 void C2AtomicParseAccess::set_memory() { 390 Node *mem = _kit->memory(_alias_idx); 391 _memory = mem; 392 } 393 394 Node* BarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val, 395 Node* new_val, const Type* value_type) const { 396 GraphKit* kit = access.kit(); 397 MemNode::MemOrd mo = access.mem_node_mo(); 398 Node* mem = access.memory(); 399 400 Node* adr = access.addr().node(); 401 const TypePtr* adr_type = access.addr().type(); 402 403 Node* load_store = NULL; 404 405 if (access.is_oop()) { 406 #ifdef _LP64 407 if (adr->bottom_type()->is_ptr_to_narrowoop()) { 408 Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop())); 409 Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop())); 410 load_store = kit->gvn().transform(new CompareAndExchangeNNode(kit->control(), mem, adr, newval_enc, oldval_enc, adr_type, value_type->make_narrowoop(), mo)); 411 } else 412 #endif 413 { 414 load_store = kit->gvn().transform(new CompareAndExchangePNode(kit->control(), mem, adr, new_val, expected_val, adr_type, value_type->is_oopptr(), mo)); 415 } 416 } else { 417 switch (access.type()) { 418 case T_BYTE: { 419 load_store = kit->gvn().transform(new CompareAndExchangeBNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo)); 420 break; 421 } 422 case T_SHORT: { 423 load_store = kit->gvn().transform(new CompareAndExchangeSNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo)); 424 break; 425 } 426 case T_INT: { 427 load_store = kit->gvn().transform(new CompareAndExchangeINode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo)); 428 break; 429 } 430 case T_LONG: { 431 load_store = kit->gvn().transform(new CompareAndExchangeLNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo)); 432 break; 433 } 434 default: 435 ShouldNotReachHere(); 436 } 437 } 438 439 access.set_raw_access(load_store); 440 pin_atomic_op(access); 441 442 #ifdef _LP64 443 if (access.is_oop() && adr->bottom_type()->is_ptr_to_narrowoop()) { 444 return kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type())); 445 } 446 #endif 447 448 return load_store; 449 } 450 451 Node* BarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val, 452 Node* new_val, const Type* value_type) const { 453 GraphKit* kit = access.kit(); 454 DecoratorSet decorators = access.decorators(); 455 MemNode::MemOrd mo = access.mem_node_mo(); 456 Node* mem = access.memory(); 457 bool is_weak_cas = (decorators & C2_WEAK_CMPXCHG) != 0; 458 Node* load_store = NULL; 459 Node* adr = access.addr().node(); 460 461 if (access.is_oop()) { 462 #ifdef _LP64 463 if (adr->bottom_type()->is_ptr_to_narrowoop()) { 464 Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop())); 465 Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop())); 466 if (is_weak_cas) { 467 load_store = kit->gvn().transform(new WeakCompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo)); 468 } else { 469 load_store = kit->gvn().transform(new CompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo)); 470 } 471 } else 472 #endif 473 { 474 if (is_weak_cas) { 475 load_store = kit->gvn().transform(new WeakCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo)); 476 } else { 477 load_store = kit->gvn().transform(new CompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo)); 478 } 479 } 480 } else { 481 switch(access.type()) { 482 case T_BYTE: { 483 if (is_weak_cas) { 484 load_store = kit->gvn().transform(new WeakCompareAndSwapBNode(kit->control(), mem, adr, new_val, expected_val, mo)); 485 } else { 486 load_store = kit->gvn().transform(new CompareAndSwapBNode(kit->control(), mem, adr, new_val, expected_val, mo)); 487 } 488 break; 489 } 490 case T_SHORT: { 491 if (is_weak_cas) { 492 load_store = kit->gvn().transform(new WeakCompareAndSwapSNode(kit->control(), mem, adr, new_val, expected_val, mo)); 493 } else { 494 load_store = kit->gvn().transform(new CompareAndSwapSNode(kit->control(), mem, adr, new_val, expected_val, mo)); 495 } 496 break; 497 } 498 case T_INT: { 499 if (is_weak_cas) { 500 load_store = kit->gvn().transform(new WeakCompareAndSwapINode(kit->control(), mem, adr, new_val, expected_val, mo)); 501 } else { 502 load_store = kit->gvn().transform(new CompareAndSwapINode(kit->control(), mem, adr, new_val, expected_val, mo)); 503 } 504 break; 505 } 506 case T_LONG: { 507 if (is_weak_cas) { 508 load_store = kit->gvn().transform(new WeakCompareAndSwapLNode(kit->control(), mem, adr, new_val, expected_val, mo)); 509 } else { 510 load_store = kit->gvn().transform(new CompareAndSwapLNode(kit->control(), mem, adr, new_val, expected_val, mo)); 511 } 512 break; 513 } 514 default: 515 ShouldNotReachHere(); 516 } 517 } 518 519 access.set_raw_access(load_store); 520 pin_atomic_op(access); 521 522 return load_store; 523 } 524 525 Node* BarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const { 526 GraphKit* kit = access.kit(); 527 Node* mem = access.memory(); 528 Node* adr = access.addr().node(); 529 const TypePtr* adr_type = access.addr().type(); 530 Node* load_store = NULL; 531 532 if (access.is_oop()) { 533 #ifdef _LP64 534 if (adr->bottom_type()->is_ptr_to_narrowoop()) { 535 Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop())); 536 load_store = kit->gvn().transform(new GetAndSetNNode(kit->control(), mem, adr, newval_enc, adr_type, value_type->make_narrowoop())); 537 } else 538 #endif 539 { 540 load_store = kit->gvn().transform(new GetAndSetPNode(kit->control(), mem, adr, new_val, adr_type, value_type->is_oopptr())); 541 } 542 } else { 543 switch (access.type()) { 544 case T_BYTE: 545 load_store = kit->gvn().transform(new GetAndSetBNode(kit->control(), mem, adr, new_val, adr_type)); 546 break; 547 case T_SHORT: 548 load_store = kit->gvn().transform(new GetAndSetSNode(kit->control(), mem, adr, new_val, adr_type)); 549 break; 550 case T_INT: 551 load_store = kit->gvn().transform(new GetAndSetINode(kit->control(), mem, adr, new_val, adr_type)); 552 break; 553 case T_LONG: 554 load_store = kit->gvn().transform(new GetAndSetLNode(kit->control(), mem, adr, new_val, adr_type)); 555 break; 556 default: 557 ShouldNotReachHere(); 558 } 559 } 560 561 access.set_raw_access(load_store); 562 pin_atomic_op(access); 563 564 #ifdef _LP64 565 if (access.is_oop() && adr->bottom_type()->is_ptr_to_narrowoop()) { 566 return kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type())); 567 } 568 #endif 569 570 return load_store; 571 } 572 573 Node* BarrierSetC2::atomic_add_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const { 574 Node* load_store = NULL; 575 GraphKit* kit = access.kit(); 576 Node* adr = access.addr().node(); 577 const TypePtr* adr_type = access.addr().type(); 578 Node* mem = access.memory(); 579 580 switch(access.type()) { 581 case T_BYTE: 582 load_store = kit->gvn().transform(new GetAndAddBNode(kit->control(), mem, adr, new_val, adr_type)); 583 break; 584 case T_SHORT: 585 load_store = kit->gvn().transform(new GetAndAddSNode(kit->control(), mem, adr, new_val, adr_type)); 586 break; 587 case T_INT: 588 load_store = kit->gvn().transform(new GetAndAddINode(kit->control(), mem, adr, new_val, adr_type)); 589 break; 590 case T_LONG: 591 load_store = kit->gvn().transform(new GetAndAddLNode(kit->control(), mem, adr, new_val, adr_type)); 592 break; 593 default: 594 ShouldNotReachHere(); 595 } 596 597 access.set_raw_access(load_store); 598 pin_atomic_op(access); 599 600 return load_store; 601 } 602 603 Node* BarrierSetC2::atomic_cmpxchg_val_at(C2AtomicParseAccess& access, Node* expected_val, 604 Node* new_val, const Type* value_type) const { 605 C2AccessFence fence(access); 606 resolve_address(access); 607 return atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type); 608 } 609 610 Node* BarrierSetC2::atomic_cmpxchg_bool_at(C2AtomicParseAccess& access, Node* expected_val, 611 Node* new_val, const Type* value_type) const { 612 C2AccessFence fence(access); 613 resolve_address(access); 614 return atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type); 615 } 616 617 Node* BarrierSetC2::atomic_xchg_at(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const { 618 C2AccessFence fence(access); 619 resolve_address(access); 620 return atomic_xchg_at_resolved(access, new_val, value_type); 621 } 622 623 Node* BarrierSetC2::atomic_add_at(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const { 624 C2AccessFence fence(access); 625 resolve_address(access); 626 return atomic_add_at_resolved(access, new_val, value_type); 627 } 628 629 void BarrierSetC2::clone(GraphKit* kit, Node* src_base, Node* dst_base, Node* countx, bool is_array) const { 630 #ifdef ASSERT 631 intptr_t src_offset; 632 Node* src = AddPNode::Ideal_base_and_offset(src_base, &kit->gvn(), src_offset); 633 intptr_t dst_offset; 634 Node* dst = AddPNode::Ideal_base_and_offset(dst_base, &kit->gvn(), dst_offset); 635 assert(src == NULL || (src_offset % BytesPerLong == 0), "expect 8 bytes alignment"); 636 assert(dst == NULL || (dst_offset % BytesPerLong == 0), "expect 8 bytes alignment"); 637 #endif 638 639 const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM; 640 641 ArrayCopyNode* ac = ArrayCopyNode::make(kit, false, src_base, NULL, dst_base, NULL, countx, true, false); 642 ac->set_clonebasic(); 643 Node* n = kit->gvn().transform(ac); 644 if (n == ac) { 645 ac->_adr_type = TypeRawPtr::BOTTOM; 646 kit->set_predefined_output_for_runtime_call(ac, ac->in(TypeFunc::Memory), raw_adr_type); 647 } else { 648 kit->set_all_memory(n); 649 } 650 } 651 652 Node* BarrierSetC2::obj_allocate(PhaseMacroExpand* macro, Node* ctrl, Node* mem, Node* toobig_false, Node* size_in_bytes, 653 Node*& i_o, Node*& needgc_ctrl, 654 Node*& fast_oop_ctrl, Node*& fast_oop_rawmem, 655 intx prefetch_lines) const { 656 657 Node* eden_top_adr; 658 Node* eden_end_adr; 659 660 macro->set_eden_pointers(eden_top_adr, eden_end_adr); 661 662 // Load Eden::end. Loop invariant and hoisted. 663 // 664 // Note: We set the control input on "eden_end" and "old_eden_top" when using 665 // a TLAB to work around a bug where these values were being moved across 666 // a safepoint. These are not oops, so they cannot be include in the oop 667 // map, but they can be changed by a GC. The proper way to fix this would 668 // be to set the raw memory state when generating a SafepointNode. However 669 // this will require extensive changes to the loop optimization in order to 670 // prevent a degradation of the optimization. 671 // See comment in memnode.hpp, around line 227 in class LoadPNode. 672 Node *eden_end = macro->make_load(ctrl, mem, eden_end_adr, 0, TypeRawPtr::BOTTOM, T_ADDRESS); 673 674 // We need a Region for the loop-back contended case. 675 enum { fall_in_path = 1, contended_loopback_path = 2 }; 676 Node *contended_region; 677 Node *contended_phi_rawmem; 678 if (UseTLAB) { 679 contended_region = toobig_false; 680 contended_phi_rawmem = mem; 681 } else { 682 contended_region = new RegionNode(3); 683 contended_phi_rawmem = new PhiNode(contended_region, Type::MEMORY, TypeRawPtr::BOTTOM); 684 // Now handle the passing-too-big test. We fall into the contended 685 // loop-back merge point. 686 contended_region ->init_req(fall_in_path, toobig_false); 687 contended_phi_rawmem->init_req(fall_in_path, mem); 688 macro->transform_later(contended_region); 689 macro->transform_later(contended_phi_rawmem); 690 } 691 692 // Load(-locked) the heap top. 693 // See note above concerning the control input when using a TLAB 694 Node *old_eden_top = UseTLAB 695 ? new LoadPNode (ctrl, contended_phi_rawmem, eden_top_adr, TypeRawPtr::BOTTOM, TypeRawPtr::BOTTOM, MemNode::unordered) 696 : new LoadPLockedNode(contended_region, contended_phi_rawmem, eden_top_adr, MemNode::acquire); 697 698 macro->transform_later(old_eden_top); 699 // Add to heap top to get a new heap top 700 Node *new_eden_top = new AddPNode(macro->top(), old_eden_top, size_in_bytes); 701 macro->transform_later(new_eden_top); 702 // Check for needing a GC; compare against heap end 703 Node *needgc_cmp = new CmpPNode(new_eden_top, eden_end); 704 macro->transform_later(needgc_cmp); 705 Node *needgc_bol = new BoolNode(needgc_cmp, BoolTest::ge); 706 macro->transform_later(needgc_bol); 707 IfNode *needgc_iff = new IfNode(contended_region, needgc_bol, PROB_UNLIKELY_MAG(4), COUNT_UNKNOWN); 708 macro->transform_later(needgc_iff); 709 710 // Plug the failing-heap-space-need-gc test into the slow-path region 711 Node *needgc_true = new IfTrueNode(needgc_iff); 712 macro->transform_later(needgc_true); 713 needgc_ctrl = needgc_true; 714 715 // No need for a GC. Setup for the Store-Conditional 716 Node *needgc_false = new IfFalseNode(needgc_iff); 717 macro->transform_later(needgc_false); 718 719 i_o = macro->prefetch_allocation(i_o, needgc_false, contended_phi_rawmem, 720 old_eden_top, new_eden_top, prefetch_lines); 721 722 Node* fast_oop = old_eden_top; 723 724 // Store (-conditional) the modified eden top back down. 725 // StorePConditional produces flags for a test PLUS a modified raw 726 // memory state. 727 if (UseTLAB) { 728 Node* store_eden_top = 729 new StorePNode(needgc_false, contended_phi_rawmem, eden_top_adr, 730 TypeRawPtr::BOTTOM, new_eden_top, MemNode::unordered); 731 macro->transform_later(store_eden_top); 732 fast_oop_ctrl = needgc_false; // No contention, so this is the fast path 733 fast_oop_rawmem = store_eden_top; 734 } else { 735 Node* store_eden_top = 736 new StorePConditionalNode(needgc_false, contended_phi_rawmem, eden_top_adr, 737 new_eden_top, fast_oop/*old_eden_top*/); 738 macro->transform_later(store_eden_top); 739 Node *contention_check = new BoolNode(store_eden_top, BoolTest::ne); 740 macro->transform_later(contention_check); 741 store_eden_top = new SCMemProjNode(store_eden_top); 742 macro->transform_later(store_eden_top); 743 744 // If not using TLABs, check to see if there was contention. 745 IfNode *contention_iff = new IfNode (needgc_false, contention_check, PROB_MIN, COUNT_UNKNOWN); 746 macro->transform_later(contention_iff); 747 Node *contention_true = new IfTrueNode(contention_iff); 748 macro->transform_later(contention_true); 749 // If contention, loopback and try again. 750 contended_region->init_req(contended_loopback_path, contention_true); 751 contended_phi_rawmem->init_req(contended_loopback_path, store_eden_top); 752 753 // Fast-path succeeded with no contention! 754 Node *contention_false = new IfFalseNode(contention_iff); 755 macro->transform_later(contention_false); 756 fast_oop_ctrl = contention_false; 757 758 // Bump total allocated bytes for this thread 759 Node* thread = new ThreadLocalNode(); 760 macro->transform_later(thread); 761 Node* alloc_bytes_adr = macro->basic_plus_adr(macro->top()/*not oop*/, thread, 762 in_bytes(JavaThread::allocated_bytes_offset())); 763 Node* alloc_bytes = macro->make_load(fast_oop_ctrl, store_eden_top, alloc_bytes_adr, 764 0, TypeLong::LONG, T_LONG); 765 #ifdef _LP64 766 Node* alloc_size = size_in_bytes; 767 #else 768 Node* alloc_size = new ConvI2LNode(size_in_bytes); 769 macro->transform_later(alloc_size); 770 #endif 771 Node* new_alloc_bytes = new AddLNode(alloc_bytes, alloc_size); 772 macro->transform_later(new_alloc_bytes); 773 fast_oop_rawmem = macro->make_store(fast_oop_ctrl, store_eden_top, alloc_bytes_adr, 774 0, new_alloc_bytes, T_LONG); 775 } 776 return fast_oop; 777 } 778 779 void BarrierSetC2::clone_barrier_at_expansion(ArrayCopyNode* ac, Node* call, PhaseIterGVN& igvn) const { 780 // no barrier 781 igvn.replace_node(ac, call); 782 }