1 /* 2 * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 #include "gc/shared/c2/barrierSetC2.hpp" 27 #include "opto/arraycopynode.hpp" 28 #include "opto/convertnode.hpp" 29 #include "opto/graphKit.hpp" 30 #include "opto/idealKit.hpp" 31 #include "opto/macro.hpp" 32 #include "opto/narrowptrnode.hpp" 33 #include "utilities/macros.hpp" 34 35 // By default this is a no-op. 36 void BarrierSetC2::resolve_address(C2Access& access) const { } 37 38 void* C2ParseAccess::barrier_set_state() const { 39 return _kit->barrier_set_state(); 40 } 41 42 PhaseGVN& C2ParseAccess::gvn() const { return _kit->gvn(); } 43 44 bool C2Access::needs_cpu_membar() const { 45 bool mismatched = (_decorators & C2_MISMATCHED) != 0; 46 bool is_unordered = (_decorators & MO_UNORDERED) != 0; 47 bool anonymous = (_decorators & C2_UNSAFE_ACCESS) != 0; 48 bool in_heap = (_decorators & IN_HEAP) != 0; 49 50 bool is_write = (_decorators & C2_WRITE_ACCESS) != 0; 51 bool is_read = (_decorators & C2_READ_ACCESS) != 0; 52 bool is_atomic = is_read && is_write; 53 54 if (is_atomic) { 55 // Atomics always need to be wrapped in CPU membars 56 return true; 57 } 58 59 if (anonymous) { 60 // We will need memory barriers unless we can determine a unique 61 // alias category for this reference. (Note: If for some reason 62 // the barriers get omitted and the unsafe reference begins to "pollute" 63 // the alias analysis of the rest of the graph, either Compile::can_alias 64 // or Compile::must_alias will throw a diagnostic assert.) 65 if (!in_heap || !is_unordered || (mismatched && !_addr.type()->isa_aryptr())) { 66 return true; 67 } 68 } 69 70 return false; 71 } 72 73 Node* BarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& val) const { 74 DecoratorSet decorators = access.decorators(); 75 76 bool mismatched = (decorators & C2_MISMATCHED) != 0; 77 bool unaligned = (decorators & C2_UNALIGNED) != 0; 78 bool unsafe = (decorators & C2_UNSAFE_ACCESS) != 0; 79 bool requires_atomic_access = (decorators & MO_UNORDERED) == 0; 80 81 bool in_native = (decorators & IN_NATIVE) != 0; 82 assert(!in_native, "not supported yet"); 83 84 MemNode::MemOrd mo = access.mem_node_mo(); 85 86 Node* store; 87 if (access.is_parse_access()) { 88 C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access); 89 90 GraphKit* kit = parse_access.kit(); 91 if (access.type() == T_DOUBLE) { 92 Node* new_val = kit->dstore_rounding(val.node()); 93 val.set_node(new_val); 94 } 95 96 store = kit->store_to_memory(kit->control(), access.addr().node(), val.node(), access.type(), 97 access.addr().type(), mo, requires_atomic_access, unaligned, mismatched, unsafe); 98 access.set_raw_access(store); 99 } else { 100 assert(!requires_atomic_access, "not yet supported"); 101 assert(access.is_opt_access(), "either parse or opt access"); 102 C2OptAccess& opt_access = static_cast<C2OptAccess&>(access); 103 Node* ctl = opt_access.ctl(); 104 MergeMemNode* mm = opt_access.mem(); 105 PhaseGVN& gvn = opt_access.gvn(); 106 const TypePtr* adr_type = access.addr().type(); 107 int alias = gvn.C->get_alias_index(adr_type); 108 Node* mem = mm->memory_at(alias); 109 110 StoreNode* st = StoreNode::make(gvn, ctl, mem, access.addr().node(), adr_type, val.node(), access.type(), mo); 111 if (unaligned) { 112 st->set_unaligned_access(); 113 } 114 if (mismatched) { 115 st->set_mismatched_access(); 116 } 117 store = gvn.transform(st); 118 if (store == st) { 119 mm->set_memory_at(alias, st); 120 } 121 } 122 return store; 123 } 124 125 Node* BarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const { 126 DecoratorSet decorators = access.decorators(); 127 128 Node* adr = access.addr().node(); 129 const TypePtr* adr_type = access.addr().type(); 130 131 bool mismatched = (decorators & C2_MISMATCHED) != 0; 132 bool requires_atomic_access = (decorators & MO_UNORDERED) == 0; 133 bool unaligned = (decorators & C2_UNALIGNED) != 0; 134 bool control_dependent = (decorators & C2_CONTROL_DEPENDENT_LOAD) != 0; 135 bool unknown_control = (decorators & C2_UNKNOWN_CONTROL_LOAD) != 0; 136 bool unsafe = (decorators & C2_UNSAFE_ACCESS) != 0; 137 138 bool in_native = (decorators & IN_NATIVE) != 0; 139 140 MemNode::MemOrd mo = access.mem_node_mo(); 141 LoadNode::ControlDependency dep = unknown_control ? LoadNode::UnknownControl : LoadNode::DependsOnlyOnTest; 142 143 Node* load; 144 if (access.is_parse_access()) { 145 C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access); 146 GraphKit* kit = parse_access.kit(); 147 Node* control = control_dependent ? kit->control() : NULL; 148 149 if (in_native) { 150 load = kit->make_load(control, adr, val_type, access.type(), mo); 151 } else { 152 load = kit->make_load(control, adr, val_type, access.type(), adr_type, mo, 153 dep, requires_atomic_access, unaligned, mismatched, unsafe); 154 } 155 access.set_raw_access(load); 156 } else { 157 assert(!requires_atomic_access, "not yet supported"); 158 assert(access.is_opt_access(), "either parse or opt access"); 159 C2OptAccess& opt_access = static_cast<C2OptAccess&>(access); 160 Node* control = control_dependent ? opt_access.ctl() : NULL; 161 MergeMemNode* mm = opt_access.mem(); 162 PhaseGVN& gvn = opt_access.gvn(); 163 Node* mem = mm->memory_at(gvn.C->get_alias_index(adr_type)); 164 load = LoadNode::make(gvn, control, mem, adr, adr_type, val_type, access.type(), mo, dep, unaligned, mismatched); 165 load = gvn.transform(load); 166 } 167 168 return load; 169 } 170 171 class C2AccessFence: public StackObj { 172 C2Access& _access; 173 Node* _leading_membar; 174 175 public: 176 C2AccessFence(C2Access& access) : 177 _access(access), _leading_membar(NULL) { 178 GraphKit* kit = NULL; 179 if (access.is_parse_access()) { 180 C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access); 181 kit = parse_access.kit(); 182 } 183 DecoratorSet decorators = access.decorators(); 184 185 bool is_write = (decorators & C2_WRITE_ACCESS) != 0; 186 bool is_read = (decorators & C2_READ_ACCESS) != 0; 187 bool is_atomic = is_read && is_write; 188 189 bool is_volatile = (decorators & MO_SEQ_CST) != 0; 190 bool is_release = (decorators & MO_RELEASE) != 0; 191 192 if (is_atomic) { 193 assert(kit != NULL, "unsupported at optimization time"); 194 // Memory-model-wise, a LoadStore acts like a little synchronized 195 // block, so needs barriers on each side. These don't translate 196 // into actual barriers on most machines, but we still need rest of 197 // compiler to respect ordering. 198 if (is_release) { 199 _leading_membar = kit->insert_mem_bar(Op_MemBarRelease); 200 } else if (is_volatile) { 201 if (SUPPORT_IRIW_FOR_NOT_MULTI_COPY_ATOMIC_CPU) { 202 _leading_membar = kit->insert_mem_bar(Op_MemBarVolatile); 203 } else { 204 _leading_membar = kit->insert_mem_bar(Op_MemBarRelease); 205 } 206 } 207 } else if (is_write) { 208 // If reference is volatile, prevent following memory ops from 209 // floating down past the volatile write. Also prevents commoning 210 // another volatile read. 211 if (is_volatile || is_release) { 212 assert(kit != NULL, "unsupported at optimization time"); 213 _leading_membar = kit->insert_mem_bar(Op_MemBarRelease); 214 } 215 } else { 216 // Memory barrier to prevent normal and 'unsafe' accesses from 217 // bypassing each other. Happens after null checks, so the 218 // exception paths do not take memory state from the memory barrier, 219 // so there's no problems making a strong assert about mixing users 220 // of safe & unsafe memory. 221 if (is_volatile && SUPPORT_IRIW_FOR_NOT_MULTI_COPY_ATOMIC_CPU) { 222 assert(kit != NULL, "unsupported at optimization time"); 223 _leading_membar = kit->insert_mem_bar(Op_MemBarVolatile); 224 } 225 } 226 227 if (access.needs_cpu_membar()) { 228 assert(kit != NULL, "unsupported at optimization time"); 229 kit->insert_mem_bar(Op_MemBarCPUOrder); 230 } 231 232 if (is_atomic) { 233 // 4984716: MemBars must be inserted before this 234 // memory node in order to avoid a false 235 // dependency which will confuse the scheduler. 236 access.set_memory(); 237 } 238 } 239 240 ~C2AccessFence() { 241 GraphKit* kit = NULL; 242 if (_access.is_parse_access()) { 243 C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(_access); 244 kit = parse_access.kit(); 245 } 246 DecoratorSet decorators = _access.decorators(); 247 248 bool is_write = (decorators & C2_WRITE_ACCESS) != 0; 249 bool is_read = (decorators & C2_READ_ACCESS) != 0; 250 bool is_atomic = is_read && is_write; 251 252 bool is_volatile = (decorators & MO_SEQ_CST) != 0; 253 bool is_acquire = (decorators & MO_ACQUIRE) != 0; 254 255 // If reference is volatile, prevent following volatiles ops from 256 // floating up before the volatile access. 257 if (_access.needs_cpu_membar()) { 258 kit->insert_mem_bar(Op_MemBarCPUOrder); 259 } 260 261 if (is_atomic) { 262 assert(kit != NULL, "unsupported at optimization time"); 263 if (is_acquire || is_volatile) { 264 Node* n = _access.raw_access(); 265 Node* mb = kit->insert_mem_bar(Op_MemBarAcquire, n); 266 if (_leading_membar != NULL) { 267 MemBarNode::set_load_store_pair(_leading_membar->as_MemBar(), mb->as_MemBar()); 268 } 269 } 270 } else if (is_write) { 271 // If not multiple copy atomic, we do the MemBarVolatile before the load. 272 if (is_volatile && !SUPPORT_IRIW_FOR_NOT_MULTI_COPY_ATOMIC_CPU) { 273 assert(kit != NULL, "unsupported at optimization time"); 274 Node* n = _access.raw_access(); 275 Node* mb = kit->insert_mem_bar(Op_MemBarVolatile, n); // Use fat membar 276 if (_leading_membar != NULL) { 277 MemBarNode::set_store_pair(_leading_membar->as_MemBar(), mb->as_MemBar()); 278 } 279 } 280 } else { 281 if (is_volatile || is_acquire) { 282 assert(kit != NULL, "unsupported at optimization time"); 283 Node* n = _access.raw_access(); 284 assert(_leading_membar == NULL || SUPPORT_IRIW_FOR_NOT_MULTI_COPY_ATOMIC_CPU, "no leading membar expected"); 285 Node* mb = kit->insert_mem_bar(Op_MemBarAcquire, n); 286 mb->as_MemBar()->set_trailing_load(); 287 } 288 } 289 } 290 }; 291 292 Node* BarrierSetC2::store_at(C2Access& access, C2AccessValue& val) const { 293 C2AccessFence fence(access); 294 resolve_address(access); 295 return store_at_resolved(access, val); 296 } 297 298 Node* BarrierSetC2::load_at(C2Access& access, const Type* val_type) const { 299 C2AccessFence fence(access); 300 resolve_address(access); 301 return load_at_resolved(access, val_type); 302 } 303 304 MemNode::MemOrd C2Access::mem_node_mo() const { 305 bool is_write = (_decorators & C2_WRITE_ACCESS) != 0; 306 bool is_read = (_decorators & C2_READ_ACCESS) != 0; 307 if ((_decorators & MO_SEQ_CST) != 0) { 308 if (is_write && is_read) { 309 // For atomic operations 310 return MemNode::seqcst; 311 } else if (is_write) { 312 return MemNode::release; 313 } else { 314 assert(is_read, "what else?"); 315 return MemNode::acquire; 316 } 317 } else if ((_decorators & MO_RELEASE) != 0) { 318 return MemNode::release; 319 } else if ((_decorators & MO_ACQUIRE) != 0) { 320 return MemNode::acquire; 321 } else if (is_write) { 322 // Volatile fields need releasing stores. 323 // Non-volatile fields also need releasing stores if they hold an 324 // object reference, because the object reference might point to 325 // a freshly created object. 326 // Conservatively release stores of object references. 327 return StoreNode::release_if_reference(_type); 328 } else { 329 return MemNode::unordered; 330 } 331 } 332 333 void C2Access::fixup_decorators() { 334 bool default_mo = (_decorators & MO_DECORATOR_MASK) == 0; 335 bool is_unordered = (_decorators & MO_UNORDERED) != 0 || default_mo; 336 bool anonymous = (_decorators & C2_UNSAFE_ACCESS) != 0; 337 338 bool is_read = (_decorators & C2_READ_ACCESS) != 0; 339 bool is_write = (_decorators & C2_WRITE_ACCESS) != 0; 340 341 if (AlwaysAtomicAccesses && is_unordered) { 342 _decorators &= ~MO_DECORATOR_MASK; // clear the MO bits 343 _decorators |= MO_RELAXED; // Force the MO_RELAXED decorator with AlwaysAtomicAccess 344 } 345 346 _decorators = AccessInternal::decorator_fixup(_decorators); 347 348 if (is_read && !is_write && anonymous) { 349 // To be valid, unsafe loads may depend on other conditions than 350 // the one that guards them: pin the Load node 351 _decorators |= C2_CONTROL_DEPENDENT_LOAD; 352 _decorators |= C2_UNKNOWN_CONTROL_LOAD; 353 const TypePtr* adr_type = _addr.type(); 354 Node* adr = _addr.node(); 355 if (!needs_cpu_membar() && adr_type->isa_instptr()) { 356 assert(adr_type->meet(TypePtr::NULL_PTR) != adr_type->remove_speculative(), "should be not null"); 357 intptr_t offset = Type::OffsetBot; 358 AddPNode::Ideal_base_and_offset(adr, &gvn(), offset); 359 if (offset >= 0) { 360 int s = Klass::layout_helper_size_in_bytes(adr_type->isa_instptr()->klass()->layout_helper()); 361 if (offset < s) { 362 // Guaranteed to be a valid access, no need to pin it 363 _decorators ^= C2_CONTROL_DEPENDENT_LOAD; 364 _decorators ^= C2_UNKNOWN_CONTROL_LOAD; 365 } 366 } 367 } 368 } 369 } 370 371 //--------------------------- atomic operations--------------------------------- 372 373 void BarrierSetC2::pin_atomic_op(C2AtomicParseAccess& access) const { 374 if (!access.needs_pinning()) { 375 return; 376 } 377 // SCMemProjNodes represent the memory state of a LoadStore. Their 378 // main role is to prevent LoadStore nodes from being optimized away 379 // when their results aren't used. 380 assert(access.is_parse_access(), "entry not supported at optimization time"); 381 C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access); 382 GraphKit* kit = parse_access.kit(); 383 Node* load_store = access.raw_access(); 384 assert(load_store != NULL, "must pin atomic op"); 385 Node* proj = kit->gvn().transform(new SCMemProjNode(load_store)); 386 kit->set_memory(proj, access.alias_idx()); 387 } 388 389 void C2AtomicParseAccess::set_memory() { 390 Node *mem = _kit->memory(_alias_idx); 391 _memory = mem; 392 } 393 394 Node* BarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val, 395 Node* new_val, const Type* value_type) const { 396 GraphKit* kit = access.kit(); 397 MemNode::MemOrd mo = access.mem_node_mo(); 398 Node* mem = access.memory(); 399 400 Node* adr = access.addr().node(); 401 const TypePtr* adr_type = access.addr().type(); 402 403 Node* load_store = NULL; 404 405 if (access.is_oop()) { 406 #ifdef _LP64 407 if (adr->bottom_type()->is_ptr_to_narrowoop()) { 408 Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop())); 409 Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop())); 410 load_store = kit->gvn().transform(new CompareAndExchangeNNode(kit->control(), mem, adr, newval_enc, oldval_enc, adr_type, value_type->make_narrowoop(), mo)); 411 } else 412 #endif 413 { 414 load_store = kit->gvn().transform(new CompareAndExchangePNode(kit->control(), mem, adr, new_val, expected_val, adr_type, value_type->is_oopptr(), mo)); 415 } 416 } else { 417 switch (access.type()) { 418 case T_BYTE: { 419 load_store = kit->gvn().transform(new CompareAndExchangeBNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo)); 420 break; 421 } 422 case T_SHORT: { 423 load_store = kit->gvn().transform(new CompareAndExchangeSNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo)); 424 break; 425 } 426 case T_INT: { 427 load_store = kit->gvn().transform(new CompareAndExchangeINode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo)); 428 break; 429 } 430 case T_LONG: { 431 load_store = kit->gvn().transform(new CompareAndExchangeLNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo)); 432 break; 433 } 434 default: 435 ShouldNotReachHere(); 436 } 437 } 438 439 access.set_raw_access(load_store); 440 pin_atomic_op(access); 441 442 #ifdef _LP64 443 if (access.is_oop() && adr->bottom_type()->is_ptr_to_narrowoop()) { 444 return kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type())); 445 } 446 #endif 447 448 return load_store; 449 } 450 451 Node* BarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val, 452 Node* new_val, const Type* value_type) const { 453 GraphKit* kit = access.kit(); 454 DecoratorSet decorators = access.decorators(); 455 MemNode::MemOrd mo = access.mem_node_mo(); 456 Node* mem = access.memory(); 457 bool is_weak_cas = (decorators & C2_WEAK_CMPXCHG) != 0; 458 Node* load_store = NULL; 459 Node* adr = access.addr().node(); 460 461 if (access.is_oop()) { 462 #ifdef _LP64 463 if (adr->bottom_type()->is_ptr_to_narrowoop()) { 464 Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop())); 465 Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop())); 466 if (is_weak_cas) { 467 load_store = kit->gvn().transform(new WeakCompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo)); 468 } else { 469 load_store = kit->gvn().transform(new CompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo)); 470 } 471 } else 472 #endif 473 { 474 if (is_weak_cas) { 475 load_store = kit->gvn().transform(new WeakCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo)); 476 } else { 477 load_store = kit->gvn().transform(new CompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo)); 478 } 479 } 480 } else { 481 switch(access.type()) { 482 case T_BYTE: { 483 if (is_weak_cas) { 484 load_store = kit->gvn().transform(new WeakCompareAndSwapBNode(kit->control(), mem, adr, new_val, expected_val, mo)); 485 } else { 486 load_store = kit->gvn().transform(new CompareAndSwapBNode(kit->control(), mem, adr, new_val, expected_val, mo)); 487 } 488 break; 489 } 490 case T_SHORT: { 491 if (is_weak_cas) { 492 load_store = kit->gvn().transform(new WeakCompareAndSwapSNode(kit->control(), mem, adr, new_val, expected_val, mo)); 493 } else { 494 load_store = kit->gvn().transform(new CompareAndSwapSNode(kit->control(), mem, adr, new_val, expected_val, mo)); 495 } 496 break; 497 } 498 case T_INT: { 499 if (is_weak_cas) { 500 load_store = kit->gvn().transform(new WeakCompareAndSwapINode(kit->control(), mem, adr, new_val, expected_val, mo)); 501 } else { 502 load_store = kit->gvn().transform(new CompareAndSwapINode(kit->control(), mem, adr, new_val, expected_val, mo)); 503 } 504 break; 505 } 506 case T_LONG: { 507 if (is_weak_cas) { 508 load_store = kit->gvn().transform(new WeakCompareAndSwapLNode(kit->control(), mem, adr, new_val, expected_val, mo)); 509 } else { 510 load_store = kit->gvn().transform(new CompareAndSwapLNode(kit->control(), mem, adr, new_val, expected_val, mo)); 511 } 512 break; 513 } 514 default: 515 ShouldNotReachHere(); 516 } 517 } 518 519 access.set_raw_access(load_store); 520 pin_atomic_op(access); 521 522 return load_store; 523 } 524 525 Node* BarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const { 526 GraphKit* kit = access.kit(); 527 Node* mem = access.memory(); 528 Node* adr = access.addr().node(); 529 const TypePtr* adr_type = access.addr().type(); 530 Node* load_store = NULL; 531 532 if (access.is_oop()) { 533 #ifdef _LP64 534 if (adr->bottom_type()->is_ptr_to_narrowoop()) { 535 Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop())); 536 load_store = kit->gvn().transform(new GetAndSetNNode(kit->control(), mem, adr, newval_enc, adr_type, value_type->make_narrowoop())); 537 } else 538 #endif 539 { 540 load_store = kit->gvn().transform(new GetAndSetPNode(kit->control(), mem, adr, new_val, adr_type, value_type->is_oopptr())); 541 } 542 } else { 543 switch (access.type()) { 544 case T_BYTE: 545 load_store = kit->gvn().transform(new GetAndSetBNode(kit->control(), mem, adr, new_val, adr_type)); 546 break; 547 case T_SHORT: 548 load_store = kit->gvn().transform(new GetAndSetSNode(kit->control(), mem, adr, new_val, adr_type)); 549 break; 550 case T_INT: 551 load_store = kit->gvn().transform(new GetAndSetINode(kit->control(), mem, adr, new_val, adr_type)); 552 break; 553 case T_LONG: 554 load_store = kit->gvn().transform(new GetAndSetLNode(kit->control(), mem, adr, new_val, adr_type)); 555 break; 556 default: 557 ShouldNotReachHere(); 558 } 559 } 560 561 access.set_raw_access(load_store); 562 pin_atomic_op(access); 563 564 #ifdef _LP64 565 if (access.is_oop() && adr->bottom_type()->is_ptr_to_narrowoop()) { 566 return kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type())); 567 } 568 #endif 569 570 return load_store; 571 } 572 573 Node* BarrierSetC2::atomic_add_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const { 574 Node* load_store = NULL; 575 GraphKit* kit = access.kit(); 576 Node* adr = access.addr().node(); 577 const TypePtr* adr_type = access.addr().type(); 578 Node* mem = access.memory(); 579 580 switch(access.type()) { 581 case T_BYTE: 582 load_store = kit->gvn().transform(new GetAndAddBNode(kit->control(), mem, adr, new_val, adr_type)); 583 break; 584 case T_SHORT: 585 load_store = kit->gvn().transform(new GetAndAddSNode(kit->control(), mem, adr, new_val, adr_type)); 586 break; 587 case T_INT: 588 load_store = kit->gvn().transform(new GetAndAddINode(kit->control(), mem, adr, new_val, adr_type)); 589 break; 590 case T_LONG: 591 load_store = kit->gvn().transform(new GetAndAddLNode(kit->control(), mem, adr, new_val, adr_type)); 592 break; 593 default: 594 ShouldNotReachHere(); 595 } 596 597 access.set_raw_access(load_store); 598 pin_atomic_op(access); 599 600 return load_store; 601 } 602 603 Node* BarrierSetC2::atomic_cmpxchg_val_at(C2AtomicParseAccess& access, Node* expected_val, 604 Node* new_val, const Type* value_type) const { 605 C2AccessFence fence(access); 606 resolve_address(access); 607 return atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type); 608 } 609 610 Node* BarrierSetC2::atomic_cmpxchg_bool_at(C2AtomicParseAccess& access, Node* expected_val, 611 Node* new_val, const Type* value_type) const { 612 C2AccessFence fence(access); 613 resolve_address(access); 614 return atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type); 615 } 616 617 Node* BarrierSetC2::atomic_xchg_at(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const { 618 C2AccessFence fence(access); 619 resolve_address(access); 620 return atomic_xchg_at_resolved(access, new_val, value_type); 621 } 622 623 Node* BarrierSetC2::atomic_add_at(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const { 624 C2AccessFence fence(access); 625 resolve_address(access); 626 return atomic_add_at_resolved(access, new_val, value_type); 627 } 628 629 void BarrierSetC2::clone(GraphKit* kit, Node* src, Node* dst, Node* size, bool is_array) const { 630 // Exclude the header but include array length to copy by 8 bytes words. 631 // Can't use base_offset_in_bytes(bt) since basic type is unknown. 632 int base_off = is_array ? arrayOopDesc::length_offset_in_bytes() : 633 instanceOopDesc::base_offset_in_bytes(); 634 // base_off: 635 // 8 - 32-bit VM 636 // 12 - 64-bit VM, compressed klass 637 // 16 - 64-bit VM, normal klass 638 if (base_off % BytesPerLong != 0) { 639 assert(UseCompressedClassPointers, ""); 640 if (is_array) { 641 // Exclude length to copy by 8 bytes words. 642 base_off += sizeof(int); 643 } else { 644 // Include klass to copy by 8 bytes words. 645 base_off = instanceOopDesc::klass_offset_in_bytes(); 646 } 647 assert(base_off % BytesPerLong == 0, "expect 8 bytes alignment"); 648 } 649 Node* src_base = kit->basic_plus_adr(src, base_off); 650 Node* dst_base = kit->basic_plus_adr(dst, base_off); 651 652 // Compute the length also, if needed: 653 Node* countx = size; 654 countx = kit->gvn().transform(new SubXNode(countx, kit->MakeConX(base_off))); 655 countx = kit->gvn().transform(new URShiftXNode(countx, kit->intcon(LogBytesPerLong) )); 656 657 const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM; 658 659 ArrayCopyNode* ac = ArrayCopyNode::make(kit, false, src_base, NULL, dst_base, NULL, countx, true, false); 660 ac->set_clonebasic(); 661 Node* n = kit->gvn().transform(ac); 662 if (n == ac) { 663 ac->_adr_type = TypeRawPtr::BOTTOM; 664 kit->set_predefined_output_for_runtime_call(ac, ac->in(TypeFunc::Memory), raw_adr_type); 665 } else { 666 kit->set_all_memory(n); 667 } 668 } 669 670 Node* BarrierSetC2::obj_allocate(PhaseMacroExpand* macro, Node* ctrl, Node* mem, Node* toobig_false, Node* size_in_bytes, 671 Node*& i_o, Node*& needgc_ctrl, 672 Node*& fast_oop_ctrl, Node*& fast_oop_rawmem, 673 intx prefetch_lines) const { 674 675 Node* eden_top_adr; 676 Node* eden_end_adr; 677 678 macro->set_eden_pointers(eden_top_adr, eden_end_adr); 679 680 // Load Eden::end. Loop invariant and hoisted. 681 // 682 // Note: We set the control input on "eden_end" and "old_eden_top" when using 683 // a TLAB to work around a bug where these values were being moved across 684 // a safepoint. These are not oops, so they cannot be include in the oop 685 // map, but they can be changed by a GC. The proper way to fix this would 686 // be to set the raw memory state when generating a SafepointNode. However 687 // this will require extensive changes to the loop optimization in order to 688 // prevent a degradation of the optimization. 689 // See comment in memnode.hpp, around line 227 in class LoadPNode. 690 Node *eden_end = macro->make_load(ctrl, mem, eden_end_adr, 0, TypeRawPtr::BOTTOM, T_ADDRESS); 691 692 // We need a Region for the loop-back contended case. 693 enum { fall_in_path = 1, contended_loopback_path = 2 }; 694 Node *contended_region; 695 Node *contended_phi_rawmem; 696 if (UseTLAB) { 697 contended_region = toobig_false; 698 contended_phi_rawmem = mem; 699 } else { 700 contended_region = new RegionNode(3); 701 contended_phi_rawmem = new PhiNode(contended_region, Type::MEMORY, TypeRawPtr::BOTTOM); 702 // Now handle the passing-too-big test. We fall into the contended 703 // loop-back merge point. 704 contended_region ->init_req(fall_in_path, toobig_false); 705 contended_phi_rawmem->init_req(fall_in_path, mem); 706 macro->transform_later(contended_region); 707 macro->transform_later(contended_phi_rawmem); 708 } 709 710 // Load(-locked) the heap top. 711 // See note above concerning the control input when using a TLAB 712 Node *old_eden_top = UseTLAB 713 ? new LoadPNode (ctrl, contended_phi_rawmem, eden_top_adr, TypeRawPtr::BOTTOM, TypeRawPtr::BOTTOM, MemNode::unordered) 714 : new LoadPLockedNode(contended_region, contended_phi_rawmem, eden_top_adr, MemNode::acquire); 715 716 macro->transform_later(old_eden_top); 717 // Add to heap top to get a new heap top 718 Node *new_eden_top = new AddPNode(macro->top(), old_eden_top, size_in_bytes); 719 macro->transform_later(new_eden_top); 720 // Check for needing a GC; compare against heap end 721 Node *needgc_cmp = new CmpPNode(new_eden_top, eden_end); 722 macro->transform_later(needgc_cmp); 723 Node *needgc_bol = new BoolNode(needgc_cmp, BoolTest::ge); 724 macro->transform_later(needgc_bol); 725 IfNode *needgc_iff = new IfNode(contended_region, needgc_bol, PROB_UNLIKELY_MAG(4), COUNT_UNKNOWN); 726 macro->transform_later(needgc_iff); 727 728 // Plug the failing-heap-space-need-gc test into the slow-path region 729 Node *needgc_true = new IfTrueNode(needgc_iff); 730 macro->transform_later(needgc_true); 731 needgc_ctrl = needgc_true; 732 733 // No need for a GC. Setup for the Store-Conditional 734 Node *needgc_false = new IfFalseNode(needgc_iff); 735 macro->transform_later(needgc_false); 736 737 i_o = macro->prefetch_allocation(i_o, needgc_false, contended_phi_rawmem, 738 old_eden_top, new_eden_top, prefetch_lines); 739 740 Node* fast_oop = old_eden_top; 741 742 // Store (-conditional) the modified eden top back down. 743 // StorePConditional produces flags for a test PLUS a modified raw 744 // memory state. 745 if (UseTLAB) { 746 Node* store_eden_top = 747 new StorePNode(needgc_false, contended_phi_rawmem, eden_top_adr, 748 TypeRawPtr::BOTTOM, new_eden_top, MemNode::unordered); 749 macro->transform_later(store_eden_top); 750 fast_oop_ctrl = needgc_false; // No contention, so this is the fast path 751 fast_oop_rawmem = store_eden_top; 752 } else { 753 Node* store_eden_top = 754 new StorePConditionalNode(needgc_false, contended_phi_rawmem, eden_top_adr, 755 new_eden_top, fast_oop/*old_eden_top*/); 756 macro->transform_later(store_eden_top); 757 Node *contention_check = new BoolNode(store_eden_top, BoolTest::ne); 758 macro->transform_later(contention_check); 759 store_eden_top = new SCMemProjNode(store_eden_top); 760 macro->transform_later(store_eden_top); 761 762 // If not using TLABs, check to see if there was contention. 763 IfNode *contention_iff = new IfNode (needgc_false, contention_check, PROB_MIN, COUNT_UNKNOWN); 764 macro->transform_later(contention_iff); 765 Node *contention_true = new IfTrueNode(contention_iff); 766 macro->transform_later(contention_true); 767 // If contention, loopback and try again. 768 contended_region->init_req(contended_loopback_path, contention_true); 769 contended_phi_rawmem->init_req(contended_loopback_path, store_eden_top); 770 771 // Fast-path succeeded with no contention! 772 Node *contention_false = new IfFalseNode(contention_iff); 773 macro->transform_later(contention_false); 774 fast_oop_ctrl = contention_false; 775 776 // Bump total allocated bytes for this thread 777 Node* thread = new ThreadLocalNode(); 778 macro->transform_later(thread); 779 Node* alloc_bytes_adr = macro->basic_plus_adr(macro->top()/*not oop*/, thread, 780 in_bytes(JavaThread::allocated_bytes_offset())); 781 Node* alloc_bytes = macro->make_load(fast_oop_ctrl, store_eden_top, alloc_bytes_adr, 782 0, TypeLong::LONG, T_LONG); 783 #ifdef _LP64 784 Node* alloc_size = size_in_bytes; 785 #else 786 Node* alloc_size = new ConvI2LNode(size_in_bytes); 787 macro->transform_later(alloc_size); 788 #endif 789 Node* new_alloc_bytes = new AddLNode(alloc_bytes, alloc_size); 790 macro->transform_later(new_alloc_bytes); 791 fast_oop_rawmem = macro->make_store(fast_oop_ctrl, store_eden_top, alloc_bytes_adr, 792 0, new_alloc_bytes, T_LONG); 793 } 794 return fast_oop; 795 } 796 797 void BarrierSetC2::clone_barrier_at_expansion(ArrayCopyNode* ac, Node* call, PhaseIterGVN& igvn) const { 798 // no barrier 799 igvn.replace_node(ac, call); 800 }