1 /* 2 * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 #include "gc/shared/c2/barrierSetC2.hpp" 27 #include "opto/arraycopynode.hpp" 28 #include "opto/convertnode.hpp" 29 #include "opto/graphKit.hpp" 30 #include "opto/idealKit.hpp" 31 #include "opto/macro.hpp" 32 #include "opto/narrowptrnode.hpp" 33 #include "opto/runtime.hpp" 34 #include "utilities/macros.hpp" 35 36 // By default this is a no-op. 37 void BarrierSetC2::resolve_address(C2Access& access) const { } 38 39 void* C2ParseAccess::barrier_set_state() const { 40 return _kit->barrier_set_state(); 41 } 42 43 PhaseGVN& C2ParseAccess::gvn() const { return _kit->gvn(); } 44 45 bool C2Access::needs_cpu_membar() const { 46 bool mismatched = (_decorators & C2_MISMATCHED) != 0; 47 bool is_unordered = (_decorators & MO_UNORDERED) != 0; 48 bool anonymous = (_decorators & C2_UNSAFE_ACCESS) != 0; 49 bool in_heap = (_decorators & IN_HEAP) != 0; 50 51 bool is_write = (_decorators & C2_WRITE_ACCESS) != 0; 52 bool is_read = (_decorators & C2_READ_ACCESS) != 0; 53 bool is_atomic = is_read && is_write; 54 55 if (is_atomic) { 56 // Atomics always need to be wrapped in CPU membars 57 return true; 58 } 59 60 if (anonymous) { 61 // We will need memory barriers unless we can determine a unique 62 // alias category for this reference. (Note: If for some reason 63 // the barriers get omitted and the unsafe reference begins to "pollute" 64 // the alias analysis of the rest of the graph, either Compile::can_alias 65 // or Compile::must_alias will throw a diagnostic assert.) 66 if (!in_heap || !is_unordered || (mismatched && !_addr.type()->isa_aryptr())) { 67 return true; 68 } 69 } 70 71 return false; 72 } 73 74 Node* BarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& val) const { 75 DecoratorSet decorators = access.decorators(); 76 77 bool mismatched = (decorators & C2_MISMATCHED) != 0; 78 bool unaligned = (decorators & C2_UNALIGNED) != 0; 79 bool unsafe = (decorators & C2_UNSAFE_ACCESS) != 0; 80 bool requires_atomic_access = (decorators & MO_UNORDERED) == 0; 81 82 bool in_native = (decorators & IN_NATIVE) != 0; 83 assert(!in_native, "not supported yet"); 84 85 MemNode::MemOrd mo = access.mem_node_mo(); 86 87 Node* store; 88 if (access.is_parse_access()) { 89 C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access); 90 91 GraphKit* kit = parse_access.kit(); 92 if (access.type() == T_DOUBLE) { 93 Node* new_val = kit->dstore_rounding(val.node()); 94 val.set_node(new_val); 95 } 96 97 store = kit->store_to_memory(kit->control(), access.addr().node(), val.node(), access.type(), 98 access.addr().type(), mo, requires_atomic_access, unaligned, mismatched, unsafe); 99 access.set_raw_access(store); 100 } else { 101 assert(!requires_atomic_access, "not yet supported"); 102 assert(access.is_opt_access(), "either parse or opt access"); 103 C2OptAccess& opt_access = static_cast<C2OptAccess&>(access); 104 Node* ctl = opt_access.ctl(); 105 MergeMemNode* mm = opt_access.mem(); 106 PhaseGVN& gvn = opt_access.gvn(); 107 const TypePtr* adr_type = access.addr().type(); 108 int alias = gvn.C->get_alias_index(adr_type); 109 Node* mem = mm->memory_at(alias); 110 111 StoreNode* st = StoreNode::make(gvn, ctl, mem, access.addr().node(), adr_type, val.node(), access.type(), mo); 112 if (unaligned) { 113 st->set_unaligned_access(); 114 } 115 if (mismatched) { 116 st->set_mismatched_access(); 117 } 118 store = gvn.transform(st); 119 if (store == st) { 120 mm->set_memory_at(alias, st); 121 } 122 } 123 return store; 124 } 125 126 Node* BarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const { 127 DecoratorSet decorators = access.decorators(); 128 129 Node* adr = access.addr().node(); 130 const TypePtr* adr_type = access.addr().type(); 131 132 bool mismatched = (decorators & C2_MISMATCHED) != 0; 133 bool requires_atomic_access = (decorators & MO_UNORDERED) == 0; 134 bool unaligned = (decorators & C2_UNALIGNED) != 0; 135 bool control_dependent = (decorators & C2_CONTROL_DEPENDENT_LOAD) != 0; 136 bool unknown_control = (decorators & C2_UNKNOWN_CONTROL_LOAD) != 0; 137 bool unsafe = (decorators & C2_UNSAFE_ACCESS) != 0; 138 139 bool in_native = (decorators & IN_NATIVE) != 0; 140 141 MemNode::MemOrd mo = access.mem_node_mo(); 142 LoadNode::ControlDependency dep = unknown_control ? LoadNode::UnknownControl : LoadNode::DependsOnlyOnTest; 143 144 Node* load; 145 if (access.is_parse_access()) { 146 C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access); 147 GraphKit* kit = parse_access.kit(); 148 Node* control = control_dependent ? kit->control() : NULL; 149 150 if (in_native) { 151 load = kit->make_load(control, adr, val_type, access.type(), mo); 152 } else { 153 load = kit->make_load(control, adr, val_type, access.type(), adr_type, mo, 154 dep, requires_atomic_access, unaligned, mismatched, unsafe); 155 } 156 access.set_raw_access(load); 157 } else { 158 assert(!requires_atomic_access, "not yet supported"); 159 assert(access.is_opt_access(), "either parse or opt access"); 160 C2OptAccess& opt_access = static_cast<C2OptAccess&>(access); 161 Node* control = control_dependent ? opt_access.ctl() : NULL; 162 MergeMemNode* mm = opt_access.mem(); 163 PhaseGVN& gvn = opt_access.gvn(); 164 Node* mem = mm->memory_at(gvn.C->get_alias_index(adr_type)); 165 load = LoadNode::make(gvn, control, mem, adr, adr_type, val_type, access.type(), mo, dep, unaligned, mismatched); 166 load = gvn.transform(load); 167 } 168 169 return load; 170 } 171 172 class C2AccessFence: public StackObj { 173 C2Access& _access; 174 Node* _leading_membar; 175 176 public: 177 C2AccessFence(C2Access& access) : 178 _access(access), _leading_membar(NULL) { 179 GraphKit* kit = NULL; 180 if (access.is_parse_access()) { 181 C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access); 182 kit = parse_access.kit(); 183 } 184 DecoratorSet decorators = access.decorators(); 185 186 bool is_write = (decorators & C2_WRITE_ACCESS) != 0; 187 bool is_read = (decorators & C2_READ_ACCESS) != 0; 188 bool is_atomic = is_read && is_write; 189 190 bool is_volatile = (decorators & MO_SEQ_CST) != 0; 191 bool is_release = (decorators & MO_RELEASE) != 0; 192 193 if (is_atomic) { 194 assert(kit != NULL, "unsupported at optimization time"); 195 // Memory-model-wise, a LoadStore acts like a little synchronized 196 // block, so needs barriers on each side. These don't translate 197 // into actual barriers on most machines, but we still need rest of 198 // compiler to respect ordering. 199 if (is_release) { 200 _leading_membar = kit->insert_mem_bar(Op_MemBarRelease); 201 } else if (is_volatile) { 202 if (support_IRIW_for_not_multiple_copy_atomic_cpu) { 203 _leading_membar = kit->insert_mem_bar(Op_MemBarVolatile); 204 } else { 205 _leading_membar = kit->insert_mem_bar(Op_MemBarRelease); 206 } 207 } 208 } else if (is_write) { 209 // If reference is volatile, prevent following memory ops from 210 // floating down past the volatile write. Also prevents commoning 211 // another volatile read. 212 if (is_volatile || is_release) { 213 assert(kit != NULL, "unsupported at optimization time"); 214 _leading_membar = kit->insert_mem_bar(Op_MemBarRelease); 215 } 216 } else { 217 // Memory barrier to prevent normal and 'unsafe' accesses from 218 // bypassing each other. Happens after null checks, so the 219 // exception paths do not take memory state from the memory barrier, 220 // so there's no problems making a strong assert about mixing users 221 // of safe & unsafe memory. 222 if (is_volatile && support_IRIW_for_not_multiple_copy_atomic_cpu) { 223 assert(kit != NULL, "unsupported at optimization time"); 224 _leading_membar = kit->insert_mem_bar(Op_MemBarVolatile); 225 } 226 } 227 228 if (access.needs_cpu_membar()) { 229 assert(kit != NULL, "unsupported at optimization time"); 230 kit->insert_mem_bar(Op_MemBarCPUOrder); 231 } 232 233 if (is_atomic) { 234 // 4984716: MemBars must be inserted before this 235 // memory node in order to avoid a false 236 // dependency which will confuse the scheduler. 237 access.set_memory(); 238 } 239 } 240 241 ~C2AccessFence() { 242 GraphKit* kit = NULL; 243 if (_access.is_parse_access()) { 244 C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(_access); 245 kit = parse_access.kit(); 246 } 247 DecoratorSet decorators = _access.decorators(); 248 249 bool is_write = (decorators & C2_WRITE_ACCESS) != 0; 250 bool is_read = (decorators & C2_READ_ACCESS) != 0; 251 bool is_atomic = is_read && is_write; 252 253 bool is_volatile = (decorators & MO_SEQ_CST) != 0; 254 bool is_acquire = (decorators & MO_ACQUIRE) != 0; 255 256 // If reference is volatile, prevent following volatiles ops from 257 // floating up before the volatile access. 258 if (_access.needs_cpu_membar()) { 259 kit->insert_mem_bar(Op_MemBarCPUOrder); 260 } 261 262 if (is_atomic) { 263 assert(kit != NULL, "unsupported at optimization time"); 264 if (is_acquire || is_volatile) { 265 Node* n = _access.raw_access(); 266 Node* mb = kit->insert_mem_bar(Op_MemBarAcquire, n); 267 if (_leading_membar != NULL) { 268 MemBarNode::set_load_store_pair(_leading_membar->as_MemBar(), mb->as_MemBar()); 269 } 270 } 271 } else if (is_write) { 272 // If not multiple copy atomic, we do the MemBarVolatile before the load. 273 if (is_volatile && !support_IRIW_for_not_multiple_copy_atomic_cpu) { 274 assert(kit != NULL, "unsupported at optimization time"); 275 Node* n = _access.raw_access(); 276 Node* mb = kit->insert_mem_bar(Op_MemBarVolatile, n); // Use fat membar 277 if (_leading_membar != NULL) { 278 MemBarNode::set_store_pair(_leading_membar->as_MemBar(), mb->as_MemBar()); 279 } 280 } 281 } else { 282 if (is_volatile || is_acquire) { 283 assert(kit != NULL, "unsupported at optimization time"); 284 Node* n = _access.raw_access(); 285 assert(_leading_membar == NULL || support_IRIW_for_not_multiple_copy_atomic_cpu, "no leading membar expected"); 286 Node* mb = kit->insert_mem_bar(Op_MemBarAcquire, n); 287 mb->as_MemBar()->set_trailing_load(); 288 } 289 } 290 } 291 }; 292 293 Node* BarrierSetC2::store_at(C2Access& access, C2AccessValue& val) const { 294 C2AccessFence fence(access); 295 resolve_address(access); 296 return store_at_resolved(access, val); 297 } 298 299 Node* BarrierSetC2::load_at(C2Access& access, const Type* val_type) const { 300 C2AccessFence fence(access); 301 resolve_address(access); 302 return load_at_resolved(access, val_type); 303 } 304 305 MemNode::MemOrd C2Access::mem_node_mo() const { 306 bool is_write = (_decorators & C2_WRITE_ACCESS) != 0; 307 bool is_read = (_decorators & C2_READ_ACCESS) != 0; 308 if ((_decorators & MO_SEQ_CST) != 0) { 309 if (is_write && is_read) { 310 // For atomic operations 311 return MemNode::seqcst; 312 } else if (is_write) { 313 return MemNode::release; 314 } else { 315 assert(is_read, "what else?"); 316 return MemNode::acquire; 317 } 318 } else if ((_decorators & MO_RELEASE) != 0) { 319 return MemNode::release; 320 } else if ((_decorators & MO_ACQUIRE) != 0) { 321 return MemNode::acquire; 322 } else if (is_write) { 323 // Volatile fields need releasing stores. 324 // Non-volatile fields also need releasing stores if they hold an 325 // object reference, because the object reference might point to 326 // a freshly created object. 327 // Conservatively release stores of object references. 328 return StoreNode::release_if_reference(_type); 329 } else { 330 return MemNode::unordered; 331 } 332 } 333 334 void C2Access::fixup_decorators() { 335 bool default_mo = (_decorators & MO_DECORATOR_MASK) == 0; 336 bool is_unordered = (_decorators & MO_UNORDERED) != 0 || default_mo; 337 bool anonymous = (_decorators & C2_UNSAFE_ACCESS) != 0; 338 339 bool is_read = (_decorators & C2_READ_ACCESS) != 0; 340 bool is_write = (_decorators & C2_WRITE_ACCESS) != 0; 341 342 if (AlwaysAtomicAccesses && is_unordered) { 343 _decorators &= ~MO_DECORATOR_MASK; // clear the MO bits 344 _decorators |= MO_RELAXED; // Force the MO_RELAXED decorator with AlwaysAtomicAccess 345 } 346 347 _decorators = AccessInternal::decorator_fixup(_decorators); 348 349 if (is_read && !is_write && anonymous) { 350 // To be valid, unsafe loads may depend on other conditions than 351 // the one that guards them: pin the Load node 352 _decorators |= C2_CONTROL_DEPENDENT_LOAD; 353 _decorators |= C2_UNKNOWN_CONTROL_LOAD; 354 const TypePtr* adr_type = _addr.type(); 355 Node* adr = _addr.node(); 356 if (!needs_cpu_membar() && adr_type->isa_instptr()) { 357 assert(adr_type->meet(TypePtr::NULL_PTR) != adr_type->remove_speculative(), "should be not null"); 358 intptr_t offset = Type::OffsetBot; 359 AddPNode::Ideal_base_and_offset(adr, &gvn(), offset); 360 if (offset >= 0) { 361 int s = Klass::layout_helper_size_in_bytes(adr_type->isa_instptr()->klass()->layout_helper()); 362 if (offset < s) { 363 // Guaranteed to be a valid access, no need to pin it 364 _decorators ^= C2_CONTROL_DEPENDENT_LOAD; 365 _decorators ^= C2_UNKNOWN_CONTROL_LOAD; 366 } 367 } 368 } 369 } 370 } 371 372 //--------------------------- atomic operations--------------------------------- 373 374 void BarrierSetC2::pin_atomic_op(C2AtomicParseAccess& access) const { 375 if (!access.needs_pinning()) { 376 return; 377 } 378 // SCMemProjNodes represent the memory state of a LoadStore. Their 379 // main role is to prevent LoadStore nodes from being optimized away 380 // when their results aren't used. 381 assert(access.is_parse_access(), "entry not supported at optimization time"); 382 C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access); 383 GraphKit* kit = parse_access.kit(); 384 Node* load_store = access.raw_access(); 385 assert(load_store != NULL, "must pin atomic op"); 386 Node* proj = kit->gvn().transform(new SCMemProjNode(load_store)); 387 kit->set_memory(proj, access.alias_idx()); 388 } 389 390 void C2AtomicParseAccess::set_memory() { 391 Node *mem = _kit->memory(_alias_idx); 392 _memory = mem; 393 } 394 395 Node* BarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val, 396 Node* new_val, const Type* value_type) const { 397 GraphKit* kit = access.kit(); 398 MemNode::MemOrd mo = access.mem_node_mo(); 399 Node* mem = access.memory(); 400 401 Node* adr = access.addr().node(); 402 const TypePtr* adr_type = access.addr().type(); 403 404 Node* load_store = NULL; 405 406 if (access.is_oop()) { 407 #ifdef _LP64 408 if (adr->bottom_type()->is_ptr_to_narrowoop()) { 409 Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop())); 410 Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop())); 411 load_store = kit->gvn().transform(new CompareAndExchangeNNode(kit->control(), mem, adr, newval_enc, oldval_enc, adr_type, value_type->make_narrowoop(), mo)); 412 } else 413 #endif 414 { 415 load_store = kit->gvn().transform(new CompareAndExchangePNode(kit->control(), mem, adr, new_val, expected_val, adr_type, value_type->is_oopptr(), mo)); 416 } 417 } else { 418 switch (access.type()) { 419 case T_BYTE: { 420 load_store = kit->gvn().transform(new CompareAndExchangeBNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo)); 421 break; 422 } 423 case T_SHORT: { 424 load_store = kit->gvn().transform(new CompareAndExchangeSNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo)); 425 break; 426 } 427 case T_INT: { 428 load_store = kit->gvn().transform(new CompareAndExchangeINode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo)); 429 break; 430 } 431 case T_LONG: { 432 load_store = kit->gvn().transform(new CompareAndExchangeLNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo)); 433 break; 434 } 435 default: 436 ShouldNotReachHere(); 437 } 438 } 439 440 access.set_raw_access(load_store); 441 pin_atomic_op(access); 442 443 #ifdef _LP64 444 if (access.is_oop() && adr->bottom_type()->is_ptr_to_narrowoop()) { 445 return kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type())); 446 } 447 #endif 448 449 return load_store; 450 } 451 452 Node* BarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val, 453 Node* new_val, const Type* value_type) const { 454 GraphKit* kit = access.kit(); 455 DecoratorSet decorators = access.decorators(); 456 MemNode::MemOrd mo = access.mem_node_mo(); 457 Node* mem = access.memory(); 458 bool is_weak_cas = (decorators & C2_WEAK_CMPXCHG) != 0; 459 Node* load_store = NULL; 460 Node* adr = access.addr().node(); 461 462 if (access.is_oop()) { 463 #ifdef _LP64 464 if (adr->bottom_type()->is_ptr_to_narrowoop()) { 465 Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop())); 466 Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop())); 467 if (is_weak_cas) { 468 load_store = kit->gvn().transform(new WeakCompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo)); 469 } else { 470 load_store = kit->gvn().transform(new CompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo)); 471 } 472 } else 473 #endif 474 { 475 if (is_weak_cas) { 476 load_store = kit->gvn().transform(new WeakCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo)); 477 } else { 478 load_store = kit->gvn().transform(new CompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo)); 479 } 480 } 481 } else { 482 switch(access.type()) { 483 case T_BYTE: { 484 if (is_weak_cas) { 485 load_store = kit->gvn().transform(new WeakCompareAndSwapBNode(kit->control(), mem, adr, new_val, expected_val, mo)); 486 } else { 487 load_store = kit->gvn().transform(new CompareAndSwapBNode(kit->control(), mem, adr, new_val, expected_val, mo)); 488 } 489 break; 490 } 491 case T_SHORT: { 492 if (is_weak_cas) { 493 load_store = kit->gvn().transform(new WeakCompareAndSwapSNode(kit->control(), mem, adr, new_val, expected_val, mo)); 494 } else { 495 load_store = kit->gvn().transform(new CompareAndSwapSNode(kit->control(), mem, adr, new_val, expected_val, mo)); 496 } 497 break; 498 } 499 case T_INT: { 500 if (is_weak_cas) { 501 load_store = kit->gvn().transform(new WeakCompareAndSwapINode(kit->control(), mem, adr, new_val, expected_val, mo)); 502 } else { 503 load_store = kit->gvn().transform(new CompareAndSwapINode(kit->control(), mem, adr, new_val, expected_val, mo)); 504 } 505 break; 506 } 507 case T_LONG: { 508 if (is_weak_cas) { 509 load_store = kit->gvn().transform(new WeakCompareAndSwapLNode(kit->control(), mem, adr, new_val, expected_val, mo)); 510 } else { 511 load_store = kit->gvn().transform(new CompareAndSwapLNode(kit->control(), mem, adr, new_val, expected_val, mo)); 512 } 513 break; 514 } 515 default: 516 ShouldNotReachHere(); 517 } 518 } 519 520 access.set_raw_access(load_store); 521 pin_atomic_op(access); 522 523 return load_store; 524 } 525 526 Node* BarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const { 527 GraphKit* kit = access.kit(); 528 Node* mem = access.memory(); 529 Node* adr = access.addr().node(); 530 const TypePtr* adr_type = access.addr().type(); 531 Node* load_store = NULL; 532 533 if (access.is_oop()) { 534 #ifdef _LP64 535 if (adr->bottom_type()->is_ptr_to_narrowoop()) { 536 Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop())); 537 load_store = kit->gvn().transform(new GetAndSetNNode(kit->control(), mem, adr, newval_enc, adr_type, value_type->make_narrowoop())); 538 } else 539 #endif 540 { 541 load_store = kit->gvn().transform(new GetAndSetPNode(kit->control(), mem, adr, new_val, adr_type, value_type->is_oopptr())); 542 } 543 } else { 544 switch (access.type()) { 545 case T_BYTE: 546 load_store = kit->gvn().transform(new GetAndSetBNode(kit->control(), mem, adr, new_val, adr_type)); 547 break; 548 case T_SHORT: 549 load_store = kit->gvn().transform(new GetAndSetSNode(kit->control(), mem, adr, new_val, adr_type)); 550 break; 551 case T_INT: 552 load_store = kit->gvn().transform(new GetAndSetINode(kit->control(), mem, adr, new_val, adr_type)); 553 break; 554 case T_LONG: 555 load_store = kit->gvn().transform(new GetAndSetLNode(kit->control(), mem, adr, new_val, adr_type)); 556 break; 557 default: 558 ShouldNotReachHere(); 559 } 560 } 561 562 access.set_raw_access(load_store); 563 pin_atomic_op(access); 564 565 #ifdef _LP64 566 if (access.is_oop() && adr->bottom_type()->is_ptr_to_narrowoop()) { 567 return kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type())); 568 } 569 #endif 570 571 return load_store; 572 } 573 574 Node* BarrierSetC2::atomic_add_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const { 575 Node* load_store = NULL; 576 GraphKit* kit = access.kit(); 577 Node* adr = access.addr().node(); 578 const TypePtr* adr_type = access.addr().type(); 579 Node* mem = access.memory(); 580 581 switch(access.type()) { 582 case T_BYTE: 583 load_store = kit->gvn().transform(new GetAndAddBNode(kit->control(), mem, adr, new_val, adr_type)); 584 break; 585 case T_SHORT: 586 load_store = kit->gvn().transform(new GetAndAddSNode(kit->control(), mem, adr, new_val, adr_type)); 587 break; 588 case T_INT: 589 load_store = kit->gvn().transform(new GetAndAddINode(kit->control(), mem, adr, new_val, adr_type)); 590 break; 591 case T_LONG: 592 load_store = kit->gvn().transform(new GetAndAddLNode(kit->control(), mem, adr, new_val, adr_type)); 593 break; 594 default: 595 ShouldNotReachHere(); 596 } 597 598 access.set_raw_access(load_store); 599 pin_atomic_op(access); 600 601 return load_store; 602 } 603 604 Node* BarrierSetC2::atomic_cmpxchg_val_at(C2AtomicParseAccess& access, Node* expected_val, 605 Node* new_val, const Type* value_type) const { 606 C2AccessFence fence(access); 607 resolve_address(access); 608 return atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type); 609 } 610 611 Node* BarrierSetC2::atomic_cmpxchg_bool_at(C2AtomicParseAccess& access, Node* expected_val, 612 Node* new_val, const Type* value_type) const { 613 C2AccessFence fence(access); 614 resolve_address(access); 615 return atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type); 616 } 617 618 Node* BarrierSetC2::atomic_xchg_at(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const { 619 C2AccessFence fence(access); 620 resolve_address(access); 621 return atomic_xchg_at_resolved(access, new_val, value_type); 622 } 623 624 Node* BarrierSetC2::atomic_add_at(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const { 625 C2AccessFence fence(access); 626 resolve_address(access); 627 return atomic_add_at_resolved(access, new_val, value_type); 628 } 629 630 void BarrierSetC2::clone(GraphKit* kit, Node* src, Node* dst, Node* size, bool is_array) const { 631 // Exclude the header but include array length to copy by 8 bytes words. 632 // Can't use base_offset_in_bytes(bt) since basic type is unknown. 633 int base_off = is_array ? arrayOopDesc::length_offset_in_bytes() : 634 instanceOopDesc::base_offset_in_bytes(); 635 // base_off: 636 // 8 - 32-bit VM 637 // 12 - 64-bit VM, compressed klass 638 // 16 - 64-bit VM, normal klass 639 if (base_off % BytesPerLong != 0) { 640 assert(UseCompressedClassPointers, ""); 641 if (is_array) { 642 // Exclude length to copy by 8 bytes words. 643 base_off += sizeof(int); 644 } else { 645 // Include klass to copy by 8 bytes words. 646 base_off = instanceOopDesc::klass_offset_in_bytes(); 647 } 648 assert(base_off % BytesPerLong == 0, "expect 8 bytes alignment"); 649 } 650 Node* src_base = kit->basic_plus_adr(src, base_off); 651 Node* dst_base = kit->basic_plus_adr(dst, base_off); 652 653 // Compute the length also, if needed: 654 Node* countx = size; 655 countx = kit->gvn().transform(new SubXNode(countx, kit->MakeConX(base_off))); 656 countx = kit->gvn().transform(new URShiftXNode(countx, kit->intcon(LogBytesPerLong) )); 657 658 const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM; 659 660 ArrayCopyNode* ac = ArrayCopyNode::make(kit, false, src_base, NULL, dst_base, NULL, countx, true, false); 661 ac->set_clonebasic(); 662 Node* n = kit->gvn().transform(ac); 663 if (n == ac) { 664 ac->_adr_type = TypeRawPtr::BOTTOM; 665 kit->set_predefined_output_for_runtime_call(ac, ac->in(TypeFunc::Memory), raw_adr_type); 666 } else { 667 kit->set_all_memory(n); 668 } 669 } 670 671 Node* BarrierSetC2::obj_allocate(PhaseMacroExpand* macro, Node* ctrl, Node* mem, Node* toobig_false, Node* size_in_bytes, 672 Node*& i_o, Node*& needgc_ctrl, 673 Node*& fast_oop_ctrl, Node*& fast_oop_rawmem, 674 intx prefetch_lines) const { 675 676 Node* eden_top_adr; 677 Node* eden_end_adr; 678 679 macro->set_eden_pointers(eden_top_adr, eden_end_adr); 680 681 // Load Eden::end. Loop invariant and hoisted. 682 // 683 // Note: We set the control input on "eden_end" and "old_eden_top" when using 684 // a TLAB to work around a bug where these values were being moved across 685 // a safepoint. These are not oops, so they cannot be include in the oop 686 // map, but they can be changed by a GC. The proper way to fix this would 687 // be to set the raw memory state when generating a SafepointNode. However 688 // this will require extensive changes to the loop optimization in order to 689 // prevent a degradation of the optimization. 690 // See comment in memnode.hpp, around line 227 in class LoadPNode. 691 Node *eden_end = macro->make_load(ctrl, mem, eden_end_adr, 0, TypeRawPtr::BOTTOM, T_ADDRESS); 692 693 // We need a Region for the loop-back contended case. 694 enum { fall_in_path = 1, contended_loopback_path = 2 }; 695 Node *contended_region; 696 Node *contended_phi_rawmem; 697 if (UseTLAB) { 698 contended_region = toobig_false; 699 contended_phi_rawmem = mem; 700 } else { 701 contended_region = new RegionNode(3); 702 contended_phi_rawmem = new PhiNode(contended_region, Type::MEMORY, TypeRawPtr::BOTTOM); 703 // Now handle the passing-too-big test. We fall into the contended 704 // loop-back merge point. 705 contended_region ->init_req(fall_in_path, toobig_false); 706 contended_phi_rawmem->init_req(fall_in_path, mem); 707 macro->transform_later(contended_region); 708 macro->transform_later(contended_phi_rawmem); 709 } 710 711 // Load(-locked) the heap top. 712 // See note above concerning the control input when using a TLAB 713 Node *old_eden_top = UseTLAB 714 ? new LoadPNode (ctrl, contended_phi_rawmem, eden_top_adr, TypeRawPtr::BOTTOM, TypeRawPtr::BOTTOM, MemNode::unordered) 715 : new LoadPLockedNode(contended_region, contended_phi_rawmem, eden_top_adr, MemNode::acquire); 716 717 macro->transform_later(old_eden_top); 718 // Add to heap top to get a new heap top 719 Node *new_eden_top = new AddPNode(macro->top(), old_eden_top, size_in_bytes); 720 macro->transform_later(new_eden_top); 721 // Check for needing a GC; compare against heap end 722 Node *needgc_cmp = new CmpPNode(new_eden_top, eden_end); 723 macro->transform_later(needgc_cmp); 724 Node *needgc_bol = new BoolNode(needgc_cmp, BoolTest::ge); 725 macro->transform_later(needgc_bol); 726 IfNode *needgc_iff = new IfNode(contended_region, needgc_bol, PROB_UNLIKELY_MAG(4), COUNT_UNKNOWN); 727 macro->transform_later(needgc_iff); 728 729 // Plug the failing-heap-space-need-gc test into the slow-path region 730 Node *needgc_true = new IfTrueNode(needgc_iff); 731 macro->transform_later(needgc_true); 732 needgc_ctrl = needgc_true; 733 734 // No need for a GC. Setup for the Store-Conditional 735 Node *needgc_false = new IfFalseNode(needgc_iff); 736 macro->transform_later(needgc_false); 737 738 i_o = macro->prefetch_allocation(i_o, needgc_false, contended_phi_rawmem, 739 old_eden_top, new_eden_top, prefetch_lines); 740 741 Node* fast_oop = old_eden_top; 742 743 // Store (-conditional) the modified eden top back down. 744 // StorePConditional produces flags for a test PLUS a modified raw 745 // memory state. 746 if (UseTLAB) { 747 Node* store_eden_top = 748 new StorePNode(needgc_false, contended_phi_rawmem, eden_top_adr, 749 TypeRawPtr::BOTTOM, new_eden_top, MemNode::unordered); 750 macro->transform_later(store_eden_top); 751 fast_oop_ctrl = needgc_false; // No contention, so this is the fast path 752 fast_oop_rawmem = store_eden_top; 753 } else { 754 Node* store_eden_top = 755 new StorePConditionalNode(needgc_false, contended_phi_rawmem, eden_top_adr, 756 new_eden_top, fast_oop/*old_eden_top*/); 757 macro->transform_later(store_eden_top); 758 Node *contention_check = new BoolNode(store_eden_top, BoolTest::ne); 759 macro->transform_later(contention_check); 760 store_eden_top = new SCMemProjNode(store_eden_top); 761 macro->transform_later(store_eden_top); 762 763 // If not using TLABs, check to see if there was contention. 764 IfNode *contention_iff = new IfNode (needgc_false, contention_check, PROB_MIN, COUNT_UNKNOWN); 765 macro->transform_later(contention_iff); 766 Node *contention_true = new IfTrueNode(contention_iff); 767 macro->transform_later(contention_true); 768 // If contention, loopback and try again. 769 contended_region->init_req(contended_loopback_path, contention_true); 770 contended_phi_rawmem->init_req(contended_loopback_path, store_eden_top); 771 772 // Fast-path succeeded with no contention! 773 Node *contention_false = new IfFalseNode(contention_iff); 774 macro->transform_later(contention_false); 775 fast_oop_ctrl = contention_false; 776 777 // Bump total allocated bytes for this thread 778 Node* thread = new ThreadLocalNode(); 779 macro->transform_later(thread); 780 Node* alloc_bytes_adr = macro->basic_plus_adr(macro->top()/*not oop*/, thread, 781 in_bytes(JavaThread::allocated_bytes_offset())); 782 Node* alloc_bytes = macro->make_load(fast_oop_ctrl, store_eden_top, alloc_bytes_adr, 783 0, TypeLong::LONG, T_LONG); 784 #ifdef _LP64 785 Node* alloc_size = size_in_bytes; 786 #else 787 Node* alloc_size = new ConvI2LNode(size_in_bytes); 788 macro->transform_later(alloc_size); 789 #endif 790 Node* new_alloc_bytes = new AddLNode(alloc_bytes, alloc_size); 791 macro->transform_later(new_alloc_bytes); 792 fast_oop_rawmem = macro->make_store(fast_oop_ctrl, store_eden_top, alloc_bytes_adr, 793 0, new_alloc_bytes, T_LONG); 794 } 795 return fast_oop; 796 } 797 798 #define XTOP LP64_ONLY(COMMA phase->top()) 799 800 void BarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac) const { 801 Node* ctrl = ac->in(TypeFunc::Control); 802 Node* mem = ac->in(TypeFunc::Memory); 803 Node* src = ac->in(ArrayCopyNode::Src); 804 Node* src_offset = ac->in(ArrayCopyNode::SrcPos); 805 Node* dest = ac->in(ArrayCopyNode::Dest); 806 Node* dest_offset = ac->in(ArrayCopyNode::DestPos); 807 Node* length = ac->in(ArrayCopyNode::Length); 808 809 assert (src_offset == NULL && dest_offset == NULL, "for clone offsets should be null"); 810 811 const char* copyfunc_name = "arraycopy"; 812 address copyfunc_addr = 813 phase->basictype2arraycopy(T_LONG, NULL, NULL, 814 true, copyfunc_name, true); 815 816 const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM; 817 const TypeFunc* call_type = OptoRuntime::fast_arraycopy_Type(); 818 819 Node* call = phase->make_leaf_call(ctrl, mem, call_type, copyfunc_addr, copyfunc_name, raw_adr_type, src, dest, length XTOP); 820 phase->transform_later(call); 821 822 phase->igvn().replace_node(ac, call); 823 }