1 /*
2 * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "gc/shared/c2/barrierSetC2.hpp"
27 #include "opto/arraycopynode.hpp"
28 #include "opto/convertnode.hpp"
29 #include "opto/graphKit.hpp"
30 #include "opto/idealKit.hpp"
31 #include "opto/macro.hpp"
32 #include "opto/narrowptrnode.hpp"
33 #include "utilities/macros.hpp"
34
35 // By default this is a no-op.
36 void BarrierSetC2::resolve_address(C2Access& access) const { }
37
38 void* C2ParseAccess::barrier_set_state() const {
39 return _kit->barrier_set_state();
40 }
41
42 PhaseGVN& C2ParseAccess::gvn() const { return _kit->gvn(); }
43
44 bool C2Access::needs_cpu_membar() const {
45 bool mismatched = (_decorators & C2_MISMATCHED) != 0;
46 bool is_unordered = (_decorators & MO_UNORDERED) != 0;
47 bool anonymous = (_decorators & C2_UNSAFE_ACCESS) != 0;
48 bool in_heap = (_decorators & IN_HEAP) != 0;
49
50 bool is_write = (_decorators & C2_WRITE_ACCESS) != 0;
51 bool is_read = (_decorators & C2_READ_ACCESS) != 0;
52 bool is_atomic = is_read && is_write;
53
54 if (is_atomic) {
55 // Atomics always need to be wrapped in CPU membars
56 return true;
57 }
58
59 if (anonymous) {
60 // We will need memory barriers unless we can determine a unique
61 // alias category for this reference. (Note: If for some reason
62 // the barriers get omitted and the unsafe reference begins to "pollute"
63 // the alias analysis of the rest of the graph, either Compile::can_alias
64 // or Compile::must_alias will throw a diagnostic assert.)
65 if (!in_heap || !is_unordered || (mismatched && !_addr.type()->isa_aryptr())) {
66 return true;
67 }
68 }
69
70 return false;
71 }
72
73 Node* BarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& val) const {
74 DecoratorSet decorators = access.decorators();
75
76 bool mismatched = (decorators & C2_MISMATCHED) != 0;
77 bool unaligned = (decorators & C2_UNALIGNED) != 0;
78 bool requires_atomic_access = (decorators & MO_UNORDERED) == 0;
79
80 bool in_native = (decorators & IN_NATIVE) != 0;
81 assert(!in_native, "not supported yet");
82
83 MemNode::MemOrd mo = access.mem_node_mo();
84
85 Node* store;
86 if (access.is_parse_access()) {
87 C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
88
89 GraphKit* kit = parse_access.kit();
90 if (access.type() == T_DOUBLE) {
91 Node* new_val = kit->dstore_rounding(val.node());
92 val.set_node(new_val);
93 }
94
95 store = kit->store_to_memory(kit->control(), access.addr().node(), val.node(), access.type(),
96 access.addr().type(), mo, requires_atomic_access, unaligned, mismatched);
97 access.set_raw_access(store);
98 } else {
99 assert(!requires_atomic_access, "not yet supported");
100 assert(access.is_opt_access(), "either parse or opt access");
101 C2OptAccess& opt_access = static_cast<C2OptAccess&>(access);
102 Node* ctl = opt_access.ctl();
103 MergeMemNode* mm = opt_access.mem();
104 PhaseGVN& gvn = opt_access.gvn();
105 const TypePtr* adr_type = access.addr().type();
106 int alias = gvn.C->get_alias_index(adr_type);
107 Node* mem = mm->memory_at(alias);
108
109 StoreNode* st = StoreNode::make(gvn, ctl, mem, access.addr().node(), adr_type, val.node(), access.type(), mo);
110 if (unaligned) {
111 st->set_unaligned_access();
112 }
113 if (mismatched) {
114 st->set_mismatched_access();
115 }
116 store = gvn.transform(st);
117 if (store == st) {
118 mm->set_memory_at(alias, st);
119 }
120 }
121 return store;
122 }
123
124 Node* BarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const {
125 DecoratorSet decorators = access.decorators();
126
127 Node* adr = access.addr().node();
128 const TypePtr* adr_type = access.addr().type();
129
130 bool mismatched = (decorators & C2_MISMATCHED) != 0;
131 bool requires_atomic_access = (decorators & MO_UNORDERED) == 0;
132 bool unaligned = (decorators & C2_UNALIGNED) != 0;
133 bool control_dependent = (decorators & C2_CONTROL_DEPENDENT_LOAD) != 0;
134 bool pinned = (decorators & C2_PINNED_LOAD) != 0;
135
136 bool in_native = (decorators & IN_NATIVE) != 0;
137
138 MemNode::MemOrd mo = access.mem_node_mo();
139 LoadNode::ControlDependency dep = pinned ? LoadNode::Pinned : LoadNode::DependsOnlyOnTest;
140
141 Node* load;
142 if (access.is_parse_access()) {
143 C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
144 GraphKit* kit = parse_access.kit();
145 Node* control = control_dependent ? kit->control() : NULL;
146
147 if (in_native) {
148 load = kit->make_load(control, adr, val_type, access.type(), mo);
149 } else {
150 load = kit->make_load(control, adr, val_type, access.type(), adr_type, mo,
151 dep, requires_atomic_access, unaligned, mismatched);
152 }
153 access.set_raw_access(load);
154 } else {
155 assert(!requires_atomic_access, "not yet supported");
156 assert(access.is_opt_access(), "either parse or opt access");
157 C2OptAccess& opt_access = static_cast<C2OptAccess&>(access);
158 Node* control = control_dependent ? opt_access.ctl() : NULL;
159 MergeMemNode* mm = opt_access.mem();
160 PhaseGVN& gvn = opt_access.gvn();
161 Node* mem = mm->memory_at(gvn.C->get_alias_index(adr_type));
162 load = LoadNode::make(gvn, control, mem, adr, adr_type, val_type, access.type(), mo, dep, unaligned, mismatched);
163 load = gvn.transform(load);
164 }
165
166 return load;
167 }
168
169 class C2AccessFence: public StackObj {
170 C2Access& _access;
171 Node* _leading_membar;
172
173 public:
174 C2AccessFence(C2Access& access) :
175 _access(access), _leading_membar(NULL) {
176 GraphKit* kit = NULL;
177 if (access.is_parse_access()) {
178 C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
179 kit = parse_access.kit();
180 }
181 DecoratorSet decorators = access.decorators();
182
183 bool is_write = (decorators & C2_WRITE_ACCESS) != 0;
184 bool is_read = (decorators & C2_READ_ACCESS) != 0;
185 bool is_atomic = is_read && is_write;
186
187 bool is_volatile = (decorators & MO_SEQ_CST) != 0;
188 bool is_release = (decorators & MO_RELEASE) != 0;
189
190 if (is_atomic) {
191 assert(kit != NULL, "unsupported at optimization time");
192 // Memory-model-wise, a LoadStore acts like a little synchronized
193 // block, so needs barriers on each side. These don't translate
194 // into actual barriers on most machines, but we still need rest of
195 // compiler to respect ordering.
196 if (is_release) {
197 _leading_membar = kit->insert_mem_bar(Op_MemBarRelease);
198 } else if (is_volatile) {
199 if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
200 _leading_membar = kit->insert_mem_bar(Op_MemBarVolatile);
201 } else {
202 _leading_membar = kit->insert_mem_bar(Op_MemBarRelease);
203 }
204 }
205 } else if (is_write) {
206 // If reference is volatile, prevent following memory ops from
207 // floating down past the volatile write. Also prevents commoning
208 // another volatile read.
209 if (is_volatile || is_release) {
210 assert(kit != NULL, "unsupported at optimization time");
211 _leading_membar = kit->insert_mem_bar(Op_MemBarRelease);
212 }
213 } else {
214 // Memory barrier to prevent normal and 'unsafe' accesses from
215 // bypassing each other. Happens after null checks, so the
216 // exception paths do not take memory state from the memory barrier,
217 // so there's no problems making a strong assert about mixing users
218 // of safe & unsafe memory.
219 if (is_volatile && support_IRIW_for_not_multiple_copy_atomic_cpu) {
220 assert(kit != NULL, "unsupported at optimization time");
221 _leading_membar = kit->insert_mem_bar(Op_MemBarVolatile);
222 }
223 }
224
225 if (access.needs_cpu_membar()) {
226 assert(kit != NULL, "unsupported at optimization time");
227 kit->insert_mem_bar(Op_MemBarCPUOrder);
228 }
229
230 if (is_atomic) {
231 // 4984716: MemBars must be inserted before this
232 // memory node in order to avoid a false
233 // dependency which will confuse the scheduler.
234 access.set_memory();
235 }
236 }
237
238 ~C2AccessFence() {
239 GraphKit* kit = NULL;
240 if (_access.is_parse_access()) {
241 C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(_access);
242 kit = parse_access.kit();
243 }
244 DecoratorSet decorators = _access.decorators();
245
246 bool is_write = (decorators & C2_WRITE_ACCESS) != 0;
247 bool is_read = (decorators & C2_READ_ACCESS) != 0;
248 bool is_atomic = is_read && is_write;
249
250 bool is_volatile = (decorators & MO_SEQ_CST) != 0;
251 bool is_acquire = (decorators & MO_ACQUIRE) != 0;
252
253 // If reference is volatile, prevent following volatiles ops from
254 // floating up before the volatile access.
255 if (_access.needs_cpu_membar()) {
256 kit->insert_mem_bar(Op_MemBarCPUOrder);
257 }
258
259 if (is_atomic) {
260 assert(kit != NULL, "unsupported at optimization time");
261 if (is_acquire || is_volatile) {
262 Node* n = _access.raw_access();
263 Node* mb = kit->insert_mem_bar(Op_MemBarAcquire, n);
264 if (_leading_membar != NULL) {
265 MemBarNode::set_load_store_pair(_leading_membar->as_MemBar(), mb->as_MemBar());
266 }
267 }
268 } else if (is_write) {
269 // If not multiple copy atomic, we do the MemBarVolatile before the load.
270 if (is_volatile && !support_IRIW_for_not_multiple_copy_atomic_cpu) {
271 assert(kit != NULL, "unsupported at optimization time");
272 Node* n = _access.raw_access();
273 Node* mb = kit->insert_mem_bar(Op_MemBarVolatile, n); // Use fat membar
274 if (_leading_membar != NULL) {
275 MemBarNode::set_store_pair(_leading_membar->as_MemBar(), mb->as_MemBar());
276 }
277 }
278 } else {
279 if (is_volatile || is_acquire) {
280 assert(kit != NULL, "unsupported at optimization time");
281 Node* n = _access.raw_access();
282 assert(_leading_membar == NULL || support_IRIW_for_not_multiple_copy_atomic_cpu, "no leading membar expected");
283 Node* mb = kit->insert_mem_bar(Op_MemBarAcquire, n);
284 mb->as_MemBar()->set_trailing_load();
285 }
286 }
287 }
288 };
289
290 Node* BarrierSetC2::store_at(C2Access& access, C2AccessValue& val) const {
291 C2AccessFence fence(access);
292 resolve_address(access);
293 return store_at_resolved(access, val);
294 }
295
296 Node* BarrierSetC2::load_at(C2Access& access, const Type* val_type) const {
297 C2AccessFence fence(access);
298 resolve_address(access);
299 return load_at_resolved(access, val_type);
300 }
301
302 MemNode::MemOrd C2Access::mem_node_mo() const {
303 bool is_write = (_decorators & C2_WRITE_ACCESS) != 0;
304 bool is_read = (_decorators & C2_READ_ACCESS) != 0;
305 if ((_decorators & MO_SEQ_CST) != 0) {
306 if (is_write && is_read) {
307 // For atomic operations
308 return MemNode::seqcst;
309 } else if (is_write) {
310 return MemNode::release;
311 } else {
312 assert(is_read, "what else?");
313 return MemNode::acquire;
314 }
315 } else if ((_decorators & MO_RELEASE) != 0) {
316 return MemNode::release;
317 } else if ((_decorators & MO_ACQUIRE) != 0) {
318 return MemNode::acquire;
319 } else if (is_write) {
320 // Volatile fields need releasing stores.
321 // Non-volatile fields also need releasing stores if they hold an
322 // object reference, because the object reference might point to
323 // a freshly created object.
324 // Conservatively release stores of object references.
325 return StoreNode::release_if_reference(_type);
326 } else {
327 return MemNode::unordered;
328 }
329 }
330
331 void C2Access::fixup_decorators() {
332 bool default_mo = (_decorators & MO_DECORATOR_MASK) == 0;
333 bool is_unordered = (_decorators & MO_UNORDERED) != 0 || default_mo;
334 bool anonymous = (_decorators & C2_UNSAFE_ACCESS) != 0;
335
336 bool is_read = (_decorators & C2_READ_ACCESS) != 0;
337 bool is_write = (_decorators & C2_WRITE_ACCESS) != 0;
338
339 if (AlwaysAtomicAccesses && is_unordered) {
340 _decorators &= ~MO_DECORATOR_MASK; // clear the MO bits
341 _decorators |= MO_RELAXED; // Force the MO_RELAXED decorator with AlwaysAtomicAccess
342 }
343
344 _decorators = AccessInternal::decorator_fixup(_decorators);
345
346 if (is_read && !is_write && anonymous) {
347 // To be valid, unsafe loads may depend on other conditions than
348 // the one that guards them: pin the Load node
349 _decorators |= C2_CONTROL_DEPENDENT_LOAD;
350 _decorators |= C2_PINNED_LOAD;
351 const TypePtr* adr_type = _addr.type();
352 Node* adr = _addr.node();
353 if (!needs_cpu_membar() && adr_type->isa_instptr()) {
354 assert(adr_type->meet(TypePtr::NULL_PTR) != adr_type->remove_speculative(), "should be not null");
355 intptr_t offset = Type::OffsetBot;
356 AddPNode::Ideal_base_and_offset(adr, &gvn(), offset);
357 if (offset >= 0) {
358 int s = Klass::layout_helper_size_in_bytes(adr_type->isa_instptr()->klass()->layout_helper());
359 if (offset < s) {
360 // Guaranteed to be a valid access, no need to pin it
361 _decorators ^= C2_CONTROL_DEPENDENT_LOAD;
362 _decorators ^= C2_PINNED_LOAD;
363 }
364 }
365 }
366 }
367 }
368
369 //--------------------------- atomic operations---------------------------------
370
371 void BarrierSetC2::pin_atomic_op(C2AtomicParseAccess& access) const {
372 if (!access.needs_pinning()) {
373 return;
374 }
375 // SCMemProjNodes represent the memory state of a LoadStore. Their
376 // main role is to prevent LoadStore nodes from being optimized away
377 // when their results aren't used.
378 assert(access.is_parse_access(), "entry not supported at optimization time");
379 C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
380 GraphKit* kit = parse_access.kit();
381 Node* load_store = access.raw_access();
382 assert(load_store != NULL, "must pin atomic op");
383 Node* proj = kit->gvn().transform(new SCMemProjNode(load_store));
384 kit->set_memory(proj, access.alias_idx());
385 }
386
387 void C2AtomicParseAccess::set_memory() {
388 Node *mem = _kit->memory(_alias_idx);
389 _memory = mem;
390 }
391
392 Node* BarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
393 Node* new_val, const Type* value_type) const {
394 GraphKit* kit = access.kit();
395 MemNode::MemOrd mo = access.mem_node_mo();
396 Node* mem = access.memory();
397
398 Node* adr = access.addr().node();
399 const TypePtr* adr_type = access.addr().type();
400
401 Node* load_store = NULL;
402
403 if (access.is_oop()) {
404 #ifdef _LP64
405 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
406 Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop()));
407 Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop()));
408 load_store = kit->gvn().transform(new CompareAndExchangeNNode(kit->control(), mem, adr, newval_enc, oldval_enc, adr_type, value_type->make_narrowoop(), mo));
409 } else
410 #endif
411 {
412 load_store = kit->gvn().transform(new CompareAndExchangePNode(kit->control(), mem, adr, new_val, expected_val, adr_type, value_type->is_oopptr(), mo));
413 }
414 } else {
415 switch (access.type()) {
416 case T_BYTE: {
417 load_store = kit->gvn().transform(new CompareAndExchangeBNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo));
418 break;
419 }
420 case T_SHORT: {
421 load_store = kit->gvn().transform(new CompareAndExchangeSNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo));
422 break;
423 }
424 case T_INT: {
425 load_store = kit->gvn().transform(new CompareAndExchangeINode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo));
426 break;
427 }
428 case T_LONG: {
429 load_store = kit->gvn().transform(new CompareAndExchangeLNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo));
430 break;
431 }
432 default:
433 ShouldNotReachHere();
434 }
435 }
436
437 access.set_raw_access(load_store);
438 pin_atomic_op(access);
439
440 #ifdef _LP64
441 if (access.is_oop() && adr->bottom_type()->is_ptr_to_narrowoop()) {
442 return kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type()));
443 }
444 #endif
445
446 return load_store;
447 }
448
449 Node* BarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
450 Node* new_val, const Type* value_type) const {
451 GraphKit* kit = access.kit();
452 DecoratorSet decorators = access.decorators();
453 MemNode::MemOrd mo = access.mem_node_mo();
454 Node* mem = access.memory();
455 bool is_weak_cas = (decorators & C2_WEAK_CMPXCHG) != 0;
456 Node* load_store = NULL;
457 Node* adr = access.addr().node();
458
459 if (access.is_oop()) {
460 #ifdef _LP64
461 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
462 Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop()));
463 Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop()));
464 if (is_weak_cas) {
465 load_store = kit->gvn().transform(new WeakCompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo));
466 } else {
467 load_store = kit->gvn().transform(new CompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo));
468 }
469 } else
470 #endif
471 {
472 if (is_weak_cas) {
473 load_store = kit->gvn().transform(new WeakCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo));
474 } else {
475 load_store = kit->gvn().transform(new CompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo));
476 }
477 }
478 } else {
479 switch(access.type()) {
480 case T_BYTE: {
481 if (is_weak_cas) {
482 load_store = kit->gvn().transform(new WeakCompareAndSwapBNode(kit->control(), mem, adr, new_val, expected_val, mo));
483 } else {
484 load_store = kit->gvn().transform(new CompareAndSwapBNode(kit->control(), mem, adr, new_val, expected_val, mo));
485 }
486 break;
487 }
488 case T_SHORT: {
489 if (is_weak_cas) {
490 load_store = kit->gvn().transform(new WeakCompareAndSwapSNode(kit->control(), mem, adr, new_val, expected_val, mo));
491 } else {
492 load_store = kit->gvn().transform(new CompareAndSwapSNode(kit->control(), mem, adr, new_val, expected_val, mo));
493 }
494 break;
495 }
496 case T_INT: {
497 if (is_weak_cas) {
498 load_store = kit->gvn().transform(new WeakCompareAndSwapINode(kit->control(), mem, adr, new_val, expected_val, mo));
499 } else {
500 load_store = kit->gvn().transform(new CompareAndSwapINode(kit->control(), mem, adr, new_val, expected_val, mo));
501 }
502 break;
503 }
504 case T_LONG: {
505 if (is_weak_cas) {
506 load_store = kit->gvn().transform(new WeakCompareAndSwapLNode(kit->control(), mem, adr, new_val, expected_val, mo));
507 } else {
508 load_store = kit->gvn().transform(new CompareAndSwapLNode(kit->control(), mem, adr, new_val, expected_val, mo));
509 }
510 break;
511 }
512 default:
513 ShouldNotReachHere();
514 }
515 }
516
517 access.set_raw_access(load_store);
518 pin_atomic_op(access);
519
520 return load_store;
521 }
522
523 Node* BarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const {
524 GraphKit* kit = access.kit();
525 Node* mem = access.memory();
526 Node* adr = access.addr().node();
527 const TypePtr* adr_type = access.addr().type();
528 Node* load_store = NULL;
529
530 if (access.is_oop()) {
531 #ifdef _LP64
532 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
533 Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop()));
534 load_store = kit->gvn().transform(new GetAndSetNNode(kit->control(), mem, adr, newval_enc, adr_type, value_type->make_narrowoop()));
535 } else
536 #endif
537 {
538 load_store = kit->gvn().transform(new GetAndSetPNode(kit->control(), mem, adr, new_val, adr_type, value_type->is_oopptr()));
539 }
540 } else {
541 switch (access.type()) {
542 case T_BYTE:
543 load_store = kit->gvn().transform(new GetAndSetBNode(kit->control(), mem, adr, new_val, adr_type));
544 break;
545 case T_SHORT:
546 load_store = kit->gvn().transform(new GetAndSetSNode(kit->control(), mem, adr, new_val, adr_type));
547 break;
548 case T_INT:
549 load_store = kit->gvn().transform(new GetAndSetINode(kit->control(), mem, adr, new_val, adr_type));
550 break;
551 case T_LONG:
552 load_store = kit->gvn().transform(new GetAndSetLNode(kit->control(), mem, adr, new_val, adr_type));
553 break;
554 default:
555 ShouldNotReachHere();
556 }
557 }
558
559 access.set_raw_access(load_store);
560 pin_atomic_op(access);
561
562 #ifdef _LP64
563 if (access.is_oop() && adr->bottom_type()->is_ptr_to_narrowoop()) {
564 return kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type()));
565 }
566 #endif
567
568 return load_store;
569 }
570
571 Node* BarrierSetC2::atomic_add_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const {
572 Node* load_store = NULL;
573 GraphKit* kit = access.kit();
574 Node* adr = access.addr().node();
575 const TypePtr* adr_type = access.addr().type();
576 Node* mem = access.memory();
577
578 switch(access.type()) {
579 case T_BYTE:
580 load_store = kit->gvn().transform(new GetAndAddBNode(kit->control(), mem, adr, new_val, adr_type));
581 break;
582 case T_SHORT:
583 load_store = kit->gvn().transform(new GetAndAddSNode(kit->control(), mem, adr, new_val, adr_type));
584 break;
585 case T_INT:
586 load_store = kit->gvn().transform(new GetAndAddINode(kit->control(), mem, adr, new_val, adr_type));
587 break;
588 case T_LONG:
589 load_store = kit->gvn().transform(new GetAndAddLNode(kit->control(), mem, adr, new_val, adr_type));
590 break;
591 default:
592 ShouldNotReachHere();
593 }
594
595 access.set_raw_access(load_store);
596 pin_atomic_op(access);
597
598 return load_store;
599 }
600
601 Node* BarrierSetC2::atomic_cmpxchg_val_at(C2AtomicParseAccess& access, Node* expected_val,
602 Node* new_val, const Type* value_type) const {
603 C2AccessFence fence(access);
604 resolve_address(access);
605 return atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type);
606 }
607
608 Node* BarrierSetC2::atomic_cmpxchg_bool_at(C2AtomicParseAccess& access, Node* expected_val,
609 Node* new_val, const Type* value_type) const {
610 C2AccessFence fence(access);
611 resolve_address(access);
612 return atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type);
613 }
614
615 Node* BarrierSetC2::atomic_xchg_at(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const {
616 C2AccessFence fence(access);
617 resolve_address(access);
618 return atomic_xchg_at_resolved(access, new_val, value_type);
619 }
620
621 Node* BarrierSetC2::atomic_add_at(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const {
622 C2AccessFence fence(access);
623 resolve_address(access);
624 return atomic_add_at_resolved(access, new_val, value_type);
625 }
626
627 void BarrierSetC2::clone(GraphKit* kit, Node* src, Node* dst, Node* size, bool is_array) const {
628 // Exclude the header but include array length to copy by 8 bytes words.
629 // Can't use base_offset_in_bytes(bt) since basic type is unknown.
630 int base_off = is_array ? arrayOopDesc::length_offset_in_bytes() :
631 instanceOopDesc::base_offset_in_bytes();
632 // base_off:
633 // 8 - 32-bit VM
634 // 12 - 64-bit VM, compressed klass
635 // 16 - 64-bit VM, normal klass
636 if (base_off % BytesPerLong != 0) {
637 assert(UseCompressedClassPointers, "");
638 if (is_array) {
639 // Exclude length to copy by 8 bytes words.
640 base_off += sizeof(int);
641 } else {
642 // Include klass to copy by 8 bytes words.
643 base_off = instanceOopDesc::klass_offset_in_bytes();
644 }
645 assert(base_off % BytesPerLong == 0, "expect 8 bytes alignment");
646 }
647 Node* src_base = kit->basic_plus_adr(src, base_off);
648 Node* dst_base = kit->basic_plus_adr(dst, base_off);
649
650 // Compute the length also, if needed:
651 Node* countx = size;
652 countx = kit->gvn().transform(new SubXNode(countx, kit->MakeConX(base_off)));
653 countx = kit->gvn().transform(new URShiftXNode(countx, kit->intcon(LogBytesPerLong) ));
654
655 const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
656
657 ArrayCopyNode* ac = ArrayCopyNode::make(kit, false, src_base, NULL, dst_base, NULL, countx, true, false);
658 ac->set_clonebasic();
659 Node* n = kit->gvn().transform(ac);
660 if (n == ac) {
661 ac->_adr_type = TypeRawPtr::BOTTOM;
662 kit->set_predefined_output_for_runtime_call(ac, ac->in(TypeFunc::Memory), raw_adr_type);
663 } else {
664 kit->set_all_memory(n);
665 }
666 }
667
668 Node* BarrierSetC2::obj_allocate(PhaseMacroExpand* macro, Node* ctrl, Node* mem, Node* toobig_false, Node* size_in_bytes,
669 Node*& i_o, Node*& needgc_ctrl,
670 Node*& fast_oop_ctrl, Node*& fast_oop_rawmem,
671 intx prefetch_lines) const {
672
673 Node* eden_top_adr;
674 Node* eden_end_adr;
675
676 macro->set_eden_pointers(eden_top_adr, eden_end_adr);
677
678 // Load Eden::end. Loop invariant and hoisted.
679 //
680 // Note: We set the control input on "eden_end" and "old_eden_top" when using
681 // a TLAB to work around a bug where these values were being moved across
682 // a safepoint. These are not oops, so they cannot be include in the oop
683 // map, but they can be changed by a GC. The proper way to fix this would
684 // be to set the raw memory state when generating a SafepointNode. However
685 // this will require extensive changes to the loop optimization in order to
686 // prevent a degradation of the optimization.
687 // See comment in memnode.hpp, around line 227 in class LoadPNode.
688 Node *eden_end = macro->make_load(ctrl, mem, eden_end_adr, 0, TypeRawPtr::BOTTOM, T_ADDRESS);
689
690 // We need a Region for the loop-back contended case.
691 enum { fall_in_path = 1, contended_loopback_path = 2 };
692 Node *contended_region;
693 Node *contended_phi_rawmem;
694 if (UseTLAB) {
695 contended_region = toobig_false;
696 contended_phi_rawmem = mem;
697 } else {
698 contended_region = new RegionNode(3);
699 contended_phi_rawmem = new PhiNode(contended_region, Type::MEMORY, TypeRawPtr::BOTTOM);
700 // Now handle the passing-too-big test. We fall into the contended
701 // loop-back merge point.
702 contended_region ->init_req(fall_in_path, toobig_false);
703 contended_phi_rawmem->init_req(fall_in_path, mem);
704 macro->transform_later(contended_region);
705 macro->transform_later(contended_phi_rawmem);
706 }
707
708 // Load(-locked) the heap top.
709 // See note above concerning the control input when using a TLAB
710 Node *old_eden_top = UseTLAB
711 ? new LoadPNode (ctrl, contended_phi_rawmem, eden_top_adr, TypeRawPtr::BOTTOM, TypeRawPtr::BOTTOM, MemNode::unordered)
712 : new LoadPLockedNode(contended_region, contended_phi_rawmem, eden_top_adr, MemNode::acquire);
713
714 macro->transform_later(old_eden_top);
715 // Add to heap top to get a new heap top
716 Node *new_eden_top = new AddPNode(macro->top(), old_eden_top, size_in_bytes);
717 macro->transform_later(new_eden_top);
718 // Check for needing a GC; compare against heap end
719 Node *needgc_cmp = new CmpPNode(new_eden_top, eden_end);
720 macro->transform_later(needgc_cmp);
721 Node *needgc_bol = new BoolNode(needgc_cmp, BoolTest::ge);
722 macro->transform_later(needgc_bol);
723 IfNode *needgc_iff = new IfNode(contended_region, needgc_bol, PROB_UNLIKELY_MAG(4), COUNT_UNKNOWN);
724 macro->transform_later(needgc_iff);
725
726 // Plug the failing-heap-space-need-gc test into the slow-path region
727 Node *needgc_true = new IfTrueNode(needgc_iff);
728 macro->transform_later(needgc_true);
729 needgc_ctrl = needgc_true;
730
731 // No need for a GC. Setup for the Store-Conditional
732 Node *needgc_false = new IfFalseNode(needgc_iff);
733 macro->transform_later(needgc_false);
734
735 i_o = macro->prefetch_allocation(i_o, needgc_false, contended_phi_rawmem,
736 old_eden_top, new_eden_top, prefetch_lines);
737
738 Node* fast_oop = old_eden_top;
739
740 // Store (-conditional) the modified eden top back down.
741 // StorePConditional produces flags for a test PLUS a modified raw
742 // memory state.
743 if (UseTLAB) {
744 Node* store_eden_top =
745 new StorePNode(needgc_false, contended_phi_rawmem, eden_top_adr,
746 TypeRawPtr::BOTTOM, new_eden_top, MemNode::unordered);
747 macro->transform_later(store_eden_top);
748 fast_oop_ctrl = needgc_false; // No contention, so this is the fast path
749 fast_oop_rawmem = store_eden_top;
750 } else {
751 Node* store_eden_top =
752 new StorePConditionalNode(needgc_false, contended_phi_rawmem, eden_top_adr,
753 new_eden_top, fast_oop/*old_eden_top*/);
754 macro->transform_later(store_eden_top);
755 Node *contention_check = new BoolNode(store_eden_top, BoolTest::ne);
756 macro->transform_later(contention_check);
757 store_eden_top = new SCMemProjNode(store_eden_top);
758 macro->transform_later(store_eden_top);
759
760 // If not using TLABs, check to see if there was contention.
761 IfNode *contention_iff = new IfNode (needgc_false, contention_check, PROB_MIN, COUNT_UNKNOWN);
762 macro->transform_later(contention_iff);
763 Node *contention_true = new IfTrueNode(contention_iff);
764 macro->transform_later(contention_true);
765 // If contention, loopback and try again.
766 contended_region->init_req(contended_loopback_path, contention_true);
767 contended_phi_rawmem->init_req(contended_loopback_path, store_eden_top);
768
769 // Fast-path succeeded with no contention!
770 Node *contention_false = new IfFalseNode(contention_iff);
771 macro->transform_later(contention_false);
772 fast_oop_ctrl = contention_false;
773
774 // Bump total allocated bytes for this thread
775 Node* thread = new ThreadLocalNode();
776 macro->transform_later(thread);
777 Node* alloc_bytes_adr = macro->basic_plus_adr(macro->top()/*not oop*/, thread,
778 in_bytes(JavaThread::allocated_bytes_offset()));
779 Node* alloc_bytes = macro->make_load(fast_oop_ctrl, store_eden_top, alloc_bytes_adr,
780 0, TypeLong::LONG, T_LONG);
781 #ifdef _LP64
782 Node* alloc_size = size_in_bytes;
783 #else
784 Node* alloc_size = new ConvI2LNode(size_in_bytes);
785 macro->transform_later(alloc_size);
786 #endif
787 Node* new_alloc_bytes = new AddLNode(alloc_bytes, alloc_size);
788 macro->transform_later(new_alloc_bytes);
789 fast_oop_rawmem = macro->make_store(fast_oop_ctrl, store_eden_top, alloc_bytes_adr,
790 0, new_alloc_bytes, T_LONG);
791 }
792 return fast_oop;
793 }
794
795 void BarrierSetC2::clone_barrier_at_expansion(ArrayCopyNode* ac, Node* call, PhaseIterGVN& igvn) const {
796 // no barrier
797 igvn.replace_node(ac, call);
798 }