18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "gc/shared/c2/barrierSetC2.hpp"
27 #include "opto/arraycopynode.hpp"
28 #include "opto/convertnode.hpp"
29 #include "opto/graphKit.hpp"
30 #include "opto/idealKit.hpp"
31 #include "opto/macro.hpp"
32 #include "opto/narrowptrnode.hpp"
33 #include "utilities/macros.hpp"
34
35 // By default this is a no-op.
36 void BarrierSetC2::resolve_address(C2Access& access) const { }
37
38 void* C2Access::barrier_set_state() const {
39 return _kit->barrier_set_state();
40 }
41
42 bool C2Access::needs_cpu_membar() const {
43 bool mismatched = (_decorators & C2_MISMATCHED) != 0;
44 bool is_unordered = (_decorators & MO_UNORDERED) != 0;
45 bool anonymous = (_decorators & C2_UNSAFE_ACCESS) != 0;
46 bool in_heap = (_decorators & IN_HEAP) != 0;
47
48 bool is_write = (_decorators & C2_WRITE_ACCESS) != 0;
49 bool is_read = (_decorators & C2_READ_ACCESS) != 0;
50 bool is_atomic = is_read && is_write;
51
52 if (is_atomic) {
53 // Atomics always need to be wrapped in CPU membars
54 return true;
55 }
56
57 if (anonymous) {
58 // We will need memory barriers unless we can determine a unique
59 // alias category for this reference. (Note: If for some reason
60 // the barriers get omitted and the unsafe reference begins to "pollute"
61 // the alias analysis of the rest of the graph, either Compile::can_alias
62 // or Compile::must_alias will throw a diagnostic assert.)
63 if (!in_heap || !is_unordered || (mismatched && !_addr.type()->isa_aryptr())) {
64 return true;
65 }
66 }
67
68 return false;
69 }
70
71 Node* BarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& val) const {
72 DecoratorSet decorators = access.decorators();
73 GraphKit* kit = access.kit();
74
75 bool mismatched = (decorators & C2_MISMATCHED) != 0;
76 bool unaligned = (decorators & C2_UNALIGNED) != 0;
77 bool requires_atomic_access = (decorators & MO_UNORDERED) == 0;
78
79 bool in_native = (decorators & IN_NATIVE) != 0;
80 assert(!in_native, "not supported yet");
81
82 if (access.type() == T_DOUBLE) {
83 Node* new_val = kit->dstore_rounding(val.node());
84 val.set_node(new_val);
85 }
86
87 MemNode::MemOrd mo = access.mem_node_mo();
88
89 Node* store = kit->store_to_memory(kit->control(), access.addr().node(), val.node(), access.type(),
90 access.addr().type(), mo, requires_atomic_access, unaligned, mismatched);
91 access.set_raw_access(store);
92 return store;
93 }
94
95 Node* BarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const {
96 DecoratorSet decorators = access.decorators();
97 GraphKit* kit = access.kit();
98
99 Node* adr = access.addr().node();
100 const TypePtr* adr_type = access.addr().type();
101
102 bool mismatched = (decorators & C2_MISMATCHED) != 0;
103 bool requires_atomic_access = (decorators & MO_UNORDERED) == 0;
104 bool unaligned = (decorators & C2_UNALIGNED) != 0;
105 bool control_dependent = (decorators & C2_CONTROL_DEPENDENT_LOAD) != 0;
106 bool pinned = (decorators & C2_PINNED_LOAD) != 0;
107
108 bool in_native = (decorators & IN_NATIVE) != 0;
109
110 MemNode::MemOrd mo = access.mem_node_mo();
111 LoadNode::ControlDependency dep = pinned ? LoadNode::Pinned : LoadNode::DependsOnlyOnTest;
112 Node* control = control_dependent ? kit->control() : NULL;
113
114 Node* load;
115 if (in_native) {
116 load = kit->make_load(control, adr, val_type, access.type(), mo);
117 } else {
118 load = kit->make_load(control, adr, val_type, access.type(), adr_type, mo,
119 dep, requires_atomic_access, unaligned, mismatched);
120 }
121 access.set_raw_access(load);
122
123 return load;
124 }
125
126 class C2AccessFence: public StackObj {
127 C2Access& _access;
128 Node* _leading_membar;
129
130 public:
131 C2AccessFence(C2Access& access) :
132 _access(access), _leading_membar(NULL) {
133 GraphKit* kit = access.kit();
134 DecoratorSet decorators = access.decorators();
135
136 bool is_write = (decorators & C2_WRITE_ACCESS) != 0;
137 bool is_read = (decorators & C2_READ_ACCESS) != 0;
138 bool is_atomic = is_read && is_write;
139
140 bool is_volatile = (decorators & MO_SEQ_CST) != 0;
141 bool is_release = (decorators & MO_RELEASE) != 0;
142
143 if (is_atomic) {
144 // Memory-model-wise, a LoadStore acts like a little synchronized
145 // block, so needs barriers on each side. These don't translate
146 // into actual barriers on most machines, but we still need rest of
147 // compiler to respect ordering.
148 if (is_release) {
149 _leading_membar = kit->insert_mem_bar(Op_MemBarRelease);
150 } else if (is_volatile) {
151 if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
152 _leading_membar = kit->insert_mem_bar(Op_MemBarVolatile);
153 } else {
154 _leading_membar = kit->insert_mem_bar(Op_MemBarRelease);
155 }
156 }
157 } else if (is_write) {
158 // If reference is volatile, prevent following memory ops from
159 // floating down past the volatile write. Also prevents commoning
160 // another volatile read.
161 if (is_volatile || is_release) {
162 _leading_membar = kit->insert_mem_bar(Op_MemBarRelease);
163 }
164 } else {
165 // Memory barrier to prevent normal and 'unsafe' accesses from
166 // bypassing each other. Happens after null checks, so the
167 // exception paths do not take memory state from the memory barrier,
168 // so there's no problems making a strong assert about mixing users
169 // of safe & unsafe memory.
170 if (is_volatile && support_IRIW_for_not_multiple_copy_atomic_cpu) {
171 _leading_membar = kit->insert_mem_bar(Op_MemBarVolatile);
172 }
173 }
174
175 if (access.needs_cpu_membar()) {
176 kit->insert_mem_bar(Op_MemBarCPUOrder);
177 }
178
179 if (is_atomic) {
180 // 4984716: MemBars must be inserted before this
181 // memory node in order to avoid a false
182 // dependency which will confuse the scheduler.
183 access.set_memory();
184 }
185 }
186
187 ~C2AccessFence() {
188 GraphKit* kit = _access.kit();
189 DecoratorSet decorators = _access.decorators();
190
191 bool is_write = (decorators & C2_WRITE_ACCESS) != 0;
192 bool is_read = (decorators & C2_READ_ACCESS) != 0;
193 bool is_atomic = is_read && is_write;
194
195 bool is_volatile = (decorators & MO_SEQ_CST) != 0;
196 bool is_acquire = (decorators & MO_ACQUIRE) != 0;
197
198 // If reference is volatile, prevent following volatiles ops from
199 // floating up before the volatile access.
200 if (_access.needs_cpu_membar()) {
201 kit->insert_mem_bar(Op_MemBarCPUOrder);
202 }
203
204 if (is_atomic) {
205 if (is_acquire || is_volatile) {
206 Node* n = _access.raw_access();
207 Node* mb = kit->insert_mem_bar(Op_MemBarAcquire, n);
208 if (_leading_membar != NULL) {
209 MemBarNode::set_load_store_pair(_leading_membar->as_MemBar(), mb->as_MemBar());
210 }
211 }
212 } else if (is_write) {
213 // If not multiple copy atomic, we do the MemBarVolatile before the load.
214 if (is_volatile && !support_IRIW_for_not_multiple_copy_atomic_cpu) {
215 Node* n = _access.raw_access();
216 Node* mb = kit->insert_mem_bar(Op_MemBarVolatile, n); // Use fat membar
217 if (_leading_membar != NULL) {
218 MemBarNode::set_store_pair(_leading_membar->as_MemBar(), mb->as_MemBar());
219 }
220 }
221 } else {
222 if (is_volatile || is_acquire) {
223 Node* n = _access.raw_access();
224 assert(_leading_membar == NULL || support_IRIW_for_not_multiple_copy_atomic_cpu, "no leading membar expected");
225 Node* mb = kit->insert_mem_bar(Op_MemBarAcquire, n);
226 mb->as_MemBar()->set_trailing_load();
227 }
228 }
229 }
230 };
231
232 Node* BarrierSetC2::store_at(C2Access& access, C2AccessValue& val) const {
233 C2AccessFence fence(access);
234 resolve_address(access);
235 return store_at_resolved(access, val);
236 }
237
238 Node* BarrierSetC2::load_at(C2Access& access, const Type* val_type) const {
239 C2AccessFence fence(access);
240 resolve_address(access);
241 return load_at_resolved(access, val_type);
242 }
278 bool is_read = (_decorators & C2_READ_ACCESS) != 0;
279 bool is_write = (_decorators & C2_WRITE_ACCESS) != 0;
280
281 if (AlwaysAtomicAccesses && is_unordered) {
282 _decorators &= ~MO_DECORATOR_MASK; // clear the MO bits
283 _decorators |= MO_RELAXED; // Force the MO_RELAXED decorator with AlwaysAtomicAccess
284 }
285
286 _decorators = AccessInternal::decorator_fixup(_decorators);
287
288 if (is_read && !is_write && anonymous) {
289 // To be valid, unsafe loads may depend on other conditions than
290 // the one that guards them: pin the Load node
291 _decorators |= C2_CONTROL_DEPENDENT_LOAD;
292 _decorators |= C2_PINNED_LOAD;
293 const TypePtr* adr_type = _addr.type();
294 Node* adr = _addr.node();
295 if (!needs_cpu_membar() && adr_type->isa_instptr()) {
296 assert(adr_type->meet(TypePtr::NULL_PTR) != adr_type->remove_speculative(), "should be not null");
297 intptr_t offset = Type::OffsetBot;
298 AddPNode::Ideal_base_and_offset(adr, &_kit->gvn(), offset);
299 if (offset >= 0) {
300 int s = Klass::layout_helper_size_in_bytes(adr_type->isa_instptr()->klass()->layout_helper());
301 if (offset < s) {
302 // Guaranteed to be a valid access, no need to pin it
303 _decorators ^= C2_CONTROL_DEPENDENT_LOAD;
304 _decorators ^= C2_PINNED_LOAD;
305 }
306 }
307 }
308 }
309 }
310
311 //--------------------------- atomic operations---------------------------------
312
313 void BarrierSetC2::pin_atomic_op(C2AtomicAccess& access) const {
314 if (!access.needs_pinning()) {
315 return;
316 }
317 // SCMemProjNodes represent the memory state of a LoadStore. Their
318 // main role is to prevent LoadStore nodes from being optimized away
319 // when their results aren't used.
320 GraphKit* kit = access.kit();
321 Node* load_store = access.raw_access();
322 assert(load_store != NULL, "must pin atomic op");
323 Node* proj = kit->gvn().transform(new SCMemProjNode(load_store));
324 kit->set_memory(proj, access.alias_idx());
325 }
326
327 void C2AtomicAccess::set_memory() {
328 Node *mem = _kit->memory(_alias_idx);
329 _memory = mem;
330 }
331
332 Node* BarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicAccess& access, Node* expected_val,
333 Node* new_val, const Type* value_type) const {
334 GraphKit* kit = access.kit();
335 MemNode::MemOrd mo = access.mem_node_mo();
336 Node* mem = access.memory();
337
338 Node* adr = access.addr().node();
339 const TypePtr* adr_type = access.addr().type();
340
341 Node* load_store = NULL;
342
343 if (access.is_oop()) {
344 #ifdef _LP64
345 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
346 Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop()));
347 Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop()));
348 load_store = kit->gvn().transform(new CompareAndExchangeNNode(kit->control(), mem, adr, newval_enc, oldval_enc, adr_type, value_type->make_narrowoop(), mo));
349 } else
350 #endif
351 {
352 load_store = kit->gvn().transform(new CompareAndExchangePNode(kit->control(), mem, adr, new_val, expected_val, adr_type, value_type->is_oopptr(), mo));
369 load_store = kit->gvn().transform(new CompareAndExchangeLNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo));
370 break;
371 }
372 default:
373 ShouldNotReachHere();
374 }
375 }
376
377 access.set_raw_access(load_store);
378 pin_atomic_op(access);
379
380 #ifdef _LP64
381 if (access.is_oop() && adr->bottom_type()->is_ptr_to_narrowoop()) {
382 return kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type()));
383 }
384 #endif
385
386 return load_store;
387 }
388
389 Node* BarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicAccess& access, Node* expected_val,
390 Node* new_val, const Type* value_type) const {
391 GraphKit* kit = access.kit();
392 DecoratorSet decorators = access.decorators();
393 MemNode::MemOrd mo = access.mem_node_mo();
394 Node* mem = access.memory();
395 bool is_weak_cas = (decorators & C2_WEAK_CMPXCHG) != 0;
396 Node* load_store = NULL;
397 Node* adr = access.addr().node();
398
399 if (access.is_oop()) {
400 #ifdef _LP64
401 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
402 Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop()));
403 Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop()));
404 if (is_weak_cas) {
405 load_store = kit->gvn().transform(new WeakCompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo));
406 } else {
407 load_store = kit->gvn().transform(new CompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo));
408 }
409 } else
443 }
444 case T_LONG: {
445 if (is_weak_cas) {
446 load_store = kit->gvn().transform(new WeakCompareAndSwapLNode(kit->control(), mem, adr, new_val, expected_val, mo));
447 } else {
448 load_store = kit->gvn().transform(new CompareAndSwapLNode(kit->control(), mem, adr, new_val, expected_val, mo));
449 }
450 break;
451 }
452 default:
453 ShouldNotReachHere();
454 }
455 }
456
457 access.set_raw_access(load_store);
458 pin_atomic_op(access);
459
460 return load_store;
461 }
462
463 Node* BarrierSetC2::atomic_xchg_at_resolved(C2AtomicAccess& access, Node* new_val, const Type* value_type) const {
464 GraphKit* kit = access.kit();
465 Node* mem = access.memory();
466 Node* adr = access.addr().node();
467 const TypePtr* adr_type = access.addr().type();
468 Node* load_store = NULL;
469
470 if (access.is_oop()) {
471 #ifdef _LP64
472 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
473 Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop()));
474 load_store = kit->gvn().transform(new GetAndSetNNode(kit->control(), mem, adr, newval_enc, adr_type, value_type->make_narrowoop()));
475 } else
476 #endif
477 {
478 load_store = kit->gvn().transform(new GetAndSetPNode(kit->control(), mem, adr, new_val, adr_type, value_type->is_oopptr()));
479 }
480 } else {
481 switch (access.type()) {
482 case T_BYTE:
483 load_store = kit->gvn().transform(new GetAndSetBNode(kit->control(), mem, adr, new_val, adr_type));
491 case T_LONG:
492 load_store = kit->gvn().transform(new GetAndSetLNode(kit->control(), mem, adr, new_val, adr_type));
493 break;
494 default:
495 ShouldNotReachHere();
496 }
497 }
498
499 access.set_raw_access(load_store);
500 pin_atomic_op(access);
501
502 #ifdef _LP64
503 if (access.is_oop() && adr->bottom_type()->is_ptr_to_narrowoop()) {
504 return kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type()));
505 }
506 #endif
507
508 return load_store;
509 }
510
511 Node* BarrierSetC2::atomic_add_at_resolved(C2AtomicAccess& access, Node* new_val, const Type* value_type) const {
512 Node* load_store = NULL;
513 GraphKit* kit = access.kit();
514 Node* adr = access.addr().node();
515 const TypePtr* adr_type = access.addr().type();
516 Node* mem = access.memory();
517
518 switch(access.type()) {
519 case T_BYTE:
520 load_store = kit->gvn().transform(new GetAndAddBNode(kit->control(), mem, adr, new_val, adr_type));
521 break;
522 case T_SHORT:
523 load_store = kit->gvn().transform(new GetAndAddSNode(kit->control(), mem, adr, new_val, adr_type));
524 break;
525 case T_INT:
526 load_store = kit->gvn().transform(new GetAndAddINode(kit->control(), mem, adr, new_val, adr_type));
527 break;
528 case T_LONG:
529 load_store = kit->gvn().transform(new GetAndAddLNode(kit->control(), mem, adr, new_val, adr_type));
530 break;
531 default:
532 ShouldNotReachHere();
533 }
534
535 access.set_raw_access(load_store);
536 pin_atomic_op(access);
537
538 return load_store;
539 }
540
541 Node* BarrierSetC2::atomic_cmpxchg_val_at(C2AtomicAccess& access, Node* expected_val,
542 Node* new_val, const Type* value_type) const {
543 C2AccessFence fence(access);
544 resolve_address(access);
545 return atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type);
546 }
547
548 Node* BarrierSetC2::atomic_cmpxchg_bool_at(C2AtomicAccess& access, Node* expected_val,
549 Node* new_val, const Type* value_type) const {
550 C2AccessFence fence(access);
551 resolve_address(access);
552 return atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type);
553 }
554
555 Node* BarrierSetC2::atomic_xchg_at(C2AtomicAccess& access, Node* new_val, const Type* value_type) const {
556 C2AccessFence fence(access);
557 resolve_address(access);
558 return atomic_xchg_at_resolved(access, new_val, value_type);
559 }
560
561 Node* BarrierSetC2::atomic_add_at(C2AtomicAccess& access, Node* new_val, const Type* value_type) const {
562 C2AccessFence fence(access);
563 resolve_address(access);
564 return atomic_add_at_resolved(access, new_val, value_type);
565 }
566
567 void BarrierSetC2::clone(GraphKit* kit, Node* src, Node* dst, Node* size, bool is_array) const {
568 // Exclude the header but include array length to copy by 8 bytes words.
569 // Can't use base_offset_in_bytes(bt) since basic type is unknown.
570 int base_off = is_array ? arrayOopDesc::length_offset_in_bytes() :
571 instanceOopDesc::base_offset_in_bytes();
572 // base_off:
573 // 8 - 32-bit VM
574 // 12 - 64-bit VM, compressed klass
575 // 16 - 64-bit VM, normal klass
576 if (base_off % BytesPerLong != 0) {
577 assert(UseCompressedClassPointers, "");
578 if (is_array) {
579 // Exclude length to copy by 8 bytes words.
580 base_off += sizeof(int);
581 } else {
582 // Include klass to copy by 8 bytes words.
583 base_off = instanceOopDesc::klass_offset_in_bytes();
584 }
585 assert(base_off % BytesPerLong == 0, "expect 8 bytes alignment");
586 }
587 Node* src_base = kit->basic_plus_adr(src, base_off);
588 Node* dst_base = kit->basic_plus_adr(dst, base_off);
589
590 // Compute the length also, if needed:
591 Node* countx = size;
592 countx = kit->gvn().transform(new SubXNode(countx, kit->MakeConX(base_off)));
593 countx = kit->gvn().transform(new URShiftXNode(countx, kit->intcon(LogBytesPerLong) ));
594
595 const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
596
597 ArrayCopyNode* ac = ArrayCopyNode::make(kit, false, src_base, NULL, dst_base, NULL, countx, false, false);
598 ac->set_clonebasic();
599 Node* n = kit->gvn().transform(ac);
600 if (n == ac) {
601 ac->_adr_type = TypeRawPtr::BOTTOM;
602 kit->set_predefined_output_for_runtime_call(ac, ac->in(TypeFunc::Memory), raw_adr_type);
603 } else {
604 kit->set_all_memory(n);
605 }
606 }
607
608 Node* BarrierSetC2::obj_allocate(PhaseMacroExpand* macro, Node* ctrl, Node* mem, Node* toobig_false, Node* size_in_bytes,
609 Node*& i_o, Node*& needgc_ctrl,
610 Node*& fast_oop_ctrl, Node*& fast_oop_rawmem,
611 intx prefetch_lines) const {
612
613 Node* eden_top_adr;
614 Node* eden_end_adr;
615
616 macro->set_eden_pointers(eden_top_adr, eden_end_adr);
617
713
714 // Bump total allocated bytes for this thread
715 Node* thread = new ThreadLocalNode();
716 macro->transform_later(thread);
717 Node* alloc_bytes_adr = macro->basic_plus_adr(macro->top()/*not oop*/, thread,
718 in_bytes(JavaThread::allocated_bytes_offset()));
719 Node* alloc_bytes = macro->make_load(fast_oop_ctrl, store_eden_top, alloc_bytes_adr,
720 0, TypeLong::LONG, T_LONG);
721 #ifdef _LP64
722 Node* alloc_size = size_in_bytes;
723 #else
724 Node* alloc_size = new ConvI2LNode(size_in_bytes);
725 macro->transform_later(alloc_size);
726 #endif
727 Node* new_alloc_bytes = new AddLNode(alloc_bytes, alloc_size);
728 macro->transform_later(new_alloc_bytes);
729 fast_oop_rawmem = macro->make_store(fast_oop_ctrl, store_eden_top, alloc_bytes_adr,
730 0, new_alloc_bytes, T_LONG);
731 }
732 return fast_oop;
733 }
|
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "gc/shared/c2/barrierSetC2.hpp"
27 #include "opto/arraycopynode.hpp"
28 #include "opto/convertnode.hpp"
29 #include "opto/graphKit.hpp"
30 #include "opto/idealKit.hpp"
31 #include "opto/macro.hpp"
32 #include "opto/narrowptrnode.hpp"
33 #include "utilities/macros.hpp"
34
35 // By default this is a no-op.
36 void BarrierSetC2::resolve_address(C2Access& access) const { }
37
38 void* C2ParseAccess::barrier_set_state() const {
39 return _kit->barrier_set_state();
40 }
41
42 PhaseGVN& C2ParseAccess::gvn() const { return _kit->gvn(); }
43
44 bool C2Access::needs_cpu_membar() const {
45 bool mismatched = (_decorators & C2_MISMATCHED) != 0;
46 bool is_unordered = (_decorators & MO_UNORDERED) != 0;
47 bool anonymous = (_decorators & C2_UNSAFE_ACCESS) != 0;
48 bool in_heap = (_decorators & IN_HEAP) != 0;
49
50 bool is_write = (_decorators & C2_WRITE_ACCESS) != 0;
51 bool is_read = (_decorators & C2_READ_ACCESS) != 0;
52 bool is_atomic = is_read && is_write;
53
54 if (is_atomic) {
55 // Atomics always need to be wrapped in CPU membars
56 return true;
57 }
58
59 if (anonymous) {
60 // We will need memory barriers unless we can determine a unique
61 // alias category for this reference. (Note: If for some reason
62 // the barriers get omitted and the unsafe reference begins to "pollute"
63 // the alias analysis of the rest of the graph, either Compile::can_alias
64 // or Compile::must_alias will throw a diagnostic assert.)
65 if (!in_heap || !is_unordered || (mismatched && !_addr.type()->isa_aryptr())) {
66 return true;
67 }
68 }
69
70 return false;
71 }
72
73 Node* BarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& val) const {
74 DecoratorSet decorators = access.decorators();
75
76 bool mismatched = (decorators & C2_MISMATCHED) != 0;
77 bool unaligned = (decorators & C2_UNALIGNED) != 0;
78 bool requires_atomic_access = (decorators & MO_UNORDERED) == 0;
79
80 bool in_native = (decorators & IN_NATIVE) != 0;
81 assert(!in_native, "not supported yet");
82
83 MemNode::MemOrd mo = access.mem_node_mo();
84
85 Node* store;
86 if (access.is_parse_access()) {
87 C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
88
89 GraphKit* kit = parse_access.kit();
90 if (access.type() == T_DOUBLE) {
91 Node* new_val = kit->dstore_rounding(val.node());
92 val.set_node(new_val);
93 }
94
95 store = kit->store_to_memory(kit->control(), access.addr().node(), val.node(), access.type(),
96 access.addr().type(), mo, requires_atomic_access, unaligned, mismatched);
97 access.set_raw_access(store);
98 } else {
99 assert(!requires_atomic_access, "not yet supported");
100 assert(access.is_opt_access(), "either parse or opt access");
101 C2OptAccess& opt_access = static_cast<C2OptAccess&>(access);
102 Node* ctl = opt_access.ctl();
103 MergeMemNode* mm = opt_access.mem();
104 PhaseGVN& gvn = opt_access.gvn();
105 const TypePtr* adr_type = access.addr().type();
106 int alias = gvn.C->get_alias_index(adr_type);
107 Node* mem = mm->memory_at(alias);
108
109 StoreNode* st = StoreNode::make(gvn, ctl, mem, access.addr().node(), adr_type, val.node(), access.type(), mo);
110 if (unaligned) {
111 st->set_unaligned_access();
112 }
113 if (mismatched) {
114 st->set_mismatched_access();
115 }
116 store = gvn.transform(st);
117 if (store == st) {
118 mm->set_memory_at(alias, st);
119 }
120 }
121 return store;
122 }
123
124 Node* BarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const {
125 DecoratorSet decorators = access.decorators();
126
127 Node* adr = access.addr().node();
128 const TypePtr* adr_type = access.addr().type();
129
130 bool mismatched = (decorators & C2_MISMATCHED) != 0;
131 bool requires_atomic_access = (decorators & MO_UNORDERED) == 0;
132 bool unaligned = (decorators & C2_UNALIGNED) != 0;
133 bool control_dependent = (decorators & C2_CONTROL_DEPENDENT_LOAD) != 0;
134 bool pinned = (decorators & C2_PINNED_LOAD) != 0;
135
136 bool in_native = (decorators & IN_NATIVE) != 0;
137
138 MemNode::MemOrd mo = access.mem_node_mo();
139 LoadNode::ControlDependency dep = pinned ? LoadNode::Pinned : LoadNode::DependsOnlyOnTest;
140
141 Node* load;
142 if (access.is_parse_access()) {
143 C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
144 GraphKit* kit = parse_access.kit();
145 Node* control = control_dependent ? kit->control() : NULL;
146
147 if (in_native) {
148 load = kit->make_load(control, adr, val_type, access.type(), mo);
149 } else {
150 load = kit->make_load(control, adr, val_type, access.type(), adr_type, mo,
151 dep, requires_atomic_access, unaligned, mismatched);
152 }
153 access.set_raw_access(load);
154 } else {
155 assert(!requires_atomic_access, "not yet supported");
156 assert(access.is_opt_access(), "either parse or opt access");
157 C2OptAccess& opt_access = static_cast<C2OptAccess&>(access);
158 Node* control = control_dependent ? opt_access.ctl() : NULL;
159 MergeMemNode* mm = opt_access.mem();
160 PhaseGVN& gvn = opt_access.gvn();
161 Node* mem = mm->memory_at(gvn.C->get_alias_index(adr_type));
162 load = LoadNode::make(gvn, control, mem, adr, adr_type, val_type, access.type(), mo, dep, unaligned, mismatched);
163 load = gvn.transform(load);
164 }
165
166 return load;
167 }
168
169 class C2AccessFence: public StackObj {
170 C2Access& _access;
171 Node* _leading_membar;
172
173 public:
174 C2AccessFence(C2Access& access) :
175 _access(access), _leading_membar(NULL) {
176 GraphKit* kit = NULL;
177 if (access.is_parse_access()) {
178 C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
179 kit = parse_access.kit();
180 }
181 DecoratorSet decorators = access.decorators();
182
183 bool is_write = (decorators & C2_WRITE_ACCESS) != 0;
184 bool is_read = (decorators & C2_READ_ACCESS) != 0;
185 bool is_atomic = is_read && is_write;
186
187 bool is_volatile = (decorators & MO_SEQ_CST) != 0;
188 bool is_release = (decorators & MO_RELEASE) != 0;
189
190 if (is_atomic) {
191 assert(kit != NULL, "unsupported at optimization time");
192 // Memory-model-wise, a LoadStore acts like a little synchronized
193 // block, so needs barriers on each side. These don't translate
194 // into actual barriers on most machines, but we still need rest of
195 // compiler to respect ordering.
196 if (is_release) {
197 _leading_membar = kit->insert_mem_bar(Op_MemBarRelease);
198 } else if (is_volatile) {
199 if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
200 _leading_membar = kit->insert_mem_bar(Op_MemBarVolatile);
201 } else {
202 _leading_membar = kit->insert_mem_bar(Op_MemBarRelease);
203 }
204 }
205 } else if (is_write) {
206 // If reference is volatile, prevent following memory ops from
207 // floating down past the volatile write. Also prevents commoning
208 // another volatile read.
209 if (is_volatile || is_release) {
210 assert(kit != NULL, "unsupported at optimization time");
211 _leading_membar = kit->insert_mem_bar(Op_MemBarRelease);
212 }
213 } else {
214 // Memory barrier to prevent normal and 'unsafe' accesses from
215 // bypassing each other. Happens after null checks, so the
216 // exception paths do not take memory state from the memory barrier,
217 // so there's no problems making a strong assert about mixing users
218 // of safe & unsafe memory.
219 if (is_volatile && support_IRIW_for_not_multiple_copy_atomic_cpu) {
220 assert(kit != NULL, "unsupported at optimization time");
221 _leading_membar = kit->insert_mem_bar(Op_MemBarVolatile);
222 }
223 }
224
225 if (access.needs_cpu_membar()) {
226 assert(kit != NULL, "unsupported at optimization time");
227 kit->insert_mem_bar(Op_MemBarCPUOrder);
228 }
229
230 if (is_atomic) {
231 // 4984716: MemBars must be inserted before this
232 // memory node in order to avoid a false
233 // dependency which will confuse the scheduler.
234 access.set_memory();
235 }
236 }
237
238 ~C2AccessFence() {
239 GraphKit* kit = NULL;
240 if (_access.is_parse_access()) {
241 C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(_access);
242 kit = parse_access.kit();
243 }
244 DecoratorSet decorators = _access.decorators();
245
246 bool is_write = (decorators & C2_WRITE_ACCESS) != 0;
247 bool is_read = (decorators & C2_READ_ACCESS) != 0;
248 bool is_atomic = is_read && is_write;
249
250 bool is_volatile = (decorators & MO_SEQ_CST) != 0;
251 bool is_acquire = (decorators & MO_ACQUIRE) != 0;
252
253 // If reference is volatile, prevent following volatiles ops from
254 // floating up before the volatile access.
255 if (_access.needs_cpu_membar()) {
256 kit->insert_mem_bar(Op_MemBarCPUOrder);
257 }
258
259 if (is_atomic) {
260 assert(kit != NULL, "unsupported at optimization time");
261 if (is_acquire || is_volatile) {
262 Node* n = _access.raw_access();
263 Node* mb = kit->insert_mem_bar(Op_MemBarAcquire, n);
264 if (_leading_membar != NULL) {
265 MemBarNode::set_load_store_pair(_leading_membar->as_MemBar(), mb->as_MemBar());
266 }
267 }
268 } else if (is_write) {
269 // If not multiple copy atomic, we do the MemBarVolatile before the load.
270 if (is_volatile && !support_IRIW_for_not_multiple_copy_atomic_cpu) {
271 assert(kit != NULL, "unsupported at optimization time");
272 Node* n = _access.raw_access();
273 Node* mb = kit->insert_mem_bar(Op_MemBarVolatile, n); // Use fat membar
274 if (_leading_membar != NULL) {
275 MemBarNode::set_store_pair(_leading_membar->as_MemBar(), mb->as_MemBar());
276 }
277 }
278 } else {
279 if (is_volatile || is_acquire) {
280 assert(kit != NULL, "unsupported at optimization time");
281 Node* n = _access.raw_access();
282 assert(_leading_membar == NULL || support_IRIW_for_not_multiple_copy_atomic_cpu, "no leading membar expected");
283 Node* mb = kit->insert_mem_bar(Op_MemBarAcquire, n);
284 mb->as_MemBar()->set_trailing_load();
285 }
286 }
287 }
288 };
289
290 Node* BarrierSetC2::store_at(C2Access& access, C2AccessValue& val) const {
291 C2AccessFence fence(access);
292 resolve_address(access);
293 return store_at_resolved(access, val);
294 }
295
296 Node* BarrierSetC2::load_at(C2Access& access, const Type* val_type) const {
297 C2AccessFence fence(access);
298 resolve_address(access);
299 return load_at_resolved(access, val_type);
300 }
336 bool is_read = (_decorators & C2_READ_ACCESS) != 0;
337 bool is_write = (_decorators & C2_WRITE_ACCESS) != 0;
338
339 if (AlwaysAtomicAccesses && is_unordered) {
340 _decorators &= ~MO_DECORATOR_MASK; // clear the MO bits
341 _decorators |= MO_RELAXED; // Force the MO_RELAXED decorator with AlwaysAtomicAccess
342 }
343
344 _decorators = AccessInternal::decorator_fixup(_decorators);
345
346 if (is_read && !is_write && anonymous) {
347 // To be valid, unsafe loads may depend on other conditions than
348 // the one that guards them: pin the Load node
349 _decorators |= C2_CONTROL_DEPENDENT_LOAD;
350 _decorators |= C2_PINNED_LOAD;
351 const TypePtr* adr_type = _addr.type();
352 Node* adr = _addr.node();
353 if (!needs_cpu_membar() && adr_type->isa_instptr()) {
354 assert(adr_type->meet(TypePtr::NULL_PTR) != adr_type->remove_speculative(), "should be not null");
355 intptr_t offset = Type::OffsetBot;
356 AddPNode::Ideal_base_and_offset(adr, &gvn(), offset);
357 if (offset >= 0) {
358 int s = Klass::layout_helper_size_in_bytes(adr_type->isa_instptr()->klass()->layout_helper());
359 if (offset < s) {
360 // Guaranteed to be a valid access, no need to pin it
361 _decorators ^= C2_CONTROL_DEPENDENT_LOAD;
362 _decorators ^= C2_PINNED_LOAD;
363 }
364 }
365 }
366 }
367 }
368
369 //--------------------------- atomic operations---------------------------------
370
371 void BarrierSetC2::pin_atomic_op(C2AtomicParseAccess& access) const {
372 if (!access.needs_pinning()) {
373 return;
374 }
375 // SCMemProjNodes represent the memory state of a LoadStore. Their
376 // main role is to prevent LoadStore nodes from being optimized away
377 // when their results aren't used.
378 assert(access.is_parse_access(), "entry not supported at optimization time");
379 C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
380 GraphKit* kit = parse_access.kit();
381 Node* load_store = access.raw_access();
382 assert(load_store != NULL, "must pin atomic op");
383 Node* proj = kit->gvn().transform(new SCMemProjNode(load_store));
384 kit->set_memory(proj, access.alias_idx());
385 }
386
387 void C2AtomicParseAccess::set_memory() {
388 Node *mem = _kit->memory(_alias_idx);
389 _memory = mem;
390 }
391
392 Node* BarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
393 Node* new_val, const Type* value_type) const {
394 GraphKit* kit = access.kit();
395 MemNode::MemOrd mo = access.mem_node_mo();
396 Node* mem = access.memory();
397
398 Node* adr = access.addr().node();
399 const TypePtr* adr_type = access.addr().type();
400
401 Node* load_store = NULL;
402
403 if (access.is_oop()) {
404 #ifdef _LP64
405 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
406 Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop()));
407 Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop()));
408 load_store = kit->gvn().transform(new CompareAndExchangeNNode(kit->control(), mem, adr, newval_enc, oldval_enc, adr_type, value_type->make_narrowoop(), mo));
409 } else
410 #endif
411 {
412 load_store = kit->gvn().transform(new CompareAndExchangePNode(kit->control(), mem, adr, new_val, expected_val, adr_type, value_type->is_oopptr(), mo));
429 load_store = kit->gvn().transform(new CompareAndExchangeLNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo));
430 break;
431 }
432 default:
433 ShouldNotReachHere();
434 }
435 }
436
437 access.set_raw_access(load_store);
438 pin_atomic_op(access);
439
440 #ifdef _LP64
441 if (access.is_oop() && adr->bottom_type()->is_ptr_to_narrowoop()) {
442 return kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type()));
443 }
444 #endif
445
446 return load_store;
447 }
448
449 Node* BarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
450 Node* new_val, const Type* value_type) const {
451 GraphKit* kit = access.kit();
452 DecoratorSet decorators = access.decorators();
453 MemNode::MemOrd mo = access.mem_node_mo();
454 Node* mem = access.memory();
455 bool is_weak_cas = (decorators & C2_WEAK_CMPXCHG) != 0;
456 Node* load_store = NULL;
457 Node* adr = access.addr().node();
458
459 if (access.is_oop()) {
460 #ifdef _LP64
461 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
462 Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop()));
463 Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop()));
464 if (is_weak_cas) {
465 load_store = kit->gvn().transform(new WeakCompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo));
466 } else {
467 load_store = kit->gvn().transform(new CompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo));
468 }
469 } else
503 }
504 case T_LONG: {
505 if (is_weak_cas) {
506 load_store = kit->gvn().transform(new WeakCompareAndSwapLNode(kit->control(), mem, adr, new_val, expected_val, mo));
507 } else {
508 load_store = kit->gvn().transform(new CompareAndSwapLNode(kit->control(), mem, adr, new_val, expected_val, mo));
509 }
510 break;
511 }
512 default:
513 ShouldNotReachHere();
514 }
515 }
516
517 access.set_raw_access(load_store);
518 pin_atomic_op(access);
519
520 return load_store;
521 }
522
523 Node* BarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const {
524 GraphKit* kit = access.kit();
525 Node* mem = access.memory();
526 Node* adr = access.addr().node();
527 const TypePtr* adr_type = access.addr().type();
528 Node* load_store = NULL;
529
530 if (access.is_oop()) {
531 #ifdef _LP64
532 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
533 Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop()));
534 load_store = kit->gvn().transform(new GetAndSetNNode(kit->control(), mem, adr, newval_enc, adr_type, value_type->make_narrowoop()));
535 } else
536 #endif
537 {
538 load_store = kit->gvn().transform(new GetAndSetPNode(kit->control(), mem, adr, new_val, adr_type, value_type->is_oopptr()));
539 }
540 } else {
541 switch (access.type()) {
542 case T_BYTE:
543 load_store = kit->gvn().transform(new GetAndSetBNode(kit->control(), mem, adr, new_val, adr_type));
551 case T_LONG:
552 load_store = kit->gvn().transform(new GetAndSetLNode(kit->control(), mem, adr, new_val, adr_type));
553 break;
554 default:
555 ShouldNotReachHere();
556 }
557 }
558
559 access.set_raw_access(load_store);
560 pin_atomic_op(access);
561
562 #ifdef _LP64
563 if (access.is_oop() && adr->bottom_type()->is_ptr_to_narrowoop()) {
564 return kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type()));
565 }
566 #endif
567
568 return load_store;
569 }
570
571 Node* BarrierSetC2::atomic_add_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const {
572 Node* load_store = NULL;
573 GraphKit* kit = access.kit();
574 Node* adr = access.addr().node();
575 const TypePtr* adr_type = access.addr().type();
576 Node* mem = access.memory();
577
578 switch(access.type()) {
579 case T_BYTE:
580 load_store = kit->gvn().transform(new GetAndAddBNode(kit->control(), mem, adr, new_val, adr_type));
581 break;
582 case T_SHORT:
583 load_store = kit->gvn().transform(new GetAndAddSNode(kit->control(), mem, adr, new_val, adr_type));
584 break;
585 case T_INT:
586 load_store = kit->gvn().transform(new GetAndAddINode(kit->control(), mem, adr, new_val, adr_type));
587 break;
588 case T_LONG:
589 load_store = kit->gvn().transform(new GetAndAddLNode(kit->control(), mem, adr, new_val, adr_type));
590 break;
591 default:
592 ShouldNotReachHere();
593 }
594
595 access.set_raw_access(load_store);
596 pin_atomic_op(access);
597
598 return load_store;
599 }
600
601 Node* BarrierSetC2::atomic_cmpxchg_val_at(C2AtomicParseAccess& access, Node* expected_val,
602 Node* new_val, const Type* value_type) const {
603 C2AccessFence fence(access);
604 resolve_address(access);
605 return atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type);
606 }
607
608 Node* BarrierSetC2::atomic_cmpxchg_bool_at(C2AtomicParseAccess& access, Node* expected_val,
609 Node* new_val, const Type* value_type) const {
610 C2AccessFence fence(access);
611 resolve_address(access);
612 return atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type);
613 }
614
615 Node* BarrierSetC2::atomic_xchg_at(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const {
616 C2AccessFence fence(access);
617 resolve_address(access);
618 return atomic_xchg_at_resolved(access, new_val, value_type);
619 }
620
621 Node* BarrierSetC2::atomic_add_at(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const {
622 C2AccessFence fence(access);
623 resolve_address(access);
624 return atomic_add_at_resolved(access, new_val, value_type);
625 }
626
627 void BarrierSetC2::clone(GraphKit* kit, Node* src, Node* dst, Node* size, bool is_array) const {
628 // Exclude the header but include array length to copy by 8 bytes words.
629 // Can't use base_offset_in_bytes(bt) since basic type is unknown.
630 int base_off = is_array ? arrayOopDesc::length_offset_in_bytes() :
631 instanceOopDesc::base_offset_in_bytes();
632 // base_off:
633 // 8 - 32-bit VM
634 // 12 - 64-bit VM, compressed klass
635 // 16 - 64-bit VM, normal klass
636 if (base_off % BytesPerLong != 0) {
637 assert(UseCompressedClassPointers, "");
638 if (is_array) {
639 // Exclude length to copy by 8 bytes words.
640 base_off += sizeof(int);
641 } else {
642 // Include klass to copy by 8 bytes words.
643 base_off = instanceOopDesc::klass_offset_in_bytes();
644 }
645 assert(base_off % BytesPerLong == 0, "expect 8 bytes alignment");
646 }
647 Node* src_base = kit->basic_plus_adr(src, base_off);
648 Node* dst_base = kit->basic_plus_adr(dst, base_off);
649
650 // Compute the length also, if needed:
651 Node* countx = size;
652 countx = kit->gvn().transform(new SubXNode(countx, kit->MakeConX(base_off)));
653 countx = kit->gvn().transform(new URShiftXNode(countx, kit->intcon(LogBytesPerLong) ));
654
655 const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
656
657 ArrayCopyNode* ac = ArrayCopyNode::make(kit, false, src_base, NULL, dst_base, NULL, countx, true, false);
658 ac->set_clonebasic();
659 Node* n = kit->gvn().transform(ac);
660 if (n == ac) {
661 ac->_adr_type = TypeRawPtr::BOTTOM;
662 kit->set_predefined_output_for_runtime_call(ac, ac->in(TypeFunc::Memory), raw_adr_type);
663 } else {
664 kit->set_all_memory(n);
665 }
666 }
667
668 Node* BarrierSetC2::obj_allocate(PhaseMacroExpand* macro, Node* ctrl, Node* mem, Node* toobig_false, Node* size_in_bytes,
669 Node*& i_o, Node*& needgc_ctrl,
670 Node*& fast_oop_ctrl, Node*& fast_oop_rawmem,
671 intx prefetch_lines) const {
672
673 Node* eden_top_adr;
674 Node* eden_end_adr;
675
676 macro->set_eden_pointers(eden_top_adr, eden_end_adr);
677
773
774 // Bump total allocated bytes for this thread
775 Node* thread = new ThreadLocalNode();
776 macro->transform_later(thread);
777 Node* alloc_bytes_adr = macro->basic_plus_adr(macro->top()/*not oop*/, thread,
778 in_bytes(JavaThread::allocated_bytes_offset()));
779 Node* alloc_bytes = macro->make_load(fast_oop_ctrl, store_eden_top, alloc_bytes_adr,
780 0, TypeLong::LONG, T_LONG);
781 #ifdef _LP64
782 Node* alloc_size = size_in_bytes;
783 #else
784 Node* alloc_size = new ConvI2LNode(size_in_bytes);
785 macro->transform_later(alloc_size);
786 #endif
787 Node* new_alloc_bytes = new AddLNode(alloc_bytes, alloc_size);
788 macro->transform_later(new_alloc_bytes);
789 fast_oop_rawmem = macro->make_store(fast_oop_ctrl, store_eden_top, alloc_bytes_adr,
790 0, new_alloc_bytes, T_LONG);
791 }
792 return fast_oop;
793 }
794
795 void BarrierSetC2::clone_barrier_at_expansion(ArrayCopyNode* ac, Node* call, PhaseIterGVN& igvn) const {
796 // no barrier
797 igvn.replace_node(ac, call);
798 }
|