1 /*
2 * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "gc/shared/c2/barrierSetC2.hpp"
27 #include "opto/arraycopynode.hpp"
28 #include "opto/convertnode.hpp"
29 #include "opto/graphKit.hpp"
30 #include "opto/idealKit.hpp"
31 #include "opto/macro.hpp"
32 #include "opto/narrowptrnode.hpp"
33 #include "utilities/macros.hpp"
34
35 // By default this is a no-op.
36 void BarrierSetC2::resolve_address(C2Access& access) const { }
37
38 void* C2Access::barrier_set_state() const {
39 return _kit->barrier_set_state();
40 }
41
42 bool C2Access::needs_cpu_membar() const {
43 bool mismatched = (_decorators & C2_MISMATCHED) != 0;
44 bool is_unordered = (_decorators & MO_UNORDERED) != 0;
45 bool anonymous = (_decorators & C2_UNSAFE_ACCESS) != 0;
46 bool in_heap = (_decorators & IN_HEAP) != 0;
47
48 bool is_write = (_decorators & C2_WRITE_ACCESS) != 0;
49 bool is_read = (_decorators & C2_READ_ACCESS) != 0;
50 bool is_atomic = is_read && is_write;
51
52 if (is_atomic) {
53 // Atomics always need to be wrapped in CPU membars
54 return true;
55 }
56
57 if (anonymous) {
58 // We will need memory barriers unless we can determine a unique
59 // alias category for this reference. (Note: If for some reason
60 // the barriers get omitted and the unsafe reference begins to "pollute"
61 // the alias analysis of the rest of the graph, either Compile::can_alias
62 // or Compile::must_alias will throw a diagnostic assert.)
63 if (!in_heap || !is_unordered || (mismatched && !_addr.type()->isa_aryptr())) {
64 return true;
65 }
66 }
67
68 return false;
69 }
70
71 Node* BarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& val) const {
72 DecoratorSet decorators = access.decorators();
73 GraphKit* kit = access.kit();
74
75 bool mismatched = (decorators & C2_MISMATCHED) != 0;
76 bool unaligned = (decorators & C2_UNALIGNED) != 0;
77 bool requires_atomic_access = (decorators & MO_UNORDERED) == 0;
78
79 bool in_native = (decorators & IN_NATIVE) != 0;
80 assert(!in_native, "not supported yet");
81
82 if (access.type() == T_DOUBLE) {
83 Node* new_val = kit->dstore_rounding(val.node());
84 val.set_node(new_val);
85 }
86
87 MemNode::MemOrd mo = access.mem_node_mo();
88
89 Node* store = kit->store_to_memory(kit->control(), access.addr().node(), val.node(), access.type(),
90 access.addr().type(), mo, requires_atomic_access, unaligned, mismatched);
91 access.set_raw_access(store);
92 return store;
93 }
94
95 Node* BarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const {
96 DecoratorSet decorators = access.decorators();
97 GraphKit* kit = access.kit();
98
99 Node* adr = access.addr().node();
100 const TypePtr* adr_type = access.addr().type();
101
102 bool mismatched = (decorators & C2_MISMATCHED) != 0;
103 bool requires_atomic_access = (decorators & MO_UNORDERED) == 0;
104 bool unaligned = (decorators & C2_UNALIGNED) != 0;
105 bool control_dependent = (decorators & C2_CONTROL_DEPENDENT_LOAD) != 0;
106 bool pinned = (decorators & C2_PINNED_LOAD) != 0;
107
108 bool in_native = (decorators & IN_NATIVE) != 0;
109
110 MemNode::MemOrd mo = access.mem_node_mo();
111 LoadNode::ControlDependency dep = pinned ? LoadNode::Pinned : LoadNode::DependsOnlyOnTest;
112 Node* control = control_dependent ? kit->control() : NULL;
113
114 Node* load;
115 if (in_native) {
116 load = kit->make_load(control, adr, val_type, access.type(), mo);
117 } else {
118 load = kit->make_load(control, adr, val_type, access.type(), adr_type, mo,
119 dep, requires_atomic_access, unaligned, mismatched);
120 }
121 access.set_raw_access(load);
122
123 return load;
124 }
125
126 class C2AccessFence: public StackObj {
127 C2Access& _access;
128 Node* _leading_membar;
129
130 public:
131 C2AccessFence(C2Access& access) :
132 _access(access), _leading_membar(NULL) {
133 GraphKit* kit = access.kit();
134 DecoratorSet decorators = access.decorators();
135
136 bool is_write = (decorators & C2_WRITE_ACCESS) != 0;
137 bool is_read = (decorators & C2_READ_ACCESS) != 0;
138 bool is_atomic = is_read && is_write;
139
140 bool is_volatile = (decorators & MO_SEQ_CST) != 0;
141 bool is_release = (decorators & MO_RELEASE) != 0;
142
143 if (is_atomic) {
144 // Memory-model-wise, a LoadStore acts like a little synchronized
145 // block, so needs barriers on each side. These don't translate
146 // into actual barriers on most machines, but we still need rest of
147 // compiler to respect ordering.
148 if (is_release) {
149 _leading_membar = kit->insert_mem_bar(Op_MemBarRelease);
150 } else if (is_volatile) {
151 if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
152 _leading_membar = kit->insert_mem_bar(Op_MemBarVolatile);
153 } else {
154 _leading_membar = kit->insert_mem_bar(Op_MemBarRelease);
155 }
156 }
157 } else if (is_write) {
158 // If reference is volatile, prevent following memory ops from
159 // floating down past the volatile write. Also prevents commoning
160 // another volatile read.
161 if (is_volatile || is_release) {
162 _leading_membar = kit->insert_mem_bar(Op_MemBarRelease);
163 }
164 } else {
165 // Memory barrier to prevent normal and 'unsafe' accesses from
166 // bypassing each other. Happens after null checks, so the
167 // exception paths do not take memory state from the memory barrier,
168 // so there's no problems making a strong assert about mixing users
169 // of safe & unsafe memory.
170 if (is_volatile && support_IRIW_for_not_multiple_copy_atomic_cpu) {
171 _leading_membar = kit->insert_mem_bar(Op_MemBarVolatile);
172 }
173 }
174
175 if (access.needs_cpu_membar()) {
176 kit->insert_mem_bar(Op_MemBarCPUOrder);
177 }
178
179 if (is_atomic) {
180 // 4984716: MemBars must be inserted before this
181 // memory node in order to avoid a false
182 // dependency which will confuse the scheduler.
183 access.set_memory();
184 }
185 }
186
187 ~C2AccessFence() {
188 GraphKit* kit = _access.kit();
189 DecoratorSet decorators = _access.decorators();
190
191 bool is_write = (decorators & C2_WRITE_ACCESS) != 0;
192 bool is_read = (decorators & C2_READ_ACCESS) != 0;
193 bool is_atomic = is_read && is_write;
194
195 bool is_volatile = (decorators & MO_SEQ_CST) != 0;
196 bool is_acquire = (decorators & MO_ACQUIRE) != 0;
197
198 // If reference is volatile, prevent following volatiles ops from
199 // floating up before the volatile access.
200 if (_access.needs_cpu_membar()) {
201 kit->insert_mem_bar(Op_MemBarCPUOrder);
202 }
203
204 if (is_atomic) {
205 if (is_acquire || is_volatile) {
206 Node* n = _access.raw_access();
207 Node* mb = kit->insert_mem_bar(Op_MemBarAcquire, n);
208 if (_leading_membar != NULL) {
209 MemBarNode::set_load_store_pair(_leading_membar->as_MemBar(), mb->as_MemBar());
210 }
211 }
212 } else if (is_write) {
213 // If not multiple copy atomic, we do the MemBarVolatile before the load.
214 if (is_volatile && !support_IRIW_for_not_multiple_copy_atomic_cpu) {
215 Node* n = _access.raw_access();
216 Node* mb = kit->insert_mem_bar(Op_MemBarVolatile, n); // Use fat membar
217 if (_leading_membar != NULL) {
218 MemBarNode::set_store_pair(_leading_membar->as_MemBar(), mb->as_MemBar());
219 }
220 }
221 } else {
222 if (is_volatile || is_acquire) {
223 Node* n = _access.raw_access();
224 assert(_leading_membar == NULL || support_IRIW_for_not_multiple_copy_atomic_cpu, "no leading membar expected");
225 Node* mb = kit->insert_mem_bar(Op_MemBarAcquire, n);
226 mb->as_MemBar()->set_trailing_load();
227 }
228 }
229 }
230 };
231
232 Node* BarrierSetC2::store_at(C2Access& access, C2AccessValue& val) const {
233 C2AccessFence fence(access);
234 resolve_address(access);
235 return store_at_resolved(access, val);
236 }
237
238 Node* BarrierSetC2::load_at(C2Access& access, const Type* val_type) const {
239 C2AccessFence fence(access);
240 resolve_address(access);
241 return load_at_resolved(access, val_type);
242 }
243
244 MemNode::MemOrd C2Access::mem_node_mo() const {
245 bool is_write = (_decorators & C2_WRITE_ACCESS) != 0;
246 bool is_read = (_decorators & C2_READ_ACCESS) != 0;
247 if ((_decorators & MO_SEQ_CST) != 0) {
248 if (is_write && is_read) {
249 // For atomic operations
250 return MemNode::seqcst;
251 } else if (is_write) {
252 return MemNode::release;
253 } else {
254 assert(is_read, "what else?");
255 return MemNode::acquire;
256 }
257 } else if ((_decorators & MO_RELEASE) != 0) {
258 return MemNode::release;
259 } else if ((_decorators & MO_ACQUIRE) != 0) {
260 return MemNode::acquire;
261 } else if (is_write) {
262 // Volatile fields need releasing stores.
263 // Non-volatile fields also need releasing stores if they hold an
264 // object reference, because the object reference might point to
265 // a freshly created object.
266 // Conservatively release stores of object references.
267 return StoreNode::release_if_reference(_type);
268 } else {
269 return MemNode::unordered;
270 }
271 }
272
273 void C2Access::fixup_decorators() {
274 bool default_mo = (_decorators & MO_DECORATOR_MASK) == 0;
275 bool is_unordered = (_decorators & MO_UNORDERED) != 0 || default_mo;
276 bool anonymous = (_decorators & C2_UNSAFE_ACCESS) != 0;
277
278 bool is_read = (_decorators & C2_READ_ACCESS) != 0;
279 bool is_write = (_decorators & C2_WRITE_ACCESS) != 0;
280
281 if (AlwaysAtomicAccesses && is_unordered) {
282 _decorators &= ~MO_DECORATOR_MASK; // clear the MO bits
283 _decorators |= MO_RELAXED; // Force the MO_RELAXED decorator with AlwaysAtomicAccess
284 }
285
286 _decorators = AccessInternal::decorator_fixup(_decorators);
287
288 if (is_read && !is_write && anonymous) {
289 // To be valid, unsafe loads may depend on other conditions than
290 // the one that guards them: pin the Load node
291 _decorators |= C2_CONTROL_DEPENDENT_LOAD;
292 _decorators |= C2_PINNED_LOAD;
293 const TypePtr* adr_type = _addr.type();
294 Node* adr = _addr.node();
295 if (!needs_cpu_membar() && adr_type->isa_instptr()) {
296 assert(adr_type->meet(TypePtr::NULL_PTR) != adr_type->remove_speculative(), "should be not null");
297 intptr_t offset = Type::OffsetBot;
298 AddPNode::Ideal_base_and_offset(adr, &_kit->gvn(), offset);
299 if (offset >= 0) {
300 int s = Klass::layout_helper_size_in_bytes(adr_type->isa_instptr()->klass()->layout_helper());
301 if (offset < s) {
302 // Guaranteed to be a valid access, no need to pin it
303 _decorators ^= C2_CONTROL_DEPENDENT_LOAD;
304 _decorators ^= C2_PINNED_LOAD;
305 }
306 }
307 }
308 }
309 }
310
311 //--------------------------- atomic operations---------------------------------
312
313 void BarrierSetC2::pin_atomic_op(C2AtomicAccess& access) const {
314 if (!access.needs_pinning()) {
315 return;
316 }
317 // SCMemProjNodes represent the memory state of a LoadStore. Their
318 // main role is to prevent LoadStore nodes from being optimized away
319 // when their results aren't used.
320 GraphKit* kit = access.kit();
321 Node* load_store = access.raw_access();
322 assert(load_store != NULL, "must pin atomic op");
323 Node* proj = kit->gvn().transform(new SCMemProjNode(load_store));
324 kit->set_memory(proj, access.alias_idx());
325 }
326
327 void C2AtomicAccess::set_memory() {
328 Node *mem = _kit->memory(_alias_idx);
329 _memory = mem;
330 }
331
332 Node* BarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicAccess& access, Node* expected_val,
333 Node* new_val, const Type* value_type) const {
334 GraphKit* kit = access.kit();
335 MemNode::MemOrd mo = access.mem_node_mo();
336 Node* mem = access.memory();
337
338 Node* adr = access.addr().node();
339 const TypePtr* adr_type = access.addr().type();
340
341 Node* load_store = NULL;
342
343 if (access.is_oop()) {
344 #ifdef _LP64
345 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
346 Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop()));
347 Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop()));
348 load_store = kit->gvn().transform(new CompareAndExchangeNNode(kit->control(), mem, adr, newval_enc, oldval_enc, adr_type, value_type->make_narrowoop(), mo));
349 } else
350 #endif
351 {
352 load_store = kit->gvn().transform(new CompareAndExchangePNode(kit->control(), mem, adr, new_val, expected_val, adr_type, value_type->is_oopptr(), mo));
353 }
354 } else {
355 switch (access.type()) {
356 case T_BYTE: {
357 load_store = kit->gvn().transform(new CompareAndExchangeBNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo));
358 break;
359 }
360 case T_SHORT: {
361 load_store = kit->gvn().transform(new CompareAndExchangeSNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo));
362 break;
363 }
364 case T_INT: {
365 load_store = kit->gvn().transform(new CompareAndExchangeINode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo));
366 break;
367 }
368 case T_LONG: {
369 load_store = kit->gvn().transform(new CompareAndExchangeLNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo));
370 break;
371 }
372 default:
373 ShouldNotReachHere();
374 }
375 }
376
377 access.set_raw_access(load_store);
378 pin_atomic_op(access);
379
380 #ifdef _LP64
381 if (access.is_oop() && adr->bottom_type()->is_ptr_to_narrowoop()) {
382 return kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type()));
383 }
384 #endif
385
386 return load_store;
387 }
388
389 Node* BarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicAccess& access, Node* expected_val,
390 Node* new_val, const Type* value_type) const {
391 GraphKit* kit = access.kit();
392 DecoratorSet decorators = access.decorators();
393 MemNode::MemOrd mo = access.mem_node_mo();
394 Node* mem = access.memory();
395 bool is_weak_cas = (decorators & C2_WEAK_CMPXCHG) != 0;
396 Node* load_store = NULL;
397 Node* adr = access.addr().node();
398
399 if (access.is_oop()) {
400 #ifdef _LP64
401 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
402 Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop()));
403 Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop()));
404 if (is_weak_cas) {
405 load_store = kit->gvn().transform(new WeakCompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo));
406 } else {
407 load_store = kit->gvn().transform(new CompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo));
408 }
409 } else
410 #endif
411 {
412 if (is_weak_cas) {
413 load_store = kit->gvn().transform(new WeakCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo));
414 } else {
415 load_store = kit->gvn().transform(new CompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo));
416 }
417 }
418 } else {
419 switch(access.type()) {
420 case T_BYTE: {
421 if (is_weak_cas) {
422 load_store = kit->gvn().transform(new WeakCompareAndSwapBNode(kit->control(), mem, adr, new_val, expected_val, mo));
423 } else {
424 load_store = kit->gvn().transform(new CompareAndSwapBNode(kit->control(), mem, adr, new_val, expected_val, mo));
425 }
426 break;
427 }
428 case T_SHORT: {
429 if (is_weak_cas) {
430 load_store = kit->gvn().transform(new WeakCompareAndSwapSNode(kit->control(), mem, adr, new_val, expected_val, mo));
431 } else {
432 load_store = kit->gvn().transform(new CompareAndSwapSNode(kit->control(), mem, adr, new_val, expected_val, mo));
433 }
434 break;
435 }
436 case T_INT: {
437 if (is_weak_cas) {
438 load_store = kit->gvn().transform(new WeakCompareAndSwapINode(kit->control(), mem, adr, new_val, expected_val, mo));
439 } else {
440 load_store = kit->gvn().transform(new CompareAndSwapINode(kit->control(), mem, adr, new_val, expected_val, mo));
441 }
442 break;
443 }
444 case T_LONG: {
445 if (is_weak_cas) {
446 load_store = kit->gvn().transform(new WeakCompareAndSwapLNode(kit->control(), mem, adr, new_val, expected_val, mo));
447 } else {
448 load_store = kit->gvn().transform(new CompareAndSwapLNode(kit->control(), mem, adr, new_val, expected_val, mo));
449 }
450 break;
451 }
452 default:
453 ShouldNotReachHere();
454 }
455 }
456
457 access.set_raw_access(load_store);
458 pin_atomic_op(access);
459
460 return load_store;
461 }
462
463 Node* BarrierSetC2::atomic_xchg_at_resolved(C2AtomicAccess& access, Node* new_val, const Type* value_type) const {
464 GraphKit* kit = access.kit();
465 Node* mem = access.memory();
466 Node* adr = access.addr().node();
467 const TypePtr* adr_type = access.addr().type();
468 Node* load_store = NULL;
469
470 if (access.is_oop()) {
471 #ifdef _LP64
472 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
473 Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop()));
474 load_store = kit->gvn().transform(new GetAndSetNNode(kit->control(), mem, adr, newval_enc, adr_type, value_type->make_narrowoop()));
475 } else
476 #endif
477 {
478 load_store = kit->gvn().transform(new GetAndSetPNode(kit->control(), mem, adr, new_val, adr_type, value_type->is_oopptr()));
479 }
480 } else {
481 switch (access.type()) {
482 case T_BYTE:
483 load_store = kit->gvn().transform(new GetAndSetBNode(kit->control(), mem, adr, new_val, adr_type));
484 break;
485 case T_SHORT:
486 load_store = kit->gvn().transform(new GetAndSetSNode(kit->control(), mem, adr, new_val, adr_type));
487 break;
488 case T_INT:
489 load_store = kit->gvn().transform(new GetAndSetINode(kit->control(), mem, adr, new_val, adr_type));
490 break;
491 case T_LONG:
492 load_store = kit->gvn().transform(new GetAndSetLNode(kit->control(), mem, adr, new_val, adr_type));
493 break;
494 default:
495 ShouldNotReachHere();
496 }
497 }
498
499 access.set_raw_access(load_store);
500 pin_atomic_op(access);
501
502 #ifdef _LP64
503 if (access.is_oop() && adr->bottom_type()->is_ptr_to_narrowoop()) {
504 return kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type()));
505 }
506 #endif
507
508 return load_store;
509 }
510
511 Node* BarrierSetC2::atomic_add_at_resolved(C2AtomicAccess& access, Node* new_val, const Type* value_type) const {
512 Node* load_store = NULL;
513 GraphKit* kit = access.kit();
514 Node* adr = access.addr().node();
515 const TypePtr* adr_type = access.addr().type();
516 Node* mem = access.memory();
517
518 switch(access.type()) {
519 case T_BYTE:
520 load_store = kit->gvn().transform(new GetAndAddBNode(kit->control(), mem, adr, new_val, adr_type));
521 break;
522 case T_SHORT:
523 load_store = kit->gvn().transform(new GetAndAddSNode(kit->control(), mem, adr, new_val, adr_type));
524 break;
525 case T_INT:
526 load_store = kit->gvn().transform(new GetAndAddINode(kit->control(), mem, adr, new_val, adr_type));
527 break;
528 case T_LONG:
529 load_store = kit->gvn().transform(new GetAndAddLNode(kit->control(), mem, adr, new_val, adr_type));
530 break;
531 default:
532 ShouldNotReachHere();
533 }
534
535 access.set_raw_access(load_store);
536 pin_atomic_op(access);
537
538 return load_store;
539 }
540
541 Node* BarrierSetC2::atomic_cmpxchg_val_at(C2AtomicAccess& access, Node* expected_val,
542 Node* new_val, const Type* value_type) const {
543 C2AccessFence fence(access);
544 resolve_address(access);
545 return atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type);
546 }
547
548 Node* BarrierSetC2::atomic_cmpxchg_bool_at(C2AtomicAccess& access, Node* expected_val,
549 Node* new_val, const Type* value_type) const {
550 C2AccessFence fence(access);
551 resolve_address(access);
552 return atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type);
553 }
554
555 Node* BarrierSetC2::atomic_xchg_at(C2AtomicAccess& access, Node* new_val, const Type* value_type) const {
556 C2AccessFence fence(access);
557 resolve_address(access);
558 return atomic_xchg_at_resolved(access, new_val, value_type);
559 }
560
561 Node* BarrierSetC2::atomic_add_at(C2AtomicAccess& access, Node* new_val, const Type* value_type) const {
562 C2AccessFence fence(access);
563 resolve_address(access);
564 return atomic_add_at_resolved(access, new_val, value_type);
565 }
566
567 void BarrierSetC2::clone(GraphKit* kit, Node* src, Node* dst, Node* size, bool is_array) const {
568 // Exclude the header but include array length to copy by 8 bytes words.
569 // Can't use base_offset_in_bytes(bt) since basic type is unknown.
570 int base_off = is_array ? arrayOopDesc::length_offset_in_bytes() :
571 instanceOopDesc::base_offset_in_bytes();
572 // base_off:
573 // 8 - 32-bit VM
574 // 12 - 64-bit VM, compressed klass
575 // 16 - 64-bit VM, normal klass
576 if (base_off % BytesPerLong != 0) {
577 assert(UseCompressedClassPointers, "");
578 if (is_array) {
579 // Exclude length to copy by 8 bytes words.
580 base_off += sizeof(int);
581 } else {
582 // Include klass to copy by 8 bytes words.
583 base_off = instanceOopDesc::klass_offset_in_bytes();
584 }
585 assert(base_off % BytesPerLong == 0, "expect 8 bytes alignment");
586 }
587 Node* src_base = kit->basic_plus_adr(src, base_off);
588 Node* dst_base = kit->basic_plus_adr(dst, base_off);
589
590 // Compute the length also, if needed:
591 Node* countx = size;
592 countx = kit->gvn().transform(new SubXNode(countx, kit->MakeConX(base_off)));
593 countx = kit->gvn().transform(new URShiftXNode(countx, kit->intcon(LogBytesPerLong) ));
594
595 const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
596
597 ArrayCopyNode* ac = ArrayCopyNode::make(kit, false, src_base, NULL, dst_base, NULL, countx, false, false);
598 ac->set_clonebasic();
599 Node* n = kit->gvn().transform(ac);
600 if (n == ac) {
601 ac->_adr_type = TypeRawPtr::BOTTOM;
602 kit->set_predefined_output_for_runtime_call(ac, ac->in(TypeFunc::Memory), raw_adr_type);
603 } else {
604 kit->set_all_memory(n);
605 }
606 }
607
608 Node* BarrierSetC2::obj_allocate(PhaseMacroExpand* macro, Node* ctrl, Node* mem, Node* toobig_false, Node* size_in_bytes,
609 Node*& i_o, Node*& needgc_ctrl,
610 Node*& fast_oop_ctrl, Node*& fast_oop_rawmem,
611 intx prefetch_lines) const {
612
613 Node* eden_top_adr;
614 Node* eden_end_adr;
615
616 macro->set_eden_pointers(eden_top_adr, eden_end_adr);
617
618 // Load Eden::end. Loop invariant and hoisted.
619 //
620 // Note: We set the control input on "eden_end" and "old_eden_top" when using
621 // a TLAB to work around a bug where these values were being moved across
622 // a safepoint. These are not oops, so they cannot be include in the oop
623 // map, but they can be changed by a GC. The proper way to fix this would
624 // be to set the raw memory state when generating a SafepointNode. However
625 // this will require extensive changes to the loop optimization in order to
626 // prevent a degradation of the optimization.
627 // See comment in memnode.hpp, around line 227 in class LoadPNode.
628 Node *eden_end = macro->make_load(ctrl, mem, eden_end_adr, 0, TypeRawPtr::BOTTOM, T_ADDRESS);
629
630 // We need a Region for the loop-back contended case.
631 enum { fall_in_path = 1, contended_loopback_path = 2 };
632 Node *contended_region;
633 Node *contended_phi_rawmem;
634 if (UseTLAB) {
635 contended_region = toobig_false;
636 contended_phi_rawmem = mem;
637 } else {
638 contended_region = new RegionNode(3);
639 contended_phi_rawmem = new PhiNode(contended_region, Type::MEMORY, TypeRawPtr::BOTTOM);
640 // Now handle the passing-too-big test. We fall into the contended
641 // loop-back merge point.
642 contended_region ->init_req(fall_in_path, toobig_false);
643 contended_phi_rawmem->init_req(fall_in_path, mem);
644 macro->transform_later(contended_region);
645 macro->transform_later(contended_phi_rawmem);
646 }
647
648 // Load(-locked) the heap top.
649 // See note above concerning the control input when using a TLAB
650 Node *old_eden_top = UseTLAB
651 ? new LoadPNode (ctrl, contended_phi_rawmem, eden_top_adr, TypeRawPtr::BOTTOM, TypeRawPtr::BOTTOM, MemNode::unordered)
652 : new LoadPLockedNode(contended_region, contended_phi_rawmem, eden_top_adr, MemNode::acquire);
653
654 macro->transform_later(old_eden_top);
655 // Add to heap top to get a new heap top
656 Node *new_eden_top = new AddPNode(macro->top(), old_eden_top, size_in_bytes);
657 macro->transform_later(new_eden_top);
658 // Check for needing a GC; compare against heap end
659 Node *needgc_cmp = new CmpPNode(new_eden_top, eden_end);
660 macro->transform_later(needgc_cmp);
661 Node *needgc_bol = new BoolNode(needgc_cmp, BoolTest::ge);
662 macro->transform_later(needgc_bol);
663 IfNode *needgc_iff = new IfNode(contended_region, needgc_bol, PROB_UNLIKELY_MAG(4), COUNT_UNKNOWN);
664 macro->transform_later(needgc_iff);
665
666 // Plug the failing-heap-space-need-gc test into the slow-path region
667 Node *needgc_true = new IfTrueNode(needgc_iff);
668 macro->transform_later(needgc_true);
669 needgc_ctrl = needgc_true;
670
671 // No need for a GC. Setup for the Store-Conditional
672 Node *needgc_false = new IfFalseNode(needgc_iff);
673 macro->transform_later(needgc_false);
674
675 i_o = macro->prefetch_allocation(i_o, needgc_false, contended_phi_rawmem,
676 old_eden_top, new_eden_top, prefetch_lines);
677
678 Node* fast_oop = old_eden_top;
679
680 // Store (-conditional) the modified eden top back down.
681 // StorePConditional produces flags for a test PLUS a modified raw
682 // memory state.
683 if (UseTLAB) {
684 Node* store_eden_top =
685 new StorePNode(needgc_false, contended_phi_rawmem, eden_top_adr,
686 TypeRawPtr::BOTTOM, new_eden_top, MemNode::unordered);
687 macro->transform_later(store_eden_top);
688 fast_oop_ctrl = needgc_false; // No contention, so this is the fast path
689 fast_oop_rawmem = store_eden_top;
690 } else {
691 Node* store_eden_top =
692 new StorePConditionalNode(needgc_false, contended_phi_rawmem, eden_top_adr,
693 new_eden_top, fast_oop/*old_eden_top*/);
694 macro->transform_later(store_eden_top);
695 Node *contention_check = new BoolNode(store_eden_top, BoolTest::ne);
696 macro->transform_later(contention_check);
697 store_eden_top = new SCMemProjNode(store_eden_top);
698 macro->transform_later(store_eden_top);
699
700 // If not using TLABs, check to see if there was contention.
701 IfNode *contention_iff = new IfNode (needgc_false, contention_check, PROB_MIN, COUNT_UNKNOWN);
702 macro->transform_later(contention_iff);
703 Node *contention_true = new IfTrueNode(contention_iff);
704 macro->transform_later(contention_true);
705 // If contention, loopback and try again.
706 contended_region->init_req(contended_loopback_path, contention_true);
707 contended_phi_rawmem->init_req(contended_loopback_path, store_eden_top);
708
709 // Fast-path succeeded with no contention!
710 Node *contention_false = new IfFalseNode(contention_iff);
711 macro->transform_later(contention_false);
712 fast_oop_ctrl = contention_false;
713
714 // Bump total allocated bytes for this thread
715 Node* thread = new ThreadLocalNode();
716 macro->transform_later(thread);
717 Node* alloc_bytes_adr = macro->basic_plus_adr(macro->top()/*not oop*/, thread,
718 in_bytes(JavaThread::allocated_bytes_offset()));
719 Node* alloc_bytes = macro->make_load(fast_oop_ctrl, store_eden_top, alloc_bytes_adr,
720 0, TypeLong::LONG, T_LONG);
721 #ifdef _LP64
722 Node* alloc_size = size_in_bytes;
723 #else
724 Node* alloc_size = new ConvI2LNode(size_in_bytes);
725 macro->transform_later(alloc_size);
726 #endif
727 Node* new_alloc_bytes = new AddLNode(alloc_bytes, alloc_size);
728 macro->transform_later(new_alloc_bytes);
729 fast_oop_rawmem = macro->make_store(fast_oop_ctrl, store_eden_top, alloc_bytes_adr,
730 0, new_alloc_bytes, T_LONG);
731 }
732 return fast_oop;
733 }