1 /*
2 * Copyright (c) 2018, 2019, Red Hat, Inc. All rights reserved.
3 *
4 * This code is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 only, as
6 * published by the Free Software Foundation.
7 *
8 * This code is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 * version 2 for more details (a copy is included in the LICENSE file that
12 * accompanied this code).
13 *
14 * You should have received a copy of the GNU General Public License version
15 * 2 along with this work; if not, write to the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 #include "precompiled.hpp"
25 #include "gc/shared/barrierSet.hpp"
26 #include "gc/shenandoah/shenandoahHeap.hpp"
27 #include "gc/shenandoah/shenandoahHeuristics.hpp"
28 #include "gc/shenandoah/shenandoahRuntime.hpp"
29 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
30 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
31 #include "gc/shenandoah/c2/shenandoahSupport.hpp"
32 #include "opto/arraycopynode.hpp"
33 #include "opto/escape.hpp"
34 #include "opto/graphKit.hpp"
35 #include "opto/idealKit.hpp"
36 #include "opto/macro.hpp"
37 #include "opto/movenode.hpp"
38 #include "opto/narrowptrnode.hpp"
39 #include "opto/rootnode.hpp"
40
41 ShenandoahBarrierSetC2* ShenandoahBarrierSetC2::bsc2() {
42 return reinterpret_cast<ShenandoahBarrierSetC2*>(BarrierSet::barrier_set()->barrier_set_c2());
43 }
44
45 ShenandoahBarrierSetC2State::ShenandoahBarrierSetC2State(Arena* comp_arena)
46 : _enqueue_barriers(new (comp_arena) GrowableArray<ShenandoahEnqueueBarrierNode*>(comp_arena, 8, 0, NULL)),
47 _load_reference_barriers(new (comp_arena) GrowableArray<ShenandoahLoadReferenceBarrierNode*>(comp_arena, 8, 0, NULL)) {
48 }
49
50 int ShenandoahBarrierSetC2State::enqueue_barriers_count() const {
51 return _enqueue_barriers->length();
52 }
53
54 ShenandoahEnqueueBarrierNode* ShenandoahBarrierSetC2State::enqueue_barrier(int idx) const {
55 return _enqueue_barriers->at(idx);
56 }
57
58 void ShenandoahBarrierSetC2State::add_enqueue_barrier(ShenandoahEnqueueBarrierNode * n) {
59 assert(!_enqueue_barriers->contains(n), "duplicate entry in barrier list");
60 _enqueue_barriers->append(n);
61 }
62
63 void ShenandoahBarrierSetC2State::remove_enqueue_barrier(ShenandoahEnqueueBarrierNode * n) {
64 if (_enqueue_barriers->contains(n)) {
65 _enqueue_barriers->remove(n);
66 }
67 }
68
69 int ShenandoahBarrierSetC2State::load_reference_barriers_count() const {
70 return _load_reference_barriers->length();
71 }
72
73 ShenandoahLoadReferenceBarrierNode* ShenandoahBarrierSetC2State::load_reference_barrier(int idx) const {
74 return _load_reference_barriers->at(idx);
75 }
76
77 void ShenandoahBarrierSetC2State::add_load_reference_barrier(ShenandoahLoadReferenceBarrierNode * n) {
78 assert(!_load_reference_barriers->contains(n), "duplicate entry in barrier list");
79 _load_reference_barriers->append(n);
80 }
81
82 void ShenandoahBarrierSetC2State::remove_load_reference_barrier(ShenandoahLoadReferenceBarrierNode * n) {
83 if (_load_reference_barriers->contains(n)) {
84 _load_reference_barriers->remove(n);
85 }
86 }
87
88 Node* ShenandoahBarrierSetC2::shenandoah_storeval_barrier(GraphKit* kit, Node* obj) const {
89 if (ShenandoahStoreValEnqueueBarrier) {
90 obj = shenandoah_enqueue_barrier(kit, obj);
91 }
92 return obj;
93 }
94
95 #define __ kit->
96
97 bool ShenandoahBarrierSetC2::satb_can_remove_pre_barrier(GraphKit* kit, PhaseTransform* phase, Node* adr,
98 BasicType bt, uint adr_idx) const {
99 intptr_t offset = 0;
100 Node* base = AddPNode::Ideal_base_and_offset(adr, phase, offset);
101 AllocateNode* alloc = AllocateNode::Ideal_allocation(base, phase);
102
103 if (offset == Type::OffsetBot) {
104 return false; // cannot unalias unless there are precise offsets
105 }
106
107 if (alloc == NULL) {
108 return false; // No allocation found
109 }
110
111 intptr_t size_in_bytes = type2aelembytes(bt);
112
113 Node* mem = __ memory(adr_idx); // start searching here...
114
115 for (int cnt = 0; cnt < 50; cnt++) {
116
117 if (mem->is_Store()) {
118
119 Node* st_adr = mem->in(MemNode::Address);
120 intptr_t st_offset = 0;
121 Node* st_base = AddPNode::Ideal_base_and_offset(st_adr, phase, st_offset);
122
123 if (st_base == NULL) {
124 break; // inscrutable pointer
125 }
126
127 // Break we have found a store with same base and offset as ours so break
128 if (st_base == base && st_offset == offset) {
129 break;
130 }
131
132 if (st_offset != offset && st_offset != Type::OffsetBot) {
133 const int MAX_STORE = BytesPerLong;
134 if (st_offset >= offset + size_in_bytes ||
135 st_offset <= offset - MAX_STORE ||
136 st_offset <= offset - mem->as_Store()->memory_size()) {
137 // Success: The offsets are provably independent.
138 // (You may ask, why not just test st_offset != offset and be done?
139 // The answer is that stores of different sizes can co-exist
140 // in the same sequence of RawMem effects. We sometimes initialize
141 // a whole 'tile' of array elements with a single jint or jlong.)
142 mem = mem->in(MemNode::Memory);
143 continue; // advance through independent store memory
144 }
145 }
146
147 if (st_base != base
148 && MemNode::detect_ptr_independence(base, alloc, st_base,
149 AllocateNode::Ideal_allocation(st_base, phase),
150 phase)) {
151 // Success: The bases are provably independent.
152 mem = mem->in(MemNode::Memory);
153 continue; // advance through independent store memory
154 }
155 } else if (mem->is_Proj() && mem->in(0)->is_Initialize()) {
156
157 InitializeNode* st_init = mem->in(0)->as_Initialize();
158 AllocateNode* st_alloc = st_init->allocation();
159
160 // Make sure that we are looking at the same allocation site.
161 // The alloc variable is guaranteed to not be null here from earlier check.
162 if (alloc == st_alloc) {
163 // Check that the initialization is storing NULL so that no previous store
164 // has been moved up and directly write a reference
165 Node* captured_store = st_init->find_captured_store(offset,
166 type2aelembytes(T_OBJECT),
167 phase);
168 if (captured_store == NULL || captured_store == st_init->zero_memory()) {
169 return true;
170 }
171 }
172 }
173
174 // Unless there is an explicit 'continue', we must bail out here,
175 // because 'mem' is an inscrutable memory state (e.g., a call).
176 break;
177 }
178
179 return false;
180 }
181
182 #undef __
183 #define __ ideal.
184
185 void ShenandoahBarrierSetC2::satb_write_barrier_pre(GraphKit* kit,
186 bool do_load,
187 Node* obj,
188 Node* adr,
189 uint alias_idx,
190 Node* val,
191 const TypeOopPtr* val_type,
192 Node* pre_val,
193 BasicType bt) const {
194 // Some sanity checks
195 // Note: val is unused in this routine.
196
197 if (do_load) {
198 // We need to generate the load of the previous value
199 assert(obj != NULL, "must have a base");
200 assert(adr != NULL, "where are loading from?");
201 assert(pre_val == NULL, "loaded already?");
202 assert(val_type != NULL, "need a type");
203
204 if (ReduceInitialCardMarks
205 && satb_can_remove_pre_barrier(kit, &kit->gvn(), adr, bt, alias_idx)) {
206 return;
207 }
208
209 } else {
210 // In this case both val_type and alias_idx are unused.
211 assert(pre_val != NULL, "must be loaded already");
212 // Nothing to be done if pre_val is null.
213 if (pre_val->bottom_type() == TypePtr::NULL_PTR) return;
214 assert(pre_val->bottom_type()->basic_type() == T_OBJECT, "or we shouldn't be here");
215 }
216 assert(bt == T_OBJECT, "or we shouldn't be here");
217
218 IdealKit ideal(kit, true);
219
220 Node* tls = __ thread(); // ThreadLocalStorage
221
222 Node* no_base = __ top();
223 Node* zero = __ ConI(0);
224 Node* zeroX = __ ConX(0);
225
226 float likely = PROB_LIKELY(0.999);
227 float unlikely = PROB_UNLIKELY(0.999);
228
229 // Offsets into the thread
230 const int index_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset());
231 const int buffer_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset());
232
233 // Now the actual pointers into the thread
234 Node* buffer_adr = __ AddP(no_base, tls, __ ConX(buffer_offset));
235 Node* index_adr = __ AddP(no_base, tls, __ ConX(index_offset));
236
237 // Now some of the values
238 Node* marking;
239 Node* gc_state = __ AddP(no_base, tls, __ ConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset())));
240 Node* ld = __ load(__ ctrl(), gc_state, TypeInt::BYTE, T_BYTE, Compile::AliasIdxRaw);
241 marking = __ AndI(ld, __ ConI(ShenandoahHeap::MARKING));
242 assert(ShenandoahBarrierC2Support::is_gc_state_load(ld), "Should match the shape");
243
244 // if (!marking)
245 __ if_then(marking, BoolTest::ne, zero, unlikely); {
246 BasicType index_bt = TypeX_X->basic_type();
247 assert(sizeof(size_t) == type2aelembytes(index_bt), "Loading G1 SATBMarkQueue::_index with wrong size.");
248 Node* index = __ load(__ ctrl(), index_adr, TypeX_X, index_bt, Compile::AliasIdxRaw);
249
250 if (do_load) {
251 // load original value
252 // alias_idx correct??
253 pre_val = __ load(__ ctrl(), adr, val_type, bt, alias_idx);
254 }
255
256 // if (pre_val != NULL)
257 __ if_then(pre_val, BoolTest::ne, kit->null()); {
258 Node* buffer = __ load(__ ctrl(), buffer_adr, TypeRawPtr::NOTNULL, T_ADDRESS, Compile::AliasIdxRaw);
259
260 // is the queue for this thread full?
261 __ if_then(index, BoolTest::ne, zeroX, likely); {
262
263 // decrement the index
264 Node* next_index = kit->gvn().transform(new SubXNode(index, __ ConX(sizeof(intptr_t))));
265
266 // Now get the buffer location we will log the previous value into and store it
267 Node *log_addr = __ AddP(no_base, buffer, next_index);
268 __ store(__ ctrl(), log_addr, pre_val, T_OBJECT, Compile::AliasIdxRaw, MemNode::unordered);
269 // update the index
270 __ store(__ ctrl(), index_adr, next_index, index_bt, Compile::AliasIdxRaw, MemNode::unordered);
271
272 } __ else_(); {
273
274 // logging buffer is full, call the runtime
275 const TypeFunc *tf = ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type();
276 __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), "shenandoah_wb_pre", pre_val, tls);
277 } __ end_if(); // (!index)
278 } __ end_if(); // (pre_val != NULL)
279 } __ end_if(); // (!marking)
280
281 // Final sync IdealKit and GraphKit.
282 kit->final_sync(ideal);
283
284 if (ShenandoahSATBBarrier && adr != NULL) {
285 Node* c = kit->control();
286 Node* call = c->in(1)->in(1)->in(1)->in(0);
287 assert(is_shenandoah_wb_pre_call(call), "shenandoah_wb_pre call expected");
288 call->add_req(adr);
289 }
290 }
291
292 bool ShenandoahBarrierSetC2::is_shenandoah_wb_pre_call(Node* call) {
293 return call->is_CallLeaf() &&
294 call->as_CallLeaf()->entry_point() == CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry);
295 }
296
297 bool ShenandoahBarrierSetC2::is_shenandoah_lrb_call(Node* call) {
298 return call->is_CallLeaf() &&
299 call->as_CallLeaf()->entry_point() == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_JRT);
300 }
301
302 bool ShenandoahBarrierSetC2::is_shenandoah_marking_if(PhaseTransform *phase, Node* n) {
303 if (n->Opcode() != Op_If) {
304 return false;
305 }
306
307 Node* bol = n->in(1);
308 assert(bol->is_Bool(), "");
309 Node* cmpx = bol->in(1);
310 if (bol->as_Bool()->_test._test == BoolTest::ne &&
311 cmpx->is_Cmp() && cmpx->in(2) == phase->intcon(0) &&
312 is_shenandoah_state_load(cmpx->in(1)->in(1)) &&
313 cmpx->in(1)->in(2)->is_Con() &&
314 cmpx->in(1)->in(2) == phase->intcon(ShenandoahHeap::MARKING)) {
315 return true;
316 }
317
318 return false;
319 }
320
321 bool ShenandoahBarrierSetC2::is_shenandoah_state_load(Node* n) {
322 if (!n->is_Load()) return false;
323 const int state_offset = in_bytes(ShenandoahThreadLocalData::gc_state_offset());
324 return n->in(2)->is_AddP() && n->in(2)->in(2)->Opcode() == Op_ThreadLocal
325 && n->in(2)->in(3)->is_Con()
326 && n->in(2)->in(3)->bottom_type()->is_intptr_t()->get_con() == state_offset;
327 }
328
329 void ShenandoahBarrierSetC2::shenandoah_write_barrier_pre(GraphKit* kit,
330 bool do_load,
331 Node* obj,
332 Node* adr,
333 uint alias_idx,
334 Node* val,
335 const TypeOopPtr* val_type,
336 Node* pre_val,
337 BasicType bt) const {
338 if (ShenandoahSATBBarrier) {
339 IdealKit ideal(kit);
340 kit->sync_kit(ideal);
341
342 satb_write_barrier_pre(kit, do_load, obj, adr, alias_idx, val, val_type, pre_val, bt);
343
344 ideal.sync_kit(kit);
345 kit->final_sync(ideal);
346 }
347 }
348
349 Node* ShenandoahBarrierSetC2::shenandoah_enqueue_barrier(GraphKit* kit, Node* pre_val) const {
350 return kit->gvn().transform(new ShenandoahEnqueueBarrierNode(pre_val));
351 }
352
353 // Helper that guards and inserts a pre-barrier.
354 void ShenandoahBarrierSetC2::insert_pre_barrier(GraphKit* kit, Node* base_oop, Node* offset,
355 Node* pre_val, bool need_mem_bar) const {
356 // We could be accessing the referent field of a reference object. If so, when G1
357 // is enabled, we need to log the value in the referent field in an SATB buffer.
358 // This routine performs some compile time filters and generates suitable
359 // runtime filters that guard the pre-barrier code.
360 // Also add memory barrier for non volatile load from the referent field
361 // to prevent commoning of loads across safepoint.
362
363 // Some compile time checks.
364
365 // If offset is a constant, is it java_lang_ref_Reference::_reference_offset?
366 const TypeX* otype = offset->find_intptr_t_type();
367 if (otype != NULL && otype->is_con() &&
368 otype->get_con() != java_lang_ref_Reference::referent_offset) {
369 // Constant offset but not the reference_offset so just return
370 return;
371 }
372
373 // We only need to generate the runtime guards for instances.
374 const TypeOopPtr* btype = base_oop->bottom_type()->isa_oopptr();
375 if (btype != NULL) {
376 if (btype->isa_aryptr()) {
377 // Array type so nothing to do
378 return;
379 }
380
381 const TypeInstPtr* itype = btype->isa_instptr();
382 if (itype != NULL) {
383 // Can the klass of base_oop be statically determined to be
384 // _not_ a sub-class of Reference and _not_ Object?
385 ciKlass* klass = itype->klass();
386 if ( klass->is_loaded() &&
387 !klass->is_subtype_of(kit->env()->Reference_klass()) &&
388 !kit->env()->Object_klass()->is_subtype_of(klass)) {
389 return;
390 }
391 }
392 }
393
394 // The compile time filters did not reject base_oop/offset so
395 // we need to generate the following runtime filters
396 //
397 // if (offset == java_lang_ref_Reference::_reference_offset) {
398 // if (instance_of(base, java.lang.ref.Reference)) {
399 // pre_barrier(_, pre_val, ...);
400 // }
401 // }
402
403 float likely = PROB_LIKELY( 0.999);
404 float unlikely = PROB_UNLIKELY(0.999);
405
406 IdealKit ideal(kit);
407
408 Node* referent_off = __ ConX(java_lang_ref_Reference::referent_offset);
409
410 __ if_then(offset, BoolTest::eq, referent_off, unlikely); {
411 // Update graphKit memory and control from IdealKit.
412 kit->sync_kit(ideal);
413
414 Node* ref_klass_con = kit->makecon(TypeKlassPtr::make(kit->env()->Reference_klass()));
415 Node* is_instof = kit->gen_instanceof(base_oop, ref_klass_con);
416
417 // Update IdealKit memory and control from graphKit.
418 __ sync_kit(kit);
419
420 Node* one = __ ConI(1);
421 // is_instof == 0 if base_oop == NULL
422 __ if_then(is_instof, BoolTest::eq, one, unlikely); {
423
424 // Update graphKit from IdeakKit.
425 kit->sync_kit(ideal);
426
427 // Use the pre-barrier to record the value in the referent field
428 satb_write_barrier_pre(kit, false /* do_load */,
429 NULL /* obj */, NULL /* adr */, max_juint /* alias_idx */, NULL /* val */, NULL /* val_type */,
430 pre_val /* pre_val */,
431 T_OBJECT);
432 if (need_mem_bar) {
433 // Add memory barrier to prevent commoning reads from this field
434 // across safepoint since GC can change its value.
435 kit->insert_mem_bar(Op_MemBarCPUOrder);
436 }
437 // Update IdealKit from graphKit.
438 __ sync_kit(kit);
439
440 } __ end_if(); // _ref_type != ref_none
441 } __ end_if(); // offset == referent_offset
442
443 // Final sync IdealKit and GraphKit.
444 kit->final_sync(ideal);
445 }
446
447 #undef __
448
449 const TypeFunc* ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type() {
450 const Type **fields = TypeTuple::fields(2);
451 fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // original field value
452 fields[TypeFunc::Parms+1] = TypeRawPtr::NOTNULL; // thread
453 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2, fields);
454
455 // create result type (range)
456 fields = TypeTuple::fields(0);
457 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0, fields);
458
459 return TypeFunc::make(domain, range);
460 }
461
462 const TypeFunc* ShenandoahBarrierSetC2::shenandoah_clone_barrier_Type() {
463 const Type **fields = TypeTuple::fields(1);
464 fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // original field value
465 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+1, fields);
466
467 // create result type (range)
468 fields = TypeTuple::fields(0);
469 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0, fields);
470
471 return TypeFunc::make(domain, range);
472 }
473
474 const TypeFunc* ShenandoahBarrierSetC2::shenandoah_load_reference_barrier_Type() {
475 const Type **fields = TypeTuple::fields(1);
476 fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // original field value
477 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+1, fields);
478
479 // create result type (range)
480 fields = TypeTuple::fields(1);
481 fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL;
482 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+1, fields);
483
484 return TypeFunc::make(domain, range);
485 }
486
487 Node* ShenandoahBarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& val) const {
488 DecoratorSet decorators = access.decorators();
489
490 const TypePtr* adr_type = access.addr().type();
491 Node* adr = access.addr().node();
492
493 bool anonymous = (decorators & ON_UNKNOWN_OOP_REF) != 0;
494 bool on_heap = (decorators & IN_HEAP) != 0;
495
496 if (!access.is_oop() || (!on_heap && !anonymous)) {
497 return BarrierSetC2::store_at_resolved(access, val);
498 }
499
500 GraphKit* kit = access.kit();
501
502 uint adr_idx = kit->C->get_alias_index(adr_type);
503 assert(adr_idx != Compile::AliasIdxTop, "use other store_to_memory factory" );
504 Node* value = val.node();
505 value = shenandoah_storeval_barrier(kit, value);
506 val.set_node(value);
507 shenandoah_write_barrier_pre(kit, true /* do_load */, /*kit->control(),*/ access.base(), adr, adr_idx, val.node(),
508 static_cast<const TypeOopPtr*>(val.type()), NULL /* pre_val */, access.type());
509 return BarrierSetC2::store_at_resolved(access, val);
510 }
511
512 Node* ShenandoahBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const {
513 DecoratorSet decorators = access.decorators();
514
515 Node* adr = access.addr().node();
516 Node* obj = access.base();
517
518 bool mismatched = (decorators & C2_MISMATCHED) != 0;
519 bool unknown = (decorators & ON_UNKNOWN_OOP_REF) != 0;
520 bool on_heap = (decorators & IN_HEAP) != 0;
521 bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
522 bool is_unordered = (decorators & MO_UNORDERED) != 0;
523 bool need_cpu_mem_bar = !is_unordered || mismatched || !on_heap;
524
525 Node* top = Compile::current()->top();
526
527 Node* offset = adr->is_AddP() ? adr->in(AddPNode::Offset) : top;
528 Node* load = BarrierSetC2::load_at_resolved(access, val_type);
529
530 if (access.is_oop()) {
531 if (ShenandoahLoadRefBarrier) {
532 load = new ShenandoahLoadReferenceBarrierNode(NULL, load);
533 load = access.kit()->gvn().transform(load);
534 }
535 }
536
537 // If we are reading the value of the referent field of a Reference
538 // object (either by using Unsafe directly or through reflection)
539 // then, if SATB is enabled, we need to record the referent in an
540 // SATB log buffer using the pre-barrier mechanism.
541 // Also we need to add memory barrier to prevent commoning reads
542 // from this field across safepoint since GC can change its value.
543 bool need_read_barrier = ShenandoahKeepAliveBarrier &&
544 (on_heap && (on_weak || (unknown && offset != top && obj != top)));
545
546 if (!access.is_oop() || !need_read_barrier) {
547 return load;
548 }
549
550 GraphKit* kit = access.kit();
551
552 if (on_weak) {
553 // Use the pre-barrier to record the value in the referent field
554 satb_write_barrier_pre(kit, false /* do_load */,
555 NULL /* obj */, NULL /* adr */, max_juint /* alias_idx */, NULL /* val */, NULL /* val_type */,
556 load /* pre_val */, T_OBJECT);
557 // Add memory barrier to prevent commoning reads from this field
558 // across safepoint since GC can change its value.
559 kit->insert_mem_bar(Op_MemBarCPUOrder);
560 } else if (unknown) {
561 // We do not require a mem bar inside pre_barrier if need_mem_bar
562 // is set: the barriers would be emitted by us.
563 insert_pre_barrier(kit, obj, offset, load, !need_cpu_mem_bar);
564 }
565
566 return load;
567 }
568
569 static void pin_atomic_op(C2AtomicAccess& access) {
570 if (!access.needs_pinning()) {
571 return;
572 }
573 // SCMemProjNodes represent the memory state of a LoadStore. Their
574 // main role is to prevent LoadStore nodes from being optimized away
575 // when their results aren't used.
576 GraphKit* kit = access.kit();
577 Node* load_store = access.raw_access();
578 assert(load_store != NULL, "must pin atomic op");
579 Node* proj = kit->gvn().transform(new SCMemProjNode(load_store));
580 kit->set_memory(proj, access.alias_idx());
581 }
582
583 Node* ShenandoahBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicAccess& access, Node* expected_val,
584 Node* new_val, const Type* value_type) const {
585 GraphKit* kit = access.kit();
586 if (access.is_oop()) {
587 new_val = shenandoah_storeval_barrier(kit, new_val);
588 shenandoah_write_barrier_pre(kit, false /* do_load */,
589 NULL, NULL, max_juint, NULL, NULL,
590 expected_val /* pre_val */, T_OBJECT);
591
592 MemNode::MemOrd mo = access.mem_node_mo();
593 Node* mem = access.memory();
594 Node* adr = access.addr().node();
595 const TypePtr* adr_type = access.addr().type();
596 Node* load_store = NULL;
597
598 #ifdef _LP64
599 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
600 Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop()));
601 Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop()));
602 if (ShenandoahCASBarrier) {
603 load_store = kit->gvn().transform(new ShenandoahCompareAndExchangeNNode(kit->control(), mem, adr, newval_enc, oldval_enc, adr_type, value_type->make_narrowoop(), mo));
604 } else {
605 load_store = kit->gvn().transform(new CompareAndExchangeNNode(kit->control(), mem, adr, newval_enc, oldval_enc, adr_type, value_type->make_narrowoop(), mo));
606 }
607 } else
608 #endif
609 {
610 if (ShenandoahCASBarrier) {
611 load_store = kit->gvn().transform(new ShenandoahCompareAndExchangePNode(kit->control(), mem, adr, new_val, expected_val, adr_type, value_type->is_oopptr(), mo));
612 } else {
613 load_store = kit->gvn().transform(new CompareAndExchangePNode(kit->control(), mem, adr, new_val, expected_val, adr_type, value_type->is_oopptr(), mo));
614 }
615 }
616
617 access.set_raw_access(load_store);
618 pin_atomic_op(access);
619
620 #ifdef _LP64
621 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
622 load_store = kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type()));
623 }
624 #endif
625 load_store = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(NULL, load_store));
626 return load_store;
627 }
628 return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type);
629 }
630
631 Node* ShenandoahBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicAccess& access, Node* expected_val,
632 Node* new_val, const Type* value_type) const {
633 GraphKit* kit = access.kit();
634 if (access.is_oop()) {
635 new_val = shenandoah_storeval_barrier(kit, new_val);
636 shenandoah_write_barrier_pre(kit, false /* do_load */,
637 NULL, NULL, max_juint, NULL, NULL,
638 expected_val /* pre_val */, T_OBJECT);
639 DecoratorSet decorators = access.decorators();
640 MemNode::MemOrd mo = access.mem_node_mo();
641 Node* mem = access.memory();
642 bool is_weak_cas = (decorators & C2_WEAK_CMPXCHG) != 0;
643 Node* load_store = NULL;
644 Node* adr = access.addr().node();
645 #ifdef _LP64
646 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
647 Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop()));
648 Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop()));
649 if (ShenandoahCASBarrier) {
650 if (is_weak_cas) {
651 load_store = kit->gvn().transform(new ShenandoahWeakCompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo));
652 } else {
653 load_store = kit->gvn().transform(new ShenandoahCompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo));
654 }
655 } else {
656 if (is_weak_cas) {
657 load_store = kit->gvn().transform(new WeakCompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo));
658 } else {
659 load_store = kit->gvn().transform(new CompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo));
660 }
661 }
662 } else
663 #endif
664 {
665 if (ShenandoahCASBarrier) {
666 if (is_weak_cas) {
667 load_store = kit->gvn().transform(new ShenandoahWeakCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo));
668 } else {
669 load_store = kit->gvn().transform(new ShenandoahCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo));
670 }
671 } else {
672 if (is_weak_cas) {
673 load_store = kit->gvn().transform(new WeakCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo));
674 } else {
675 load_store = kit->gvn().transform(new CompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo));
676 }
677 }
678 }
679 access.set_raw_access(load_store);
680 pin_atomic_op(access);
681 return load_store;
682 }
683 return BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type);
684 }
685
686 Node* ShenandoahBarrierSetC2::atomic_xchg_at_resolved(C2AtomicAccess& access, Node* val, const Type* value_type) const {
687 GraphKit* kit = access.kit();
688 if (access.is_oop()) {
689 val = shenandoah_storeval_barrier(kit, val);
690 }
691 Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, val, value_type);
692 if (access.is_oop()) {
693 result = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(NULL, result));
694 shenandoah_write_barrier_pre(kit, false /* do_load */,
695 NULL, NULL, max_juint, NULL, NULL,
696 result /* pre_val */, T_OBJECT);
697 }
698 return result;
699 }
700
701 void ShenandoahBarrierSetC2::clone(GraphKit* kit, Node* src, Node* dst, Node* size, bool is_array) const {
702 // TODO: Implement using proper barriers.
703 BarrierSetC2::clone(kit, src, dst, size, is_array);
704 }
705
706 // Support for GC barriers emitted during parsing
707 bool ShenandoahBarrierSetC2::is_gc_barrier_node(Node* node) const {
708 if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) return true;
709 if (node->Opcode() != Op_CallLeaf && node->Opcode() != Op_CallLeafNoFP) {
710 return false;
711 }
712 CallLeafNode *call = node->as_CallLeaf();
713 if (call->_name == NULL) {
714 return false;
715 }
716
717 return strcmp(call->_name, "shenandoah_clone_barrier") == 0 ||
718 strcmp(call->_name, "shenandoah_cas_obj") == 0 ||
719 strcmp(call->_name, "shenandoah_wb_pre") == 0;
720 }
721
722 Node* ShenandoahBarrierSetC2::step_over_gc_barrier(Node* c) const {
723 if (c->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
724 return c->in(ShenandoahLoadReferenceBarrierNode::ValueIn);
725 }
726 if (c->Opcode() == Op_ShenandoahEnqueueBarrier) {
727 c = c->in(1);
728 }
729 return c;
730 }
731
732 bool ShenandoahBarrierSetC2::expand_barriers(Compile* C, PhaseIterGVN& igvn) const {
733 return !ShenandoahBarrierC2Support::expand(C, igvn);
734 }
735
736 bool ShenandoahBarrierSetC2::optimize_loops(PhaseIdealLoop* phase, LoopOptsMode mode, VectorSet& visited, Node_Stack& nstack, Node_List& worklist) const {
737 if (mode == LoopOptsShenandoahExpand) {
738 assert(UseShenandoahGC, "only for shenandoah");
739 ShenandoahBarrierC2Support::pin_and_expand(phase);
740 return true;
741 } else if (mode == LoopOptsShenandoahPostExpand) {
742 assert(UseShenandoahGC, "only for shenandoah");
743 visited.Clear();
744 ShenandoahBarrierC2Support::optimize_after_expansion(visited, nstack, worklist, phase);
745 return true;
746 }
747 return false;
748 }
749
750 bool ShenandoahBarrierSetC2::array_copy_requires_gc_barriers(BasicType type) const {
751 return false;
752 }
753
754 bool ShenandoahBarrierSetC2::clone_needs_postbarrier(ArrayCopyNode *ac, PhaseIterGVN& igvn) {
755 Node* src = ac->in(ArrayCopyNode::Src);
756 const TypeOopPtr* src_type = igvn.type(src)->is_oopptr();
757 if (src_type->isa_instptr() != NULL) {
758 ciInstanceKlass* ik = src_type->klass()->as_instance_klass();
759 if ((src_type->klass_is_exact() || (!ik->is_interface() && !ik->has_subklass())) && !ik->has_injected_fields()) {
760 if (ik->has_object_fields()) {
761 return true;
762 } else {
763 if (!src_type->klass_is_exact()) {
764 igvn.C->dependencies()->assert_leaf_type(ik);
765 }
766 }
767 } else {
768 return true;
769 }
770 } else if (src_type->isa_aryptr()) {
771 BasicType src_elem = src_type->klass()->as_array_klass()->element_type()->basic_type();
772 if (src_elem == T_OBJECT || src_elem == T_ARRAY) {
773 return true;
774 }
775 } else {
776 return true;
777 }
778 return false;
779 }
780
781 // Support for macro expanded GC barriers
782 void ShenandoahBarrierSetC2::register_potential_barrier_node(Node* node) const {
783 if (node->Opcode() == Op_ShenandoahEnqueueBarrier) {
784 state()->add_enqueue_barrier((ShenandoahEnqueueBarrierNode*) node);
785 }
786 if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
787 state()->add_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
788 }
789 }
790
791 void ShenandoahBarrierSetC2::unregister_potential_barrier_node(Node* node) const {
792 if (node->Opcode() == Op_ShenandoahEnqueueBarrier) {
793 state()->remove_enqueue_barrier((ShenandoahEnqueueBarrierNode*) node);
794 }
795 if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
796 state()->remove_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
797 }
798 }
799
800 void ShenandoahBarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* n) const {
801 if (is_shenandoah_wb_pre_call(n)) {
802 shenandoah_eliminate_wb_pre(n, ¯o->igvn());
803 }
804 }
805
806 void ShenandoahBarrierSetC2::shenandoah_eliminate_wb_pre(Node* call, PhaseIterGVN* igvn) const {
807 assert(UseShenandoahGC && is_shenandoah_wb_pre_call(call), "");
808 Node* c = call->as_Call()->proj_out(TypeFunc::Control);
809 c = c->unique_ctrl_out();
810 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
811 c = c->unique_ctrl_out();
812 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
813 Node* iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
814 assert(iff->is_If(), "expect test");
815 if (!is_shenandoah_marking_if(igvn, iff)) {
816 c = c->unique_ctrl_out();
817 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
818 iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
819 assert(is_shenandoah_marking_if(igvn, iff), "expect marking test");
820 }
821 Node* cmpx = iff->in(1)->in(1);
822 igvn->replace_node(cmpx, igvn->makecon(TypeInt::CC_EQ));
823 igvn->rehash_node_delayed(call);
824 call->del_req(call->req()-1);
825 }
826
827 void ShenandoahBarrierSetC2::enqueue_useful_gc_barrier(Unique_Node_List &worklist, Node* node) const {
828 if (node->Opcode() == Op_AddP && ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(node)) {
829 worklist.push(node);
830 }
831 }
832
833 void ShenandoahBarrierSetC2::eliminate_useless_gc_barriers(Unique_Node_List &useful) const {
834 for (uint i = 0; i < useful.size(); i++) {
835 Node* n = useful.at(i);
836 if (n->Opcode() == Op_AddP && ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(n)) {
837 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
838 Compile::current()->record_for_igvn(n->fast_out(i));
839 }
840 }
841 }
842 for (int i = state()->enqueue_barriers_count() - 1; i >= 0; i--) {
843 ShenandoahEnqueueBarrierNode* n = state()->enqueue_barrier(i);
844 if (!useful.member(n)) {
845 state()->remove_enqueue_barrier(n);
846 }
847 }
848 for (int i = state()->load_reference_barriers_count() - 1; i >= 0; i--) {
849 ShenandoahLoadReferenceBarrierNode* n = state()->load_reference_barrier(i);
850 if (!useful.member(n)) {
851 state()->remove_load_reference_barrier(n);
852 }
853 }
854 }
855
856 void ShenandoahBarrierSetC2::add_users_to_worklist(Unique_Node_List* worklist) const {}
857
858 void* ShenandoahBarrierSetC2::create_barrier_state(Arena* comp_arena) const {
859 return new(comp_arena) ShenandoahBarrierSetC2State(comp_arena);
860 }
861
862 ShenandoahBarrierSetC2State* ShenandoahBarrierSetC2::state() const {
863 return reinterpret_cast<ShenandoahBarrierSetC2State*>(Compile::current()->barrier_set_state());
864 }
865
866 // If the BarrierSetC2 state has kept macro nodes in its compilation unit state to be
867 // expanded later, then now is the time to do so.
868 bool ShenandoahBarrierSetC2::expand_macro_nodes(PhaseMacroExpand* macro) const { return false; }
869
870 #ifdef ASSERT
871 void ShenandoahBarrierSetC2::verify_gc_barriers(bool post_parse) const {
872 if (ShenandoahVerifyOptoBarriers && !post_parse) {
873 ShenandoahBarrierC2Support::verify(Compile::current()->root());
874 }
875 }
876 #endif
877
878 Node* ShenandoahBarrierSetC2::ideal_node(PhaseGVN* phase, Node* n, bool can_reshape) const {
879 if (is_shenandoah_wb_pre_call(n)) {
880 uint cnt = ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type()->domain()->cnt();
881 if (n->req() > cnt) {
882 Node* addp = n->in(cnt);
883 if (has_only_shenandoah_wb_pre_uses(addp)) {
884 n->del_req(cnt);
885 if (can_reshape) {
886 phase->is_IterGVN()->_worklist.push(addp);
887 }
888 return n;
889 }
890 }
891 }
892 return NULL;
893 }
894
895 bool ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(Node* n) {
896 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
897 Node* u = n->fast_out(i);
898 if (!is_shenandoah_wb_pre_call(u)) {
899 return false;
900 }
901 }
902 return n->outcnt() > 0;
903 }