1 /*
2 * Copyright (c) 2018, Red Hat, Inc. All rights reserved.
3 *
4 * This code is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 only, as
6 * published by the Free Software Foundation.
7 *
8 * This code is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 * version 2 for more details (a copy is included in the LICENSE file that
12 * accompanied this code).
13 *
14 * You should have received a copy of the GNU General Public License version
15 * 2 along with this work; if not, write to the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 #include "precompiled.hpp"
25 #include "gc/shenandoah/shenandoahHeap.hpp"
26 #include "gc/shenandoah/shenandoahHeuristics.hpp"
27 #include "gc/shenandoah/shenandoahRuntime.hpp"
28 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
29 #include "gc/shenandoah/c2/shenandoahSupport.hpp"
30 #include "opto/arraycopynode.hpp"
31 #include "opto/escape.hpp"
32 #include "opto/graphKit.hpp"
33 #include "opto/idealKit.hpp"
34 #include "opto/macro.hpp"
35 #include "opto/movenode.hpp"
36 #include "opto/narrowptrnode.hpp"
37 #include "opto/rootnode.hpp"
38
39 ShenandoahBarrierSetC2* ShenandoahBarrierSetC2::bsc2() {
40 return reinterpret_cast<ShenandoahBarrierSetC2*>(BarrierSet::barrier_set()->barrier_set_c2());
41 }
42
43 ShenandoahBarrierSetC2State::ShenandoahBarrierSetC2State(Arena* comp_arena)
44 : _shenandoah_barriers(new (comp_arena) GrowableArray<ShenandoahWriteBarrierNode*>(comp_arena, 8, 0, NULL)) {
45 }
46
47 int ShenandoahBarrierSetC2State::shenandoah_barriers_count() const {
48 return _shenandoah_barriers->length();
49 }
50
51 ShenandoahWriteBarrierNode* ShenandoahBarrierSetC2State::shenandoah_barrier(int idx) const {
52 return _shenandoah_barriers->at(idx);
53 }
54
55 void ShenandoahBarrierSetC2State::add_shenandoah_barrier(ShenandoahWriteBarrierNode * n) {
56 assert(!_shenandoah_barriers->contains(n), "duplicate entry in barrier list");
57 _shenandoah_barriers->append(n);
58 }
59
60 void ShenandoahBarrierSetC2State::remove_shenandoah_barrier(ShenandoahWriteBarrierNode * n) {
61 if (_shenandoah_barriers->contains(n)) {
62 _shenandoah_barriers->remove(n);
63 }
64 }
65
66 #define __ kit->
67
68 Node* ShenandoahBarrierSetC2::shenandoah_read_barrier(GraphKit* kit, Node* obj) const {
69 if (ShenandoahReadBarrier) {
70 obj = shenandoah_read_barrier_impl(kit, obj, false, true, true);
71 }
72 return obj;
73 }
74
75 Node* ShenandoahBarrierSetC2::shenandoah_storeval_barrier(GraphKit* kit, Node* obj) const {
76 if (ShenandoahStoreValEnqueueBarrier) {
77 obj = shenandoah_write_barrier(kit, obj);
78 obj = shenandoah_enqueue_barrier(kit, obj);
79 }
80 if (ShenandoahStoreValReadBarrier) {
81 obj = shenandoah_read_barrier_impl(kit, obj, true, false, false);
82 }
83 return obj;
84 }
85
86 Node* ShenandoahBarrierSetC2::shenandoah_read_barrier_acmp(GraphKit* kit, Node* obj) {
87 return shenandoah_read_barrier_impl(kit, obj, true, true, false);
88 }
89
90 Node* ShenandoahBarrierSetC2::shenandoah_read_barrier_impl(GraphKit* kit, Node* obj, bool use_ctrl, bool use_mem, bool allow_fromspace) const {
91 const Type* obj_type = obj->bottom_type();
92 if (obj_type->higher_equal(TypePtr::NULL_PTR)) {
93 return obj;
94 }
95 const TypePtr* adr_type = ShenandoahBarrierNode::brooks_pointer_type(obj_type);
96 Node* mem = use_mem ? __ memory(adr_type) : __ immutable_memory();
97
98 if (! ShenandoahBarrierNode::needs_barrier(&__ gvn(), NULL, obj, mem, allow_fromspace)) {
99 // We know it is null, no barrier needed.
100 return obj;
101 }
102
103 if (obj_type->meet(TypePtr::NULL_PTR) == obj_type->remove_speculative()) {
104
105 // We don't know if it's null or not. Need null-check.
106 enum { _not_null_path = 1, _null_path, PATH_LIMIT };
107 RegionNode* region = new RegionNode(PATH_LIMIT);
108 Node* phi = new PhiNode(region, obj_type);
109 Node* null_ctrl = __ top();
110 Node* not_null_obj = __ null_check_oop(obj, &null_ctrl);
111
112 region->init_req(_null_path, null_ctrl);
113 phi ->init_req(_null_path, __ zerocon(T_OBJECT));
114
115 Node* ctrl = use_ctrl ? __ control() : NULL;
116 ShenandoahReadBarrierNode* rb = new ShenandoahReadBarrierNode(ctrl, mem, not_null_obj, allow_fromspace);
117 Node* n = __ gvn().transform(rb);
118
119 region->init_req(_not_null_path, __ control());
120 phi ->init_req(_not_null_path, n);
121
122 __ set_control(__ gvn().transform(region));
123 __ record_for_igvn(region);
124 return __ gvn().transform(phi);
125
126 } else {
127 // We know it is not null. Simple barrier is sufficient.
128 Node* ctrl = use_ctrl ? __ control() : NULL;
129 ShenandoahReadBarrierNode* rb = new ShenandoahReadBarrierNode(ctrl, mem, obj, allow_fromspace);
130 Node* n = __ gvn().transform(rb);
131 __ record_for_igvn(n);
132 return n;
133 }
134 }
135
136 Node* ShenandoahBarrierSetC2::shenandoah_write_barrier_helper(GraphKit* kit, Node* obj, const TypePtr* adr_type) const {
137 ShenandoahWriteBarrierNode* wb = new ShenandoahWriteBarrierNode(kit->C, kit->control(), kit->memory(adr_type), obj);
138 Node* n = __ gvn().transform(wb);
139 if (n == wb) { // New barrier needs memory projection.
140 Node* proj = __ gvn().transform(new ShenandoahWBMemProjNode(n));
141 __ set_memory(proj, adr_type);
142 }
143 return n;
144 }
145
146 Node* ShenandoahBarrierSetC2::shenandoah_write_barrier(GraphKit* kit, Node* obj) const {
147 if (ShenandoahWriteBarrier) {
148 obj = shenandoah_write_barrier_impl(kit, obj);
149 }
150 return obj;
151 }
152
153 Node* ShenandoahBarrierSetC2::shenandoah_write_barrier_impl(GraphKit* kit, Node* obj) const {
154 if (! ShenandoahBarrierNode::needs_barrier(&__ gvn(), NULL, obj, NULL, true)) {
155 return obj;
156 }
157 const Type* obj_type = obj->bottom_type();
158 const TypePtr* adr_type = ShenandoahBarrierNode::brooks_pointer_type(obj_type);
159 Node* n = shenandoah_write_barrier_helper(kit, obj, adr_type);
160 __ record_for_igvn(n);
161 return n;
162 }
163
164 bool ShenandoahBarrierSetC2::satb_can_remove_pre_barrier(GraphKit* kit, PhaseTransform* phase, Node* adr,
165 BasicType bt, uint adr_idx) const {
166 intptr_t offset = 0;
167 Node* base = AddPNode::Ideal_base_and_offset(adr, phase, offset);
168 AllocateNode* alloc = AllocateNode::Ideal_allocation(base, phase);
169
170 if (offset == Type::OffsetBot) {
171 return false; // cannot unalias unless there are precise offsets
172 }
173
174 if (alloc == NULL) {
175 return false; // No allocation found
176 }
177
178 intptr_t size_in_bytes = type2aelembytes(bt);
179
180 Node* mem = __ memory(adr_idx); // start searching here...
181
182 for (int cnt = 0; cnt < 50; cnt++) {
183
184 if (mem->is_Store()) {
185
186 Node* st_adr = mem->in(MemNode::Address);
187 intptr_t st_offset = 0;
188 Node* st_base = AddPNode::Ideal_base_and_offset(st_adr, phase, st_offset);
189
190 if (st_base == NULL) {
191 break; // inscrutable pointer
192 }
193
194 // Break we have found a store with same base and offset as ours so break
195 if (st_base == base && st_offset == offset) {
196 break;
197 }
198
199 if (st_offset != offset && st_offset != Type::OffsetBot) {
200 const int MAX_STORE = BytesPerLong;
201 if (st_offset >= offset + size_in_bytes ||
202 st_offset <= offset - MAX_STORE ||
203 st_offset <= offset - mem->as_Store()->memory_size()) {
204 // Success: The offsets are provably independent.
205 // (You may ask, why not just test st_offset != offset and be done?
206 // The answer is that stores of different sizes can co-exist
207 // in the same sequence of RawMem effects. We sometimes initialize
208 // a whole 'tile' of array elements with a single jint or jlong.)
209 mem = mem->in(MemNode::Memory);
210 continue; // advance through independent store memory
211 }
212 }
213
214 if (st_base != base
215 && MemNode::detect_ptr_independence(base, alloc, st_base,
216 AllocateNode::Ideal_allocation(st_base, phase),
217 phase)) {
218 // Success: The bases are provably independent.
219 mem = mem->in(MemNode::Memory);
220 continue; // advance through independent store memory
221 }
222 } else if (mem->is_Proj() && mem->in(0)->is_Initialize()) {
223
224 InitializeNode* st_init = mem->in(0)->as_Initialize();
225 AllocateNode* st_alloc = st_init->allocation();
226
227 // Make sure that we are looking at the same allocation site.
228 // The alloc variable is guaranteed to not be null here from earlier check.
229 if (alloc == st_alloc) {
230 // Check that the initialization is storing NULL so that no previous store
231 // has been moved up and directly write a reference
232 Node* captured_store = st_init->find_captured_store(offset,
233 type2aelembytes(T_OBJECT),
234 phase);
235 if (captured_store == NULL || captured_store == st_init->zero_memory()) {
236 return true;
237 }
238 }
239 }
240
241 // Unless there is an explicit 'continue', we must bail out here,
242 // because 'mem' is an inscrutable memory state (e.g., a call).
243 break;
244 }
245
246 return false;
247 }
248
249 #undef __
250 #define __ ideal.
251
252 void ShenandoahBarrierSetC2::satb_write_barrier_pre(GraphKit* kit,
253 bool do_load,
254 Node* obj,
255 Node* adr,
256 uint alias_idx,
257 Node* val,
258 const TypeOopPtr* val_type,
259 Node* pre_val,
260 BasicType bt) const {
261 // Some sanity checks
262 // Note: val is unused in this routine.
263
264 if (do_load) {
265 // We need to generate the load of the previous value
266 assert(obj != NULL, "must have a base");
267 assert(adr != NULL, "where are loading from?");
268 assert(pre_val == NULL, "loaded already?");
269 assert(val_type != NULL, "need a type");
270
271 if (ReduceInitialCardMarks
272 && satb_can_remove_pre_barrier(kit, &kit->gvn(), adr, bt, alias_idx)) {
273 return;
274 }
275
276 } else {
277 // In this case both val_type and alias_idx are unused.
278 assert(pre_val != NULL, "must be loaded already");
279 // Nothing to be done if pre_val is null.
280 if (pre_val->bottom_type() == TypePtr::NULL_PTR) return;
281 assert(pre_val->bottom_type()->basic_type() == T_OBJECT, "or we shouldn't be here");
282 }
283 assert(bt == T_OBJECT, "or we shouldn't be here");
284
285 IdealKit ideal(kit, true);
286
287 Node* tls = __ thread(); // ThreadLocalStorage
288
289 Node* no_base = __ top();
290 Node* zero = __ ConI(0);
291 Node* zeroX = __ ConX(0);
292
293 float likely = PROB_LIKELY(0.999);
294 float unlikely = PROB_UNLIKELY(0.999);
295
296 // Offsets into the thread
297 const int index_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset());
298 const int buffer_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset());
299
300 // Now the actual pointers into the thread
301 Node* buffer_adr = __ AddP(no_base, tls, __ ConX(buffer_offset));
302 Node* index_adr = __ AddP(no_base, tls, __ ConX(index_offset));
303
304 // Now some of the values
305 Node* marking;
306 Node* gc_state = __ AddP(no_base, tls, __ ConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset())));
307 Node* ld = __ load(__ ctrl(), gc_state, TypeInt::BYTE, T_BYTE, Compile::AliasIdxRaw);
308 marking = __ AndI(ld, __ ConI(ShenandoahHeap::MARKING));
309 assert(ShenandoahWriteBarrierNode::is_gc_state_load(ld), "Should match the shape");
310
311 // if (!marking)
312 __ if_then(marking, BoolTest::ne, zero, unlikely); {
313 BasicType index_bt = TypeX_X->basic_type();
314 assert(sizeof(size_t) == type2aelembytes(index_bt), "Loading G1 SATBMarkQueue::_index with wrong size.");
315 Node* index = __ load(__ ctrl(), index_adr, TypeX_X, index_bt, Compile::AliasIdxRaw);
316
317 if (do_load) {
318 // load original value
319 // alias_idx correct??
320 pre_val = __ load(__ ctrl(), adr, val_type, bt, alias_idx);
321 }
322
323 // if (pre_val != NULL)
324 __ if_then(pre_val, BoolTest::ne, kit->null()); {
325 Node* buffer = __ load(__ ctrl(), buffer_adr, TypeRawPtr::NOTNULL, T_ADDRESS, Compile::AliasIdxRaw);
326
327 // is the queue for this thread full?
328 __ if_then(index, BoolTest::ne, zeroX, likely); {
329
330 // decrement the index
331 Node* next_index = kit->gvn().transform(new SubXNode(index, __ ConX(sizeof(intptr_t))));
332
333 // Now get the buffer location we will log the previous value into and store it
334 Node *log_addr = __ AddP(no_base, buffer, next_index);
335 __ store(__ ctrl(), log_addr, pre_val, T_OBJECT, Compile::AliasIdxRaw, MemNode::unordered);
336 // update the index
337 __ store(__ ctrl(), index_adr, next_index, index_bt, Compile::AliasIdxRaw, MemNode::unordered);
338
339 } __ else_(); {
340
341 // logging buffer is full, call the runtime
342 const TypeFunc *tf = ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type();
343 __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), "shenandoah_wb_pre", pre_val, tls);
344 } __ end_if(); // (!index)
345 } __ end_if(); // (pre_val != NULL)
346 } __ end_if(); // (!marking)
347
348 // Final sync IdealKit and GraphKit.
349 kit->final_sync(ideal);
350
351 if (ShenandoahSATBBarrier && adr != NULL) {
352 Node* c = kit->control();
353 Node* call = c->in(1)->in(1)->in(1)->in(0);
354 assert(is_shenandoah_wb_pre_call(call), "shenandoah_wb_pre call expected");
355 call->add_req(adr);
356 }
357 }
358
359 bool ShenandoahBarrierSetC2::is_shenandoah_wb_pre_call(Node* call) {
360 return call->is_CallLeaf() &&
361 call->as_CallLeaf()->entry_point() == CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry);
362 }
363
364 bool ShenandoahBarrierSetC2::is_shenandoah_wb_call(Node* call) {
365 return call->is_CallLeaf() &&
366 call->as_CallLeaf()->entry_point() == CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_JRT);
367 }
368
369 bool ShenandoahBarrierSetC2::is_shenandoah_marking_if(PhaseTransform *phase, Node* n) {
370 if (n->Opcode() != Op_If) {
371 return false;
372 }
373
374 Node* bol = n->in(1);
375 assert(bol->is_Bool(), "");
376 Node* cmpx = bol->in(1);
377 if (bol->as_Bool()->_test._test == BoolTest::ne &&
378 cmpx->is_Cmp() && cmpx->in(2) == phase->intcon(0) &&
379 is_shenandoah_state_load(cmpx->in(1)->in(1)) &&
380 cmpx->in(1)->in(2)->is_Con() &&
381 cmpx->in(1)->in(2) == phase->intcon(ShenandoahHeap::MARKING)) {
382 return true;
383 }
384
385 return false;
386 }
387
388 bool ShenandoahBarrierSetC2::is_shenandoah_state_load(Node* n) {
389 if (!n->is_Load()) return false;
390 const int state_offset = in_bytes(ShenandoahThreadLocalData::gc_state_offset());
391 return n->in(2)->is_AddP() && n->in(2)->in(2)->Opcode() == Op_ThreadLocal
392 && n->in(2)->in(3)->is_Con()
393 && n->in(2)->in(3)->bottom_type()->is_intptr_t()->get_con() == state_offset;
394 }
395
396 void ShenandoahBarrierSetC2::shenandoah_write_barrier_pre(GraphKit* kit,
397 bool do_load,
398 Node* obj,
399 Node* adr,
400 uint alias_idx,
401 Node* val,
402 const TypeOopPtr* val_type,
403 Node* pre_val,
404 BasicType bt) const {
405 if (ShenandoahSATBBarrier) {
406 IdealKit ideal(kit);
407 kit->sync_kit(ideal);
408
409 satb_write_barrier_pre(kit, do_load, obj, adr, alias_idx, val, val_type, pre_val, bt);
410
411 ideal.sync_kit(kit);
412 kit->final_sync(ideal);
413 }
414 }
415
416 Node* ShenandoahBarrierSetC2::shenandoah_enqueue_barrier(GraphKit* kit, Node* pre_val) const {
417 return kit->gvn().transform(new ShenandoahEnqueueBarrierNode(pre_val));
418 }
419
420 // Helper that guards and inserts a pre-barrier.
421 void ShenandoahBarrierSetC2::insert_pre_barrier(GraphKit* kit, Node* base_oop, Node* offset,
422 Node* pre_val, bool need_mem_bar) const {
423 // We could be accessing the referent field of a reference object. If so, when G1
424 // is enabled, we need to log the value in the referent field in an SATB buffer.
425 // This routine performs some compile time filters and generates suitable
426 // runtime filters that guard the pre-barrier code.
427 // Also add memory barrier for non volatile load from the referent field
428 // to prevent commoning of loads across safepoint.
429
430 // Some compile time checks.
431
432 // If offset is a constant, is it java_lang_ref_Reference::_reference_offset?
433 const TypeX* otype = offset->find_intptr_t_type();
434 if (otype != NULL && otype->is_con() &&
435 otype->get_con() != java_lang_ref_Reference::referent_offset) {
436 // Constant offset but not the reference_offset so just return
437 return;
438 }
439
440 // We only need to generate the runtime guards for instances.
441 const TypeOopPtr* btype = base_oop->bottom_type()->isa_oopptr();
442 if (btype != NULL) {
443 if (btype->isa_aryptr()) {
444 // Array type so nothing to do
445 return;
446 }
447
448 const TypeInstPtr* itype = btype->isa_instptr();
449 if (itype != NULL) {
450 // Can the klass of base_oop be statically determined to be
451 // _not_ a sub-class of Reference and _not_ Object?
452 ciKlass* klass = itype->klass();
453 if ( klass->is_loaded() &&
454 !klass->is_subtype_of(kit->env()->Reference_klass()) &&
455 !kit->env()->Object_klass()->is_subtype_of(klass)) {
456 return;
457 }
458 }
459 }
460
461 // The compile time filters did not reject base_oop/offset so
462 // we need to generate the following runtime filters
463 //
464 // if (offset == java_lang_ref_Reference::_reference_offset) {
465 // if (instance_of(base, java.lang.ref.Reference)) {
466 // pre_barrier(_, pre_val, ...);
467 // }
468 // }
469
470 float likely = PROB_LIKELY( 0.999);
471 float unlikely = PROB_UNLIKELY(0.999);
472
473 IdealKit ideal(kit);
474
475 Node* referent_off = __ ConX(java_lang_ref_Reference::referent_offset);
476
477 __ if_then(offset, BoolTest::eq, referent_off, unlikely); {
478 // Update graphKit memory and control from IdealKit.
479 kit->sync_kit(ideal);
480
481 Node* ref_klass_con = kit->makecon(TypeKlassPtr::make(kit->env()->Reference_klass()));
482 Node* is_instof = kit->gen_instanceof(base_oop, ref_klass_con);
483
484 // Update IdealKit memory and control from graphKit.
485 __ sync_kit(kit);
486
487 Node* one = __ ConI(1);
488 // is_instof == 0 if base_oop == NULL
489 __ if_then(is_instof, BoolTest::eq, one, unlikely); {
490
491 // Update graphKit from IdeakKit.
492 kit->sync_kit(ideal);
493
494 // Use the pre-barrier to record the value in the referent field
495 satb_write_barrier_pre(kit, false /* do_load */,
496 NULL /* obj */, NULL /* adr */, max_juint /* alias_idx */, NULL /* val */, NULL /* val_type */,
497 pre_val /* pre_val */,
498 T_OBJECT);
499 if (need_mem_bar) {
500 // Add memory barrier to prevent commoning reads from this field
501 // across safepoint since GC can change its value.
502 kit->insert_mem_bar(Op_MemBarCPUOrder);
503 }
504 // Update IdealKit from graphKit.
505 __ sync_kit(kit);
506
507 } __ end_if(); // _ref_type != ref_none
508 } __ end_if(); // offset == referent_offset
509
510 // Final sync IdealKit and GraphKit.
511 kit->final_sync(ideal);
512 }
513
514 #undef __
515
516 const TypeFunc* ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type() {
517 const Type **fields = TypeTuple::fields(2);
518 fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // original field value
519 fields[TypeFunc::Parms+1] = TypeRawPtr::NOTNULL; // thread
520 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2, fields);
521
522 // create result type (range)
523 fields = TypeTuple::fields(0);
524 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0, fields);
525
526 return TypeFunc::make(domain, range);
527 }
528
529 const TypeFunc* ShenandoahBarrierSetC2::shenandoah_clone_barrier_Type() {
530 const Type **fields = TypeTuple::fields(1);
531 fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // original field value
532 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+1, fields);
533
534 // create result type (range)
535 fields = TypeTuple::fields(0);
536 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0, fields);
537
538 return TypeFunc::make(domain, range);
539 }
540
541 const TypeFunc* ShenandoahBarrierSetC2::shenandoah_write_barrier_Type() {
542 const Type **fields = TypeTuple::fields(1);
543 fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // original field value
544 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+1, fields);
545
546 // create result type (range)
547 fields = TypeTuple::fields(1);
548 fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL;
549 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+1, fields);
550
551 return TypeFunc::make(domain, range);
552 }
553
554 void ShenandoahBarrierSetC2::resolve_address(C2Access& access) const {
555 const TypePtr* adr_type = access.addr().type();
556
557 if ((access.decorators() & IN_NATIVE) == 0 && (adr_type->isa_instptr() || adr_type->isa_aryptr())) {
558 int off = adr_type->is_ptr()->offset();
559 int base_off = adr_type->isa_instptr() ? instanceOopDesc::base_offset_in_bytes() :
560 arrayOopDesc::base_offset_in_bytes(adr_type->is_aryptr()->elem()->array_element_basic_type());
561 assert(off != Type::OffsetTop, "unexpected offset");
562 if (off == Type::OffsetBot || off >= base_off) {
563 DecoratorSet decorators = access.decorators();
564 bool is_write = (decorators & C2_WRITE_ACCESS) != 0;
565 GraphKit* kit = NULL;
566 if (access.is_parse_access()) {
567 C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
568 kit = parse_access.kit();
569 }
570 Node* adr = access.addr().node();
571 assert(adr->is_AddP(), "unexpected address shape");
572 Node* base = adr->in(AddPNode::Base);
573
574 if (is_write) {
575 if (kit != NULL) {
576 base = shenandoah_write_barrier(kit, base);
577 } else {
578 assert(access.is_opt_access(), "either parse or opt access");
579 assert((access.decorators() & C2_ARRAY_COPY) != 0, "can be skipped for clone");
580 }
581 } else {
582 if (adr_type->isa_instptr()) {
583 Compile* C = access.gvn().C;
584 ciField* field = C->alias_type(adr_type)->field();
585
586 // Insert read barrier for Shenandoah.
587 if (field != NULL &&
588 ((ShenandoahOptimizeStaticFinals && field->is_static() && field->is_final()) ||
589 (ShenandoahOptimizeInstanceFinals && !field->is_static() && field->is_final()) ||
590 (ShenandoahOptimizeStableFinals && field->is_stable()))) {
591 // Skip the barrier for special fields
592 } else {
593 if (kit != NULL) {
594 base = shenandoah_read_barrier(kit, base);
595 } else {
596 assert(access.is_opt_access(), "either parse or opt access");
597 assert((access.decorators() & C2_ARRAY_COPY) != 0, "can be skipped for arraycopy");
598 }
599 }
600 } else {
601 if (kit != NULL) {
602 base = shenandoah_read_barrier(kit, base);
603 } else {
604 assert(access.is_opt_access(), "either parse or opt access");
605 assert((access.decorators() & C2_ARRAY_COPY) != 0, "can be skipped for arraycopy");
606 }
607 }
608 }
609 if (base != adr->in(AddPNode::Base)) {
610 assert(kit != NULL, "no barrier should have been added");
611
612 Node* address = adr->in(AddPNode::Address);
613
614 if (address->is_AddP()) {
615 assert(address->in(AddPNode::Base) == adr->in(AddPNode::Base), "unexpected address shape");
616 assert(!address->in(AddPNode::Address)->is_AddP(), "unexpected address shape");
617 assert(address->in(AddPNode::Address) == adr->in(AddPNode::Base), "unexpected address shape");
618 address = address->clone();
619 address->set_req(AddPNode::Base, base);
620 address->set_req(AddPNode::Address, base);
621 address = kit->gvn().transform(address);
622 } else {
623 assert(address == adr->in(AddPNode::Base), "unexpected address shape");
624 address = base;
625 }
626 adr = adr->clone();
627 adr->set_req(AddPNode::Base, base);
628 adr->set_req(AddPNode::Address, address);
629 adr = kit->gvn().transform(adr);
630 access.addr().set_node(adr);
631 }
632 }
633 }
634 }
635
636 Node* ShenandoahBarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& val) const {
637 DecoratorSet decorators = access.decorators();
638
639 const TypePtr* adr_type = access.addr().type();
640 Node* adr = access.addr().node();
641
642 bool anonymous = (decorators & ON_UNKNOWN_OOP_REF) != 0;
643 bool on_heap = (decorators & IN_HEAP) != 0;
644
645 if (!access.is_oop() || (!on_heap && !anonymous)) {
646 return BarrierSetC2::store_at_resolved(access, val);
647 }
648
649 if (access.is_parse_access()) {
650 C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
651 GraphKit* kit = parse_access.kit();
652
653 uint adr_idx = kit->C->get_alias_index(adr_type);
654 assert(adr_idx != Compile::AliasIdxTop, "use other store_to_memory factory" );
655 Node* value = val.node();
656 value = shenandoah_storeval_barrier(kit, value);
657 val.set_node(value);
658 shenandoah_write_barrier_pre(kit, true /* do_load */, /*kit->control(),*/ access.base(), adr, adr_idx, val.node(),
659 static_cast<const TypeOopPtr*>(val.type()), NULL /* pre_val */, access.type());
660 } else {
661 assert(access.is_opt_access(), "only for optimization passes");
662 assert(((decorators & C2_TIGHLY_COUPLED_ALLOC) != 0 || !ShenandoahSATBBarrier) && (decorators & C2_ARRAY_COPY) != 0, "unexpected caller of this code");
663 C2OptAccess& opt_access = static_cast<C2OptAccess&>(access);
664 PhaseGVN& gvn = opt_access.gvn();
665 MergeMemNode* mm = opt_access.mem();
666
667 if (ShenandoahStoreValReadBarrier) {
668 RegionNode* region = new RegionNode(3);
669 const Type* v_t = gvn.type(val.node());
670 Node* phi = new PhiNode(region, v_t->isa_oopptr() ? v_t->is_oopptr()->cast_to_nonconst() : v_t);
671 Node* cmp = gvn.transform(new CmpPNode(val.node(), gvn.zerocon(T_OBJECT)));
672 Node* bol = gvn.transform(new BoolNode(cmp, BoolTest::ne));
673 IfNode* iff = new IfNode(opt_access.ctl(), bol, PROB_LIKELY_MAG(3), COUNT_UNKNOWN);
674
675 gvn.transform(iff);
676 if (gvn.is_IterGVN()) {
677 gvn.is_IterGVN()->_worklist.push(iff);
678 } else {
679 gvn.record_for_igvn(iff);
680 }
681
682 Node* null_true = gvn.transform(new IfFalseNode(iff));
683 Node* null_false = gvn.transform(new IfTrueNode(iff));
684 region->init_req(1, null_true);
685 region->init_req(2, null_false);
686 phi->init_req(1, gvn.zerocon(T_OBJECT));
687 Node* cast = new CastPPNode(val.node(), gvn.type(val.node())->join_speculative(TypePtr::NOTNULL));
688 cast->set_req(0, null_false);
689 cast = gvn.transform(cast);
690 Node* rb = gvn.transform(new ShenandoahReadBarrierNode(null_false, gvn.C->immutable_memory(), cast, false));
691 phi->init_req(2, rb);
692 opt_access.set_ctl(gvn.transform(region));
693 val.set_node(gvn.transform(phi));
694 }
695 if (ShenandoahStoreValEnqueueBarrier) {
696 const TypePtr* adr_type = ShenandoahBarrierNode::brooks_pointer_type(gvn.type(val.node()));
697 int alias = gvn.C->get_alias_index(adr_type);
698 Node* wb = new ShenandoahWriteBarrierNode(gvn.C, opt_access.ctl(), mm->memory_at(alias), val.node());
699 Node* wb_transformed = gvn.transform(wb);
700 Node* enqueue = gvn.transform(new ShenandoahEnqueueBarrierNode(wb_transformed));
701 if (wb_transformed == wb) {
702 Node* proj = gvn.transform(new ShenandoahWBMemProjNode(wb));
703 mm->set_memory_at(alias, proj);
704 }
705 val.set_node(enqueue);
706 }
707 }
708 return BarrierSetC2::store_at_resolved(access, val);
709 }
710
711 Node* ShenandoahBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const {
712 DecoratorSet decorators = access.decorators();
713
714 Node* adr = access.addr().node();
715 Node* obj = access.base();
716
717 bool mismatched = (decorators & C2_MISMATCHED) != 0;
718 bool unknown = (decorators & ON_UNKNOWN_OOP_REF) != 0;
719 bool on_heap = (decorators & IN_HEAP) != 0;
720 bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
721 bool is_unordered = (decorators & MO_UNORDERED) != 0;
722 bool need_cpu_mem_bar = !is_unordered || mismatched || !on_heap;
723
724 Node* top = Compile::current()->top();
725
726 Node* offset = adr->is_AddP() ? adr->in(AddPNode::Offset) : top;
727 Node* load = BarrierSetC2::load_at_resolved(access, val_type);
728
729 // If we are reading the value of the referent field of a Reference
730 // object (either by using Unsafe directly or through reflection)
731 // then, if SATB is enabled, we need to record the referent in an
732 // SATB log buffer using the pre-barrier mechanism.
733 // Also we need to add memory barrier to prevent commoning reads
734 // from this field across safepoint since GC can change its value.
735 bool need_read_barrier = ShenandoahKeepAliveBarrier &&
736 (on_heap && (on_weak || (unknown && offset != top && obj != top)));
737
738 if (!access.is_oop() || !need_read_barrier) {
739 return load;
740 }
741
742 assert(access.is_parse_access(), "entry not supported at optimization time");
743 C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
744 GraphKit* kit = parse_access.kit();
745
746 if (on_weak) {
747 // Use the pre-barrier to record the value in the referent field
748 satb_write_barrier_pre(kit, false /* do_load */,
749 NULL /* obj */, NULL /* adr */, max_juint /* alias_idx */, NULL /* val */, NULL /* val_type */,
750 load /* pre_val */, T_OBJECT);
751 // Add memory barrier to prevent commoning reads from this field
752 // across safepoint since GC can change its value.
753 kit->insert_mem_bar(Op_MemBarCPUOrder);
754 } else if (unknown) {
755 // We do not require a mem bar inside pre_barrier if need_mem_bar
756 // is set: the barriers would be emitted by us.
757 insert_pre_barrier(kit, obj, offset, load, !need_cpu_mem_bar);
758 }
759
760 return load;
761 }
762
763 Node* ShenandoahBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
764 Node* new_val, const Type* value_type) const {
765 GraphKit* kit = access.kit();
766 if (access.is_oop()) {
767 new_val = shenandoah_storeval_barrier(kit, new_val);
768 shenandoah_write_barrier_pre(kit, false /* do_load */,
769 NULL, NULL, max_juint, NULL, NULL,
770 expected_val /* pre_val */, T_OBJECT);
771
772 MemNode::MemOrd mo = access.mem_node_mo();
773 Node* mem = access.memory();
774 Node* adr = access.addr().node();
775 const TypePtr* adr_type = access.addr().type();
776 Node* load_store = NULL;
777
778 #ifdef _LP64
779 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
780 Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop()));
781 Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop()));
782 load_store = kit->gvn().transform(new ShenandoahCompareAndExchangeNNode(kit->control(), mem, adr, newval_enc, oldval_enc, adr_type, value_type->make_narrowoop(), mo));
783 } else
784 #endif
785 {
786 load_store = kit->gvn().transform(new ShenandoahCompareAndExchangePNode(kit->control(), mem, adr, new_val, expected_val, adr_type, value_type->is_oopptr(), mo));
787 }
788
789 access.set_raw_access(load_store);
790 pin_atomic_op(access);
791
792 #ifdef _LP64
793 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
794 return kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type()));
795 }
796 #endif
797 return load_store;
798 }
799 return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type);
800 }
801
802 Node* ShenandoahBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
803 Node* new_val, const Type* value_type) const {
804 GraphKit* kit = access.kit();
805 if (access.is_oop()) {
806 new_val = shenandoah_storeval_barrier(kit, new_val);
807 shenandoah_write_barrier_pre(kit, false /* do_load */,
808 NULL, NULL, max_juint, NULL, NULL,
809 expected_val /* pre_val */, T_OBJECT);
810 DecoratorSet decorators = access.decorators();
811 MemNode::MemOrd mo = access.mem_node_mo();
812 Node* mem = access.memory();
813 bool is_weak_cas = (decorators & C2_WEAK_CMPXCHG) != 0;
814 Node* load_store = NULL;
815 Node* adr = access.addr().node();
816 #ifdef _LP64
817 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
818 Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop()));
819 Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop()));
820 if (is_weak_cas) {
821 load_store = kit->gvn().transform(new ShenandoahWeakCompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo));
822 } else {
823 load_store = kit->gvn().transform(new ShenandoahCompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo));
824 }
825 } else
826 #endif
827 {
828 if (is_weak_cas) {
829 load_store = kit->gvn().transform(new ShenandoahWeakCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo));
830 } else {
831 load_store = kit->gvn().transform(new ShenandoahCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo));
832 }
833 }
834 access.set_raw_access(load_store);
835 pin_atomic_op(access);
836 return load_store;
837 }
838 return BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type);
839 }
840
841 Node* ShenandoahBarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* val, const Type* value_type) const {
842 GraphKit* kit = access.kit();
843 if (access.is_oop()) {
844 val = shenandoah_storeval_barrier(kit, val);
845 }
846 Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, val, value_type);
847 if (access.is_oop()) {
848 shenandoah_write_barrier_pre(kit, false /* do_load */,
849 NULL, NULL, max_juint, NULL, NULL,
850 result /* pre_val */, T_OBJECT);
851 }
852 return result;
853 }
854
855 void ShenandoahBarrierSetC2::clone(GraphKit* kit, Node* src, Node* dst, Node* size, bool is_array) const {
856 assert(!src->is_AddP(), "unexpected input");
857 src = shenandoah_read_barrier(kit, src);
858 BarrierSetC2::clone(kit, src, dst, size, is_array);
859 }
860
861 Node* ShenandoahBarrierSetC2::resolve(GraphKit* kit, Node* n, DecoratorSet decorators) const {
862 bool is_write = decorators & ACCESS_WRITE;
863 if (is_write) {
864 return shenandoah_write_barrier(kit, n);
865 } else {
866 return shenandoah_read_barrier(kit, n);
867 }
868 }
869
870 Node* ShenandoahBarrierSetC2::obj_allocate(PhaseMacroExpand* macro, Node* ctrl, Node* mem, Node* toobig_false, Node* size_in_bytes,
871 Node*& i_o, Node*& needgc_ctrl,
872 Node*& fast_oop_ctrl, Node*& fast_oop_rawmem,
873 intx prefetch_lines) const {
874 PhaseIterGVN& igvn = macro->igvn();
875
876 // Allocate several words more for the Shenandoah brooks pointer.
877 size_in_bytes = new AddXNode(size_in_bytes, igvn.MakeConX(ShenandoahBrooksPointer::byte_size()));
878 macro->transform_later(size_in_bytes);
879
880 Node* fast_oop = BarrierSetC2::obj_allocate(macro, ctrl, mem, toobig_false, size_in_bytes,
881 i_o, needgc_ctrl, fast_oop_ctrl, fast_oop_rawmem,
882 prefetch_lines);
883
884 // Bump up object for Shenandoah brooks pointer.
885 fast_oop = new AddPNode(macro->top(), fast_oop, igvn.MakeConX(ShenandoahBrooksPointer::byte_size()));
886 macro->transform_later(fast_oop);
887
888 // Initialize Shenandoah brooks pointer to point to the object itself.
889 fast_oop_rawmem = macro->make_store(fast_oop_ctrl, fast_oop_rawmem, fast_oop, ShenandoahBrooksPointer::byte_offset(), fast_oop, T_OBJECT);
890
891 return fast_oop;
892 }
893
894 // Support for GC barriers emitted during parsing
895 bool ShenandoahBarrierSetC2::is_gc_barrier_node(Node* node) const {
896 if (node->Opcode() != Op_CallLeaf && node->Opcode() != Op_CallLeafNoFP) {
897 return false;
898 }
899 CallLeafNode *call = node->as_CallLeaf();
900 if (call->_name == NULL) {
901 return false;
902 }
903
904 return strcmp(call->_name, "shenandoah_clone_barrier") == 0 ||
905 strcmp(call->_name, "shenandoah_cas_obj") == 0 ||
906 strcmp(call->_name, "shenandoah_wb_pre") == 0;
907 }
908
909 Node* ShenandoahBarrierSetC2::step_over_gc_barrier(Node* c) const {
910 return ShenandoahBarrierNode::skip_through_barrier(c);
911 }
912
913 bool ShenandoahBarrierSetC2::array_copy_requires_gc_barriers(bool tightly_coupled_alloc, BasicType type, bool is_clone, ArrayCopyPhase phase) const {
914 bool is_oop = type == T_OBJECT || type == T_ARRAY;
915 if (!is_oop) {
916 return false;
917 }
918
919 if (tightly_coupled_alloc) {
920 if (phase == Optimization) {
921 return false;
922 }
923 return !is_clone;
924 }
925 if (phase == Optimization) {
926 return !ShenandoahStoreValEnqueueBarrier;
927 }
928 return true;
929 }
930
931 bool ShenandoahBarrierSetC2::clone_needs_postbarrier(ArrayCopyNode *ac, PhaseIterGVN& igvn) {
932 Node* src = ac->in(ArrayCopyNode::Src);
933 const TypeOopPtr* src_type = igvn.type(src)->is_oopptr();
934 if (src_type->isa_instptr() != NULL) {
935 ciInstanceKlass* ik = src_type->klass()->as_instance_klass();
936 if ((src_type->klass_is_exact() || (!ik->is_interface() && !ik->has_subklass())) && !ik->has_injected_fields()) {
937 if (ik->has_object_fields()) {
938 return true;
939 } else {
940 if (!src_type->klass_is_exact()) {
941 igvn.C->dependencies()->assert_leaf_type(ik);
942 }
943 }
944 } else {
945 return true;
946 }
947 } else if (src_type->isa_aryptr()) {
948 BasicType src_elem = src_type->klass()->as_array_klass()->element_type()->basic_type();
949 if (src_elem == T_OBJECT || src_elem == T_ARRAY) {
950 return true;
951 }
952 } else {
953 return true;
954 }
955 return false;
956 }
957
958 void ShenandoahBarrierSetC2::clone_barrier_at_expansion(ArrayCopyNode* ac, Node* call, PhaseIterGVN& igvn) const {
959 assert(ac->is_clonebasic(), "no other kind of arraycopy here");
960
961 if (!clone_needs_postbarrier(ac, igvn)) {
962 BarrierSetC2::clone_barrier_at_expansion(ac, call, igvn);
963 return;
964 }
965
966 const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
967 Node* c = new ProjNode(call,TypeFunc::Control);
968 c = igvn.transform(c);
969 Node* m = new ProjNode(call, TypeFunc::Memory);
970 c = igvn.transform(m);
971
972 Node* dest = ac->in(ArrayCopyNode::Dest);
973 assert(dest->is_AddP(), "bad input");
974 Node* barrier_call = new CallLeafNode(ShenandoahBarrierSetC2::shenandoah_clone_barrier_Type(),
975 CAST_FROM_FN_PTR(address, ShenandoahRuntime::shenandoah_clone_barrier),
976 "shenandoah_clone_barrier", raw_adr_type);
977 barrier_call->init_req(TypeFunc::Control, c);
978 barrier_call->init_req(TypeFunc::I_O , igvn.C->top());
979 barrier_call->init_req(TypeFunc::Memory , m);
980 barrier_call->init_req(TypeFunc::ReturnAdr, igvn.C->top());
981 barrier_call->init_req(TypeFunc::FramePtr, igvn.C->top());
982 barrier_call->init_req(TypeFunc::Parms+0, dest->in(AddPNode::Base));
983
984 barrier_call = igvn.transform(barrier_call);
985 c = new ProjNode(barrier_call,TypeFunc::Control);
986 c = igvn.transform(c);
987 m = new ProjNode(barrier_call, TypeFunc::Memory);
988 m = igvn.transform(m);
989
990 Node* out_c = ac->proj_out(TypeFunc::Control);
991 Node* out_m = ac->proj_out(TypeFunc::Memory);
992 igvn.replace_node(out_c, c);
993 igvn.replace_node(out_m, m);
994 }
995
996
997 // Support for macro expanded GC barriers
998 void ShenandoahBarrierSetC2::register_potential_barrier_node(Node* node) const {
999 if (node->Opcode() == Op_ShenandoahWriteBarrier) {
1000 state()->add_shenandoah_barrier((ShenandoahWriteBarrierNode*) node);
1001 }
1002 }
1003
1004 void ShenandoahBarrierSetC2::unregister_potential_barrier_node(Node* node) const {
1005 if (node->Opcode() == Op_ShenandoahWriteBarrier) {
1006 state()->remove_shenandoah_barrier((ShenandoahWriteBarrierNode*) node);
1007 }
1008 }
1009
1010 void ShenandoahBarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* n) const {
1011 if (is_shenandoah_wb_pre_call(n)) {
1012 shenandoah_eliminate_wb_pre(n, ¯o->igvn());
1013 }
1014 }
1015
1016 void ShenandoahBarrierSetC2::shenandoah_eliminate_wb_pre(Node* call, PhaseIterGVN* igvn) const {
1017 assert(UseShenandoahGC && is_shenandoah_wb_pre_call(call), "");
1018 Node* c = call->as_Call()->proj_out(TypeFunc::Control);
1019 c = c->unique_ctrl_out();
1020 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
1021 c = c->unique_ctrl_out();
1022 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
1023 Node* iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
1024 assert(iff->is_If(), "expect test");
1025 if (!is_shenandoah_marking_if(igvn, iff)) {
1026 c = c->unique_ctrl_out();
1027 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
1028 iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
1029 assert(is_shenandoah_marking_if(igvn, iff), "expect marking test");
1030 }
1031 Node* cmpx = iff->in(1)->in(1);
1032 igvn->replace_node(cmpx, igvn->makecon(TypeInt::CC_EQ));
1033 igvn->rehash_node_delayed(call);
1034 call->del_req(call->req()-1);
1035 }
1036
1037 void ShenandoahBarrierSetC2::enqueue_useful_gc_barrier(PhaseIterGVN* igvn, Node* node) const {
1038 if (node->Opcode() == Op_AddP && ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(node)) {
1039 igvn->add_users_to_worklist(node);
1040 }
1041 }
1042
1043 void ShenandoahBarrierSetC2::eliminate_useless_gc_barriers(Unique_Node_List &useful, Compile* C) const {
1044 for (uint i = 0; i < useful.size(); i++) {
1045 Node* n = useful.at(i);
1046 if (n->Opcode() == Op_AddP && ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(n)) {
1047 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
1048 C->record_for_igvn(n->fast_out(i));
1049 }
1050 }
1051 }
1052 for (int i = state()->shenandoah_barriers_count()-1; i >= 0; i--) {
1053 ShenandoahWriteBarrierNode* n = state()->shenandoah_barrier(i);
1054 if (!useful.member(n)) {
1055 state()->remove_shenandoah_barrier(n);
1056 }
1057 }
1058
1059 }
1060
1061 bool ShenandoahBarrierSetC2::has_special_unique_user(const Node* node) const {
1062 assert(node->outcnt() == 1, "match only for unique out");
1063 Node* n = node->unique_out();
1064 return node->Opcode() == Op_ShenandoahWriteBarrier && n->Opcode() == Op_ShenandoahWBMemProj;
1065 }
1066
1067 void ShenandoahBarrierSetC2::add_users_to_worklist(Unique_Node_List* worklist) const {}
1068
1069 void* ShenandoahBarrierSetC2::create_barrier_state(Arena* comp_arena) const {
1070 return new(comp_arena) ShenandoahBarrierSetC2State(comp_arena);
1071 }
1072
1073 ShenandoahBarrierSetC2State* ShenandoahBarrierSetC2::state() const {
1074 return reinterpret_cast<ShenandoahBarrierSetC2State*>(Compile::current()->barrier_set_state());
1075 }
1076
1077 // If the BarrierSetC2 state has kept macro nodes in its compilation unit state to be
1078 // expanded later, then now is the time to do so.
1079 bool ShenandoahBarrierSetC2::expand_macro_nodes(PhaseMacroExpand* macro) const { return false; }
1080
1081 #ifdef ASSERT
1082 void ShenandoahBarrierSetC2::verify_gc_barriers(Compile* compile, CompilePhase phase) const {
1083 if (ShenandoahVerifyOptoBarriers && phase == BarrierSetC2::BeforeExpand) {
1084 ShenandoahBarrierNode::verify(Compile::current()->root());
1085 } else if (phase == BarrierSetC2::BeforeCodeGen) {
1086 // Verify G1 pre-barriers
1087 const int marking_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_active_offset());
1088
1089 ResourceArea *area = Thread::current()->resource_area();
1090 Unique_Node_List visited(area);
1091 Node_List worklist(area);
1092 // We're going to walk control flow backwards starting from the Root
1093 worklist.push(compile->root());
1094 while (worklist.size() > 0) {
1095 Node *x = worklist.pop();
1096 if (x == NULL || x == compile->top()) continue;
1097 if (visited.member(x)) {
1098 continue;
1099 } else {
1100 visited.push(x);
1101 }
1102
1103 if (x->is_Region()) {
1104 for (uint i = 1; i < x->req(); i++) {
1105 worklist.push(x->in(i));
1106 }
1107 } else {
1108 worklist.push(x->in(0));
1109 // We are looking for the pattern:
1110 // /->ThreadLocal
1111 // If->Bool->CmpI->LoadB->AddP->ConL(marking_offset)
1112 // \->ConI(0)
1113 // We want to verify that the If and the LoadB have the same control
1114 // See GraphKit::g1_write_barrier_pre()
1115 if (x->is_If()) {
1116 IfNode *iff = x->as_If();
1117 if (iff->in(1)->is_Bool() && iff->in(1)->in(1)->is_Cmp()) {
1118 CmpNode *cmp = iff->in(1)->in(1)->as_Cmp();
1119 if (cmp->Opcode() == Op_CmpI && cmp->in(2)->is_Con() && cmp->in(2)->bottom_type()->is_int()->get_con() == 0
1120 && cmp->in(1)->is_Load()) {
1121 LoadNode *load = cmp->in(1)->as_Load();
1122 if (load->Opcode() == Op_LoadB && load->in(2)->is_AddP() && load->in(2)->in(2)->Opcode() == Op_ThreadLocal
1123 && load->in(2)->in(3)->is_Con()
1124 && load->in(2)->in(3)->bottom_type()->is_intptr_t()->get_con() == marking_offset) {
1125
1126 Node *if_ctrl = iff->in(0);
1127 Node *load_ctrl = load->in(0);
1128
1129 if (if_ctrl != load_ctrl) {
1130 // Skip possible CProj->NeverBranch in infinite loops
1131 if ((if_ctrl->is_Proj() && if_ctrl->Opcode() == Op_CProj)
1132 && (if_ctrl->in(0)->is_MultiBranch() && if_ctrl->in(0)->Opcode() == Op_NeverBranch)) {
1133 if_ctrl = if_ctrl->in(0)->in(0);
1134 }
1135 }
1136 assert(load_ctrl != NULL && if_ctrl == load_ctrl, "controls must match");
1137 }
1138 }
1139 }
1140 }
1141 }
1142 }
1143 }
1144 }
1145 #endif
1146
1147 Node* ShenandoahBarrierSetC2::ideal_node(PhaseGVN* phase, Node* n, bool can_reshape) const {
1148 if (is_shenandoah_wb_pre_call(n)) {
1149 uint cnt = ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type()->domain()->cnt();
1150 if (n->req() > cnt) {
1151 Node* addp = n->in(cnt);
1152 if (has_only_shenandoah_wb_pre_uses(addp)) {
1153 n->del_req(cnt);
1154 if (can_reshape) {
1155 phase->is_IterGVN()->_worklist.push(addp);
1156 }
1157 return n;
1158 }
1159 }
1160 }
1161 if (n->Opcode() == Op_CmpP) {
1162 Node* in1 = n->in(1);
1163 Node* in2 = n->in(2);
1164 if (in1->bottom_type() == TypePtr::NULL_PTR) {
1165 in2 = step_over_gc_barrier(in2);
1166 }
1167 if (in2->bottom_type() == TypePtr::NULL_PTR) {
1168 in1 = step_over_gc_barrier(in1);
1169 }
1170 PhaseIterGVN* igvn = phase->is_IterGVN();
1171 if (in1 != n->in(1)) {
1172 if (igvn != NULL) {
1173 n->set_req_X(1, in1, igvn);
1174 } else {
1175 n->set_req(1, in1);
1176 }
1177 assert(in2 == n->in(2), "only one change");
1178 return n;
1179 }
1180 if (in2 != n->in(2)) {
1181 if (igvn != NULL) {
1182 n->set_req_X(2, in2, igvn);
1183 } else {
1184 n->set_req(2, in2);
1185 }
1186 return n;
1187 }
1188 } else if (can_reshape &&
1189 n->Opcode() == Op_If &&
1190 ShenandoahWriteBarrierNode::is_heap_stable_test(n) &&
1191 n->in(0) != NULL) {
1192 Node* dom = n->in(0);
1193 Node* prev_dom = n;
1194 int op = n->Opcode();
1195 int dist = 16;
1196 // Search up the dominator tree for another heap stable test
1197 while (dom->Opcode() != op || // Not same opcode?
1198 !ShenandoahWriteBarrierNode::is_heap_stable_test(dom) || // Not same input 1?
1199 prev_dom->in(0) != dom) { // One path of test does not dominate?
1200 if (dist < 0) return NULL;
1201
1202 dist--;
1203 prev_dom = dom;
1204 dom = IfNode::up_one_dom(dom);
1205 if (!dom) return NULL;
1206 }
1207
1208 // Check that we did not follow a loop back to ourselves
1209 if (n == dom) {
1210 return NULL;
1211 }
1212
1213 return n->as_If()->dominated_by(prev_dom, phase->is_IterGVN());
1214 }
1215
1216 return NULL;
1217 }
1218
1219 Node* ShenandoahBarrierSetC2::identity_node(PhaseGVN* phase, Node* n) const {
1220 if (n->is_Load()) {
1221 Node *mem = n->in(MemNode::Memory);
1222 Node *value = n->as_Load()->can_see_stored_value(mem, phase);
1223 if (value) {
1224 PhaseIterGVN *igvn = phase->is_IterGVN();
1225 if (igvn != NULL &&
1226 value->is_Phi() &&
1227 value->req() > 2 &&
1228 value->in(1) != NULL &&
1229 value->in(1)->is_ShenandoahBarrier()) {
1230 if (igvn->_worklist.member(value) ||
1231 igvn->_worklist.member(value->in(0)) ||
1232 (value->in(0)->in(1) != NULL &&
1233 value->in(0)->in(1)->is_IfProj() &&
1234 (igvn->_worklist.member(value->in(0)->in(1)) ||
1235 (value->in(0)->in(1)->in(0) != NULL &&
1236 igvn->_worklist.member(value->in(0)->in(1)->in(0)))))) {
1237 igvn->_worklist.push(n);
1238 return n;
1239 }
1240 }
1241 // (This works even when value is a Con, but LoadNode::Value
1242 // usually runs first, producing the singleton type of the Con.)
1243 Node *value_no_barrier = step_over_gc_barrier(value->Opcode() == Op_EncodeP ? value->in(1) : value);
1244 if (value->Opcode() == Op_EncodeP) {
1245 if (value_no_barrier != value->in(1)) {
1246 Node *encode = value->clone();
1247 encode->set_req(1, value_no_barrier);
1248 encode = phase->transform(encode);
1249 return encode;
1250 }
1251 } else {
1252 return value_no_barrier;
1253 }
1254 }
1255 }
1256 return n;
1257 }
1258
1259 bool ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(Node* n) {
1260 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
1261 Node* u = n->fast_out(i);
1262 if (!is_shenandoah_wb_pre_call(u)) {
1263 return false;
1264 }
1265 }
1266 return n->outcnt() > 0;
1267 }
1268
1269 bool ShenandoahBarrierSetC2::flatten_gc_alias_type(const TypePtr*& adr_type) const {
1270 int offset = adr_type->offset();
1271 if (offset == ShenandoahBrooksPointer::byte_offset()) {
1272 if (adr_type->isa_aryptr()) {
1273 adr_type = TypeAryPtr::make(adr_type->ptr(), adr_type->isa_aryptr()->ary(), adr_type->isa_aryptr()->klass(), false, offset);
1274 } else if (adr_type->isa_instptr()) {
1275 adr_type = TypeInstPtr::make(adr_type->ptr(), ciEnv::current()->Object_klass(), false, NULL, offset);
1276 }
1277 return true;
1278 } else {
1279 return false;
1280 }
1281 }
1282
1283 bool ShenandoahBarrierSetC2::final_graph_reshaping(Compile* compile, Node* n, uint opcode) const {
1284 switch (opcode) {
1285 case Op_CallLeaf:
1286 case Op_CallLeafNoFP: {
1287 assert (n->is_Call(), "");
1288 CallNode *call = n->as_Call();
1289 if (ShenandoahBarrierSetC2::is_shenandoah_wb_pre_call(call)) {
1290 uint cnt = ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type()->domain()->cnt();
1291 if (call->req() > cnt) {
1292 assert(call->req() == cnt + 1, "only one extra input");
1293 Node *addp = call->in(cnt);
1294 assert(!ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(addp), "useless address computation?");
1295 call->del_req(cnt);
1296 }
1297 }
1298 return false;
1299 }
1300 case Op_ShenandoahCompareAndSwapP:
1301 case Op_ShenandoahCompareAndSwapN:
1302 case Op_ShenandoahWeakCompareAndSwapN:
1303 case Op_ShenandoahWeakCompareAndSwapP:
1304 case Op_ShenandoahCompareAndExchangeP:
1305 case Op_ShenandoahCompareAndExchangeN:
1306 #ifdef ASSERT
1307 if( VerifyOptoOopOffsets ) {
1308 MemNode* mem = n->as_Mem();
1309 // Check to see if address types have grounded out somehow.
1310 const TypeInstPtr *tp = mem->in(MemNode::Address)->bottom_type()->isa_instptr();
1311 ciInstanceKlass *k = tp->klass()->as_instance_klass();
1312 bool oop_offset_is_sane = k->contains_field_offset(tp->offset());
1313 assert( !tp || oop_offset_is_sane, "" );
1314 }
1315 #endif
1316 return true;
1317 case Op_ShenandoahReadBarrier:
1318 return true;
1319 case Op_ShenandoahWriteBarrier:
1320 assert(false, "should have been expanded already");
1321 return true;
1322 default:
1323 return false;
1324 }
1325 }
1326
1327 #ifdef ASSERT
1328 bool ShenandoahBarrierSetC2::verify_gc_alias_type(const TypePtr* adr_type, int offset) const {
1329 if (offset == ShenandoahBrooksPointer::byte_offset() &&
1330 (adr_type->base() == Type::AryPtr || adr_type->base() == Type::OopPtr)) {
1331 return true;
1332 } else {
1333 return false;
1334 }
1335 }
1336 #endif
1337
1338 bool ShenandoahBarrierSetC2::escape_add_to_con_graph(ConnectionGraph* conn_graph, PhaseGVN* gvn, Unique_Node_List* delayed_worklist, Node* n, uint opcode) const {
1339 switch (opcode) {
1340 case Op_ShenandoahCompareAndExchangeP:
1341 case Op_ShenandoahCompareAndExchangeN:
1342 conn_graph->add_objload_to_connection_graph(n, delayed_worklist);
1343 // fallthrough
1344 case Op_ShenandoahWeakCompareAndSwapP:
1345 case Op_ShenandoahWeakCompareAndSwapN:
1346 case Op_ShenandoahCompareAndSwapP:
1347 case Op_ShenandoahCompareAndSwapN:
1348 conn_graph->add_to_congraph_unsafe_access(n, opcode, delayed_worklist);
1349 return true;
1350 case Op_StoreP: {
1351 Node* adr = n->in(MemNode::Address);
1352 const Type* adr_type = gvn->type(adr);
1353 // Pointer stores in G1 barriers looks like unsafe access.
1354 // Ignore such stores to be able scalar replace non-escaping
1355 // allocations.
1356 if (adr_type->isa_rawptr() && adr->is_AddP()) {
1357 Node* base = conn_graph->get_addp_base(adr);
1358 if (base->Opcode() == Op_LoadP &&
1359 base->in(MemNode::Address)->is_AddP()) {
1360 adr = base->in(MemNode::Address);
1361 Node* tls = conn_graph->get_addp_base(adr);
1362 if (tls->Opcode() == Op_ThreadLocal) {
1363 int offs = (int) gvn->find_intptr_t_con(adr->in(AddPNode::Offset), Type::OffsetBot);
1364 const int buf_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset());
1365 if (offs == buf_offset) {
1366 return true; // Pre barrier previous oop value store.
1367 }
1368 }
1369 }
1370 }
1371 return false;
1372 }
1373 case Op_ShenandoahReadBarrier:
1374 case Op_ShenandoahWriteBarrier:
1375 // Barriers 'pass through' its arguments. I.e. what goes in, comes out.
1376 // It doesn't escape.
1377 conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(ShenandoahBarrierNode::ValueIn), delayed_worklist);
1378 break;
1379 case Op_ShenandoahEnqueueBarrier:
1380 conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(1), delayed_worklist);
1381 break;
1382 default:
1383 // Nothing
1384 break;
1385 }
1386 return false;
1387 }
1388
1389 bool ShenandoahBarrierSetC2::escape_add_final_edges(ConnectionGraph* conn_graph, PhaseGVN* gvn, Node* n, uint opcode) const {
1390 switch (opcode) {
1391 case Op_ShenandoahCompareAndExchangeP:
1392 case Op_ShenandoahCompareAndExchangeN: {
1393 Node *adr = n->in(MemNode::Address);
1394 conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, adr, NULL);
1395 // fallthrough
1396 }
1397 case Op_ShenandoahCompareAndSwapP:
1398 case Op_ShenandoahCompareAndSwapN:
1399 case Op_ShenandoahWeakCompareAndSwapP:
1400 case Op_ShenandoahWeakCompareAndSwapN:
1401 return conn_graph->add_final_edges_unsafe_access(n, opcode);
1402 case Op_ShenandoahReadBarrier:
1403 case Op_ShenandoahWriteBarrier:
1404 // Barriers 'pass through' its arguments. I.e. what goes in, comes out.
1405 // It doesn't escape.
1406 conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(ShenandoahBarrierNode::ValueIn), NULL);
1407 return true;
1408 case Op_ShenandoahEnqueueBarrier:
1409 conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(1), NULL);
1410 return true;
1411 default:
1412 // Nothing
1413 break;
1414 }
1415 return false;
1416 }
1417
1418 bool ShenandoahBarrierSetC2::escape_has_out_with_unsafe_object(Node* n) const {
1419 return n->has_out_with(Op_ShenandoahCompareAndExchangeP) || n->has_out_with(Op_ShenandoahCompareAndExchangeN) ||
1420 n->has_out_with(Op_ShenandoahCompareAndSwapP, Op_ShenandoahCompareAndSwapN, Op_ShenandoahWeakCompareAndSwapP, Op_ShenandoahWeakCompareAndSwapN);
1421
1422 }
1423
1424 bool ShenandoahBarrierSetC2::escape_is_barrier_node(Node* n) const {
1425 return n->is_ShenandoahBarrier();
1426 }
1427
1428 bool ShenandoahBarrierSetC2::matcher_find_shared_visit(Matcher* matcher, Matcher::MStack& mstack, Node* n, uint opcode, bool& mem_op, int& mem_addr_idx) const {
1429 switch (opcode) {
1430 case Op_ShenandoahReadBarrier:
1431 if (n->in(ShenandoahBarrierNode::ValueIn)->is_DecodeNarrowPtr()) {
1432 matcher->set_shared(n->in(ShenandoahBarrierNode::ValueIn)->in(1));
1433 }
1434 matcher->set_shared(n);
1435 return true;
1436 default:
1437 break;
1438 }
1439 return false;
1440 }
1441
1442 bool ShenandoahBarrierSetC2::matcher_find_shared_post_visit(Matcher* matcher, Node* n, uint opcode) const {
1443 switch (opcode) {
1444 case Op_ShenandoahCompareAndExchangeP:
1445 case Op_ShenandoahCompareAndExchangeN:
1446 case Op_ShenandoahWeakCompareAndSwapP:
1447 case Op_ShenandoahWeakCompareAndSwapN:
1448 case Op_ShenandoahCompareAndSwapP:
1449 case Op_ShenandoahCompareAndSwapN: { // Convert trinary to binary-tree
1450 Node* newval = n->in(MemNode::ValueIn);
1451 Node* oldval = n->in(LoadStoreConditionalNode::ExpectedIn);
1452 Node* pair = new BinaryNode(oldval, newval);
1453 n->set_req(MemNode::ValueIn,pair);
1454 n->del_req(LoadStoreConditionalNode::ExpectedIn);
1455 return true;
1456 }
1457 default:
1458 break;
1459 }
1460 return false;
1461 }
1462
1463 bool ShenandoahBarrierSetC2::matcher_is_store_load_barrier(Node* x, uint xop) const {
1464 return xop == Op_ShenandoahCompareAndExchangeP ||
1465 xop == Op_ShenandoahCompareAndExchangeN ||
1466 xop == Op_ShenandoahWeakCompareAndSwapP ||
1467 xop == Op_ShenandoahWeakCompareAndSwapN ||
1468 xop == Op_ShenandoahCompareAndSwapN ||
1469 xop == Op_ShenandoahCompareAndSwapP;
1470 }
1471
1472 void ShenandoahBarrierSetC2::igvn_add_users_to_worklist(PhaseIterGVN* igvn, Node* use) const {
1473 if (use->is_ShenandoahBarrier()) {
1474 for (DUIterator_Fast i2max, i2 = use->fast_outs(i2max); i2 < i2max; i2++) {
1475 Node* u = use->fast_out(i2);
1476 Node* cmp = use->find_out_with(Op_CmpP);
1477 if (u->Opcode() == Op_CmpP) {
1478 igvn->_worklist.push(cmp);
1479 }
1480 }
1481 }
1482 }
1483
1484 void ShenandoahBarrierSetC2::ccp_analyze(PhaseCCP* ccp, Unique_Node_List& worklist, Node* use) const {
1485 if (use->is_ShenandoahBarrier()) {
1486 for (DUIterator_Fast i2max, i2 = use->fast_outs(i2max); i2 < i2max; i2++) {
1487 Node* p = use->fast_out(i2);
1488 if (p->Opcode() == Op_AddP) {
1489 for (DUIterator_Fast i3max, i3 = p->fast_outs(i3max); i3 < i3max; i3++) {
1490 Node* q = p->fast_out(i3);
1491 if (q->is_Load()) {
1492 if(q->bottom_type() != ccp->type(q)) {
1493 worklist.push(q);
1494 }
1495 }
1496 }
1497 }
1498 }
1499 }
1500 }