1 /*
2 * Copyright (c) 2018, Red Hat, Inc. All rights reserved.
3 *
4 * This code is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 only, as
6 * published by the Free Software Foundation.
7 *
8 * This code is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 * version 2 for more details (a copy is included in the LICENSE file that
12 * accompanied this code).
13 *
14 * You should have received a copy of the GNU General Public License version
15 * 2 along with this work; if not, write to the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 #include "precompiled.hpp"
25 #include "gc/shenandoah/shenandoahHeap.hpp"
26 #include "gc/shenandoah/shenandoahHeuristics.hpp"
27 #include "gc/shenandoah/shenandoahRuntime.hpp"
28 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
29 #include "gc/shenandoah/c2/shenandoahSupport.hpp"
30 #include "opto/arraycopynode.hpp"
31 #include "opto/escape.hpp"
32 #include "opto/graphKit.hpp"
33 #include "opto/idealKit.hpp"
34 #include "opto/macro.hpp"
35 #include "opto/movenode.hpp"
36 #include "opto/narrowptrnode.hpp"
37 #include "opto/rootnode.hpp"
38
39 ShenandoahBarrierSetC2* ShenandoahBarrierSetC2::bsc2() {
40 return reinterpret_cast<ShenandoahBarrierSetC2*>(BarrierSet::barrier_set()->barrier_set_c2());
41 }
42
43 ShenandoahBarrierSetC2State::ShenandoahBarrierSetC2State(Arena* comp_arena)
44 : _shenandoah_barriers(new (comp_arena) GrowableArray<ShenandoahWriteBarrierNode*>(comp_arena, 8, 0, NULL)) {
45 }
46
47 int ShenandoahBarrierSetC2State::shenandoah_barriers_count() const {
48 return _shenandoah_barriers->length();
49 }
50
51 ShenandoahWriteBarrierNode* ShenandoahBarrierSetC2State::shenandoah_barrier(int idx) const {
52 return _shenandoah_barriers->at(idx);
53 }
54
55 void ShenandoahBarrierSetC2State::add_shenandoah_barrier(ShenandoahWriteBarrierNode * n) {
56 assert(!_shenandoah_barriers->contains(n), "duplicate entry in barrier list");
57 _shenandoah_barriers->append(n);
58 }
59
60 void ShenandoahBarrierSetC2State::remove_shenandoah_barrier(ShenandoahWriteBarrierNode * n) {
61 if (_shenandoah_barriers->contains(n)) {
62 _shenandoah_barriers->remove(n);
63 }
64 }
65
66 #define __ kit->
67
68 Node* ShenandoahBarrierSetC2::shenandoah_read_barrier(GraphKit* kit, Node* obj) const {
69 if (ShenandoahReadBarrier) {
70 obj = shenandoah_read_barrier_impl(kit, obj, false, true, true);
71 }
72 return obj;
73 }
74
75 Node* ShenandoahBarrierSetC2::shenandoah_storeval_barrier(GraphKit* kit, Node* obj) const {
76 if (ShenandoahStoreValEnqueueBarrier) {
77 obj = shenandoah_write_barrier(kit, obj);
78 obj = shenandoah_enqueue_barrier(kit, obj);
79 }
80 if (ShenandoahStoreValReadBarrier) {
81 obj = shenandoah_read_barrier_impl(kit, obj, true, false, false);
82 }
83 return obj;
84 }
85
86 Node* ShenandoahBarrierSetC2::shenandoah_read_barrier_acmp(GraphKit* kit, Node* obj) {
87 return shenandoah_read_barrier_impl(kit, obj, true, true, false);
88 }
89
90 Node* ShenandoahBarrierSetC2::shenandoah_read_barrier_impl(GraphKit* kit, Node* obj, bool use_ctrl, bool use_mem, bool allow_fromspace) const {
91 const Type* obj_type = obj->bottom_type();
92 if (obj_type->higher_equal(TypePtr::NULL_PTR)) {
93 return obj;
94 }
95 const TypePtr* adr_type = ShenandoahBarrierNode::brooks_pointer_type(obj_type);
96 Node* mem = use_mem ? __ memory(adr_type) : __ immutable_memory();
97
98 if (! ShenandoahBarrierNode::needs_barrier(&__ gvn(), NULL, obj, mem, allow_fromspace)) {
99 // We know it is null, no barrier needed.
100 return obj;
101 }
102
103 if (obj_type->meet(TypePtr::NULL_PTR) == obj_type->remove_speculative()) {
104
105 // We don't know if it's null or not. Need null-check.
106 enum { _not_null_path = 1, _null_path, PATH_LIMIT };
107 RegionNode* region = new RegionNode(PATH_LIMIT);
108 Node* phi = new PhiNode(region, obj_type);
109 Node* null_ctrl = __ top();
110 Node* not_null_obj = __ null_check_oop(obj, &null_ctrl);
111
112 region->init_req(_null_path, null_ctrl);
113 phi ->init_req(_null_path, __ zerocon(T_OBJECT));
114
115 Node* ctrl = use_ctrl ? __ control() : NULL;
116 ShenandoahReadBarrierNode* rb = new ShenandoahReadBarrierNode(ctrl, mem, not_null_obj, allow_fromspace);
117 Node* n = __ gvn().transform(rb);
118
119 region->init_req(_not_null_path, __ control());
120 phi ->init_req(_not_null_path, n);
121
122 __ set_control(__ gvn().transform(region));
123 __ record_for_igvn(region);
124 return __ gvn().transform(phi);
125
126 } else {
127 // We know it is not null. Simple barrier is sufficient.
128 Node* ctrl = use_ctrl ? __ control() : NULL;
129 ShenandoahReadBarrierNode* rb = new ShenandoahReadBarrierNode(ctrl, mem, obj, allow_fromspace);
130 Node* n = __ gvn().transform(rb);
131 __ record_for_igvn(n);
132 return n;
133 }
134 }
135
136 Node* ShenandoahBarrierSetC2::shenandoah_write_barrier_helper(GraphKit* kit, Node* obj, const TypePtr* adr_type) const {
137 ShenandoahWriteBarrierNode* wb = new ShenandoahWriteBarrierNode(kit->C, kit->control(), kit->memory(adr_type), obj);
138 Node* n = __ gvn().transform(wb);
139 if (n == wb) { // New barrier needs memory projection.
140 Node* proj = __ gvn().transform(new ShenandoahWBMemProjNode(n));
141 __ set_memory(proj, adr_type);
142 }
143 return n;
144 }
145
146 Node* ShenandoahBarrierSetC2::shenandoah_write_barrier(GraphKit* kit, Node* obj) const {
147 if (ShenandoahWriteBarrier) {
148 obj = shenandoah_write_barrier_impl(kit, obj);
149 }
150 return obj;
151 }
152
153 Node* ShenandoahBarrierSetC2::shenandoah_write_barrier_impl(GraphKit* kit, Node* obj) const {
154 if (! ShenandoahBarrierNode::needs_barrier(&__ gvn(), NULL, obj, NULL, true)) {
155 return obj;
156 }
157 const Type* obj_type = obj->bottom_type();
158 const TypePtr* adr_type = ShenandoahBarrierNode::brooks_pointer_type(obj_type);
159 Node* n = shenandoah_write_barrier_helper(kit, obj, adr_type);
160 __ record_for_igvn(n);
161 return n;
162 }
163
164 bool ShenandoahBarrierSetC2::satb_can_remove_pre_barrier(GraphKit* kit, PhaseTransform* phase, Node* adr,
165 BasicType bt, uint adr_idx) const {
166 intptr_t offset = 0;
167 Node* base = AddPNode::Ideal_base_and_offset(adr, phase, offset);
168 AllocateNode* alloc = AllocateNode::Ideal_allocation(base, phase);
169
170 if (offset == Type::OffsetBot) {
171 return false; // cannot unalias unless there are precise offsets
172 }
173
174 if (alloc == NULL) {
175 return false; // No allocation found
176 }
177
178 intptr_t size_in_bytes = type2aelembytes(bt);
179
180 Node* mem = __ memory(adr_idx); // start searching here...
181
182 for (int cnt = 0; cnt < 50; cnt++) {
183
184 if (mem->is_Store()) {
185
186 Node* st_adr = mem->in(MemNode::Address);
187 intptr_t st_offset = 0;
188 Node* st_base = AddPNode::Ideal_base_and_offset(st_adr, phase, st_offset);
189
190 if (st_base == NULL) {
191 break; // inscrutable pointer
192 }
193
194 // Break we have found a store with same base and offset as ours so break
195 if (st_base == base && st_offset == offset) {
196 break;
197 }
198
199 if (st_offset != offset && st_offset != Type::OffsetBot) {
200 const int MAX_STORE = BytesPerLong;
201 if (st_offset >= offset + size_in_bytes ||
202 st_offset <= offset - MAX_STORE ||
203 st_offset <= offset - mem->as_Store()->memory_size()) {
204 // Success: The offsets are provably independent.
205 // (You may ask, why not just test st_offset != offset and be done?
206 // The answer is that stores of different sizes can co-exist
207 // in the same sequence of RawMem effects. We sometimes initialize
208 // a whole 'tile' of array elements with a single jint or jlong.)
209 mem = mem->in(MemNode::Memory);
210 continue; // advance through independent store memory
211 }
212 }
213
214 if (st_base != base
215 && MemNode::detect_ptr_independence(base, alloc, st_base,
216 AllocateNode::Ideal_allocation(st_base, phase),
217 phase)) {
218 // Success: The bases are provably independent.
219 mem = mem->in(MemNode::Memory);
220 continue; // advance through independent store memory
221 }
222 } else if (mem->is_Proj() && mem->in(0)->is_Initialize()) {
223
224 InitializeNode* st_init = mem->in(0)->as_Initialize();
225 AllocateNode* st_alloc = st_init->allocation();
226
227 // Make sure that we are looking at the same allocation site.
228 // The alloc variable is guaranteed to not be null here from earlier check.
229 if (alloc == st_alloc) {
230 // Check that the initialization is storing NULL so that no previous store
231 // has been moved up and directly write a reference
232 Node* captured_store = st_init->find_captured_store(offset,
233 type2aelembytes(T_OBJECT),
234 phase);
235 if (captured_store == NULL || captured_store == st_init->zero_memory()) {
236 return true;
237 }
238 }
239 }
240
241 // Unless there is an explicit 'continue', we must bail out here,
242 // because 'mem' is an inscrutable memory state (e.g., a call).
243 break;
244 }
245
246 return false;
247 }
248
249 #undef __
250 #define __ ideal.
251
252 void ShenandoahBarrierSetC2::satb_write_barrier_pre(GraphKit* kit,
253 bool do_load,
254 Node* obj,
255 Node* adr,
256 uint alias_idx,
257 Node* val,
258 const TypeOopPtr* val_type,
259 Node* pre_val,
260 BasicType bt) const {
261 // Some sanity checks
262 // Note: val is unused in this routine.
263
264 if (do_load) {
265 // We need to generate the load of the previous value
266 assert(obj != NULL, "must have a base");
267 assert(adr != NULL, "where are loading from?");
268 assert(pre_val == NULL, "loaded already?");
269 assert(val_type != NULL, "need a type");
270
271 if (ReduceInitialCardMarks
272 && satb_can_remove_pre_barrier(kit, &kit->gvn(), adr, bt, alias_idx)) {
273 return;
274 }
275
276 } else {
277 // In this case both val_type and alias_idx are unused.
278 assert(pre_val != NULL, "must be loaded already");
279 // Nothing to be done if pre_val is null.
280 if (pre_val->bottom_type() == TypePtr::NULL_PTR) return;
281 assert(pre_val->bottom_type()->basic_type() == T_OBJECT, "or we shouldn't be here");
282 }
283 assert(bt == T_OBJECT, "or we shouldn't be here");
284
285 IdealKit ideal(kit, true);
286
287 Node* tls = __ thread(); // ThreadLocalStorage
288
289 Node* no_base = __ top();
290 Node* zero = __ ConI(0);
291 Node* zeroX = __ ConX(0);
292
293 float likely = PROB_LIKELY(0.999);
294 float unlikely = PROB_UNLIKELY(0.999);
295
296 // Offsets into the thread
297 const int index_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset());
298 const int buffer_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset());
299
300 // Now the actual pointers into the thread
301 Node* buffer_adr = __ AddP(no_base, tls, __ ConX(buffer_offset));
302 Node* index_adr = __ AddP(no_base, tls, __ ConX(index_offset));
303
304 // Now some of the values
305 Node* marking;
306 Node* gc_state = __ AddP(no_base, tls, __ ConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset())));
307 Node* ld = __ load(__ ctrl(), gc_state, TypeInt::BYTE, T_BYTE, Compile::AliasIdxRaw);
308 marking = __ AndI(ld, __ ConI(ShenandoahHeap::MARKING));
309 assert(ShenandoahWriteBarrierNode::is_gc_state_load(ld), "Should match the shape");
310
311 // if (!marking)
312 __ if_then(marking, BoolTest::ne, zero, unlikely); {
313 BasicType index_bt = TypeX_X->basic_type();
314 assert(sizeof(size_t) == type2aelembytes(index_bt), "Loading G1 SATBMarkQueue::_index with wrong size.");
315 Node* index = __ load(__ ctrl(), index_adr, TypeX_X, index_bt, Compile::AliasIdxRaw);
316
317 if (do_load) {
318 // load original value
319 // alias_idx correct??
320 pre_val = __ load(__ ctrl(), adr, val_type, bt, alias_idx);
321 }
322
323 // if (pre_val != NULL)
324 __ if_then(pre_val, BoolTest::ne, kit->null()); {
325 Node* buffer = __ load(__ ctrl(), buffer_adr, TypeRawPtr::NOTNULL, T_ADDRESS, Compile::AliasIdxRaw);
326
327 // is the queue for this thread full?
328 __ if_then(index, BoolTest::ne, zeroX, likely); {
329
330 // decrement the index
331 Node* next_index = kit->gvn().transform(new SubXNode(index, __ ConX(sizeof(intptr_t))));
332
333 // Now get the buffer location we will log the previous value into and store it
334 Node *log_addr = __ AddP(no_base, buffer, next_index);
335 __ store(__ ctrl(), log_addr, pre_val, T_OBJECT, Compile::AliasIdxRaw, MemNode::unordered);
336 // update the index
337 __ store(__ ctrl(), index_adr, next_index, index_bt, Compile::AliasIdxRaw, MemNode::unordered);
338
339 } __ else_(); {
340
341 // logging buffer is full, call the runtime
342 const TypeFunc *tf = ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type();
343 __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), "shenandoah_wb_pre", pre_val, tls);
344 } __ end_if(); // (!index)
345 } __ end_if(); // (pre_val != NULL)
346 } __ end_if(); // (!marking)
347
348 // Final sync IdealKit and GraphKit.
349 kit->final_sync(ideal);
350
351 if (ShenandoahSATBBarrier && adr != NULL) {
352 Node* c = kit->control();
353 Node* call = c->in(1)->in(1)->in(1)->in(0);
354 assert(is_shenandoah_wb_pre_call(call), "shenandoah_wb_pre call expected");
355 call->add_req(adr);
356 }
357 }
358
359 bool ShenandoahBarrierSetC2::is_shenandoah_wb_pre_call(Node* call) {
360 return call->is_CallLeaf() &&
361 call->as_CallLeaf()->entry_point() == CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry);
362 }
363
364 bool ShenandoahBarrierSetC2::is_shenandoah_wb_call(Node* call) {
365 return call->is_CallLeaf() &&
366 call->as_CallLeaf()->entry_point() == CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_JRT);
367 }
368
369 bool ShenandoahBarrierSetC2::is_shenandoah_marking_if(PhaseTransform *phase, Node* n) {
370 if (n->Opcode() != Op_If) {
371 return false;
372 }
373
374 Node* bol = n->in(1);
375 assert(bol->is_Bool(), "");
376 Node* cmpx = bol->in(1);
377 if (bol->as_Bool()->_test._test == BoolTest::ne &&
378 cmpx->is_Cmp() && cmpx->in(2) == phase->intcon(0) &&
379 is_shenandoah_state_load(cmpx->in(1)->in(1)) &&
380 cmpx->in(1)->in(2)->is_Con() &&
381 cmpx->in(1)->in(2) == phase->intcon(ShenandoahHeap::MARKING)) {
382 return true;
383 }
384
385 return false;
386 }
387
388 bool ShenandoahBarrierSetC2::is_shenandoah_state_load(Node* n) {
389 if (!n->is_Load()) return false;
390 const int state_offset = in_bytes(ShenandoahThreadLocalData::gc_state_offset());
391 return n->in(2)->is_AddP() && n->in(2)->in(2)->Opcode() == Op_ThreadLocal
392 && n->in(2)->in(3)->is_Con()
393 && n->in(2)->in(3)->bottom_type()->is_intptr_t()->get_con() == state_offset;
394 }
395
396 void ShenandoahBarrierSetC2::shenandoah_write_barrier_pre(GraphKit* kit,
397 bool do_load,
398 Node* obj,
399 Node* adr,
400 uint alias_idx,
401 Node* val,
402 const TypeOopPtr* val_type,
403 Node* pre_val,
404 BasicType bt) const {
405 if (ShenandoahSATBBarrier) {
406 IdealKit ideal(kit);
407 kit->sync_kit(ideal);
408
409 satb_write_barrier_pre(kit, do_load, obj, adr, alias_idx, val, val_type, pre_val, bt);
410
411 ideal.sync_kit(kit);
412 kit->final_sync(ideal);
413 }
414 }
415
416 Node* ShenandoahBarrierSetC2::shenandoah_enqueue_barrier(GraphKit* kit, Node* pre_val) const {
417 return kit->gvn().transform(new ShenandoahEnqueueBarrierNode(pre_val));
418 }
419
420 // Helper that guards and inserts a pre-barrier.
421 void ShenandoahBarrierSetC2::insert_pre_barrier(GraphKit* kit, Node* base_oop, Node* offset,
422 Node* pre_val, bool need_mem_bar) const {
423 // We could be accessing the referent field of a reference object. If so, when G1
424 // is enabled, we need to log the value in the referent field in an SATB buffer.
425 // This routine performs some compile time filters and generates suitable
426 // runtime filters that guard the pre-barrier code.
427 // Also add memory barrier for non volatile load from the referent field
428 // to prevent commoning of loads across safepoint.
429
430 // Some compile time checks.
431
432 // If offset is a constant, is it java_lang_ref_Reference::_reference_offset?
433 const TypeX* otype = offset->find_intptr_t_type();
434 if (otype != NULL && otype->is_con() &&
435 otype->get_con() != java_lang_ref_Reference::referent_offset) {
436 // Constant offset but not the reference_offset so just return
437 return;
438 }
439
440 // We only need to generate the runtime guards for instances.
441 const TypeOopPtr* btype = base_oop->bottom_type()->isa_oopptr();
442 if (btype != NULL) {
443 if (btype->isa_aryptr()) {
444 // Array type so nothing to do
445 return;
446 }
447
448 const TypeInstPtr* itype = btype->isa_instptr();
449 if (itype != NULL) {
450 // Can the klass of base_oop be statically determined to be
451 // _not_ a sub-class of Reference and _not_ Object?
452 ciKlass* klass = itype->klass();
453 if ( klass->is_loaded() &&
454 !klass->is_subtype_of(kit->env()->Reference_klass()) &&
455 !kit->env()->Object_klass()->is_subtype_of(klass)) {
456 return;
457 }
458 }
459 }
460
461 // The compile time filters did not reject base_oop/offset so
462 // we need to generate the following runtime filters
463 //
464 // if (offset == java_lang_ref_Reference::_reference_offset) {
465 // if (instance_of(base, java.lang.ref.Reference)) {
466 // pre_barrier(_, pre_val, ...);
467 // }
468 // }
469
470 float likely = PROB_LIKELY( 0.999);
471 float unlikely = PROB_UNLIKELY(0.999);
472
473 IdealKit ideal(kit);
474
475 Node* referent_off = __ ConX(java_lang_ref_Reference::referent_offset);
476
477 __ if_then(offset, BoolTest::eq, referent_off, unlikely); {
478 // Update graphKit memory and control from IdealKit.
479 kit->sync_kit(ideal);
480
481 Node* ref_klass_con = kit->makecon(TypeKlassPtr::make(kit->env()->Reference_klass()));
482 Node* is_instof = kit->gen_instanceof(base_oop, ref_klass_con);
483
484 // Update IdealKit memory and control from graphKit.
485 __ sync_kit(kit);
486
487 Node* one = __ ConI(1);
488 // is_instof == 0 if base_oop == NULL
489 __ if_then(is_instof, BoolTest::eq, one, unlikely); {
490
491 // Update graphKit from IdeakKit.
492 kit->sync_kit(ideal);
493
494 // Use the pre-barrier to record the value in the referent field
495 satb_write_barrier_pre(kit, false /* do_load */,
496 NULL /* obj */, NULL /* adr */, max_juint /* alias_idx */, NULL /* val */, NULL /* val_type */,
497 pre_val /* pre_val */,
498 T_OBJECT);
499 if (need_mem_bar) {
500 // Add memory barrier to prevent commoning reads from this field
501 // across safepoint since GC can change its value.
502 kit->insert_mem_bar(Op_MemBarCPUOrder);
503 }
504 // Update IdealKit from graphKit.
505 __ sync_kit(kit);
506
507 } __ end_if(); // _ref_type != ref_none
508 } __ end_if(); // offset == referent_offset
509
510 // Final sync IdealKit and GraphKit.
511 kit->final_sync(ideal);
512 }
513
514 #undef __
515
516 const TypeFunc* ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type() {
517 const Type **fields = TypeTuple::fields(2);
518 fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // original field value
519 fields[TypeFunc::Parms+1] = TypeRawPtr::NOTNULL; // thread
520 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2, fields);
521
522 // create result type (range)
523 fields = TypeTuple::fields(0);
524 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0, fields);
525
526 return TypeFunc::make(domain, range);
527 }
528
529 const TypeFunc* ShenandoahBarrierSetC2::shenandoah_clone_barrier_Type() {
530 const Type **fields = TypeTuple::fields(1);
531 fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // original field value
532 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+1, fields);
533
534 // create result type (range)
535 fields = TypeTuple::fields(0);
536 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0, fields);
537
538 return TypeFunc::make(domain, range);
539 }
540
541 const TypeFunc* ShenandoahBarrierSetC2::shenandoah_write_barrier_Type() {
542 const Type **fields = TypeTuple::fields(1);
543 fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // original field value
544 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+1, fields);
545
546 // create result type (range)
547 fields = TypeTuple::fields(1);
548 fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL;
549 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+1, fields);
550
551 return TypeFunc::make(domain, range);
552 }
553
554 void ShenandoahBarrierSetC2::resolve_address(C2Access& access) const {
555 const TypePtr* adr_type = access.addr().type();
556
557 if ((access.decorators() & IN_NATIVE) == 0 && (adr_type->isa_instptr() || adr_type->isa_aryptr())) {
558 int off = adr_type->is_ptr()->offset();
559 int base_off = adr_type->isa_instptr() ? instanceOopDesc::base_offset_in_bytes() :
560 arrayOopDesc::base_offset_in_bytes(adr_type->is_aryptr()->elem()->array_element_basic_type());
561 assert(off != Type::OffsetTop, "unexpected offset");
562 if (off == Type::OffsetBot || off >= base_off) {
563 DecoratorSet decorators = access.decorators();
564 bool is_write = (decorators & C2_WRITE_ACCESS) != 0;
565 GraphKit* kit = NULL;
566 if (access.is_parse_access()) {
567 C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
568 kit = parse_access.kit();
569 }
570 Node* adr = access.addr().node();
571 assert(adr->is_AddP(), "unexpected address shape");
572 Node* base = adr->in(AddPNode::Base);
573
574 if (is_write) {
575 if (kit != NULL) {
576 base = shenandoah_write_barrier(kit, base);
577 } else {
578 assert(access.is_opt_access(), "either parse or opt access");
579 assert((access.decorators() & C2_ARRAY_COPY) != 0, "can be skipped for clone");
580 }
581 } else {
582 if (adr_type->isa_instptr()) {
583 Compile* C = access.gvn().C;
584 ciField* field = C->alias_type(adr_type)->field();
585
586 // Insert read barrier for Shenandoah.
587 if (field != NULL &&
588 ((ShenandoahOptimizeStaticFinals && field->is_static() && field->is_final()) ||
589 (ShenandoahOptimizeInstanceFinals && !field->is_static() && field->is_final()) ||
590 (ShenandoahOptimizeStableFinals && field->is_stable()))) {
591 // Skip the barrier for special fields
592 } else {
593 if (kit != NULL) {
594 base = shenandoah_read_barrier(kit, base);
595 } else {
596 assert(access.is_opt_access(), "either parse or opt access");
597 assert((access.decorators() & C2_ARRAY_COPY) != 0, "can be skipped for arraycopy");
598 }
599 }
600 } else {
601 if (kit != NULL) {
602 base = shenandoah_read_barrier(kit, base);
603 } else {
604 assert(access.is_opt_access(), "either parse or opt access");
605 assert((access.decorators() & C2_ARRAY_COPY) != 0, "can be skipped for arraycopy");
606 }
607 }
608 }
609 if (base != adr->in(AddPNode::Base)) {
610 assert(kit != NULL, "no barrier should have been added");
611
612 Node* address = adr->in(AddPNode::Address);
613
614 if (address->is_AddP()) {
615 assert(address->in(AddPNode::Base) == adr->in(AddPNode::Base), "unexpected address shape");
616 assert(!address->in(AddPNode::Address)->is_AddP(), "unexpected address shape");
617 assert(address->in(AddPNode::Address) == adr->in(AddPNode::Base), "unexpected address shape");
618 address = address->clone();
619 address->set_req(AddPNode::Base, base);
620 address->set_req(AddPNode::Address, base);
621 address = kit->gvn().transform(address);
622 } else {
623 assert(address == adr->in(AddPNode::Base), "unexpected address shape");
624 address = base;
625 }
626 adr = adr->clone();
627 adr->set_req(AddPNode::Base, base);
628 adr->set_req(AddPNode::Address, address);
629 adr = kit->gvn().transform(adr);
630 access.addr().set_node(adr);
631 }
632 }
633 }
634 }
635
636 Node* ShenandoahBarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& val) const {
637 DecoratorSet decorators = access.decorators();
638
639 const TypePtr* adr_type = access.addr().type();
640 Node* adr = access.addr().node();
641
642 bool anonymous = (decorators & ON_UNKNOWN_OOP_REF) != 0;
643 bool on_heap = (decorators & IN_HEAP) != 0;
644
645 if (!access.is_oop() || (!on_heap && !anonymous)) {
646 return BarrierSetC2::store_at_resolved(access, val);
647 }
648
649 if (access.is_parse_access()) {
650 C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
651 GraphKit* kit = parse_access.kit();
652
653 uint adr_idx = kit->C->get_alias_index(adr_type);
654 assert(adr_idx != Compile::AliasIdxTop, "use other store_to_memory factory" );
655 Node* value = val.node();
656 value = shenandoah_storeval_barrier(kit, value);
657 val.set_node(value);
658 shenandoah_write_barrier_pre(kit, true /* do_load */, /*kit->control(),*/ access.base(), adr, adr_idx, val.node(),
659 static_cast<const TypeOopPtr*>(val.type()), NULL /* pre_val */, access.type());
660 } else {
661 assert(access.is_opt_access(), "only for optimization passes");
662 assert(((decorators & C2_TIGHLY_COUPLED_ALLOC) != 0 || !ShenandoahSATBBarrier) && (decorators & C2_ARRAY_COPY) != 0, "unexpected caller of this code");
663 C2OptAccess& opt_access = static_cast<C2OptAccess&>(access);
664 PhaseGVN& gvn = opt_access.gvn();
665 MergeMemNode* mm = opt_access.mem();
666
667 if (ShenandoahStoreValReadBarrier) {
668 RegionNode* region = new RegionNode(3);
669 const Type* v_t = gvn.type(val.node());
670 Node* phi = new PhiNode(region, v_t->isa_oopptr() ? v_t->is_oopptr()->cast_to_nonconst() : v_t);
671 Node* cmp = gvn.transform(new CmpPNode(val.node(), gvn.zerocon(T_OBJECT)));
672 Node* bol = gvn.transform(new BoolNode(cmp, BoolTest::ne));
673 IfNode* iff = new IfNode(opt_access.ctl(), bol, PROB_LIKELY_MAG(3), COUNT_UNKNOWN);
674
675 gvn.transform(iff);
676 if (gvn.is_IterGVN()) {
677 gvn.is_IterGVN()->_worklist.push(iff);
678 } else {
679 gvn.record_for_igvn(iff);
680 }
681
682 Node* null_true = gvn.transform(new IfFalseNode(iff));
683 Node* null_false = gvn.transform(new IfTrueNode(iff));
684 region->init_req(1, null_true);
685 region->init_req(2, null_false);
686 phi->init_req(1, gvn.zerocon(T_OBJECT));
687 Node* cast = new CastPPNode(val.node(), gvn.type(val.node())->join_speculative(TypePtr::NOTNULL));
688 cast->set_req(0, null_false);
689 cast = gvn.transform(cast);
690 Node* rb = gvn.transform(new ShenandoahReadBarrierNode(null_false, gvn.C->immutable_memory(), cast, false));
691 phi->init_req(2, rb);
692 opt_access.set_ctl(gvn.transform(region));
693 val.set_node(gvn.transform(phi));
694 }
695 if (ShenandoahStoreValEnqueueBarrier) {
696 const TypePtr* adr_type = ShenandoahBarrierNode::brooks_pointer_type(gvn.type(val.node()));
697 int alias = gvn.C->get_alias_index(adr_type);
698 Node* wb = new ShenandoahWriteBarrierNode(gvn.C, opt_access.ctl(), mm->memory_at(alias), val.node());
699 Node* wb_transformed = gvn.transform(wb);
700 Node* enqueue = gvn.transform(new ShenandoahEnqueueBarrierNode(wb_transformed));
701 if (wb_transformed == wb) {
702 Node* proj = gvn.transform(new ShenandoahWBMemProjNode(wb));
703 mm->set_memory_at(alias, proj);
704 }
705 val.set_node(enqueue);
706 }
707 }
708 return BarrierSetC2::store_at_resolved(access, val);
709 }
710
711 Node* ShenandoahBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const {
712 DecoratorSet decorators = access.decorators();
713
714 Node* adr = access.addr().node();
715 Node* obj = access.base();
716
717 bool mismatched = (decorators & C2_MISMATCHED) != 0;
718 bool unknown = (decorators & ON_UNKNOWN_OOP_REF) != 0;
719 bool on_heap = (decorators & IN_HEAP) != 0;
720 bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
721 bool is_unordered = (decorators & MO_UNORDERED) != 0;
722 bool need_cpu_mem_bar = !is_unordered || mismatched || !on_heap;
723
724 Node* top = Compile::current()->top();
725
726 Node* offset = adr->is_AddP() ? adr->in(AddPNode::Offset) : top;
727 Node* load = BarrierSetC2::load_at_resolved(access, val_type);
728
729 // If we are reading the value of the referent field of a Reference
730 // object (either by using Unsafe directly or through reflection)
731 // then, if SATB is enabled, we need to record the referent in an
732 // SATB log buffer using the pre-barrier mechanism.
733 // Also we need to add memory barrier to prevent commoning reads
734 // from this field across safepoint since GC can change its value.
735 bool need_read_barrier = ShenandoahKeepAliveBarrier &&
736 (on_heap && (on_weak || (unknown && offset != top && obj != top)));
737
738 if (!access.is_oop() || !need_read_barrier) {
739 return load;
740 }
741
742 assert(access.is_parse_access(), "entry not supported at optimization time");
743 C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
744 GraphKit* kit = parse_access.kit();
745
746 if (on_weak) {
747 // Use the pre-barrier to record the value in the referent field
748 satb_write_barrier_pre(kit, false /* do_load */,
749 NULL /* obj */, NULL /* adr */, max_juint /* alias_idx */, NULL /* val */, NULL /* val_type */,
750 load /* pre_val */, T_OBJECT);
751 // Add memory barrier to prevent commoning reads from this field
752 // across safepoint since GC can change its value.
753 kit->insert_mem_bar(Op_MemBarCPUOrder);
754 } else if (unknown) {
755 // We do not require a mem bar inside pre_barrier if need_mem_bar
756 // is set: the barriers would be emitted by us.
757 insert_pre_barrier(kit, obj, offset, load, !need_cpu_mem_bar);
758 }
759
760 return load;
761 }
762
763 Node* ShenandoahBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
764 Node* new_val, const Type* value_type) const {
765 GraphKit* kit = access.kit();
766 if (access.is_oop()) {
767 new_val = shenandoah_storeval_barrier(kit, new_val);
768 shenandoah_write_barrier_pre(kit, false /* do_load */,
769 NULL, NULL, max_juint, NULL, NULL,
770 expected_val /* pre_val */, T_OBJECT);
771
772 MemNode::MemOrd mo = access.mem_node_mo();
773 Node* mem = access.memory();
774 Node* adr = access.addr().node();
775 const TypePtr* adr_type = access.addr().type();
776 Node* load_store = NULL;
777
778 #ifdef _LP64
779 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
780 Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop()));
781 Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop()));
782 load_store = kit->gvn().transform(new ShenandoahCompareAndExchangeNNode(kit->control(), mem, adr, newval_enc, oldval_enc, adr_type, value_type->make_narrowoop(), mo));
783 } else
784 #endif
785 {
786 load_store = kit->gvn().transform(new ShenandoahCompareAndExchangePNode(kit->control(), mem, adr, new_val, expected_val, adr_type, value_type->is_oopptr(), mo));
787 }
788
789 access.set_raw_access(load_store);
790 pin_atomic_op(access);
791
792 #ifdef _LP64
793 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
794 return kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type()));
795 }
796 #endif
797 return load_store;
798 }
799 return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type);
800 }
801
802 Node* ShenandoahBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
803 Node* new_val, const Type* value_type) const {
804 GraphKit* kit = access.kit();
805 if (access.is_oop()) {
806 new_val = shenandoah_storeval_barrier(kit, new_val);
807 shenandoah_write_barrier_pre(kit, false /* do_load */,
808 NULL, NULL, max_juint, NULL, NULL,
809 expected_val /* pre_val */, T_OBJECT);
810 DecoratorSet decorators = access.decorators();
811 MemNode::MemOrd mo = access.mem_node_mo();
812 Node* mem = access.memory();
813 bool is_weak_cas = (decorators & C2_WEAK_CMPXCHG) != 0;
814 Node* load_store = NULL;
815 Node* adr = access.addr().node();
816 #ifdef _LP64
817 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
818 Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop()));
819 Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop()));
820 if (is_weak_cas) {
821 load_store = kit->gvn().transform(new ShenandoahWeakCompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo));
822 } else {
823 load_store = kit->gvn().transform(new ShenandoahCompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo));
824 }
825 } else
826 #endif
827 {
828 if (is_weak_cas) {
829 load_store = kit->gvn().transform(new ShenandoahWeakCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo));
830 } else {
831 load_store = kit->gvn().transform(new ShenandoahCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo));
832 }
833 }
834 access.set_raw_access(load_store);
835 pin_atomic_op(access);
836 return load_store;
837 }
838 return BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type);
839 }
840
841 Node* ShenandoahBarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* val, const Type* value_type) const {
842 GraphKit* kit = access.kit();
843 if (access.is_oop()) {
844 val = shenandoah_storeval_barrier(kit, val);
845 }
846 Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, val, value_type);
847 if (access.is_oop()) {
848 shenandoah_write_barrier_pre(kit, false /* do_load */,
849 NULL, NULL, max_juint, NULL, NULL,
850 result /* pre_val */, T_OBJECT);
851 }
852 return result;
853 }
854
855 void ShenandoahBarrierSetC2::clone(GraphKit* kit, Node* src, Node* dst, Node* size, bool is_array) const {
856 assert(!src->is_AddP(), "unexpected input");
857 src = shenandoah_read_barrier(kit, src);
858 BarrierSetC2::clone(kit, src, dst, size, is_array);
859 }
860
861 Node* ShenandoahBarrierSetC2::resolve(GraphKit* kit, Node* n, DecoratorSet decorators) const {
862 bool is_write = decorators & ACCESS_WRITE;
863 if (is_write) {
864 return shenandoah_write_barrier(kit, n);
865 } else {
866 return shenandoah_read_barrier(kit, n);
867 }
868 }
869
870 Node* ShenandoahBarrierSetC2::obj_allocate(PhaseMacroExpand* macro, Node* ctrl, Node* mem, Node* toobig_false, Node* size_in_bytes,
871 Node*& i_o, Node*& needgc_ctrl,
872 Node*& fast_oop_ctrl, Node*& fast_oop_rawmem,
873 intx prefetch_lines) const {
874 PhaseIterGVN& igvn = macro->igvn();
875
876 // Allocate several words more for the Shenandoah brooks pointer.
877 size_in_bytes = new AddXNode(size_in_bytes, igvn.MakeConX(ShenandoahBrooksPointer::byte_size()));
878 macro->transform_later(size_in_bytes);
879
880 Node* fast_oop = BarrierSetC2::obj_allocate(macro, ctrl, mem, toobig_false, size_in_bytes,
881 i_o, needgc_ctrl, fast_oop_ctrl, fast_oop_rawmem,
882 prefetch_lines);
883
884 // Bump up object for Shenandoah brooks pointer.
885 fast_oop = new AddPNode(macro->top(), fast_oop, igvn.MakeConX(ShenandoahBrooksPointer::byte_size()));
886 macro->transform_later(fast_oop);
887
888 // Initialize Shenandoah brooks pointer to point to the object itself.
889 fast_oop_rawmem = macro->make_store(fast_oop_ctrl, fast_oop_rawmem, fast_oop, ShenandoahBrooksPointer::byte_offset(), fast_oop, T_OBJECT);
890
891 return fast_oop;
892 }
893
894 // Support for GC barriers emitted during parsing
895 bool ShenandoahBarrierSetC2::is_gc_barrier_node(Node* node) const {
896 if (node->Opcode() != Op_CallLeaf && node->Opcode() != Op_CallLeafNoFP) {
897 return false;
898 }
899 CallLeafNode *call = node->as_CallLeaf();
900 if (call->_name == NULL) {
901 return false;
902 }
903
904 return strcmp(call->_name, "shenandoah_clone_barrier") == 0 ||
905 strcmp(call->_name, "shenandoah_cas_obj") == 0 ||
906 strcmp(call->_name, "shenandoah_wb_pre") == 0;
907 }
908
909 Node* ShenandoahBarrierSetC2::step_over_gc_barrier(Node* c) const {
910 return ShenandoahBarrierNode::skip_through_barrier(c);
911 }
912
913 bool ShenandoahBarrierSetC2::expand_barriers(Compile* C, PhaseIterGVN& igvn) const {
914 return !ShenandoahWriteBarrierNode::expand(C, igvn);
915 }
916
917 bool ShenandoahBarrierSetC2::optimize_loops(PhaseIdealLoop* phase, LoopOptsMode mode, VectorSet& visited, Node_Stack& nstack, Node_List& worklist) const {
918 if (mode == LoopOptsShenandoahExpand) {
919 assert(UseShenandoahGC, "only for shenandoah");
920 ShenandoahWriteBarrierNode::pin_and_expand(phase);
921 return true;
922 } else if (mode == LoopOptsShenandoahPostExpand) {
923 assert(UseShenandoahGC, "only for shenandoah");
924 visited.Clear();
925 ShenandoahWriteBarrierNode::optimize_after_expansion(visited, nstack, worklist, phase);
926 return true;
927 }
928 GrowableArray<MemoryGraphFixer*> memory_graph_fixers;
929 ShenandoahWriteBarrierNode::optimize_before_expansion(phase, memory_graph_fixers, false);
930 return false;
931 }
932
933 bool ShenandoahBarrierSetC2::array_copy_requires_gc_barriers(bool tightly_coupled_alloc, BasicType type, bool is_clone, ArrayCopyPhase phase) const {
934 bool is_oop = type == T_OBJECT || type == T_ARRAY;
935 if (!is_oop) {
936 return false;
937 }
938
939 if (tightly_coupled_alloc) {
940 if (phase == Optimization) {
941 return false;
942 }
943 return !is_clone;
944 }
945 if (phase == Optimization) {
946 return !ShenandoahStoreValEnqueueBarrier;
947 }
948 return true;
949 }
950
951 bool ShenandoahBarrierSetC2::clone_needs_postbarrier(ArrayCopyNode *ac, PhaseIterGVN& igvn) {
952 Node* src = ac->in(ArrayCopyNode::Src);
953 const TypeOopPtr* src_type = igvn.type(src)->is_oopptr();
954 if (src_type->isa_instptr() != NULL) {
955 ciInstanceKlass* ik = src_type->klass()->as_instance_klass();
956 if ((src_type->klass_is_exact() || (!ik->is_interface() && !ik->has_subklass())) && !ik->has_injected_fields()) {
957 if (ik->has_object_fields()) {
958 return true;
959 } else {
960 if (!src_type->klass_is_exact()) {
961 igvn.C->dependencies()->assert_leaf_type(ik);
962 }
963 }
964 } else {
965 return true;
966 }
967 } else if (src_type->isa_aryptr()) {
968 BasicType src_elem = src_type->klass()->as_array_klass()->element_type()->basic_type();
969 if (src_elem == T_OBJECT || src_elem == T_ARRAY) {
970 return true;
971 }
972 } else {
973 return true;
974 }
975 return false;
976 }
977
978 void ShenandoahBarrierSetC2::clone_barrier_at_expansion(ArrayCopyNode* ac, Node* call, PhaseIterGVN& igvn) const {
979 assert(ac->is_clonebasic(), "no other kind of arraycopy here");
980
981 if (!clone_needs_postbarrier(ac, igvn)) {
982 BarrierSetC2::clone_barrier_at_expansion(ac, call, igvn);
983 return;
984 }
985
986 const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
987 Node* c = new ProjNode(call,TypeFunc::Control);
988 c = igvn.transform(c);
989 Node* m = new ProjNode(call, TypeFunc::Memory);
990 c = igvn.transform(m);
991
992 Node* dest = ac->in(ArrayCopyNode::Dest);
993 assert(dest->is_AddP(), "bad input");
994 Node* barrier_call = new CallLeafNode(ShenandoahBarrierSetC2::shenandoah_clone_barrier_Type(),
995 CAST_FROM_FN_PTR(address, ShenandoahRuntime::shenandoah_clone_barrier),
996 "shenandoah_clone_barrier", raw_adr_type);
997 barrier_call->init_req(TypeFunc::Control, c);
998 barrier_call->init_req(TypeFunc::I_O , igvn.C->top());
999 barrier_call->init_req(TypeFunc::Memory , m);
1000 barrier_call->init_req(TypeFunc::ReturnAdr, igvn.C->top());
1001 barrier_call->init_req(TypeFunc::FramePtr, igvn.C->top());
1002 barrier_call->init_req(TypeFunc::Parms+0, dest->in(AddPNode::Base));
1003
1004 barrier_call = igvn.transform(barrier_call);
1005 c = new ProjNode(barrier_call,TypeFunc::Control);
1006 c = igvn.transform(c);
1007 m = new ProjNode(barrier_call, TypeFunc::Memory);
1008 m = igvn.transform(m);
1009
1010 Node* out_c = ac->proj_out(TypeFunc::Control);
1011 Node* out_m = ac->proj_out(TypeFunc::Memory);
1012 igvn.replace_node(out_c, c);
1013 igvn.replace_node(out_m, m);
1014 }
1015
1016
1017 // Support for macro expanded GC barriers
1018 void ShenandoahBarrierSetC2::register_potential_barrier_node(Node* node) const {
1019 if (node->Opcode() == Op_ShenandoahWriteBarrier) {
1020 state()->add_shenandoah_barrier((ShenandoahWriteBarrierNode*) node);
1021 }
1022 }
1023
1024 void ShenandoahBarrierSetC2::unregister_potential_barrier_node(Node* node) const {
1025 if (node->Opcode() == Op_ShenandoahWriteBarrier) {
1026 state()->remove_shenandoah_barrier((ShenandoahWriteBarrierNode*) node);
1027 }
1028 }
1029
1030 void ShenandoahBarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* n) const {
1031 if (is_shenandoah_wb_pre_call(n)) {
1032 shenandoah_eliminate_wb_pre(n, ¯o->igvn());
1033 }
1034 }
1035
1036 void ShenandoahBarrierSetC2::shenandoah_eliminate_wb_pre(Node* call, PhaseIterGVN* igvn) const {
1037 assert(UseShenandoahGC && is_shenandoah_wb_pre_call(call), "");
1038 Node* c = call->as_Call()->proj_out(TypeFunc::Control);
1039 c = c->unique_ctrl_out();
1040 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
1041 c = c->unique_ctrl_out();
1042 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
1043 Node* iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
1044 assert(iff->is_If(), "expect test");
1045 if (!is_shenandoah_marking_if(igvn, iff)) {
1046 c = c->unique_ctrl_out();
1047 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
1048 iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
1049 assert(is_shenandoah_marking_if(igvn, iff), "expect marking test");
1050 }
1051 Node* cmpx = iff->in(1)->in(1);
1052 igvn->replace_node(cmpx, igvn->makecon(TypeInt::CC_EQ));
1053 igvn->rehash_node_delayed(call);
1054 call->del_req(call->req()-1);
1055 }
1056
1057 void ShenandoahBarrierSetC2::enqueue_useful_gc_barrier(PhaseIterGVN* igvn, Node* node) const {
1058 if (node->Opcode() == Op_AddP && ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(node)) {
1059 igvn->add_users_to_worklist(node);
1060 }
1061 }
1062
1063 void ShenandoahBarrierSetC2::eliminate_useless_gc_barriers(Unique_Node_List &useful, Compile* C) const {
1064 for (uint i = 0; i < useful.size(); i++) {
1065 Node* n = useful.at(i);
1066 if (n->Opcode() == Op_AddP && ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(n)) {
1067 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
1068 C->record_for_igvn(n->fast_out(i));
1069 }
1070 }
1071 }
1072 for (int i = state()->shenandoah_barriers_count()-1; i >= 0; i--) {
1073 ShenandoahWriteBarrierNode* n = state()->shenandoah_barrier(i);
1074 if (!useful.member(n)) {
1075 state()->remove_shenandoah_barrier(n);
1076 }
1077 }
1078
1079 }
1080
1081 bool ShenandoahBarrierSetC2::has_special_unique_user(const Node* node) const {
1082 assert(node->outcnt() == 1, "match only for unique out");
1083 Node* n = node->unique_out();
1084 return node->Opcode() == Op_ShenandoahWriteBarrier && n->Opcode() == Op_ShenandoahWBMemProj;
1085 }
1086
1087 void ShenandoahBarrierSetC2::add_users_to_worklist(Unique_Node_List* worklist) const {}
1088
1089 void* ShenandoahBarrierSetC2::create_barrier_state(Arena* comp_arena) const {
1090 return new(comp_arena) ShenandoahBarrierSetC2State(comp_arena);
1091 }
1092
1093 ShenandoahBarrierSetC2State* ShenandoahBarrierSetC2::state() const {
1094 return reinterpret_cast<ShenandoahBarrierSetC2State*>(Compile::current()->barrier_set_state());
1095 }
1096
1097 // If the BarrierSetC2 state has kept macro nodes in its compilation unit state to be
1098 // expanded later, then now is the time to do so.
1099 bool ShenandoahBarrierSetC2::expand_macro_nodes(PhaseMacroExpand* macro) const { return false; }
1100
1101 #ifdef ASSERT
1102 void ShenandoahBarrierSetC2::verify_gc_barriers(Compile* compile, CompilePhase phase) const {
1103 if (ShenandoahVerifyOptoBarriers && phase == BarrierSetC2::BeforeExpand) {
1104 ShenandoahBarrierNode::verify(Compile::current()->root());
1105 } else if (phase == BarrierSetC2::BeforeCodeGen) {
1106 // Verify G1 pre-barriers
1107 const int marking_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_active_offset());
1108
1109 ResourceArea *area = Thread::current()->resource_area();
1110 Unique_Node_List visited(area);
1111 Node_List worklist(area);
1112 // We're going to walk control flow backwards starting from the Root
1113 worklist.push(compile->root());
1114 while (worklist.size() > 0) {
1115 Node *x = worklist.pop();
1116 if (x == NULL || x == compile->top()) continue;
1117 if (visited.member(x)) {
1118 continue;
1119 } else {
1120 visited.push(x);
1121 }
1122
1123 if (x->is_Region()) {
1124 for (uint i = 1; i < x->req(); i++) {
1125 worklist.push(x->in(i));
1126 }
1127 } else {
1128 worklist.push(x->in(0));
1129 // We are looking for the pattern:
1130 // /->ThreadLocal
1131 // If->Bool->CmpI->LoadB->AddP->ConL(marking_offset)
1132 // \->ConI(0)
1133 // We want to verify that the If and the LoadB have the same control
1134 // See GraphKit::g1_write_barrier_pre()
1135 if (x->is_If()) {
1136 IfNode *iff = x->as_If();
1137 if (iff->in(1)->is_Bool() && iff->in(1)->in(1)->is_Cmp()) {
1138 CmpNode *cmp = iff->in(1)->in(1)->as_Cmp();
1139 if (cmp->Opcode() == Op_CmpI && cmp->in(2)->is_Con() && cmp->in(2)->bottom_type()->is_int()->get_con() == 0
1140 && cmp->in(1)->is_Load()) {
1141 LoadNode *load = cmp->in(1)->as_Load();
1142 if (load->Opcode() == Op_LoadB && load->in(2)->is_AddP() && load->in(2)->in(2)->Opcode() == Op_ThreadLocal
1143 && load->in(2)->in(3)->is_Con()
1144 && load->in(2)->in(3)->bottom_type()->is_intptr_t()->get_con() == marking_offset) {
1145
1146 Node *if_ctrl = iff->in(0);
1147 Node *load_ctrl = load->in(0);
1148
1149 if (if_ctrl != load_ctrl) {
1150 // Skip possible CProj->NeverBranch in infinite loops
1151 if ((if_ctrl->is_Proj() && if_ctrl->Opcode() == Op_CProj)
1152 && (if_ctrl->in(0)->is_MultiBranch() && if_ctrl->in(0)->Opcode() == Op_NeverBranch)) {
1153 if_ctrl = if_ctrl->in(0)->in(0);
1154 }
1155 }
1156 assert(load_ctrl != NULL && if_ctrl == load_ctrl, "controls must match");
1157 }
1158 }
1159 }
1160 }
1161 }
1162 }
1163 }
1164 }
1165 #endif
1166
1167 Node* ShenandoahBarrierSetC2::ideal_node(PhaseGVN* phase, Node* n, bool can_reshape) const {
1168 if (is_shenandoah_wb_pre_call(n)) {
1169 uint cnt = ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type()->domain()->cnt();
1170 if (n->req() > cnt) {
1171 Node* addp = n->in(cnt);
1172 if (has_only_shenandoah_wb_pre_uses(addp)) {
1173 n->del_req(cnt);
1174 if (can_reshape) {
1175 phase->is_IterGVN()->_worklist.push(addp);
1176 }
1177 return n;
1178 }
1179 }
1180 }
1181 if (n->Opcode() == Op_CmpP) {
1182 Node* in1 = n->in(1);
1183 Node* in2 = n->in(2);
1184 if (in1->bottom_type() == TypePtr::NULL_PTR) {
1185 in2 = step_over_gc_barrier(in2);
1186 }
1187 if (in2->bottom_type() == TypePtr::NULL_PTR) {
1188 in1 = step_over_gc_barrier(in1);
1189 }
1190 PhaseIterGVN* igvn = phase->is_IterGVN();
1191 if (in1 != n->in(1)) {
1192 if (igvn != NULL) {
1193 n->set_req_X(1, in1, igvn);
1194 } else {
1195 n->set_req(1, in1);
1196 }
1197 assert(in2 == n->in(2), "only one change");
1198 return n;
1199 }
1200 if (in2 != n->in(2)) {
1201 if (igvn != NULL) {
1202 n->set_req_X(2, in2, igvn);
1203 } else {
1204 n->set_req(2, in2);
1205 }
1206 return n;
1207 }
1208 } else if (can_reshape &&
1209 n->Opcode() == Op_If &&
1210 ShenandoahWriteBarrierNode::is_heap_stable_test(n) &&
1211 n->in(0) != NULL) {
1212 Node* dom = n->in(0);
1213 Node* prev_dom = n;
1214 int op = n->Opcode();
1215 int dist = 16;
1216 // Search up the dominator tree for another heap stable test
1217 while (dom->Opcode() != op || // Not same opcode?
1218 !ShenandoahWriteBarrierNode::is_heap_stable_test(dom) || // Not same input 1?
1219 prev_dom->in(0) != dom) { // One path of test does not dominate?
1220 if (dist < 0) return NULL;
1221
1222 dist--;
1223 prev_dom = dom;
1224 dom = IfNode::up_one_dom(dom);
1225 if (!dom) return NULL;
1226 }
1227
1228 // Check that we did not follow a loop back to ourselves
1229 if (n == dom) {
1230 return NULL;
1231 }
1232
1233 return n->as_If()->dominated_by(prev_dom, phase->is_IterGVN());
1234 }
1235
1236 return NULL;
1237 }
1238
1239 Node* ShenandoahBarrierSetC2::identity_node(PhaseGVN* phase, Node* n) const {
1240 if (n->is_Load()) {
1241 Node *mem = n->in(MemNode::Memory);
1242 Node *value = n->as_Load()->can_see_stored_value(mem, phase);
1243 if (value) {
1244 PhaseIterGVN *igvn = phase->is_IterGVN();
1245 if (igvn != NULL &&
1246 value->is_Phi() &&
1247 value->req() > 2 &&
1248 value->in(1) != NULL &&
1249 value->in(1)->is_ShenandoahBarrier()) {
1250 if (igvn->_worklist.member(value) ||
1251 igvn->_worklist.member(value->in(0)) ||
1252 (value->in(0)->in(1) != NULL &&
1253 value->in(0)->in(1)->is_IfProj() &&
1254 (igvn->_worklist.member(value->in(0)->in(1)) ||
1255 (value->in(0)->in(1)->in(0) != NULL &&
1256 igvn->_worklist.member(value->in(0)->in(1)->in(0)))))) {
1257 igvn->_worklist.push(n);
1258 return n;
1259 }
1260 }
1261 // (This works even when value is a Con, but LoadNode::Value
1262 // usually runs first, producing the singleton type of the Con.)
1263 Node *value_no_barrier = step_over_gc_barrier(value->Opcode() == Op_EncodeP ? value->in(1) : value);
1264 if (value->Opcode() == Op_EncodeP) {
1265 if (value_no_barrier != value->in(1)) {
1266 Node *encode = value->clone();
1267 encode->set_req(1, value_no_barrier);
1268 encode = phase->transform(encode);
1269 return encode;
1270 }
1271 } else {
1272 return value_no_barrier;
1273 }
1274 }
1275 }
1276 return n;
1277 }
1278
1279 bool ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(Node* n) {
1280 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
1281 Node* u = n->fast_out(i);
1282 if (!is_shenandoah_wb_pre_call(u)) {
1283 return false;
1284 }
1285 }
1286 return n->outcnt() > 0;
1287 }
1288
1289 bool ShenandoahBarrierSetC2::flatten_gc_alias_type(const TypePtr*& adr_type) const {
1290 int offset = adr_type->offset();
1291 if (offset == ShenandoahBrooksPointer::byte_offset()) {
1292 if (adr_type->isa_aryptr()) {
1293 adr_type = TypeAryPtr::make(adr_type->ptr(), adr_type->isa_aryptr()->ary(), adr_type->isa_aryptr()->klass(), false, offset);
1294 } else if (adr_type->isa_instptr()) {
1295 adr_type = TypeInstPtr::make(adr_type->ptr(), ciEnv::current()->Object_klass(), false, NULL, offset);
1296 }
1297 return true;
1298 } else {
1299 return false;
1300 }
1301 }
1302
1303 bool ShenandoahBarrierSetC2::final_graph_reshaping(Compile* compile, Node* n, uint opcode) const {
1304 switch (opcode) {
1305 case Op_CallLeaf:
1306 case Op_CallLeafNoFP: {
1307 assert (n->is_Call(), "");
1308 CallNode *call = n->as_Call();
1309 if (ShenandoahBarrierSetC2::is_shenandoah_wb_pre_call(call)) {
1310 uint cnt = ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type()->domain()->cnt();
1311 if (call->req() > cnt) {
1312 assert(call->req() == cnt + 1, "only one extra input");
1313 Node *addp = call->in(cnt);
1314 assert(!ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(addp), "useless address computation?");
1315 call->del_req(cnt);
1316 }
1317 }
1318 return false;
1319 }
1320 case Op_ShenandoahCompareAndSwapP:
1321 case Op_ShenandoahCompareAndSwapN:
1322 case Op_ShenandoahWeakCompareAndSwapN:
1323 case Op_ShenandoahWeakCompareAndSwapP:
1324 case Op_ShenandoahCompareAndExchangeP:
1325 case Op_ShenandoahCompareAndExchangeN:
1326 #ifdef ASSERT
1327 if( VerifyOptoOopOffsets ) {
1328 MemNode* mem = n->as_Mem();
1329 // Check to see if address types have grounded out somehow.
1330 const TypeInstPtr *tp = mem->in(MemNode::Address)->bottom_type()->isa_instptr();
1331 ciInstanceKlass *k = tp->klass()->as_instance_klass();
1332 bool oop_offset_is_sane = k->contains_field_offset(tp->offset());
1333 assert( !tp || oop_offset_is_sane, "" );
1334 }
1335 #endif
1336 return true;
1337 case Op_ShenandoahReadBarrier:
1338 return true;
1339 case Op_ShenandoahWriteBarrier:
1340 assert(false, "should have been expanded already");
1341 return true;
1342 default:
1343 return false;
1344 }
1345 }
1346
1347 #ifdef ASSERT
1348 bool ShenandoahBarrierSetC2::verify_gc_alias_type(const TypePtr* adr_type, int offset) const {
1349 if (offset == ShenandoahBrooksPointer::byte_offset() &&
1350 (adr_type->base() == Type::AryPtr || adr_type->base() == Type::OopPtr)) {
1351 return true;
1352 } else {
1353 return false;
1354 }
1355 }
1356 #endif
1357
1358 bool ShenandoahBarrierSetC2::escape_add_to_con_graph(ConnectionGraph* conn_graph, PhaseGVN* gvn, Unique_Node_List* delayed_worklist, Node* n, uint opcode) const {
1359 switch (opcode) {
1360 case Op_ShenandoahCompareAndExchangeP:
1361 case Op_ShenandoahCompareAndExchangeN:
1362 conn_graph->add_objload_to_connection_graph(n, delayed_worklist);
1363 // fallthrough
1364 case Op_ShenandoahWeakCompareAndSwapP:
1365 case Op_ShenandoahWeakCompareAndSwapN:
1366 case Op_ShenandoahCompareAndSwapP:
1367 case Op_ShenandoahCompareAndSwapN:
1368 conn_graph->add_to_congraph_unsafe_access(n, opcode, delayed_worklist);
1369 return true;
1370 case Op_StoreP: {
1371 Node* adr = n->in(MemNode::Address);
1372 const Type* adr_type = gvn->type(adr);
1373 // Pointer stores in G1 barriers looks like unsafe access.
1374 // Ignore such stores to be able scalar replace non-escaping
1375 // allocations.
1376 if (adr_type->isa_rawptr() && adr->is_AddP()) {
1377 Node* base = conn_graph->get_addp_base(adr);
1378 if (base->Opcode() == Op_LoadP &&
1379 base->in(MemNode::Address)->is_AddP()) {
1380 adr = base->in(MemNode::Address);
1381 Node* tls = conn_graph->get_addp_base(adr);
1382 if (tls->Opcode() == Op_ThreadLocal) {
1383 int offs = (int) gvn->find_intptr_t_con(adr->in(AddPNode::Offset), Type::OffsetBot);
1384 const int buf_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset());
1385 if (offs == buf_offset) {
1386 return true; // Pre barrier previous oop value store.
1387 }
1388 }
1389 }
1390 }
1391 return false;
1392 }
1393 case Op_ShenandoahReadBarrier:
1394 case Op_ShenandoahWriteBarrier:
1395 // Barriers 'pass through' its arguments. I.e. what goes in, comes out.
1396 // It doesn't escape.
1397 conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(ShenandoahBarrierNode::ValueIn), delayed_worklist);
1398 break;
1399 case Op_ShenandoahEnqueueBarrier:
1400 conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(1), delayed_worklist);
1401 break;
1402 default:
1403 // Nothing
1404 break;
1405 }
1406 return false;
1407 }
1408
1409 bool ShenandoahBarrierSetC2::escape_add_final_edges(ConnectionGraph* conn_graph, PhaseGVN* gvn, Node* n, uint opcode) const {
1410 switch (opcode) {
1411 case Op_ShenandoahCompareAndExchangeP:
1412 case Op_ShenandoahCompareAndExchangeN: {
1413 Node *adr = n->in(MemNode::Address);
1414 conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, adr, NULL);
1415 // fallthrough
1416 }
1417 case Op_ShenandoahCompareAndSwapP:
1418 case Op_ShenandoahCompareAndSwapN:
1419 case Op_ShenandoahWeakCompareAndSwapP:
1420 case Op_ShenandoahWeakCompareAndSwapN:
1421 return conn_graph->add_final_edges_unsafe_access(n, opcode);
1422 case Op_ShenandoahReadBarrier:
1423 case Op_ShenandoahWriteBarrier:
1424 // Barriers 'pass through' its arguments. I.e. what goes in, comes out.
1425 // It doesn't escape.
1426 conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(ShenandoahBarrierNode::ValueIn), NULL);
1427 return true;
1428 case Op_ShenandoahEnqueueBarrier:
1429 conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(1), NULL);
1430 return true;
1431 default:
1432 // Nothing
1433 break;
1434 }
1435 return false;
1436 }
1437
1438 bool ShenandoahBarrierSetC2::escape_has_out_with_unsafe_object(Node* n) const {
1439 return n->has_out_with(Op_ShenandoahCompareAndExchangeP) || n->has_out_with(Op_ShenandoahCompareAndExchangeN) ||
1440 n->has_out_with(Op_ShenandoahCompareAndSwapP, Op_ShenandoahCompareAndSwapN, Op_ShenandoahWeakCompareAndSwapP, Op_ShenandoahWeakCompareAndSwapN);
1441
1442 }
1443
1444 bool ShenandoahBarrierSetC2::escape_is_barrier_node(Node* n) const {
1445 return n->is_ShenandoahBarrier();
1446 }
1447
1448 bool ShenandoahBarrierSetC2::matcher_find_shared_visit(Matcher* matcher, Matcher::MStack& mstack, Node* n, uint opcode, bool& mem_op, int& mem_addr_idx) const {
1449 switch (opcode) {
1450 case Op_ShenandoahReadBarrier:
1451 if (n->in(ShenandoahBarrierNode::ValueIn)->is_DecodeNarrowPtr()) {
1452 matcher->set_shared(n->in(ShenandoahBarrierNode::ValueIn)->in(1));
1453 }
1454 matcher->set_shared(n);
1455 return true;
1456 default:
1457 break;
1458 }
1459 return false;
1460 }
1461
1462 bool ShenandoahBarrierSetC2::matcher_find_shared_post_visit(Matcher* matcher, Node* n, uint opcode) const {
1463 switch (opcode) {
1464 case Op_ShenandoahCompareAndExchangeP:
1465 case Op_ShenandoahCompareAndExchangeN:
1466 case Op_ShenandoahWeakCompareAndSwapP:
1467 case Op_ShenandoahWeakCompareAndSwapN:
1468 case Op_ShenandoahCompareAndSwapP:
1469 case Op_ShenandoahCompareAndSwapN: { // Convert trinary to binary-tree
1470 Node* newval = n->in(MemNode::ValueIn);
1471 Node* oldval = n->in(LoadStoreConditionalNode::ExpectedIn);
1472 Node* pair = new BinaryNode(oldval, newval);
1473 n->set_req(MemNode::ValueIn,pair);
1474 n->del_req(LoadStoreConditionalNode::ExpectedIn);
1475 return true;
1476 }
1477 default:
1478 break;
1479 }
1480 return false;
1481 }
1482
1483 bool ShenandoahBarrierSetC2::matcher_is_store_load_barrier(Node* x, uint xop) const {
1484 return xop == Op_ShenandoahCompareAndExchangeP ||
1485 xop == Op_ShenandoahCompareAndExchangeN ||
1486 xop == Op_ShenandoahWeakCompareAndSwapP ||
1487 xop == Op_ShenandoahWeakCompareAndSwapN ||
1488 xop == Op_ShenandoahCompareAndSwapN ||
1489 xop == Op_ShenandoahCompareAndSwapP;
1490 }
1491
1492 void ShenandoahBarrierSetC2::igvn_add_users_to_worklist(PhaseIterGVN* igvn, Node* use) const {
1493 if (use->is_ShenandoahBarrier()) {
1494 for (DUIterator_Fast i2max, i2 = use->fast_outs(i2max); i2 < i2max; i2++) {
1495 Node* u = use->fast_out(i2);
1496 Node* cmp = use->find_out_with(Op_CmpP);
1497 if (u->Opcode() == Op_CmpP) {
1498 igvn->_worklist.push(cmp);
1499 }
1500 }
1501 }
1502 }
1503
1504 void ShenandoahBarrierSetC2::ccp_analyze(PhaseCCP* ccp, Unique_Node_List& worklist, Node* use) const {
1505 if (use->is_ShenandoahBarrier()) {
1506 for (DUIterator_Fast i2max, i2 = use->fast_outs(i2max); i2 < i2max; i2++) {
1507 Node* p = use->fast_out(i2);
1508 if (p->Opcode() == Op_AddP) {
1509 for (DUIterator_Fast i3max, i3 = p->fast_outs(i3max); i3 < i3max; i3++) {
1510 Node* q = p->fast_out(i3);
1511 if (q->is_Load()) {
1512 if(q->bottom_type() != ccp->type(q)) {
1513 worklist.push(q);
1514 }
1515 }
1516 }
1517 }
1518 }
1519 }
1520 }
1521
1522 Node* ShenandoahBarrierSetC2::split_if_pre(PhaseIdealLoop* phase, Node* n) const {
1523 if (n->Opcode() == Op_ShenandoahReadBarrier) {
1524 ((ShenandoahReadBarrierNode*)n)->try_move(phase);
1525 } else if (n->Opcode() == Op_ShenandoahWriteBarrier) {
1526 return ((ShenandoahWriteBarrierNode*)n)->try_split_thru_phi(phase);
1527 }
1528
1529 return NULL;
1530 }
1531
1532 bool ShenandoahBarrierSetC2::build_loop_late_post(PhaseIdealLoop* phase, Node* n) const {
1533 return ShenandoahBarrierNode::build_loop_late_post(phase, n);
1534 }
1535
1536 bool ShenandoahBarrierSetC2::sink_node(PhaseIdealLoop* phase, Node* n, Node* x, Node* x_ctrl, Node* n_ctrl) const {
1537 if (n->is_ShenandoahBarrier()) {
1538 return x->as_ShenandoahBarrier()->sink_node(phase, x_ctrl, n_ctrl);
1539 }
1540 return false;
1541 }