1 /*
2 * Copyright (c) 2015, 2019, Red Hat, Inc. All rights reserved.
3 *
4 * This code is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 only, as
6 * published by the Free Software Foundation.
7 *
8 * This code is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 * version 2 for more details (a copy is included in the LICENSE file that
12 * accompanied this code).
13 *
14 * You should have received a copy of the GNU General Public License version
15 * 2 along with this work; if not, write to the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 #include "precompiled.hpp"
25
26 #include "gc/shenandoah/c2/shenandoahSupport.hpp"
27 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
28 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
29 #include "gc/shenandoah/shenandoahBrooksPointer.hpp"
30 #include "gc/shenandoah/shenandoahHeap.hpp"
31 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
32 #include "gc/shenandoah/shenandoahRuntime.hpp"
33 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
34 #include "opto/arraycopynode.hpp"
35 #include "opto/block.hpp"
36 #include "opto/callnode.hpp"
37 #include "opto/castnode.hpp"
38 #include "opto/movenode.hpp"
39 #include "opto/phaseX.hpp"
40 #include "opto/rootnode.hpp"
41 #include "opto/runtime.hpp"
42 #include "opto/subnode.hpp"
43
44 Node* ShenandoahBarrierNode::skip_through_barrier(Node* n) {
45 if (!UseShenandoahGC) {
46 return n;
47 }
48 if (n == NULL) {
49 return NULL;
50 }
51 if (n->Opcode() == Op_ShenandoahEnqueueBarrier) {
52 n = n->in(1);
53 }
54
55 if (n->is_ShenandoahBarrier()) {
56 return n->in(ValueIn);
57 } else if (n->is_Phi() &&
58 n->req() == 3 &&
59 n->in(1) != NULL &&
60 n->in(1)->is_ShenandoahBarrier() &&
61 n->in(2) != NULL &&
62 n->in(2)->bottom_type() == TypePtr::NULL_PTR &&
63 n->in(0) != NULL &&
64 n->in(0)->in(1) != NULL &&
65 n->in(0)->in(1)->is_IfProj() &&
66 n->in(0)->in(2) != NULL &&
67 n->in(0)->in(2)->is_IfProj() &&
68 n->in(0)->in(1)->in(0) != NULL &&
69 n->in(0)->in(1)->in(0) == n->in(0)->in(2)->in(0) &&
70 n->in(1)->in(ValueIn)->Opcode() == Op_CastPP) {
71 Node* iff = n->in(0)->in(1)->in(0);
72 Node* res = n->in(1)->in(ValueIn)->in(1);
73 if (iff->is_If() &&
74 iff->in(1) != NULL &&
75 iff->in(1)->is_Bool() &&
76 iff->in(1)->as_Bool()->_test._test == BoolTest::ne &&
77 iff->in(1)->in(1) != NULL &&
78 iff->in(1)->in(1)->Opcode() == Op_CmpP &&
79 iff->in(1)->in(1)->in(1) != NULL &&
80 iff->in(1)->in(1)->in(1) == res &&
81 iff->in(1)->in(1)->in(2) != NULL &&
82 iff->in(1)->in(1)->in(2)->bottom_type() == TypePtr::NULL_PTR) {
83 return res;
84 }
85 }
86 return n;
87 }
88
89 bool ShenandoahBarrierNode::needs_barrier(PhaseGVN* phase, ShenandoahBarrierNode* orig, Node* n, Node* rb_mem, bool allow_fromspace) {
90 Unique_Node_List visited;
91 return needs_barrier_impl(phase, orig, n, rb_mem, allow_fromspace, visited);
92 }
93
94 bool ShenandoahBarrierNode::needs_barrier_impl(PhaseGVN* phase, ShenandoahBarrierNode* orig, Node* n, Node* rb_mem, bool allow_fromspace, Unique_Node_List &visited) {
95 if (visited.member(n)) {
96 return false; // Been there.
97 }
98 visited.push(n);
99
100 if (n->is_Allocate()) {
101 return false;
102 }
103
104 if (n->is_Call()) {
105 return true;
106 }
107
108 const Type* type = phase->type(n);
109 if (type == Type::TOP) {
110 return false;
111 }
112 if (type->make_ptr()->higher_equal(TypePtr::NULL_PTR)) {
113 return false;
114 }
115 if (type->make_oopptr() && type->make_oopptr()->const_oop() != NULL) {
116 return false;
117 }
118
119 if (ShenandoahOptimizeStableFinals) {
120 const TypeAryPtr* ary = type->isa_aryptr();
121 if (ary && ary->is_stable() && allow_fromspace) {
122 return false;
123 }
124 }
125
126 if (n->is_CheckCastPP() || n->is_ConstraintCast() || n->Opcode() == Op_ShenandoahEnqueueBarrier) {
127 return needs_barrier_impl(phase, orig, n->in(1), rb_mem, allow_fromspace, visited);
128 }
129 if (n->is_Parm()) {
130 return true;
131 }
132 if (n->is_Proj()) {
133 return needs_barrier_impl(phase, orig, n->in(0), rb_mem, allow_fromspace, visited);
134 }
135
136 if (n->Opcode() == Op_ShenandoahWBMemProj) {
137 return needs_barrier_impl(phase, orig, n->in(ShenandoahWBMemProjNode::WriteBarrier), rb_mem, allow_fromspace, visited);
138 }
139 if (n->is_Phi()) {
140 bool need_barrier = false;
141 for (uint i = 1; i < n->req() && ! need_barrier; i++) {
142 Node* input = n->in(i);
143 if (input == NULL) {
144 need_barrier = true; // Phi not complete yet?
145 } else if (needs_barrier_impl(phase, orig, input, rb_mem, allow_fromspace, visited)) {
146 need_barrier = true;
147 }
148 }
149 return need_barrier;
150 }
151 if (n->is_CMove()) {
152 return needs_barrier_impl(phase, orig, n->in(CMoveNode::IfFalse), rb_mem, allow_fromspace, visited) ||
153 needs_barrier_impl(phase, orig, n->in(CMoveNode::IfTrue ), rb_mem, allow_fromspace, visited);
154 }
155 if (n->Opcode() == Op_CreateEx) {
156 return true;
157 }
158 if (n->Opcode() == Op_ShenandoahWriteBarrier) {
159 return false;
160 }
161 if (n->Opcode() == Op_ShenandoahReadBarrier) {
162 if (rb_mem == n->in(Memory)) {
163 return false;
164 } else {
165 return true;
166 }
167 }
168
169 if (n->Opcode() == Op_LoadP ||
170 n->Opcode() == Op_LoadN ||
171 n->Opcode() == Op_GetAndSetP ||
172 n->Opcode() == Op_CompareAndExchangeP ||
173 n->Opcode() == Op_ShenandoahCompareAndExchangeP ||
174 n->Opcode() == Op_GetAndSetN ||
175 n->Opcode() == Op_CompareAndExchangeN ||
176 n->Opcode() == Op_ShenandoahCompareAndExchangeN) {
177 return true;
178 }
179 if (n->Opcode() == Op_DecodeN ||
180 n->Opcode() == Op_EncodeP) {
181 return needs_barrier_impl(phase, orig, n->in(1), rb_mem, allow_fromspace, visited);
182 }
183
184 #ifdef ASSERT
185 tty->print("need barrier on?: "); n->dump();
186 ShouldNotReachHere();
187 #endif
188 return true;
189 }
190
191 bool ShenandoahReadBarrierNode::dominates_memory_rb_impl(PhaseGVN* phase,
192 Node* b1,
193 Node* b2,
194 Node* current,
195 bool linear) {
196 ResourceMark rm;
197 VectorSet visited(Thread::current()->resource_area());
198 Node_Stack phis(0);
199
200 for(int i = 0; i < 10; i++) {
201 if (current == NULL) {
202 return false;
203 } else if (visited.test_set(current->_idx) || current->is_top() || current == b1) {
204 current = NULL;
205 while (phis.is_nonempty() && current == NULL) {
206 uint idx = phis.index();
207 Node* phi = phis.node();
208 if (idx >= phi->req()) {
209 phis.pop();
210 } else {
211 current = phi->in(idx);
212 phis.set_index(idx+1);
213 }
214 }
215 if (current == NULL) {
216 return true;
217 }
218 } else if (current == phase->C->immutable_memory()) {
219 return false;
220 } else if (current->isa_Phi()) {
221 if (!linear) {
222 return false;
223 }
224 phis.push(current, 2);
225 current = current->in(1);
226 } else if (current->Opcode() == Op_ShenandoahWriteBarrier) {
227 const Type* in_type = current->bottom_type();
228 const Type* this_type = b2->bottom_type();
229 if (is_independent(in_type, this_type)) {
230 current = current->in(Memory);
231 } else {
232 return false;
233 }
234 } else if (current->Opcode() == Op_ShenandoahWBMemProj) {
235 current = current->in(ShenandoahWBMemProjNode::WriteBarrier);
236 } else if (current->is_Proj()) {
237 current = current->in(0);
238 } else if (current->is_Call()) {
239 return false; // TODO: Maybe improve by looking at the call's memory effects?
240 } else if (current->is_MemBar()) {
241 return false; // TODO: Do we need to stop at *any* membar?
242 } else if (current->is_MergeMem()) {
243 const TypePtr* adr_type = brooks_pointer_type(phase->type(b2));
244 uint alias_idx = phase->C->get_alias_index(adr_type);
245 current = current->as_MergeMem()->memory_at(alias_idx);
246 } else {
247 #ifdef ASSERT
248 current->dump();
249 #endif
250 ShouldNotReachHere();
251 return false;
252 }
253 }
254 return false;
255 }
256
257 bool ShenandoahReadBarrierNode::is_independent(Node* mem) {
258 if (mem->is_Phi() || mem->is_Proj() || mem->is_MergeMem()) {
259 return true;
260 } else if (mem->Opcode() == Op_ShenandoahWBMemProj) {
261 return true;
262 } else if (mem->Opcode() == Op_ShenandoahWriteBarrier) {
263 const Type* mem_type = mem->bottom_type();
264 const Type* this_type = bottom_type();
265 if (is_independent(mem_type, this_type)) {
266 return true;
267 } else {
268 return false;
269 }
270 } else if (mem->is_Call() || mem->is_MemBar()) {
271 return false;
272 }
273 #ifdef ASSERT
274 mem->dump();
275 #endif
276 ShouldNotReachHere();
277 return true;
278 }
279
280 bool ShenandoahReadBarrierNode::dominates_memory_rb(PhaseGVN* phase, Node* b1, Node* b2, bool linear) {
281 return dominates_memory_rb_impl(phase, b1->in(Memory), b2, b2->in(Memory), linear);
282 }
283
284 bool ShenandoahReadBarrierNode::is_independent(const Type* in_type, const Type* this_type) {
285 assert(in_type->isa_oopptr(), "expect oop ptr");
286 assert(this_type->isa_oopptr(), "expect oop ptr");
287
288 ciKlass* in_kls = in_type->is_oopptr()->klass();
289 ciKlass* this_kls = this_type->is_oopptr()->klass();
290 if (in_kls != NULL && this_kls != NULL &&
291 in_kls->is_loaded() && this_kls->is_loaded() &&
292 (!in_kls->is_subclass_of(this_kls)) &&
293 (!this_kls->is_subclass_of(in_kls))) {
294 return true;
295 }
296 return false;
297 }
298
299 Node* ShenandoahReadBarrierNode::Ideal(PhaseGVN *phase, bool can_reshape) {
300 if (! can_reshape) {
301 return NULL;
302 }
303
304 if (in(Memory) == phase->C->immutable_memory()) return NULL;
305
306 // If memory input is a MergeMem, take the appropriate slice out of it.
307 Node* mem_in = in(Memory);
308 if (mem_in->isa_MergeMem()) {
309 const TypePtr* adr_type = brooks_pointer_type(bottom_type());
310 uint alias_idx = phase->C->get_alias_index(adr_type);
311 mem_in = mem_in->as_MergeMem()->memory_at(alias_idx);
312 set_req(Memory, mem_in);
313 return this;
314 }
315
316 Node* input = in(Memory);
317 if (input->Opcode() == Op_ShenandoahWBMemProj) {
318 ResourceMark rm;
319 VectorSet seen(Thread::current()->resource_area());
320 Node* n = in(Memory);
321 while (n->Opcode() == Op_ShenandoahWBMemProj &&
322 n->in(ShenandoahWBMemProjNode::WriteBarrier) != NULL &&
323 n->in(ShenandoahWBMemProjNode::WriteBarrier)->Opcode() == Op_ShenandoahWriteBarrier &&
324 n->in(ShenandoahWBMemProjNode::WriteBarrier)->in(Memory) != NULL) {
325 if (seen.test_set(n->_idx)) {
326 return NULL; // loop
327 }
328 n = n->in(ShenandoahWBMemProjNode::WriteBarrier)->in(Memory);
329 }
330
331 Node* wb = input->in(ShenandoahWBMemProjNode::WriteBarrier);
332 const Type* in_type = phase->type(wb);
333 // is_top() test not sufficient here: we can come here after CCP
334 // in a dead branch of the graph that has not yet been removed.
335 if (in_type == Type::TOP) return NULL; // Dead path.
336 assert(wb->Opcode() == Op_ShenandoahWriteBarrier, "expect write barrier");
337 if (is_independent(in_type, _type)) {
338 phase->igvn_rehash_node_delayed(wb);
339 set_req(Memory, wb->in(Memory));
340 if (can_reshape && input->outcnt() == 0) {
341 phase->is_IterGVN()->_worklist.push(input);
342 }
343 return this;
344 }
345 }
346 return NULL;
347 }
348
349 ShenandoahWriteBarrierNode::ShenandoahWriteBarrierNode(Compile* C, Node* ctrl, Node* mem, Node* obj)
350 : ShenandoahBarrierNode(ctrl, mem, obj, false) {
351 assert(UseShenandoahGC && ShenandoahWriteBarrier, "should be enabled");
352 ShenandoahBarrierSetC2::bsc2()->state()->add_shenandoah_barrier(this);
353 }
354
355 Node* ShenandoahWriteBarrierNode::Identity(PhaseGVN* phase) {
356 assert(in(0) != NULL, "should have control");
357 PhaseIterGVN* igvn = phase->is_IterGVN();
358 Node* mem_in = in(Memory);
359 Node* mem_proj = NULL;
360
361 if (igvn != NULL) {
362 mem_proj = find_out_with(Op_ShenandoahWBMemProj);
363 if (mem_in == mem_proj) {
364 return this;
365 }
366 }
367
368 Node* replacement = Identity_impl(phase);
369 if (igvn != NULL) {
370 if (replacement != NULL && replacement != this && mem_proj != NULL) {
371 igvn->replace_node(mem_proj, mem_in);
372 }
373 }
374 return replacement;
375 }
376
377 Node* ShenandoahWriteBarrierNode::Ideal(PhaseGVN *phase, bool can_reshape) {
378 assert(in(0) != NULL, "should have control");
379 if (!can_reshape) {
380 return NULL;
381 }
382
383 Node* mem_in = in(Memory);
384
385 if (mem_in->isa_MergeMem()) {
386 const TypePtr* adr_type = brooks_pointer_type(bottom_type());
387 uint alias_idx = phase->C->get_alias_index(adr_type);
388 mem_in = mem_in->as_MergeMem()->memory_at(alias_idx);
389 set_req(Memory, mem_in);
390 return this;
391 }
392
393 Node* val = in(ValueIn);
394 if (val->is_ShenandoahBarrier()) {
395 set_req(ValueIn, val->in(ValueIn));
396 return this;
397 }
398
399 return NULL;
400 }
401
402 bool ShenandoahWriteBarrierNode::expand(Compile* C, PhaseIterGVN& igvn, int& loop_opts_cnt) {
403 if (UseShenandoahGC) {
404 if (ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count() > 0 || (!ShenandoahWriteBarrier && ShenandoahStoreValEnqueueBarrier)) {
405 bool attempt_more_loopopts = ShenandoahLoopOptsAfterExpansion;
406 C->clear_major_progress();
407 PhaseIdealLoop ideal_loop(igvn, LoopOptsShenandoahExpand);
408 if (C->failing()) return false;
409 PhaseIdealLoop::verify(igvn);
410 DEBUG_ONLY(ShenandoahBarrierNode::verify_raw_mem(C->root());)
411 if (attempt_more_loopopts) {
412 C->set_major_progress();
413 if (!C->optimize_loops(loop_opts_cnt, igvn, LoopOptsShenandoahPostExpand)) {
414 return false;
415 }
416 C->clear_major_progress();
417 }
418 }
419 }
420 return true;
421 }
422
423 bool ShenandoahWriteBarrierNode::is_heap_state_test(Node* iff, int mask) {
424 if (!UseShenandoahGC) {
425 return false;
426 }
427 assert(iff->is_If(), "bad input");
428 if (iff->Opcode() != Op_If) {
429 return false;
430 }
431 Node* bol = iff->in(1);
432 if (!bol->is_Bool() || bol->as_Bool()->_test._test != BoolTest::ne) {
433 return false;
434 }
435 Node* cmp = bol->in(1);
436 if (cmp->Opcode() != Op_CmpI) {
437 return false;
438 }
439 Node* in1 = cmp->in(1);
440 Node* in2 = cmp->in(2);
441 if (in2->find_int_con(-1) != 0) {
442 return false;
443 }
444 if (in1->Opcode() != Op_AndI) {
445 return false;
446 }
447 in2 = in1->in(2);
448 if (in2->find_int_con(-1) != mask) {
449 return false;
450 }
451 in1 = in1->in(1);
452
453 return is_gc_state_load(in1);
454 }
455
456 bool ShenandoahWriteBarrierNode::is_heap_stable_test(Node* iff) {
457 return is_heap_state_test(iff, ShenandoahHeap::HAS_FORWARDED);
458 }
459
460 bool ShenandoahWriteBarrierNode::is_gc_state_load(Node *n) {
461 if (!UseShenandoahGC) {
462 return false;
463 }
464 if (n->Opcode() != Op_LoadB && n->Opcode() != Op_LoadUB) {
465 return false;
466 }
467 Node* addp = n->in(MemNode::Address);
468 if (!addp->is_AddP()) {
469 return false;
470 }
471 Node* base = addp->in(AddPNode::Address);
472 Node* off = addp->in(AddPNode::Offset);
473 if (base->Opcode() != Op_ThreadLocal) {
474 return false;
475 }
476 if (off->find_intptr_t_con(-1) != in_bytes(ShenandoahThreadLocalData::gc_state_offset())) {
477 return false;
478 }
479 return true;
480 }
481
482 bool ShenandoahWriteBarrierNode::has_safepoint_between(Node* start, Node* stop, PhaseIdealLoop *phase) {
483 assert(phase->is_dominator(stop, start), "bad inputs");
484 ResourceMark rm;
485 Unique_Node_List wq;
486 wq.push(start);
487 for (uint next = 0; next < wq.size(); next++) {
488 Node *m = wq.at(next);
489 if (m == stop) {
490 continue;
491 }
492 if (m->is_SafePoint() && !m->is_CallLeaf()) {
493 return true;
494 }
495 if (m->is_Region()) {
496 for (uint i = 1; i < m->req(); i++) {
497 wq.push(m->in(i));
498 }
499 } else {
500 wq.push(m->in(0));
501 }
502 }
503 return false;
504 }
505
506 bool ShenandoahWriteBarrierNode::try_common_gc_state_load(Node *n, PhaseIdealLoop *phase) {
507 assert(is_gc_state_load(n), "inconsistent");
508 Node* addp = n->in(MemNode::Address);
509 Node* dominator = NULL;
510 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
511 Node* u = addp->fast_out(i);
512 assert(is_gc_state_load(u), "inconsistent");
513 if (u != n && phase->is_dominator(u->in(0), n->in(0))) {
514 if (dominator == NULL) {
515 dominator = u;
516 } else {
517 if (phase->dom_depth(u->in(0)) < phase->dom_depth(dominator->in(0))) {
518 dominator = u;
519 }
520 }
521 }
522 }
523 if (dominator == NULL || has_safepoint_between(n->in(0), dominator->in(0), phase)) {
524 return false;
525 }
526 phase->igvn().replace_node(n, dominator);
527
528 return true;
529 }
530
531 bool ShenandoahBarrierNode::dominates_memory_impl(PhaseGVN* phase,
532 Node* b1,
533 Node* b2,
534 Node* current,
535 bool linear) {
536 ResourceMark rm;
537 VectorSet visited(Thread::current()->resource_area());
538 Node_Stack phis(0);
539
540 for(int i = 0; i < 10; i++) {
541 if (current == NULL) {
542 return false;
543 } else if (visited.test_set(current->_idx) || current->is_top() || current == b1) {
544 current = NULL;
545 while (phis.is_nonempty() && current == NULL) {
546 uint idx = phis.index();
547 Node* phi = phis.node();
548 if (idx >= phi->req()) {
549 phis.pop();
550 } else {
551 current = phi->in(idx);
552 phis.set_index(idx+1);
553 }
554 }
555 if (current == NULL) {
556 return true;
557 }
558 } else if (current == b2) {
559 return false;
560 } else if (current == phase->C->immutable_memory()) {
561 return false;
562 } else if (current->isa_Phi()) {
563 if (!linear) {
564 return false;
565 }
566 phis.push(current, 2);
567 current = current->in(1);
568 } else if (current->Opcode() == Op_ShenandoahWriteBarrier) {
569 current = current->in(Memory);
570 } else if (current->Opcode() == Op_ShenandoahWBMemProj) {
571 current = current->in(ShenandoahWBMemProjNode::WriteBarrier);
572 } else if (current->is_Proj()) {
573 current = current->in(0);
574 } else if (current->is_Call()) {
575 current = current->in(TypeFunc::Memory);
576 } else if (current->is_MemBar()) {
577 current = current->in(TypeFunc::Memory);
578 } else if (current->is_MergeMem()) {
579 const TypePtr* adr_type = brooks_pointer_type(phase->type(b2));
580 uint alias_idx = phase->C->get_alias_index(adr_type);
581 current = current->as_MergeMem()->memory_at(alias_idx);
582 } else {
583 #ifdef ASSERT
584 current->dump();
585 #endif
586 ShouldNotReachHere();
587 return false;
588 }
589 }
590 return false;
591 }
592
593 /**
594 * Determines if b1 dominates b2 through memory inputs. It returns true if:
595 * - b1 can be reached by following each branch in b2's memory input (through phis, etc)
596 * - or we get back to b2 (i.e. through a loop) without seeing b1
597 * In all other cases, (in particular, if we reach immutable_memory without having seen b1)
598 * we return false.
599 */
600 bool ShenandoahBarrierNode::dominates_memory(PhaseGVN* phase, Node* b1, Node* b2, bool linear) {
601 return dominates_memory_impl(phase, b1, b2, b2->in(Memory), linear);
602 }
603
604 Node* ShenandoahBarrierNode::Identity_impl(PhaseGVN* phase) {
605 Node* n = in(ValueIn);
606
607 Node* rb_mem = Opcode() == Op_ShenandoahReadBarrier ? in(Memory) : NULL;
608 if (! needs_barrier(phase, this, n, rb_mem, _allow_fromspace)) {
609 return n;
610 }
611
612 // Try to find a write barrier sibling with identical inputs that we can fold into.
613 for (DUIterator i = n->outs(); n->has_out(i); i++) {
614 Node* sibling = n->out(i);
615 if (sibling == this) {
616 continue;
617 }
618 if (sibling->Opcode() != Op_ShenandoahWriteBarrier) {
619 continue;
620 }
621
622 assert(sibling->in(ValueIn) == in(ValueIn), "sanity");
623 assert(sibling->Opcode() == Op_ShenandoahWriteBarrier, "sanity");
624
625 if (dominates_memory(phase, sibling, this, phase->is_IterGVN() == NULL)) {
626 return sibling;
627 }
628 }
629 return this;
630 }
631
632 #ifndef PRODUCT
633 void ShenandoahBarrierNode::dump_spec(outputStream *st) const {
634 const TypePtr* adr = adr_type();
635 if (adr == NULL) {
636 return;
637 }
638 st->print(" @");
639 adr->dump_on(st);
640 st->print(" (");
641 Compile::current()->alias_type(adr)->adr_type()->dump_on(st);
642 st->print(") ");
643 }
644 #endif
645
646 Node* ShenandoahReadBarrierNode::Identity(PhaseGVN* phase) {
647 Node* id = Identity_impl(phase);
648
649 if (id == this && phase->is_IterGVN()) {
650 Node* n = in(ValueIn);
651 // No success in super call. Try to combine identical read barriers.
652 for (DUIterator i = n->outs(); n->has_out(i); i++) {
653 Node* sibling = n->out(i);
654 if (sibling == this || sibling->Opcode() != Op_ShenandoahReadBarrier) {
655 continue;
656 }
657 assert(sibling->in(ValueIn) == in(ValueIn), "sanity");
658 if (phase->is_IterGVN()->hash_find(sibling) &&
659 sibling->bottom_type() == bottom_type() &&
660 sibling->in(Control) == in(Control) &&
661 dominates_memory_rb(phase, sibling, this, phase->is_IterGVN() == NULL)) {
662 return sibling;
663 }
664 }
665 }
666 return id;
667 }
668
669 const Type* ShenandoahBarrierNode::Value(PhaseGVN* phase) const {
670 // Either input is TOP ==> the result is TOP
671 const Type *t1 = phase->type(in(Memory));
672 if (t1 == Type::TOP) return Type::TOP;
673 const Type *t2 = phase->type(in(ValueIn));
674 if( t2 == Type::TOP ) return Type::TOP;
675
676 if (t2 == TypePtr::NULL_PTR) {
677 return _type;
678 }
679
680 const Type* type = t2->is_oopptr()->cast_to_nonconst();
681 return type;
682 }
683
684 uint ShenandoahBarrierNode::hash() const {
685 return TypeNode::hash() + _allow_fromspace;
686 }
687
688 uint ShenandoahBarrierNode::cmp(const Node& n) const {
689 return _allow_fromspace == ((ShenandoahBarrierNode&) n)._allow_fromspace
690 && TypeNode::cmp(n);
691 }
692
693 uint ShenandoahBarrierNode::size_of() const {
694 return sizeof(*this);
695 }
696
697 Node* ShenandoahWBMemProjNode::Identity(PhaseGVN* phase) {
698 Node* wb = in(WriteBarrier);
699 if (wb->is_top()) return phase->C->top(); // Dead path.
700
701 assert(wb->Opcode() == Op_ShenandoahWriteBarrier, "expect write barrier");
702 PhaseIterGVN* igvn = phase->is_IterGVN();
703 // We can't do the below unless the graph is fully constructed.
704 if (igvn == NULL) {
705 return this;
706 }
707
708 // If the mem projection has no barrier users, it's not needed anymore.
709 if (wb->outcnt() == 1) {
710 return wb->in(ShenandoahBarrierNode::Memory);
711 }
712
713 return this;
714 }
715
716 #ifdef ASSERT
717 bool ShenandoahBarrierNode::verify_helper(Node* in, Node_Stack& phis, VectorSet& visited, verify_type t, bool trace, Unique_Node_List& barriers_used) {
718 assert(phis.size() == 0, "");
719
720 while (true) {
721 if (in->bottom_type() == TypePtr::NULL_PTR) {
722 if (trace) {tty->print_cr("NULL");}
723 } else if (!in->bottom_type()->make_ptr()->make_oopptr()) {
724 if (trace) {tty->print_cr("Non oop");}
725 } else if (t == ShenandoahLoad && ShenandoahOptimizeStableFinals &&
726 in->bottom_type()->make_ptr()->isa_aryptr() &&
727 in->bottom_type()->make_ptr()->is_aryptr()->is_stable()) {
728 if (trace) {tty->print_cr("Stable array load");}
729 } else {
730 if (in->is_ConstraintCast()) {
731 in = in->in(1);
732 continue;
733 } else if (in->is_AddP()) {
734 assert(!in->in(AddPNode::Address)->is_top(), "no raw memory access");
735 in = in->in(AddPNode::Address);
736 continue;
737 } else if (in->is_Con()) {
738 if (trace) {tty->print("Found constant"); in->dump();}
739 } else if (in->is_ShenandoahBarrier()) {
740 if (t == ShenandoahOopStore) {
741 if (in->Opcode() != Op_ShenandoahWriteBarrier) {
742 return false;
743 }
744 uint i = 0;
745 for (; i < phis.size(); i++) {
746 Node* n = phis.node_at(i);
747 if (n->Opcode() == Op_ShenandoahEnqueueBarrier) {
748 break;
749 }
750 }
751 if (i == phis.size()) {
752 return false;
753 }
754 } else if (t == ShenandoahStore && in->Opcode() != Op_ShenandoahWriteBarrier) {
755 return false;
756 }
757 barriers_used.push(in);
758 if (trace) {tty->print("Found barrier"); in->dump();}
759 } else if (in->Opcode() == Op_ShenandoahEnqueueBarrier) {
760 if (t != ShenandoahOopStore) {
761 return false;
762 }
763 if (trace) {tty->print("Found enqueue barrier"); in->dump();}
764 phis.push(in, in->req());
765 in = in->in(1);
766 continue;
767 } else if (in->is_Proj() && in->in(0)->is_Allocate()) {
768 if (trace) {tty->print("Found alloc"); in->in(0)->dump();}
769 } else if (in->is_Phi()) {
770 if (!visited.test_set(in->_idx)) {
771 if (trace) {tty->print("Pushed phi:"); in->dump();}
772 phis.push(in, 2);
773 in = in->in(1);
774 continue;
775 }
776 if (trace) {tty->print("Already seen phi:"); in->dump();}
777 } else if (in->Opcode() == Op_CMoveP || in->Opcode() == Op_CMoveN) {
778 if (!visited.test_set(in->_idx)) {
779 if (trace) {tty->print("Pushed cmovep:"); in->dump();}
780 phis.push(in, CMoveNode::IfTrue);
781 in = in->in(CMoveNode::IfFalse);
782 continue;
783 }
784 if (trace) {tty->print("Already seen cmovep:"); in->dump();}
785 } else if (in->Opcode() == Op_EncodeP || in->Opcode() == Op_DecodeN) {
786 in = in->in(1);
787 continue;
788 } else {
789 return false;
790 }
791 }
792 bool cont = false;
793 while (phis.is_nonempty()) {
794 uint idx = phis.index();
795 Node* phi = phis.node();
796 if (idx >= phi->req()) {
797 if (trace) {tty->print("Popped phi:"); phi->dump();}
798 phis.pop();
799 continue;
800 }
801 if (trace) {tty->print("Next entry(%d) for phi:", idx); phi->dump();}
802 in = phi->in(idx);
803 phis.set_index(idx+1);
804 cont = true;
805 break;
806 }
807 if (!cont) {
808 break;
809 }
810 }
811 return true;
812 }
813
814 void ShenandoahBarrierNode::report_verify_failure(const char *msg, Node *n1, Node *n2) {
815 if (n1 != NULL) {
816 n1->dump(+10);
817 }
818 if (n2 != NULL) {
819 n2->dump(+10);
820 }
821 fatal("%s", msg);
822 }
823
824 void ShenandoahBarrierNode::verify(RootNode* root) {
825 ResourceMark rm;
826 Unique_Node_List wq;
827 GrowableArray<Node*> barriers;
828 Unique_Node_List barriers_used;
829 Node_Stack phis(0);
830 VectorSet visited(Thread::current()->resource_area());
831 const bool trace = false;
832 const bool verify_no_useless_barrier = false;
833
834 wq.push(root);
835 for (uint next = 0; next < wq.size(); next++) {
836 Node *n = wq.at(next);
837 if (n->is_Load()) {
838 const bool trace = false;
839 if (trace) {tty->print("Verifying"); n->dump();}
840 if (n->Opcode() == Op_LoadRange || n->Opcode() == Op_LoadKlass || n->Opcode() == Op_LoadNKlass) {
841 if (trace) {tty->print_cr("Load range/klass");}
842 } else {
843 const TypePtr* adr_type = n->as_Load()->adr_type();
844
845 if (adr_type->isa_oopptr() && adr_type->is_oopptr()->offset() == oopDesc::mark_offset_in_bytes()) {
846 if (trace) {tty->print_cr("Mark load");}
847 } else if (adr_type->isa_instptr() &&
848 adr_type->is_instptr()->klass()->is_subtype_of(Compile::current()->env()->Reference_klass()) &&
849 adr_type->is_instptr()->offset() == java_lang_ref_Reference::referent_offset) {
850 if (trace) {tty->print_cr("Reference.get()");}
851 } else {
852 bool verify = true;
853 if (adr_type->isa_instptr()) {
854 const TypeInstPtr* tinst = adr_type->is_instptr();
855 ciKlass* k = tinst->klass();
856 assert(k->is_instance_klass(), "");
857 ciInstanceKlass* ik = (ciInstanceKlass*)k;
858 int offset = adr_type->offset();
859
860 if ((ik->debug_final_field_at(offset) && ShenandoahOptimizeInstanceFinals) ||
861 (ik->debug_stable_field_at(offset) && ShenandoahOptimizeStableFinals)) {
862 if (trace) {tty->print_cr("Final/stable");}
863 verify = false;
864 } else if (k == ciEnv::current()->Class_klass() &&
865 tinst->const_oop() != NULL &&
866 tinst->offset() >= (ik->size_helper() * wordSize)) {
867 ciInstanceKlass* k = tinst->const_oop()->as_instance()->java_lang_Class_klass()->as_instance_klass();
868 ciField* field = k->get_field_by_offset(tinst->offset(), true);
869 if ((ShenandoahOptimizeStaticFinals && field->is_final()) ||
870 (ShenandoahOptimizeStableFinals && field->is_stable())) {
871 verify = false;
872 }
873 }
874 }
875
876 if (verify && !ShenandoahBarrierNode::verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahLoad, trace, barriers_used)) {
877 report_verify_failure("Shenandoah verification: Load should have barriers", n);
878 }
879 }
880 }
881 } else if (n->is_Store()) {
882 const bool trace = false;
883
884 if (trace) {tty->print("Verifying"); n->dump();}
885 if (n->in(MemNode::ValueIn)->bottom_type()->make_oopptr()) {
886 Node* adr = n->in(MemNode::Address);
887 bool verify = true;
888
889 if (adr->is_AddP() && adr->in(AddPNode::Base)->is_top()) {
890 adr = adr->in(AddPNode::Address);
891 if (adr->is_AddP()) {
892 assert(adr->in(AddPNode::Base)->is_top(), "");
893 adr = adr->in(AddPNode::Address);
894 if (adr->Opcode() == Op_LoadP &&
895 adr->in(MemNode::Address)->in(AddPNode::Base)->is_top() &&
896 adr->in(MemNode::Address)->in(AddPNode::Address)->Opcode() == Op_ThreadLocal &&
897 adr->in(MemNode::Address)->in(AddPNode::Offset)->find_intptr_t_con(-1) == in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset())) {
898 if (trace) {tty->print_cr("SATB prebarrier");}
899 verify = false;
900 }
901 }
902 }
903
904 if (verify && !ShenandoahBarrierNode::verify_helper(n->in(MemNode::ValueIn), phis, visited, ShenandoahStoreValEnqueueBarrier ? ShenandoahOopStore : ShenandoahValue, trace, barriers_used)) {
905 report_verify_failure("Shenandoah verification: Store should have barriers", n);
906 }
907 }
908 if (!ShenandoahBarrierNode::verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahStore, trace, barriers_used)) {
909 report_verify_failure("Shenandoah verification: Store (address) should have barriers", n);
910 }
911 } else if (n->Opcode() == Op_CmpP) {
912 const bool trace = false;
913
914 Node* in1 = n->in(1);
915 Node* in2 = n->in(2);
916 if (in1->bottom_type()->isa_oopptr()) {
917 if (trace) {tty->print("Verifying"); n->dump();}
918
919 bool mark_inputs = false;
920 if (in1->bottom_type() == TypePtr::NULL_PTR || in2->bottom_type() == TypePtr::NULL_PTR ||
921 (in1->is_Con() || in2->is_Con())) {
922 if (trace) {tty->print_cr("Comparison against a constant");}
923 mark_inputs = true;
924 } else if ((in1->is_CheckCastPP() && in1->in(1)->is_Proj() && in1->in(1)->in(0)->is_Allocate()) ||
925 (in2->is_CheckCastPP() && in2->in(1)->is_Proj() && in2->in(1)->in(0)->is_Allocate())) {
926 if (trace) {tty->print_cr("Comparison with newly alloc'ed object");}
927 mark_inputs = true;
928 } else {
929 assert(in2->bottom_type()->isa_oopptr(), "");
930
931 if (!ShenandoahBarrierNode::verify_helper(in1, phis, visited, ShenandoahStore, trace, barriers_used) ||
932 !ShenandoahBarrierNode::verify_helper(in2, phis, visited, ShenandoahStore, trace, barriers_used)) {
933 report_verify_failure("Shenandoah verification: Cmp should have barriers", n);
934 }
935 }
936 if (verify_no_useless_barrier &&
937 mark_inputs &&
938 (!ShenandoahBarrierNode::verify_helper(in1, phis, visited, ShenandoahValue, trace, barriers_used) ||
939 !ShenandoahBarrierNode::verify_helper(in2, phis, visited, ShenandoahValue, trace, barriers_used))) {
940 phis.clear();
941 visited.Reset();
942 }
943 }
944 } else if (n->is_LoadStore()) {
945 if (n->in(MemNode::ValueIn)->bottom_type()->make_ptr() &&
946 !ShenandoahBarrierNode::verify_helper(n->in(MemNode::ValueIn), phis, visited, ShenandoahStoreValEnqueueBarrier ? ShenandoahOopStore : ShenandoahValue, trace, barriers_used)) {
947 report_verify_failure("Shenandoah verification: LoadStore (value) should have barriers", n);
948 }
949
950 if (n->in(MemNode::Address)->bottom_type()->make_oopptr() && !ShenandoahBarrierNode::verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahStore, trace, barriers_used)) {
951 report_verify_failure("Shenandoah verification: LoadStore (address) should have barriers", n);
952 }
953 } else if (n->Opcode() == Op_CallLeafNoFP || n->Opcode() == Op_CallLeaf) {
954 CallNode* call = n->as_Call();
955
956 static struct {
957 const char* name;
958 struct {
959 int pos;
960 verify_type t;
961 } args[6];
962 } calls[] = {
963 "aescrypt_encryptBlock",
964 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad },
965 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
966 "aescrypt_decryptBlock",
967 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad },
968 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
969 "multiplyToLen",
970 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad }, { TypeFunc::Parms+4, ShenandoahStore },
971 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
972 "squareToLen",
973 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad }, { -1, ShenandoahNone},
974 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
975 "montgomery_multiply",
976 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad },
977 { TypeFunc::Parms+6, ShenandoahStore }, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
978 "montgomery_square",
979 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+5, ShenandoahStore },
980 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
981 "mulAdd",
982 { { TypeFunc::Parms, ShenandoahStore }, { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone},
983 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
984 "vectorizedMismatch",
985 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone},
986 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
987 "updateBytesCRC32",
988 { { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone},
989 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
990 "updateBytesAdler32",
991 { { TypeFunc::Parms+1, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone},
992 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
993 "updateBytesCRC32C",
994 { { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahLoad}, { -1, ShenandoahNone},
995 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
996 "counterMode_AESCrypt",
997 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad },
998 { TypeFunc::Parms+3, ShenandoahStore }, { TypeFunc::Parms+5, ShenandoahStore }, { TypeFunc::Parms+6, ShenandoahStore } },
999 "cipherBlockChaining_encryptAESCrypt",
1000 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad },
1001 { TypeFunc::Parms+3, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
1002 "cipherBlockChaining_decryptAESCrypt",
1003 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad },
1004 { TypeFunc::Parms+3, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
1005 "shenandoah_clone_barrier",
1006 { { TypeFunc::Parms, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone},
1007 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
1008 "ghash_processBlocks",
1009 { { TypeFunc::Parms, ShenandoahStore }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad },
1010 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
1011 "sha1_implCompress",
1012 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
1013 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
1014 "sha256_implCompress",
1015 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
1016 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
1017 "sha512_implCompress",
1018 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
1019 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
1020 "sha1_implCompressMB",
1021 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
1022 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
1023 "sha256_implCompressMB",
1024 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
1025 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
1026 "sha512_implCompressMB",
1027 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
1028 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
1029 "encodeBlock",
1030 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahStore }, { -1, ShenandoahNone },
1031 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
1032 };
1033
1034 if (call->is_call_to_arraycopystub()) {
1035 Node* dest = NULL;
1036 const TypeTuple* args = n->as_Call()->_tf->domain();
1037 for (uint i = TypeFunc::Parms, j = 0; i < args->cnt(); i++) {
1038 if (args->field_at(i)->isa_ptr()) {
1039 j++;
1040 if (j == 2) {
1041 dest = n->in(i);
1042 break;
1043 }
1044 }
1045 }
1046 if (!ShenandoahBarrierNode::verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahLoad, trace, barriers_used) ||
1047 !ShenandoahBarrierNode::verify_helper(dest, phis, visited, ShenandoahStore, trace, barriers_used)) {
1048 report_verify_failure("Shenandoah verification: ArrayCopy should have barriers", n);
1049 }
1050 } else if (strlen(call->_name) > 5 &&
1051 !strcmp(call->_name + strlen(call->_name) - 5, "_fill")) {
1052 if (!ShenandoahBarrierNode::verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahStore, trace, barriers_used)) {
1053 report_verify_failure("Shenandoah verification: _fill should have barriers", n);
1054 }
1055 } else if (!strcmp(call->_name, "shenandoah_wb_pre")) {
1056 // skip
1057 } else {
1058 const int calls_len = sizeof(calls) / sizeof(calls[0]);
1059 int i = 0;
1060 for (; i < calls_len; i++) {
1061 if (!strcmp(calls[i].name, call->_name)) {
1062 break;
1063 }
1064 }
1065 if (i != calls_len) {
1066 const uint args_len = sizeof(calls[0].args) / sizeof(calls[0].args[0]);
1067 for (uint j = 0; j < args_len; j++) {
1068 int pos = calls[i].args[j].pos;
1069 if (pos == -1) {
1070 break;
1071 }
1072 if (!ShenandoahBarrierNode::verify_helper(call->in(pos), phis, visited, calls[i].args[j].t, trace, barriers_used)) {
1073 report_verify_failure("Shenandoah verification: intrinsic calls should have barriers", n);
1074 }
1075 }
1076 for (uint j = TypeFunc::Parms; j < call->req(); j++) {
1077 if (call->in(j)->bottom_type()->make_ptr() &&
1078 call->in(j)->bottom_type()->make_ptr()->isa_oopptr()) {
1079 uint k = 0;
1080 for (; k < args_len && calls[i].args[k].pos != (int)j; k++);
1081 if (k == args_len) {
1082 fatal("arg %d for call %s not covered", j, call->_name);
1083 }
1084 }
1085 }
1086 } else {
1087 for (uint j = TypeFunc::Parms; j < call->req(); j++) {
1088 if (call->in(j)->bottom_type()->make_ptr() &&
1089 call->in(j)->bottom_type()->make_ptr()->isa_oopptr()) {
1090 fatal("%s not covered", call->_name);
1091 }
1092 }
1093 }
1094 }
1095 } else if (n->is_ShenandoahBarrier()) {
1096 assert(!barriers.contains(n), "");
1097 assert(n->Opcode() != Op_ShenandoahWriteBarrier || n->find_out_with(Op_ShenandoahWBMemProj) != NULL, "bad shenandoah write barrier");
1098 assert(n->Opcode() != Op_ShenandoahWriteBarrier || n->outcnt() > 1, "bad shenandoah write barrier");
1099 barriers.push(n);
1100 } else if (n->Opcode() == Op_ShenandoahEnqueueBarrier) {
1101 // skip
1102 } else if (n->Opcode() == Op_ShenandoahWBMemProj) {
1103 assert(n->in(0) == NULL && n->in(ShenandoahWBMemProjNode::WriteBarrier)->Opcode() == Op_ShenandoahWriteBarrier, "strange ShenandoahWBMemProj");
1104 } else if (n->is_AddP()
1105 || n->is_Phi()
1106 || n->is_ConstraintCast()
1107 || n->Opcode() == Op_Return
1108 || n->Opcode() == Op_CMoveP
1109 || n->Opcode() == Op_CMoveN
1110 || n->Opcode() == Op_Rethrow
1111 || n->is_MemBar()
1112 || n->Opcode() == Op_Conv2B
1113 || n->Opcode() == Op_SafePoint
1114 || n->is_CallJava()
1115 || n->Opcode() == Op_Unlock
1116 || n->Opcode() == Op_EncodeP
1117 || n->Opcode() == Op_DecodeN) {
1118 // nothing to do
1119 } else {
1120 static struct {
1121 int opcode;
1122 struct {
1123 int pos;
1124 verify_type t;
1125 } inputs[2];
1126 } others[] = {
1127 Op_FastLock,
1128 { { 1, ShenandoahLoad }, { -1, ShenandoahNone} },
1129 Op_Lock,
1130 { { TypeFunc::Parms, ShenandoahLoad }, { -1, ShenandoahNone} },
1131 Op_ArrayCopy,
1132 { { ArrayCopyNode::Src, ShenandoahLoad }, { ArrayCopyNode::Dest, ShenandoahStore } },
1133 Op_StrCompressedCopy,
1134 { { 2, ShenandoahLoad }, { 3, ShenandoahStore } },
1135 Op_StrInflatedCopy,
1136 { { 2, ShenandoahLoad }, { 3, ShenandoahStore } },
1137 Op_AryEq,
1138 { { 2, ShenandoahLoad }, { 3, ShenandoahLoad } },
1139 Op_StrIndexOf,
1140 { { 2, ShenandoahLoad }, { 4, ShenandoahLoad } },
1141 Op_StrComp,
1142 { { 2, ShenandoahLoad }, { 4, ShenandoahLoad } },
1143 Op_StrEquals,
1144 { { 2, ShenandoahLoad }, { 3, ShenandoahLoad } },
1145 Op_EncodeISOArray,
1146 { { 2, ShenandoahLoad }, { 3, ShenandoahStore } },
1147 Op_HasNegatives,
1148 { { 2, ShenandoahLoad }, { -1, ShenandoahNone} },
1149 Op_CastP2X,
1150 { { 1, ShenandoahLoad }, { -1, ShenandoahNone} },
1151 Op_StrIndexOfChar,
1152 { { 2, ShenandoahLoad }, { -1, ShenandoahNone } },
1153 };
1154
1155 const int others_len = sizeof(others) / sizeof(others[0]);
1156 int i = 0;
1157 for (; i < others_len; i++) {
1158 if (others[i].opcode == n->Opcode()) {
1159 break;
1160 }
1161 }
1162 uint stop = n->is_Call() ? n->as_Call()->tf()->domain()->cnt() : n->req();
1163 if (i != others_len) {
1164 const uint inputs_len = sizeof(others[0].inputs) / sizeof(others[0].inputs[0]);
1165 for (uint j = 0; j < inputs_len; j++) {
1166 int pos = others[i].inputs[j].pos;
1167 if (pos == -1) {
1168 break;
1169 }
1170 if (!ShenandoahBarrierNode::verify_helper(n->in(pos), phis, visited, others[i].inputs[j].t, trace, barriers_used)) {
1171 report_verify_failure("Shenandoah verification: intrinsic calls should have barriers", n);
1172 }
1173 }
1174 for (uint j = 1; j < stop; j++) {
1175 if (n->in(j) != NULL && n->in(j)->bottom_type()->make_ptr() &&
1176 n->in(j)->bottom_type()->make_ptr()->make_oopptr()) {
1177 uint k = 0;
1178 for (; k < inputs_len && others[i].inputs[k].pos != (int)j; k++);
1179 if (k == inputs_len) {
1180 fatal("arg %d for node %s not covered", j, n->Name());
1181 }
1182 }
1183 }
1184 } else {
1185 for (uint j = 1; j < stop; j++) {
1186 if (n->in(j) != NULL && n->in(j)->bottom_type()->make_ptr() &&
1187 n->in(j)->bottom_type()->make_ptr()->make_oopptr()) {
1188 fatal("%s not covered", n->Name());
1189 }
1190 }
1191 }
1192 }
1193
1194 if (n->is_SafePoint()) {
1195 SafePointNode* sfpt = n->as_SafePoint();
1196 if (verify_no_useless_barrier && sfpt->jvms() != NULL) {
1197 for (uint i = sfpt->jvms()->scloff(); i < sfpt->jvms()->endoff(); i++) {
1198 if (!ShenandoahBarrierNode::verify_helper(sfpt->in(i), phis, visited, ShenandoahLoad, trace, barriers_used)) {
1199 phis.clear();
1200 visited.Reset();
1201 }
1202 }
1203 }
1204 }
1205 for( uint i = 0; i < n->len(); ++i ) {
1206 Node *m = n->in(i);
1207 if (m == NULL) continue;
1208
1209 // In most cases, inputs should be known to be non null. If it's
1210 // not the case, it could be a missing cast_not_null() in an
1211 // intrinsic or support might be needed in AddPNode::Ideal() to
1212 // avoid a NULL+offset input.
1213 if (!(n->is_Phi() ||
1214 (n->is_SafePoint() && (!n->is_CallRuntime() || !strcmp(n->as_Call()->_name, "shenandoah_wb_pre") || !strcmp(n->as_Call()->_name, "unsafe_arraycopy"))) ||
1215 n->Opcode() == Op_CmpP ||
1216 n->Opcode() == Op_CmpN ||
1217 (n->Opcode() == Op_StoreP && i == StoreNode::ValueIn) ||
1218 (n->Opcode() == Op_StoreN && i == StoreNode::ValueIn) ||
1219 n->is_ConstraintCast() ||
1220 n->Opcode() == Op_Return ||
1221 n->Opcode() == Op_Conv2B ||
1222 n->is_AddP() ||
1223 n->Opcode() == Op_CMoveP ||
1224 n->Opcode() == Op_CMoveN ||
1225 n->Opcode() == Op_Rethrow ||
1226 n->is_MemBar() ||
1227 n->is_Mem() ||
1228 n->Opcode() == Op_AryEq ||
1229 n->Opcode() == Op_SCMemProj ||
1230 n->Opcode() == Op_EncodeP ||
1231 n->Opcode() == Op_DecodeN ||
1232 n->Opcode() == Op_ShenandoahWriteBarrier ||
1233 n->Opcode() == Op_ShenandoahWBMemProj ||
1234 n->Opcode() == Op_ShenandoahEnqueueBarrier)) {
1235 if (m->bottom_type()->make_oopptr() && m->bottom_type()->make_oopptr()->meet(TypePtr::NULL_PTR) == m->bottom_type()) {
1236 report_verify_failure("Shenandoah verification: null input", n, m);
1237 }
1238 }
1239
1240 wq.push(m);
1241 }
1242 }
1243
1244 if (verify_no_useless_barrier) {
1245 for (int i = 0; i < barriers.length(); i++) {
1246 Node* n = barriers.at(i);
1247 if (!barriers_used.member(n)) {
1248 tty->print("XXX useless barrier"); n->dump(-2);
1249 ShouldNotReachHere();
1250 }
1251 }
1252 }
1253 }
1254 #endif
1255
1256 bool ShenandoahBarrierNode::is_dominator_same_ctrl(Node*c, Node* d, Node* n, PhaseIdealLoop* phase) {
1257 // That both nodes have the same control is not sufficient to prove
1258 // domination, verify that there's no path from d to n
1259 ResourceMark rm;
1260 Unique_Node_List wq;
1261 wq.push(d);
1262 for (uint next = 0; next < wq.size(); next++) {
1263 Node *m = wq.at(next);
1264 if (m == n) {
1265 return false;
1266 }
1267 if (m->is_Phi() && m->in(0)->is_Loop()) {
1268 assert(phase->ctrl_or_self(m->in(LoopNode::EntryControl)) != c, "following loop entry should lead to new control");
1269 } else {
1270 for (uint i = 0; i < m->req(); i++) {
1271 if (m->in(i) != NULL && phase->ctrl_or_self(m->in(i)) == c) {
1272 wq.push(m->in(i));
1273 }
1274 }
1275 }
1276 }
1277 return true;
1278 }
1279
1280 bool ShenandoahBarrierNode::is_dominator(Node *d_c, Node *n_c, Node* d, Node* n, PhaseIdealLoop* phase) {
1281 if (d_c != n_c) {
1282 return phase->is_dominator(d_c, n_c);
1283 }
1284 return is_dominator_same_ctrl(d_c, d, n, phase);
1285 }
1286
1287 Node* next_mem(Node* mem, int alias) {
1288 Node* res = NULL;
1289 if (mem->is_Proj()) {
1290 res = mem->in(0);
1291 } else if (mem->is_SafePoint() || mem->is_MemBar()) {
1292 res = mem->in(TypeFunc::Memory);
1293 } else if (mem->is_Phi()) {
1294 res = mem->in(1);
1295 } else if (mem->is_ShenandoahBarrier()) {
1296 res = mem->in(ShenandoahBarrierNode::Memory);
1297 } else if (mem->is_MergeMem()) {
1298 res = mem->as_MergeMem()->memory_at(alias);
1299 } else if (mem->is_Store() || mem->is_LoadStore() || mem->is_ClearArray()) {
1300 assert(alias = Compile::AliasIdxRaw, "following raw memory can't lead to a barrier");
1301 res = mem->in(MemNode::Memory);
1302 } else if (mem->Opcode() == Op_ShenandoahWBMemProj) {
1303 res = mem->in(ShenandoahWBMemProjNode::WriteBarrier);
1304 } else {
1305 #ifdef ASSERT
1306 mem->dump();
1307 #endif
1308 ShouldNotReachHere();
1309 }
1310 return res;
1311 }
1312
1313 Node* ShenandoahBarrierNode::no_branches(Node* c, Node* dom, bool allow_one_proj, PhaseIdealLoop* phase) {
1314 Node* iffproj = NULL;
1315 while (c != dom) {
1316 Node* next = phase->idom(c);
1317 assert(next->unique_ctrl_out() == c || c->is_Proj() || c->is_Region(), "multiple control flow out but no proj or region?");
1318 if (c->is_Region()) {
1319 ResourceMark rm;
1320 Unique_Node_List wq;
1321 wq.push(c);
1322 for (uint i = 0; i < wq.size(); i++) {
1323 Node *n = wq.at(i);
1324 if (n == next) {
1325 continue;
1326 }
1327 if (n->is_Region()) {
1328 for (uint j = 1; j < n->req(); j++) {
1329 wq.push(n->in(j));
1330 }
1331 } else {
1332 wq.push(n->in(0));
1333 }
1334 }
1335 for (uint i = 0; i < wq.size(); i++) {
1336 Node *n = wq.at(i);
1337 assert(n->is_CFG(), "");
1338 if (n->is_Multi()) {
1339 for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) {
1340 Node* u = n->fast_out(j);
1341 if (u->is_CFG()) {
1342 if (!wq.member(u) && !u->as_Proj()->is_uncommon_trap_proj(Deoptimization::Reason_none)) {
1343 return NodeSentinel;
1344 }
1345 }
1346 }
1347 }
1348 }
1349 } else if (c->is_Proj()) {
1350 if (c->is_IfProj()) {
1351 if (c->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none) != NULL) {
1352 // continue;
1353 } else {
1354 if (!allow_one_proj) {
1355 return NodeSentinel;
1356 }
1357 if (iffproj == NULL) {
1358 iffproj = c;
1359 } else {
1360 return NodeSentinel;
1361 }
1362 }
1363 } else if (c->Opcode() == Op_JumpProj) {
1364 return NodeSentinel; // unsupported
1365 } else if (c->Opcode() == Op_CatchProj) {
1366 return NodeSentinel; // unsupported
1367 } else if (c->Opcode() == Op_CProj && next->Opcode() == Op_NeverBranch) {
1368 return NodeSentinel; // unsupported
1369 } else {
1370 assert(next->unique_ctrl_out() == c, "unsupported branch pattern");
1371 }
1372 }
1373 c = next;
1374 }
1375 return iffproj;
1376 }
1377
1378 #ifdef ASSERT
1379 void ShenandoahWriteBarrierNode::memory_dominates_all_paths_helper(Node* c, Node* rep_ctrl, Unique_Node_List& controls, PhaseIdealLoop* phase) {
1380 const bool trace = false;
1381 if (trace) { tty->print("X control is"); c->dump(); }
1382
1383 uint start = controls.size();
1384 controls.push(c);
1385 for (uint i = start; i < controls.size(); i++) {
1386 Node *n = controls.at(i);
1387
1388 if (trace) { tty->print("X from"); n->dump(); }
1389
1390 if (n == rep_ctrl) {
1391 continue;
1392 }
1393
1394 if (n->is_Proj()) {
1395 Node* n_dom = n->in(0);
1396 IdealLoopTree* n_dom_loop = phase->get_loop(n_dom);
1397 if (n->is_IfProj() && n_dom->outcnt() == 2) {
1398 n_dom_loop = phase->get_loop(n_dom->as_If()->proj_out(n->as_Proj()->_con == 0 ? 1 : 0));
1399 }
1400 if (n_dom_loop != phase->ltree_root()) {
1401 Node* tail = n_dom_loop->tail();
1402 if (tail->is_Region()) {
1403 for (uint j = 1; j < tail->req(); j++) {
1404 if (phase->is_dominator(n_dom, tail->in(j)) && !phase->is_dominator(n, tail->in(j))) {
1405 assert(phase->is_dominator(rep_ctrl, tail->in(j)), "why are we here?");
1406 // entering loop from below, mark backedge
1407 if (trace) { tty->print("X pushing backedge"); tail->in(j)->dump(); }
1408 controls.push(tail->in(j));
1409 //assert(n->in(0) == n_dom, "strange flow control");
1410 }
1411 }
1412 } else if (phase->get_loop(n) != n_dom_loop && phase->is_dominator(n_dom, tail)) {
1413 // entering loop from below, mark backedge
1414 if (trace) { tty->print("X pushing backedge"); tail->dump(); }
1415 controls.push(tail);
1416 //assert(n->in(0) == n_dom, "strange flow control");
1417 }
1418 }
1419 }
1420
1421 if (n->is_Loop()) {
1422 Node* c = n->in(LoopNode::EntryControl);
1423 if (trace) { tty->print("X pushing"); c->dump(); }
1424 controls.push(c);
1425 } else if (n->is_Region()) {
1426 for (uint i = 1; i < n->req(); i++) {
1427 Node* c = n->in(i);
1428 if (trace) { tty->print("X pushing"); c->dump(); }
1429 controls.push(c);
1430 }
1431 } else {
1432 Node* c = n->in(0);
1433 if (trace) { tty->print("X pushing"); c->dump(); }
1434 controls.push(c);
1435 }
1436 }
1437 }
1438
1439 bool ShenandoahWriteBarrierNode::memory_dominates_all_paths(Node* mem, Node* rep_ctrl, int alias, PhaseIdealLoop* phase) {
1440 const bool trace = false;
1441 if (trace) {
1442 tty->print("XXX mem is"); mem->dump();
1443 tty->print("XXX rep ctrl is"); rep_ctrl->dump();
1444 tty->print_cr("XXX alias is %d", alias);
1445 }
1446 ResourceMark rm;
1447 Unique_Node_List wq;
1448 Unique_Node_List controls;
1449 wq.push(mem);
1450 for (uint next = 0; next < wq.size(); next++) {
1451 Node *nn = wq.at(next);
1452 if (trace) { tty->print("XX from mem"); nn->dump(); }
1453 assert(nn->bottom_type() == Type::MEMORY, "memory only");
1454
1455 if (nn->is_Phi()) {
1456 Node* r = nn->in(0);
1457 for (DUIterator_Fast jmax, j = r->fast_outs(jmax); j < jmax; j++) {
1458 Node* u = r->fast_out(j);
1459 if (u->is_Phi() && u->bottom_type() == Type::MEMORY && u != nn &&
1460 (u->adr_type() == TypePtr::BOTTOM || phase->C->get_alias_index(u->adr_type()) == alias)) {
1461 if (trace) { tty->print("XX Next mem (other phi)"); u->dump(); }
1462 wq.push(u);
1463 }
1464 }
1465 }
1466
1467 for (DUIterator_Fast imax, i = nn->fast_outs(imax); i < imax; i++) {
1468 Node* use = nn->fast_out(i);
1469
1470 if (trace) { tty->print("XX use %p", use->adr_type()); use->dump(); }
1471 if (use->is_CFG() && use->in(TypeFunc::Memory) == nn) {
1472 Node* c = use->in(0);
1473 if (phase->is_dominator(rep_ctrl, c)) {
1474 memory_dominates_all_paths_helper(c, rep_ctrl, controls, phase);
1475 } else if (use->is_CallStaticJava() && use->as_CallStaticJava()->uncommon_trap_request() != 0 && c->is_Region()) {
1476 Node* region = c;
1477 if (trace) { tty->print("XX unc region"); region->dump(); }
1478 for (uint j = 1; j < region->req(); j++) {
1479 if (phase->is_dominator(rep_ctrl, region->in(j))) {
1480 if (trace) { tty->print("XX unc follows"); region->in(j)->dump(); }
1481 memory_dominates_all_paths_helper(region->in(j), rep_ctrl, controls, phase);
1482 }
1483 }
1484 }
1485 //continue;
1486 } else if (use->is_Phi()) {
1487 assert(use->bottom_type() == Type::MEMORY, "bad phi");
1488 if ((use->adr_type() == TypePtr::BOTTOM) ||
1489 phase->C->get_alias_index(use->adr_type()) == alias) {
1490 for (uint j = 1; j < use->req(); j++) {
1491 if (use->in(j) == nn) {
1492 Node* c = use->in(0)->in(j);
1493 if (phase->is_dominator(rep_ctrl, c)) {
1494 memory_dominates_all_paths_helper(c, rep_ctrl, controls, phase);
1495 }
1496 }
1497 }
1498 }
1499 // continue;
1500 }
1501
1502 if (use->is_MergeMem()) {
1503 if (use->as_MergeMem()->memory_at(alias) == nn) {
1504 if (trace) { tty->print("XX Next mem"); use->dump(); }
1505 // follow the memory edges
1506 wq.push(use);
1507 }
1508 } else if (use->is_Phi()) {
1509 assert(use->bottom_type() == Type::MEMORY, "bad phi");
1510 if ((use->adr_type() == TypePtr::BOTTOM) ||
1511 phase->C->get_alias_index(use->adr_type()) == alias) {
1512 if (trace) { tty->print("XX Next mem"); use->dump(); }
1513 // follow the memory edges
1514 wq.push(use);
1515 }
1516 } else if (use->bottom_type() == Type::MEMORY &&
1517 (use->adr_type() == TypePtr::BOTTOM || phase->C->get_alias_index(use->adr_type()) == alias)) {
1518 if (trace) { tty->print("XX Next mem"); use->dump(); }
1519 // follow the memory edges
1520 wq.push(use);
1521 } else if ((use->is_SafePoint() || use->is_MemBar()) &&
1522 (use->adr_type() == TypePtr::BOTTOM || phase->C->get_alias_index(use->adr_type()) == alias)) {
1523 for (DUIterator_Fast jmax, j = use->fast_outs(jmax); j < jmax; j++) {
1524 Node* u = use->fast_out(j);
1525 if (u->bottom_type() == Type::MEMORY) {
1526 if (trace) { tty->print("XX Next mem"); u->dump(); }
1527 // follow the memory edges
1528 wq.push(u);
1529 }
1530 }
1531 } else if (use->Opcode() == Op_ShenandoahWriteBarrier && phase->C->get_alias_index(use->adr_type()) == alias) {
1532 Node* m = use->find_out_with(Op_ShenandoahWBMemProj);
1533 if (m != NULL) {
1534 if (trace) { tty->print("XX Next mem"); m->dump(); }
1535 // follow the memory edges
1536 wq.push(m);
1537 }
1538 }
1539 }
1540 }
1541
1542 if (controls.size() == 0) {
1543 return false;
1544 }
1545
1546 for (uint i = 0; i < controls.size(); i++) {
1547 Node *n = controls.at(i);
1548
1549 if (trace) { tty->print("X checking"); n->dump(); }
1550
1551 if (n->unique_ctrl_out() != NULL) {
1552 continue;
1553 }
1554
1555 if (n->Opcode() == Op_NeverBranch) {
1556 Node* taken = n->as_Multi()->proj_out(0);
1557 if (!controls.member(taken)) {
1558 if (trace) { tty->print("X not seen"); taken->dump(); }
1559 return false;
1560 }
1561 continue;
1562 }
1563
1564 for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) {
1565 Node* u = n->fast_out(j);
1566
1567 if (u->is_CFG()) {
1568 if (!controls.member(u)) {
1569 if (u->is_Proj() && u->as_Proj()->is_uncommon_trap_proj(Deoptimization::Reason_none)) {
1570 if (trace) { tty->print("X not seen but unc"); u->dump(); }
1571 } else {
1572 Node* c = u;
1573 do {
1574 c = c->unique_ctrl_out();
1575 } while (c != NULL && c->is_Region());
1576 if (c != NULL && c->Opcode() == Op_Halt) {
1577 if (trace) { tty->print("X not seen but halt"); c->dump(); }
1578 } else {
1579 if (trace) { tty->print("X not seen"); u->dump(); }
1580 return false;
1581 }
1582 }
1583 } else {
1584 if (trace) { tty->print("X seen"); u->dump(); }
1585 }
1586 }
1587 }
1588 }
1589 return true;
1590 }
1591 #endif
1592
1593 Node* ShenandoahBarrierNode::dom_mem(Node* mem, Node*& mem_ctrl, Node* n, Node* rep_ctrl, int alias, PhaseIdealLoop* phase) {
1594 ResourceMark rm;
1595 VectorSet wq(Thread::current()->resource_area());
1596 wq.set(mem->_idx);
1597 mem_ctrl = phase->get_ctrl(mem);
1598 while (!is_dominator(mem_ctrl, rep_ctrl, mem, n, phase)) {
1599 mem = next_mem(mem, alias);
1600 if (wq.test_set(mem->_idx)) {
1601 return NULL; // hit an unexpected loop
1602 }
1603 mem_ctrl = phase->ctrl_or_self(mem);
1604 }
1605 if (mem->is_MergeMem()) {
1606 mem = mem->as_MergeMem()->memory_at(alias);
1607 mem_ctrl = phase->ctrl_or_self(mem);
1608 }
1609 return mem;
1610 }
1611
1612 Node* ShenandoahBarrierNode::dom_mem(Node* mem, Node* ctrl, int alias, Node*& mem_ctrl, PhaseIdealLoop* phase) {
1613 ResourceMark rm;
1614 VectorSet wq(Thread::current()->resource_area());
1615 wq.set(mem->_idx);
1616 mem_ctrl = phase->ctrl_or_self(mem);
1617 while (!phase->is_dominator(mem_ctrl, ctrl) || mem_ctrl == ctrl) {
1618 mem = next_mem(mem, alias);
1619 if (wq.test_set(mem->_idx)) {
1620 return NULL;
1621 }
1622 mem_ctrl = phase->ctrl_or_self(mem);
1623 }
1624 if (mem->is_MergeMem()) {
1625 mem = mem->as_MergeMem()->memory_at(alias);
1626 mem_ctrl = phase->ctrl_or_self(mem);
1627 }
1628 return mem;
1629 }
1630
1631 static void disconnect_barrier_mem(Node* wb, PhaseIterGVN& igvn) {
1632 Node* mem_in = wb->in(ShenandoahBarrierNode::Memory);
1633 Node* proj = wb->find_out_with(Op_ShenandoahWBMemProj);
1634
1635 for (DUIterator_Last imin, i = proj->last_outs(imin); i >= imin; ) {
1636 Node* u = proj->last_out(i);
1637 igvn.rehash_node_delayed(u);
1638 int nb = u->replace_edge(proj, mem_in);
1639 assert(nb > 0, "no replacement?");
1640 i -= nb;
1641 }
1642 }
1643
1644 Node* ShenandoahWriteBarrierNode::move_above_predicates(LoopNode* cl, Node* val_ctrl, PhaseIdealLoop* phase) {
1645 Node* entry = cl->skip_strip_mined(-1)->in(LoopNode::EntryControl);
1646 Node* above_pred = phase->skip_all_loop_predicates(entry);
1647 Node* ctrl = entry;
1648 while (ctrl != above_pred) {
1649 Node* next = ctrl->in(0);
1650 if (!phase->is_dominator(val_ctrl, next)) {
1651 break;
1652 }
1653 ctrl = next;
1654 }
1655 return ctrl;
1656 }
1657
1658 static MemoryGraphFixer* find_fixer(GrowableArray<MemoryGraphFixer*>& memory_graph_fixers, int alias) {
1659 for (int i = 0; i < memory_graph_fixers.length(); i++) {
1660 if (memory_graph_fixers.at(i)->alias() == alias) {
1661 return memory_graph_fixers.at(i);
1662 }
1663 }
1664 return NULL;
1665 }
1666
1667 static MemoryGraphFixer* create_fixer(GrowableArray<MemoryGraphFixer*>& memory_graph_fixers, int alias, PhaseIdealLoop* phase, bool include_lsm) {
1668 assert(find_fixer(memory_graph_fixers, alias) == NULL, "none should exist yet");
1669 MemoryGraphFixer* fixer = new MemoryGraphFixer(alias, include_lsm, phase);
1670 memory_graph_fixers.push(fixer);
1671 return fixer;
1672 }
1673
1674 void ShenandoahWriteBarrierNode::try_move_before_loop_helper(LoopNode* cl, Node* val_ctrl, GrowableArray<MemoryGraphFixer*>& memory_graph_fixers, PhaseIdealLoop* phase, bool include_lsm, Unique_Node_List& uses) {
1675 assert(cl->is_Loop(), "bad control");
1676 Node* ctrl = move_above_predicates(cl, val_ctrl, phase);
1677 Node* mem_ctrl = NULL;
1678 int alias = phase->C->get_alias_index(adr_type());
1679
1680 MemoryGraphFixer* fixer = find_fixer(memory_graph_fixers, alias);
1681 if (fixer == NULL) {
1682 fixer = create_fixer(memory_graph_fixers, alias, phase, include_lsm);
1683 }
1684
1685 Node* proj = find_out_with(Op_ShenandoahWBMemProj);
1686
1687 fixer->remove(proj);
1688 Node* mem = fixer->find_mem(ctrl, NULL);
1689
1690 assert(!ShenandoahVerifyOptoBarriers || memory_dominates_all_paths(mem, ctrl, alias, phase), "can't fix the memory graph");
1691
1692 phase->set_ctrl_and_loop(this, ctrl);
1693 phase->igvn().replace_input_of(this, Control, ctrl);
1694
1695 disconnect_barrier_mem(this, phase->igvn());
1696
1697 phase->igvn().replace_input_of(this, Memory, mem);
1698 phase->set_ctrl_and_loop(proj, ctrl);
1699
1700 fixer->fix_mem(ctrl, ctrl, mem, mem, proj, uses);
1701 assert(proj->outcnt() > 0, "disconnected write barrier");
1702 }
1703
1704 LoopNode* ShenandoahWriteBarrierNode::try_move_before_pre_loop(Node* c, Node* val_ctrl, PhaseIdealLoop* phase) {
1705 // A write barrier between a pre and main loop can get in the way of
1706 // vectorization. Move it above the pre loop if possible
1707 CountedLoopNode* cl = NULL;
1708 if (c->is_IfFalse() &&
1709 c->in(0)->is_CountedLoopEnd()) {
1710 cl = c->in(0)->as_CountedLoopEnd()->loopnode();
1711 } else if (c->is_IfProj() &&
1712 c->in(0)->is_If() &&
1713 c->in(0)->in(0)->is_IfFalse() &&
1714 c->in(0)->in(0)->in(0)->is_CountedLoopEnd()) {
1715 cl = c->in(0)->in(0)->in(0)->as_CountedLoopEnd()->loopnode();
1716 }
1717 if (cl != NULL &&
1718 cl->is_pre_loop() &&
1719 val_ctrl != cl &&
1720 phase->is_dominator(val_ctrl, cl)) {
1721 return cl;
1722 }
1723 return NULL;
1724 }
1725
1726 void ShenandoahWriteBarrierNode::try_move_before_loop(GrowableArray<MemoryGraphFixer*>& memory_graph_fixers, PhaseIdealLoop* phase, bool include_lsm, Unique_Node_List& uses) {
1727 Node *n_ctrl = phase->get_ctrl(this);
1728 IdealLoopTree *n_loop = phase->get_loop(n_ctrl);
1729 Node* val = in(ValueIn);
1730 Node* val_ctrl = phase->get_ctrl(val);
1731 if (n_loop != phase->ltree_root() && !n_loop->_irreducible) {
1732 IdealLoopTree *val_loop = phase->get_loop(val_ctrl);
1733 Node* mem = in(Memory);
1734 IdealLoopTree *mem_loop = phase->get_loop(phase->get_ctrl(mem));
1735 if (!n_loop->is_member(val_loop) &&
1736 n_loop->is_member(mem_loop)) {
1737 Node* n_loop_head = n_loop->_head;
1738
1739 if (n_loop_head->is_Loop()) {
1740 LoopNode* loop = n_loop_head->as_Loop();
1741 if (n_loop_head->is_CountedLoop() && n_loop_head->as_CountedLoop()->is_main_loop()) {
1742 LoopNode* res = try_move_before_pre_loop(n_loop_head->in(LoopNode::EntryControl), val_ctrl, phase);
1743 if (res != NULL) {
1744 loop = res;
1745 }
1746 }
1747
1748 try_move_before_loop_helper(loop, val_ctrl, memory_graph_fixers, phase, include_lsm, uses);
1749 }
1750 }
1751 }
1752 LoopNode* ctrl = try_move_before_pre_loop(in(0), val_ctrl, phase);
1753 if (ctrl != NULL) {
1754 try_move_before_loop_helper(ctrl, val_ctrl, memory_graph_fixers, phase, include_lsm, uses);
1755 }
1756 }
1757
1758 Node* ShenandoahWriteBarrierNode::would_subsume(ShenandoahBarrierNode* other, PhaseIdealLoop* phase) {
1759 Node* val = in(ValueIn);
1760 Node* val_ctrl = phase->get_ctrl(val);
1761 Node* other_mem = other->in(Memory);
1762 Node* other_ctrl = phase->get_ctrl(other);
1763 Node* this_ctrl = phase->get_ctrl(this);
1764 IdealLoopTree* this_loop = phase->get_loop(this_ctrl);
1765 IdealLoopTree* other_loop = phase->get_loop(other_ctrl);
1766
1767 Node* ctrl = phase->dom_lca(other_ctrl, this_ctrl);
1768
1769 if (ctrl->is_Proj() &&
1770 ctrl->in(0)->is_Call() &&
1771 ctrl->unique_ctrl_out() != NULL &&
1772 ctrl->unique_ctrl_out()->Opcode() == Op_Catch &&
1773 !phase->is_dominator(val_ctrl, ctrl->in(0)->in(0))) {
1774 return NULL;
1775 }
1776
1777 IdealLoopTree* loop = phase->get_loop(ctrl);
1778
1779 // We don't want to move a write barrier in a loop
1780 // If the LCA is in a inner loop, try a control out of loop if possible
1781 while (!loop->is_member(this_loop) && (other->Opcode() != Op_ShenandoahWriteBarrier || !loop->is_member(other_loop))) {
1782 ctrl = phase->idom(ctrl);
1783 if (ctrl->is_MultiBranch()) {
1784 ctrl = ctrl->in(0);
1785 }
1786 if (ctrl != val_ctrl && phase->is_dominator(ctrl, val_ctrl)) {
1787 return NULL;
1788 }
1789 loop = phase->get_loop(ctrl);
1790 }
1791
1792 if (ShenandoahDontIncreaseWBFreq) {
1793 Node* this_iffproj = no_branches(this_ctrl, ctrl, true, phase);
1794 if (other->Opcode() == Op_ShenandoahWriteBarrier) {
1795 Node* other_iffproj = no_branches(other_ctrl, ctrl, true, phase);
1796 if (other_iffproj == NULL || this_iffproj == NULL) {
1797 return ctrl;
1798 } else if (other_iffproj != NodeSentinel && this_iffproj != NodeSentinel &&
1799 other_iffproj->in(0) == this_iffproj->in(0)) {
1800 return ctrl;
1801 }
1802 } else if (this_iffproj == NULL) {
1803 return ctrl;
1804 }
1805 return NULL;
1806 }
1807
1808 return ctrl;
1809 }
1810
1811 void ShenandoahWriteBarrierNode::optimize_before_expansion(PhaseIdealLoop* phase, GrowableArray<MemoryGraphFixer*> memory_graph_fixers, bool include_lsm) {
1812 bool progress = false;
1813 Unique_Node_List uses;
1814 do {
1815 progress = false;
1816 for (int i = 0; i < ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count(); i++) {
1817 ShenandoahWriteBarrierNode* wb = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barrier(i);
1818
1819 wb->try_move_before_loop(memory_graph_fixers, phase, include_lsm, uses);
1820
1821 Node* val = wb->in(ValueIn);
1822
1823 for (DUIterator_Fast jmax, j = val->fast_outs(jmax); j < jmax; j++) {
1824 Node* u = val->fast_out(j);
1825 if (u != wb && u->is_ShenandoahBarrier()) {
1826 Node* rep_ctrl = wb->would_subsume(u->as_ShenandoahBarrier(), phase);
1827
1828 if (rep_ctrl != NULL) {
1829 Node* other = u;
1830 Node* val_ctrl = phase->get_ctrl(val);
1831 if (rep_ctrl->is_Proj() &&
1832 rep_ctrl->in(0)->is_Call() &&
1833 rep_ctrl->unique_ctrl_out() != NULL &&
1834 rep_ctrl->unique_ctrl_out()->Opcode() == Op_Catch) {
1835 rep_ctrl = rep_ctrl->in(0)->in(0);
1836
1837 assert(phase->is_dominator(val_ctrl, rep_ctrl), "bad control");
1838 } else {
1839 LoopNode* c = ShenandoahWriteBarrierNode::try_move_before_pre_loop(rep_ctrl, val_ctrl, phase);
1840 if (c != NULL) {
1841 rep_ctrl = ShenandoahWriteBarrierNode::move_above_predicates(c, val_ctrl, phase);
1842 } else {
1843 while (rep_ctrl->is_IfProj()) {
1844 CallStaticJavaNode* unc = rep_ctrl->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none);
1845 if (unc != NULL) {
1846 int req = unc->uncommon_trap_request();
1847 Deoptimization::DeoptReason trap_reason = Deoptimization::trap_request_reason(req);
1848 if ((trap_reason == Deoptimization::Reason_loop_limit_check ||
1849 trap_reason == Deoptimization::Reason_predicate ||
1850 trap_reason == Deoptimization::Reason_profile_predicate) &&
1851 phase->is_dominator(val_ctrl, rep_ctrl->in(0)->in(0))) {
1852 rep_ctrl = rep_ctrl->in(0)->in(0);
1853 continue;
1854 }
1855 }
1856 break;
1857 }
1858 }
1859 }
1860
1861 Node* wb_ctrl = phase->get_ctrl(wb);
1862 Node* other_ctrl = phase->get_ctrl(other);
1863 int alias = phase->C->get_alias_index(wb->adr_type());
1864 MemoryGraphFixer* fixer = find_fixer(memory_graph_fixers, alias);;
1865 if (!is_dominator(wb_ctrl, other_ctrl, wb, other, phase)) {
1866 if (fixer == NULL) {
1867 fixer = create_fixer(memory_graph_fixers, alias, phase, include_lsm);
1868 }
1869 Node* mem = fixer->find_mem(rep_ctrl, phase->get_ctrl(other) == rep_ctrl ? other : NULL);
1870
1871 if (mem->has_out_with(Op_Lock) || mem->has_out_with(Op_Unlock)) {
1872 continue;
1873 }
1874
1875 Node* wb_proj = wb->find_out_with(Op_ShenandoahWBMemProj);
1876 fixer->remove(wb_proj);
1877 Node* mem_for_ctrl = fixer->find_mem(rep_ctrl, NULL);
1878
1879 if (wb->in(Memory) != mem) {
1880 disconnect_barrier_mem(wb, phase->igvn());
1881 phase->igvn().replace_input_of(wb, Memory, mem);
1882 }
1883 if (rep_ctrl != wb_ctrl) {
1884 phase->set_ctrl_and_loop(wb, rep_ctrl);
1885 phase->igvn().replace_input_of(wb, Control, rep_ctrl);
1886 phase->set_ctrl_and_loop(wb_proj, rep_ctrl);
1887 progress = true;
1888 }
1889
1890 fixer->fix_mem(rep_ctrl, rep_ctrl, mem, mem_for_ctrl, wb_proj, uses);
1891
1892 assert(!ShenandoahVerifyOptoBarriers || ShenandoahWriteBarrierNode::memory_dominates_all_paths(mem, rep_ctrl, alias, phase), "can't fix the memory graph");
1893 }
1894
1895 if (other->Opcode() == Op_ShenandoahWriteBarrier) {
1896 Node* other_proj = other->find_out_with(Op_ShenandoahWBMemProj);
1897 if (fixer != NULL) {
1898 fixer->remove(other_proj);
1899 }
1900 phase->igvn().replace_node(other_proj, other->in(Memory));
1901 }
1902 phase->igvn().replace_node(other, wb);
1903 --j; --jmax;
1904 }
1905 }
1906 }
1907 }
1908 } while(progress);
1909 }
1910
1911 void ShenandoahReadBarrierNode::try_move(Node *n_ctrl, PhaseIdealLoop* phase) {
1912 Node* mem = in(MemNode::Memory);
1913 int alias = phase->C->get_alias_index(adr_type());
1914 const bool trace = false;
1915
1916 #ifdef ASSERT
1917 if (trace) { tty->print("Trying to move mem of"); dump(); }
1918 #endif
1919
1920 Node* new_mem = mem;
1921
1922 ResourceMark rm;
1923 VectorSet seen(Thread::current()->resource_area());
1924 Node_List phis;
1925
1926 for (;;) {
1927 #ifdef ASSERT
1928 if (trace) { tty->print("Looking for dominator from"); mem->dump(); }
1929 #endif
1930 if (mem->is_Proj() && mem->in(0)->is_Start()) {
1931 if (new_mem != in(MemNode::Memory)) {
1932 #ifdef ASSERT
1933 if (trace) { tty->print("XXX Setting mem to"); new_mem->dump(); tty->print(" for "); dump(); }
1934 #endif
1935 phase->igvn().replace_input_of(this, MemNode::Memory, new_mem);
1936 }
1937 return;
1938 }
1939
1940 Node* candidate = mem;
1941 do {
1942 if (!is_independent(mem)) {
1943 if (trace) { tty->print_cr("Not independent"); }
1944 if (new_mem != in(MemNode::Memory)) {
1945 #ifdef ASSERT
1946 if (trace) { tty->print("XXX Setting mem to"); new_mem->dump(); tty->print(" for "); dump(); }
1947 #endif
1948 phase->igvn().replace_input_of(this, MemNode::Memory, new_mem);
1949 }
1950 return;
1951 }
1952 if (seen.test_set(mem->_idx)) {
1953 if (trace) { tty->print_cr("Already seen"); }
1954 ShouldNotReachHere();
1955 // Strange graph
1956 if (new_mem != in(MemNode::Memory)) {
1957 #ifdef ASSERT
1958 if (trace) { tty->print("XXX Setting mem to"); new_mem->dump(); tty->print(" for "); dump(); }
1959 #endif
1960 phase->igvn().replace_input_of(this, MemNode::Memory, new_mem);
1961 }
1962 return;
1963 }
1964 if (mem->is_Phi()) {
1965 phis.push(mem);
1966 }
1967 mem = next_mem(mem, alias);
1968 if (mem->bottom_type() == Type::MEMORY) {
1969 candidate = mem;
1970 }
1971 assert(is_dominator(phase->ctrl_or_self(mem), n_ctrl, mem, this, phase) == phase->is_dominator(phase->ctrl_or_self(mem), n_ctrl), "strange dominator");
1972 #ifdef ASSERT
1973 if (trace) { tty->print("Next mem is"); mem->dump(); }
1974 #endif
1975 } while (mem->bottom_type() != Type::MEMORY || !phase->is_dominator(phase->ctrl_or_self(mem), n_ctrl));
1976
1977 assert(mem->bottom_type() == Type::MEMORY, "bad mem");
1978
1979 bool not_dom = false;
1980 for (uint i = 0; i < phis.size() && !not_dom; i++) {
1981 Node* nn = phis.at(i);
1982
1983 #ifdef ASSERT
1984 if (trace) { tty->print("Looking from phi"); nn->dump(); }
1985 #endif
1986 assert(nn->is_Phi(), "phis only");
1987 for (uint j = 2; j < nn->req() && !not_dom; j++) {
1988 Node* m = nn->in(j);
1989 #ifdef ASSERT
1990 if (trace) { tty->print("Input %d is", j); m->dump(); }
1991 #endif
1992 while (m != mem && !seen.test_set(m->_idx)) {
1993 if (is_dominator(phase->ctrl_or_self(m), phase->ctrl_or_self(mem), m, mem, phase)) {
1994 not_dom = true;
1995 // Scheduling anomaly
1996 #ifdef ASSERT
1997 if (trace) { tty->print("Giving up"); m->dump(); }
1998 #endif
1999 break;
2000 }
2001 if (!is_independent(m)) {
2002 if (trace) { tty->print_cr("Not independent"); }
2003 if (new_mem != in(MemNode::Memory)) {
2004 #ifdef ASSERT
2005 if (trace) { tty->print("XXX Setting mem to"); new_mem->dump(); tty->print(" for "); dump(); }
2006 #endif
2007 phase->igvn().replace_input_of(this, MemNode::Memory, new_mem);
2008 }
2009 return;
2010 }
2011 if (m->is_Phi()) {
2012 phis.push(m);
2013 }
2014 m = next_mem(m, alias);
2015 #ifdef ASSERT
2016 if (trace) { tty->print("Next mem is"); m->dump(); }
2017 #endif
2018 }
2019 }
2020 }
2021 if (!not_dom) {
2022 new_mem = mem;
2023 phis.clear();
2024 } else {
2025 seen.Clear();
2026 }
2027 }
2028 }
2029
2030 CallStaticJavaNode* ShenandoahWriteBarrierNode::pin_and_expand_null_check(PhaseIterGVN& igvn) {
2031 Node* val = in(ValueIn);
2032
2033 const Type* val_t = igvn.type(val);
2034
2035 if (val_t->meet(TypePtr::NULL_PTR) != val_t &&
2036 val->Opcode() == Op_CastPP &&
2037 val->in(0) != NULL &&
2038 val->in(0)->Opcode() == Op_IfTrue &&
2039 val->in(0)->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none) &&
2040 val->in(0)->in(0)->is_If() &&
2041 val->in(0)->in(0)->in(1)->Opcode() == Op_Bool &&
2042 val->in(0)->in(0)->in(1)->as_Bool()->_test._test == BoolTest::ne &&
2043 val->in(0)->in(0)->in(1)->in(1)->Opcode() == Op_CmpP &&
2044 val->in(0)->in(0)->in(1)->in(1)->in(1) == val->in(1) &&
2045 val->in(0)->in(0)->in(1)->in(1)->in(2)->bottom_type() == TypePtr::NULL_PTR) {
2046 assert(val->in(0)->in(0)->in(1)->in(1)->in(1) == val->in(1), "");
2047 CallStaticJavaNode* unc = val->in(0)->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none);
2048 return unc;
2049 }
2050 return NULL;
2051 }
2052
2053 void ShenandoahWriteBarrierNode::pin_and_expand_move_barrier(PhaseIdealLoop* phase, GrowableArray<MemoryGraphFixer*>& memory_graph_fixers, Unique_Node_List& uses) {
2054 Node* unc = pin_and_expand_null_check(phase->igvn());
2055 Node* val = in(ValueIn);
2056
2057 if (unc != NULL) {
2058 Node* ctrl = phase->get_ctrl(this);
2059 Node* unc_ctrl = val->in(0);
2060
2061 // Don't move write barrier in a loop
2062 IdealLoopTree* loop = phase->get_loop(ctrl);
2063 IdealLoopTree* unc_loop = phase->get_loop(unc_ctrl);
2064
2065 if (!unc_loop->is_member(loop)) {
2066 return;
2067 }
2068
2069 Node* branch = no_branches(ctrl, unc_ctrl, false, phase);
2070 assert(branch == NULL || branch == NodeSentinel, "was not looking for a branch");
2071 if (branch == NodeSentinel) {
2072 return;
2073 }
2074
2075 RegionNode* r = new RegionNode(3);
2076 IfNode* iff = unc_ctrl->in(0)->as_If();
2077
2078 Node* ctrl_use = unc_ctrl->unique_ctrl_out();
2079 Node* unc_ctrl_clone = unc_ctrl->clone();
2080 phase->register_control(unc_ctrl_clone, loop, iff);
2081 Node* c = unc_ctrl_clone;
2082 Node* new_cast = clone_null_check(c, val, unc_ctrl_clone, phase);
2083 r->init_req(1, new_cast->in(0)->in(0)->as_If()->proj_out(0));
2084
2085 phase->igvn().replace_input_of(unc_ctrl, 0, c->in(0));
2086 phase->set_idom(unc_ctrl, c->in(0), phase->dom_depth(unc_ctrl));
2087 phase->lazy_replace(c, unc_ctrl);
2088 c = NULL;;
2089 phase->igvn().replace_input_of(val, 0, unc_ctrl_clone);
2090 phase->set_ctrl(val, unc_ctrl_clone);
2091
2092 IfNode* new_iff = new_cast->in(0)->in(0)->as_If();
2093 fix_null_check(unc, unc_ctrl_clone, r, uses, phase);
2094 Node* iff_proj = iff->proj_out(0);
2095 r->init_req(2, iff_proj);
2096 phase->register_control(r, phase->ltree_root(), iff);
2097
2098 Node* new_bol = new_iff->in(1)->clone();
2099 Node* new_cmp = new_bol->in(1)->clone();
2100 assert(new_cmp->Opcode() == Op_CmpP, "broken");
2101 assert(new_cmp->in(1) == val->in(1), "broken");
2102 new_bol->set_req(1, new_cmp);
2103 new_cmp->set_req(1, this);
2104 phase->register_new_node(new_bol, new_iff->in(0));
2105 phase->register_new_node(new_cmp, new_iff->in(0));
2106 phase->igvn().replace_input_of(new_iff, 1, new_bol);
2107 phase->igvn().replace_input_of(new_cast, 1, this);
2108
2109 for (DUIterator_Fast imax, i = this->fast_outs(imax); i < imax; i++) {
2110 Node* u = this->fast_out(i);
2111 if (u == new_cast || u->Opcode() == Op_ShenandoahWBMemProj || u == new_cmp) {
2112 continue;
2113 }
2114 phase->igvn().rehash_node_delayed(u);
2115 int nb = u->replace_edge(this, new_cast);
2116 assert(nb > 0, "no update?");
2117 --i; imax -= nb;
2118 }
2119
2120 for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) {
2121 Node* u = val->fast_out(i);
2122 if (u == this) {
2123 continue;
2124 }
2125 phase->igvn().rehash_node_delayed(u);
2126 int nb = u->replace_edge(val, new_cast);
2127 assert(nb > 0, "no update?");
2128 --i; imax -= nb;
2129 }
2130
2131 Node* new_ctrl = unc_ctrl_clone;
2132
2133 int alias = phase->C->get_alias_index(adr_type());
2134 MemoryGraphFixer* fixer = find_fixer(memory_graph_fixers, alias);
2135 if (fixer == NULL) {
2136 fixer = create_fixer(memory_graph_fixers, alias, phase, true);
2137 }
2138
2139 Node* proj = find_out_with(Op_ShenandoahWBMemProj);
2140 fixer->remove(proj);
2141 Node* mem = fixer->find_mem(new_ctrl, NULL);
2142
2143 if (in(Memory) != mem) {
2144 disconnect_barrier_mem(this, phase->igvn());
2145 phase->igvn().replace_input_of(this, Memory, mem);
2146 }
2147
2148 phase->set_ctrl_and_loop(this, new_ctrl);
2149 phase->igvn().replace_input_of(this, Control, new_ctrl);
2150 phase->set_ctrl_and_loop(proj, new_ctrl);
2151
2152 fixer->fix_mem(new_ctrl, new_ctrl, mem, mem, proj, uses);
2153 }
2154 }
2155
2156 void ShenandoahWriteBarrierNode::pin_and_expand_helper(PhaseIdealLoop* phase) {
2157 Node* val = in(ValueIn);
2158 CallStaticJavaNode* unc = pin_and_expand_null_check(phase->igvn());
2159 Node* rep = this;
2160 Node* ctrl = phase->get_ctrl(this);
2161 if (unc != NULL && val->in(0) == ctrl) {
2162 Node* unc_ctrl = val->in(0);
2163 IfNode* other_iff = unc_ctrl->unique_ctrl_out()->as_If();
2164 ProjNode* other_unc_ctrl = other_iff->proj_out(1);
2165 Node* cast = NULL;
2166 for (DUIterator_Fast imax, i = other_unc_ctrl->fast_outs(imax); i < imax && cast == NULL; i++) {
2167 Node* u = other_unc_ctrl->fast_out(i);
2168 if (u->Opcode() == Op_CastPP && u->in(1) == this) {
2169 cast = u;
2170 }
2171 }
2172 assert(other_unc_ctrl->is_uncommon_trap_if_pattern(Deoptimization::Reason_none) == unc, "broken");
2173 rep = cast;
2174 }
2175
2176 // Replace all uses of barrier's input that are dominated by ctrl
2177 // with the value returned by the barrier: no need to keep both
2178 // live.
2179 for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) {
2180 Node* u = val->fast_out(i);
2181 if (u != this) {
2182 if (u->is_Phi()) {
2183 int nb = 0;
2184 for (uint j = 1; j < u->req(); j++) {
2185 if (u->in(j) == val) {
2186 Node* c = u->in(0)->in(j);
2187 if (phase->is_dominator(ctrl, c)) {
2188 phase->igvn().replace_input_of(u, j, rep);
2189 nb++;
2190 }
2191 }
2192 }
2193 if (nb > 0) {
2194 imax -= nb;
2195 --i;
2196 }
2197 } else {
2198 Node* c = phase->ctrl_or_self(u);
2199 if (is_dominator(ctrl, c, this, u, phase)) {
2200 phase->igvn().rehash_node_delayed(u);
2201 int nb = u->replace_edge(val, rep);
2202 assert(nb > 0, "no update?");
2203 --i, imax -= nb;
2204 }
2205 }
2206 }
2207 }
2208 }
2209
2210 Node* ShenandoahWriteBarrierNode::find_bottom_mem(Node* ctrl, PhaseIdealLoop* phase) {
2211 Node* mem = NULL;
2212 Node* c = ctrl;
2213 do {
2214 if (c->is_Region()) {
2215 Node* phi_bottom = NULL;
2216 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax && mem == NULL; i++) {
2217 Node* u = c->fast_out(i);
2218 if (u->is_Phi() && u->bottom_type() == Type::MEMORY) {
2219 if (u->adr_type() == TypePtr::BOTTOM) {
2220 mem = u;
2221 }
2222 }
2223 }
2224 } else {
2225 if (c->is_Call() && c->as_Call()->adr_type() != NULL) {
2226 CallProjections projs;
2227 c->as_Call()->extract_projections(&projs, true, false);
2228 if (projs.fallthrough_memproj != NULL) {
2229 if (projs.fallthrough_memproj->adr_type() == TypePtr::BOTTOM) {
2230 if (projs.catchall_memproj == NULL) {
2231 mem = projs.fallthrough_memproj;
2232 } else {
2233 if (phase->is_dominator(projs.fallthrough_catchproj, ctrl)) {
2234 mem = projs.fallthrough_memproj;
2235 } else {
2236 assert(phase->is_dominator(projs.catchall_catchproj, ctrl), "one proj must dominate barrier");
2237 mem = projs.catchall_memproj;
2238 }
2239 }
2240 }
2241 } else {
2242 Node* proj = c->as_Call()->proj_out(TypeFunc::Memory);
2243 if (proj != NULL &&
2244 proj->adr_type() == TypePtr::BOTTOM) {
2245 mem = proj;
2246 }
2247 }
2248 } else {
2249 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) {
2250 Node* u = c->fast_out(i);
2251 if (u->is_Proj() &&
2252 u->bottom_type() == Type::MEMORY &&
2253 u->adr_type() == TypePtr::BOTTOM) {
2254 assert(c->is_SafePoint() || c->is_MemBar() || c->is_Start(), "");
2255 assert(mem == NULL, "only one proj");
2256 mem = u;
2257 }
2258 }
2259 assert(!c->is_Call() || c->as_Call()->adr_type() != NULL || mem == NULL, "no mem projection expected");
2260 }
2261 }
2262 c = phase->idom(c);
2263 } while (mem == NULL);
2264 return mem;
2265 }
2266
2267 void ShenandoahWriteBarrierNode::follow_barrier_uses(Node* n, Node* ctrl, Unique_Node_List& uses, PhaseIdealLoop* phase) {
2268 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
2269 Node* u = n->fast_out(i);
2270 if (!u->is_CFG() && phase->get_ctrl(u) == ctrl && (!u->is_Phi() || !u->in(0)->is_Loop() || u->in(LoopNode::LoopBackControl) != n)) {
2271 uses.push(u);
2272 }
2273 }
2274 }
2275
2276 static void hide_strip_mined_loop(OuterStripMinedLoopNode* outer, CountedLoopNode* inner, PhaseIdealLoop* phase) {
2277 OuterStripMinedLoopEndNode* le = inner->outer_loop_end();
2278 Node* new_outer = new LoopNode(outer->in(LoopNode::EntryControl), outer->in(LoopNode::LoopBackControl));
2279 phase->register_control(new_outer, phase->get_loop(outer), outer->in(LoopNode::EntryControl));
2280 Node* new_le = new IfNode(le->in(0), le->in(1), le->_prob, le->_fcnt);
2281 phase->register_control(new_le, phase->get_loop(le), le->in(0));
2282 phase->lazy_replace(outer, new_outer);
2283 phase->lazy_replace(le, new_le);
2284 inner->clear_strip_mined();
2285 }
2286
2287 void ShenandoahWriteBarrierNode::test_heap_stable(Node*& ctrl, Node* raw_mem, Node*& heap_stable_ctrl,
2288 PhaseIdealLoop* phase) {
2289 IdealLoopTree* loop = phase->get_loop(ctrl);
2290 Node* thread = new ThreadLocalNode();
2291 phase->register_new_node(thread, ctrl);
2292 Node* offset = phase->igvn().MakeConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
2293 phase->set_ctrl(offset, phase->C->root());
2294 Node* gc_state_addr = new AddPNode(phase->C->top(), thread, offset);
2295 phase->register_new_node(gc_state_addr, ctrl);
2296 uint gc_state_idx = Compile::AliasIdxRaw;
2297 const TypePtr* gc_state_adr_type = NULL; // debug-mode-only argument
2298 debug_only(gc_state_adr_type = phase->C->get_adr_type(gc_state_idx));
2299
2300 Node* gc_state = new LoadBNode(ctrl, raw_mem, gc_state_addr, gc_state_adr_type, TypeInt::BYTE, MemNode::unordered);
2301 phase->register_new_node(gc_state, ctrl);
2302 Node* heap_stable_and = new AndINode(gc_state, phase->igvn().intcon(ShenandoahHeap::HAS_FORWARDED));
2303 phase->register_new_node(heap_stable_and, ctrl);
2304 Node* heap_stable_cmp = new CmpINode(heap_stable_and, phase->igvn().zerocon(T_INT));
2305 phase->register_new_node(heap_stable_cmp, ctrl);
2306 Node* heap_stable_test = new BoolNode(heap_stable_cmp, BoolTest::ne);
2307 phase->register_new_node(heap_stable_test, ctrl);
2308 IfNode* heap_stable_iff = new IfNode(ctrl, heap_stable_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN);
2309 phase->register_control(heap_stable_iff, loop, ctrl);
2310
2311 heap_stable_ctrl = new IfFalseNode(heap_stable_iff);
2312 phase->register_control(heap_stable_ctrl, loop, heap_stable_iff);
2313 ctrl = new IfTrueNode(heap_stable_iff);
2314 phase->register_control(ctrl, loop, heap_stable_iff);
2315
2316 assert(is_heap_stable_test(heap_stable_iff), "Should match the shape");
2317 }
2318
2319 void ShenandoahWriteBarrierNode::test_null(Node*& ctrl, Node* val, Node*& null_ctrl, PhaseIdealLoop* phase) {
2320 const Type* val_t = phase->igvn().type(val);
2321 if (val_t->meet(TypePtr::NULL_PTR) == val_t) {
2322 IdealLoopTree* loop = phase->get_loop(ctrl);
2323 Node* null_cmp = new CmpPNode(val, phase->igvn().zerocon(T_OBJECT));
2324 phase->register_new_node(null_cmp, ctrl);
2325 Node* null_test = new BoolNode(null_cmp, BoolTest::ne);
2326 phase->register_new_node(null_test, ctrl);
2327 IfNode* null_iff = new IfNode(ctrl, null_test, PROB_LIKELY(0.999), COUNT_UNKNOWN);
2328 phase->register_control(null_iff, loop, ctrl);
2329 ctrl = new IfTrueNode(null_iff);
2330 phase->register_control(ctrl, loop, null_iff);
2331 null_ctrl = new IfFalseNode(null_iff);
2332 phase->register_control(null_ctrl, loop, null_iff);
2333 }
2334 }
2335
2336 Node* ShenandoahWriteBarrierNode::clone_null_check(Node*& c, Node* val, Node* unc_ctrl, PhaseIdealLoop* phase) {
2337 IdealLoopTree *loop = phase->get_loop(c);
2338 Node* iff = unc_ctrl->in(0);
2339 assert(iff->is_If(), "broken");
2340 Node* new_iff = iff->clone();
2341 new_iff->set_req(0, c);
2342 phase->register_control(new_iff, loop, c);
2343 Node* iffalse = new IfFalseNode(new_iff->as_If());
2344 phase->register_control(iffalse, loop, new_iff);
2345 Node* iftrue = new IfTrueNode(new_iff->as_If());
2346 phase->register_control(iftrue, loop, new_iff);
2347 c = iftrue;
2348 const Type *t = phase->igvn().type(val);
2349 assert(val->Opcode() == Op_CastPP, "expect cast to non null here");
2350 Node* uncasted_val = val->in(1);
2351 val = new CastPPNode(uncasted_val, t);
2352 val->init_req(0, c);
2353 phase->register_new_node(val, c);
2354 return val;
2355 }
2356
2357 void ShenandoahWriteBarrierNode::fix_null_check(Node* unc, Node* unc_ctrl, Node* new_unc_ctrl,
2358 Unique_Node_List& uses, PhaseIdealLoop* phase) {
2359 IfNode* iff = unc_ctrl->in(0)->as_If();
2360 Node* proj = iff->proj_out(0);
2361 assert(proj != unc_ctrl, "bad projection");
2362 Node* use = proj->unique_ctrl_out();
2363
2364 assert(use == unc || use->is_Region(), "what else?");
2365
2366 uses.clear();
2367 if (use == unc) {
2368 phase->set_idom(use, new_unc_ctrl, phase->dom_depth(use));
2369 for (uint i = 1; i < unc->req(); i++) {
2370 Node* n = unc->in(i);
2371 if (phase->has_ctrl(n) && phase->get_ctrl(n) == proj) {
2372 uses.push(n);
2373 }
2374 }
2375 } else {
2376 assert(use->is_Region(), "what else?");
2377 uint idx = 1;
2378 for (; use->in(idx) != proj; idx++);
2379 for (DUIterator_Fast imax, i = use->fast_outs(imax); i < imax; i++) {
2380 Node* u = use->fast_out(i);
2381 if (u->is_Phi() && phase->get_ctrl(u->in(idx)) == proj) {
2382 uses.push(u->in(idx));
2383 }
2384 }
2385 }
2386 for(uint next = 0; next < uses.size(); next++ ) {
2387 Node *n = uses.at(next);
2388 assert(phase->get_ctrl(n) == proj, "bad control");
2389 phase->set_ctrl_and_loop(n, new_unc_ctrl);
2390 if (n->in(0) == proj) {
2391 phase->igvn().replace_input_of(n, 0, new_unc_ctrl);
2392 }
2393 for (uint i = 0; i < n->req(); i++) {
2394 Node* m = n->in(i);
2395 if (m != NULL && phase->has_ctrl(m) && phase->get_ctrl(m) == proj) {
2396 uses.push(m);
2397 }
2398 }
2399 }
2400
2401 phase->igvn().rehash_node_delayed(use);
2402 int nb = use->replace_edge(proj, new_unc_ctrl);
2403 assert(nb == 1, "only use expected");
2404 }
2405
2406 void ShenandoahWriteBarrierNode::in_cset_fast_test(Node*& ctrl, Node*& not_cset_ctrl, Node* val, Node* raw_mem, PhaseIdealLoop* phase) {
2407 IdealLoopTree *loop = phase->get_loop(ctrl);
2408 Node* raw_rbtrue = new CastP2XNode(ctrl, val);
2409 phase->register_new_node(raw_rbtrue, ctrl);
2410 Node* cset_offset = new URShiftXNode(raw_rbtrue, phase->igvn().intcon(ShenandoahHeapRegion::region_size_bytes_shift_jint()));
2411 phase->register_new_node(cset_offset, ctrl);
2412 Node* in_cset_fast_test_base_addr = phase->igvn().makecon(TypeRawPtr::make(ShenandoahHeap::in_cset_fast_test_addr()));
2413 phase->set_ctrl(in_cset_fast_test_base_addr, phase->C->root());
2414 Node* in_cset_fast_test_adr = new AddPNode(phase->C->top(), in_cset_fast_test_base_addr, cset_offset);
2415 phase->register_new_node(in_cset_fast_test_adr, ctrl);
2416 uint in_cset_fast_test_idx = Compile::AliasIdxRaw;
2417 const TypePtr* in_cset_fast_test_adr_type = NULL; // debug-mode-only argument
2418 debug_only(in_cset_fast_test_adr_type = phase->C->get_adr_type(in_cset_fast_test_idx));
2419 Node* in_cset_fast_test_load = new LoadBNode(ctrl, raw_mem, in_cset_fast_test_adr, in_cset_fast_test_adr_type, TypeInt::BYTE, MemNode::unordered);
2420 phase->register_new_node(in_cset_fast_test_load, ctrl);
2421 Node* in_cset_fast_test_cmp = new CmpINode(in_cset_fast_test_load, phase->igvn().zerocon(T_INT));
2422 phase->register_new_node(in_cset_fast_test_cmp, ctrl);
2423 Node* in_cset_fast_test_test = new BoolNode(in_cset_fast_test_cmp, BoolTest::eq);
2424 phase->register_new_node(in_cset_fast_test_test, ctrl);
2425 IfNode* in_cset_fast_test_iff = new IfNode(ctrl, in_cset_fast_test_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN);
2426 phase->register_control(in_cset_fast_test_iff, loop, ctrl);
2427
2428 not_cset_ctrl = new IfTrueNode(in_cset_fast_test_iff);
2429 phase->register_control(not_cset_ctrl, loop, in_cset_fast_test_iff);
2430
2431 ctrl = new IfFalseNode(in_cset_fast_test_iff);
2432 phase->register_control(ctrl, loop, in_cset_fast_test_iff);
2433 }
2434
2435 void ShenandoahWriteBarrierNode::call_wb_stub(Node*& ctrl, Node*& val, Node*& result_mem,
2436 Node* raw_mem, Node* wb_mem,
2437 int alias,
2438 PhaseIdealLoop* phase) {
2439 IdealLoopTree*loop = phase->get_loop(ctrl);
2440 const TypePtr* obj_type = phase->igvn().type(val)->is_oopptr()->cast_to_nonconst();
2441
2442 // The slow path stub consumes and produces raw memory in addition
2443 // to the existing memory edges
2444 Node* base = find_bottom_mem(ctrl, phase);
2445
2446 MergeMemNode* mm = MergeMemNode::make(base);
2447 mm->set_memory_at(alias, wb_mem);
2448 mm->set_memory_at(Compile::AliasIdxRaw, raw_mem);
2449 phase->register_new_node(mm, ctrl);
2450
2451 Node* call = new CallLeafNode(ShenandoahBarrierSetC2::shenandoah_write_barrier_Type(), CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_JRT), "shenandoah_write_barrier", TypeRawPtr::BOTTOM);
2452 call->init_req(TypeFunc::Control, ctrl);
2453 call->init_req(TypeFunc::I_O, phase->C->top());
2454 call->init_req(TypeFunc::Memory, mm);
2455 call->init_req(TypeFunc::FramePtr, phase->C->top());
2456 call->init_req(TypeFunc::ReturnAdr, phase->C->top());
2457 call->init_req(TypeFunc::Parms, val);
2458 phase->register_control(call, loop, ctrl);
2459 ctrl = new ProjNode(call, TypeFunc::Control);
2460 phase->register_control(ctrl, loop, call);
2461 result_mem = new ProjNode(call, TypeFunc::Memory);
2462 phase->register_new_node(result_mem, call);
2463 val = new ProjNode(call, TypeFunc::Parms);
2464 phase->register_new_node(val, call);
2465 val = new CheckCastPPNode(ctrl, val, obj_type);
2466 phase->register_new_node(val, ctrl);
2467 }
2468
2469 void ShenandoahWriteBarrierNode::fix_ctrl(Node* barrier, Node* region, const MemoryGraphFixer& fixer, Unique_Node_List& uses, Unique_Node_List& uses_to_ignore, uint last, PhaseIdealLoop* phase) {
2470 Node* ctrl = phase->get_ctrl(barrier);
2471 Node* init_raw_mem = fixer.find_mem(ctrl, barrier);
2472
2473 // Update the control of all nodes that should be after the
2474 // barrier control flow
2475 uses.clear();
2476 // Every node that is control dependent on the barrier's input
2477 // control will be after the expanded barrier. The raw memory (if
2478 // its memory is control dependent on the barrier's input control)
2479 // must stay above the barrier.
2480 uses_to_ignore.clear();
2481 if (phase->has_ctrl(init_raw_mem) && phase->get_ctrl(init_raw_mem) == ctrl && !init_raw_mem->is_Phi()) {
2482 uses_to_ignore.push(init_raw_mem);
2483 }
2484 for (uint next = 0; next < uses_to_ignore.size(); next++) {
2485 Node *n = uses_to_ignore.at(next);
2486 for (uint i = 0; i < n->req(); i++) {
2487 Node* in = n->in(i);
2488 if (in != NULL && phase->has_ctrl(in) && phase->get_ctrl(in) == ctrl) {
2489 uses_to_ignore.push(in);
2490 }
2491 }
2492 }
2493 for (DUIterator_Fast imax, i = ctrl->fast_outs(imax); i < imax; i++) {
2494 Node* u = ctrl->fast_out(i);
2495 if (u->_idx < last &&
2496 u != barrier &&
2497 !uses_to_ignore.member(u) &&
2498 (u->in(0) != ctrl || (!u->is_Region() && !u->is_Phi())) &&
2499 (ctrl->Opcode() != Op_CatchProj || u->Opcode() != Op_CreateEx)) {
2500 Node* old_c = phase->ctrl_or_self(u);
2501 Node* c = old_c;
2502 if (c != ctrl ||
2503 is_dominator_same_ctrl(old_c, barrier, u, phase) ||
2504 ShenandoahBarrierSetC2::is_shenandoah_state_load(u)) {
2505 phase->igvn().rehash_node_delayed(u);
2506 int nb = u->replace_edge(ctrl, region);
2507 if (u->is_CFG()) {
2508 if (phase->idom(u) == ctrl) {
2509 phase->set_idom(u, region, phase->dom_depth(region));
2510 }
2511 } else if (phase->get_ctrl(u) == ctrl) {
2512 assert(u != init_raw_mem, "should leave input raw mem above the barrier");
2513 uses.push(u);
2514 }
2515 assert(nb == 1, "more than 1 ctrl input?");
2516 --i, imax -= nb;
2517 }
2518 }
2519 }
2520 }
2521
2522 void ShenandoahWriteBarrierNode::pin_and_expand(PhaseIdealLoop* phase) {
2523 Node_List enqueue_barriers;
2524 if (ShenandoahStoreValEnqueueBarrier) {
2525 Unique_Node_List wq;
2526 wq.push(phase->C->root());
2527 for (uint i = 0; i < wq.size(); i++) {
2528 Node* n = wq.at(i);
2529 if (n->Opcode() == Op_ShenandoahEnqueueBarrier) {
2530 enqueue_barriers.push(n);
2531 }
2532 for (uint i = 0; i < n->req(); i++) {
2533 Node* in = n->in(i);
2534 if (in != NULL) {
2535 wq.push(in);
2536 }
2537 }
2538 }
2539 }
2540
2541 const bool trace = false;
2542
2543 // Collect raw memory state at CFG points in the entire graph and
2544 // record it in memory_nodes. Optimize the raw memory graph in the
2545 // process. Optimizing the memory graph also makes the memory graph
2546 // simpler.
2547 GrowableArray<MemoryGraphFixer*> memory_graph_fixers;
2548
2549 // Let's try to common write barriers again
2550 optimize_before_expansion(phase, memory_graph_fixers, true);
2551
2552 Unique_Node_List uses;
2553 for (int i = 0; i < ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count(); i++) {
2554 ShenandoahWriteBarrierNode* wb = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barrier(i);
2555 Node* ctrl = phase->get_ctrl(wb);
2556
2557 Node* val = wb->in(ValueIn);
2558 if (ctrl->is_Proj() && ctrl->in(0)->is_CallJava()) {
2559 assert(is_dominator(phase->get_ctrl(val), ctrl->in(0)->in(0), val, ctrl->in(0), phase), "can't move");
2560 phase->set_ctrl(wb, ctrl->in(0)->in(0));
2561 } else if (ctrl->is_CallRuntime()) {
2562 assert(is_dominator(phase->get_ctrl(val), ctrl->in(0), val, ctrl, phase), "can't move");
2563 phase->set_ctrl(wb, ctrl->in(0));
2564 }
2565
2566 assert(wb->Opcode() == Op_ShenandoahWriteBarrier, "only for write barriers");
2567 // Look for a null check that dominates this barrier and move the
2568 // barrier right after the null check to enable implicit null
2569 // checks
2570 wb->pin_and_expand_move_barrier(phase, memory_graph_fixers, uses);
2571
2572 wb->pin_and_expand_helper(phase);
2573 }
2574
2575 for (uint i = 0; i < enqueue_barriers.size(); i++) {
2576 Node* barrier = enqueue_barriers.at(i);
2577 Node* ctrl = phase->get_ctrl(barrier);
2578 IdealLoopTree* loop = phase->get_loop(ctrl);
2579 if (loop->_head->is_OuterStripMinedLoop()) {
2580 // Expanding a barrier here will break loop strip mining
2581 // verification. Transform the loop so the loop nest doesn't
2582 // appear as strip mined.
2583 OuterStripMinedLoopNode* outer = loop->_head->as_OuterStripMinedLoop();
2584 hide_strip_mined_loop(outer, outer->unique_ctrl_out()->as_CountedLoop(), phase);
2585 }
2586 }
2587
2588 for (int i = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count(); i > 0; i--) {
2589 int cnt = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count();
2590 ShenandoahWriteBarrierNode* wb = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barrier(i-1);
2591 Node* ctrl = phase->get_ctrl(wb);
2592 IdealLoopTree* loop = phase->get_loop(ctrl);
2593 if (loop->_head->is_OuterStripMinedLoop()) {
2594 // Expanding a barrier here will break loop strip mining
2595 // verification. Transform the loop so the loop nest doesn't
2596 // appear as strip mined.
2597 OuterStripMinedLoopNode* outer = loop->_head->as_OuterStripMinedLoop();
2598 hide_strip_mined_loop(outer, outer->unique_ctrl_out()->as_CountedLoop(), phase);
2599 }
2600 }
2601
2602 MemoryGraphFixer fixer(Compile::AliasIdxRaw, true, phase);
2603 Unique_Node_List uses_to_ignore;
2604 for (uint i = 0; i < enqueue_barriers.size(); i++) {
2605 Node* barrier = enqueue_barriers.at(i);
2606 Node* pre_val = barrier->in(1);
2607
2608 if (phase->igvn().type(pre_val)->higher_equal(TypePtr::NULL_PTR)) {
2609 ShouldNotReachHere();
2610 continue;
2611 }
2612
2613 Node* ctrl = phase->get_ctrl(barrier);
2614
2615 if (ctrl->is_Proj() && ctrl->in(0)->is_CallJava()) {
2616 assert(is_dominator(phase->get_ctrl(pre_val), ctrl->in(0)->in(0), pre_val, ctrl->in(0), phase), "can't move");
2617 ctrl = ctrl->in(0)->in(0);
2618 phase->set_ctrl(barrier, ctrl);
2619 } else if (ctrl->is_CallRuntime()) {
2620 assert(is_dominator(phase->get_ctrl(pre_val), ctrl->in(0), pre_val, ctrl, phase), "can't move");
2621 ctrl = ctrl->in(0);
2622 phase->set_ctrl(barrier, ctrl);
2623 }
2624
2625 Node* init_ctrl = ctrl;
2626 IdealLoopTree* loop = phase->get_loop(ctrl);
2627 Node* raw_mem = fixer.find_mem(ctrl, barrier);
2628 Node* init_raw_mem = raw_mem;
2629 Node* raw_mem_for_ctrl = fixer.find_mem(ctrl, NULL);
2630 Node* heap_stable_ctrl = NULL;
2631 Node* null_ctrl = NULL;
2632 uint last = phase->C->unique();
2633
2634 enum { _heap_stable = 1, _heap_unstable, PATH_LIMIT };
2635 Node* region = new RegionNode(PATH_LIMIT);
2636 Node* phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);
2637
2638 enum { _fast_path = 1, _slow_path, _null_path, PATH_LIMIT2 };
2639 Node* region2 = new RegionNode(PATH_LIMIT2);
2640 Node* phi2 = PhiNode::make(region2, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);
2641
2642 // Stable path.
2643 test_heap_stable(ctrl, raw_mem, heap_stable_ctrl, phase);
2644 region->init_req(_heap_stable, heap_stable_ctrl);
2645 phi->init_req(_heap_stable, raw_mem);
2646
2647 // Null path
2648 Node* reg2_ctrl = NULL;
2649 test_null(ctrl, pre_val, null_ctrl, phase);
2650 if (null_ctrl != NULL) {
2651 reg2_ctrl = null_ctrl->in(0);
2652 region2->init_req(_null_path, null_ctrl);
2653 phi2->init_req(_null_path, raw_mem);
2654 } else {
2655 region2->del_req(_null_path);
2656 phi2->del_req(_null_path);
2657 }
2658
2659 const int index_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset());
2660 const int buffer_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset());
2661 Node* thread = new ThreadLocalNode();
2662 phase->register_new_node(thread, ctrl);
2663 Node* buffer_adr = new AddPNode(phase->C->top(), thread, phase->igvn().MakeConX(buffer_offset));
2664 phase->register_new_node(buffer_adr, ctrl);
2665 Node* index_adr = new AddPNode(phase->C->top(), thread, phase->igvn().MakeConX(index_offset));
2666 phase->register_new_node(index_adr, ctrl);
2667
2668 BasicType index_bt = TypeX_X->basic_type();
2669 assert(sizeof(size_t) == type2aelembytes(index_bt), "Loading G1 SATBMarkQueue::_index with wrong size.");
2670 const TypePtr* adr_type = TypeRawPtr::BOTTOM;
2671 Node* index = new LoadXNode(ctrl, raw_mem, index_adr, adr_type, TypeX_X, MemNode::unordered);
2672 phase->register_new_node(index, ctrl);
2673 Node* index_cmp = new CmpXNode(index, phase->igvn().MakeConX(0));
2674 phase->register_new_node(index_cmp, ctrl);
2675 Node* index_test = new BoolNode(index_cmp, BoolTest::ne);
2676 phase->register_new_node(index_test, ctrl);
2677 IfNode* queue_full_iff = new IfNode(ctrl, index_test, PROB_LIKELY(0.999), COUNT_UNKNOWN);
2678 if (reg2_ctrl == NULL) reg2_ctrl = queue_full_iff;
2679 phase->register_control(queue_full_iff, loop, ctrl);
2680 Node* not_full = new IfTrueNode(queue_full_iff);
2681 phase->register_control(not_full, loop, queue_full_iff);
2682 Node* full = new IfFalseNode(queue_full_iff);
2683 phase->register_control(full, loop, queue_full_iff);
2684
2685 ctrl = not_full;
2686
2687 Node* next_index = new SubXNode(index, phase->igvn().MakeConX(sizeof(intptr_t)));
2688 phase->register_new_node(next_index, ctrl);
2689
2690 Node* buffer = new LoadPNode(ctrl, raw_mem, buffer_adr, adr_type, TypeRawPtr::NOTNULL, MemNode::unordered);
2691 phase->register_new_node(buffer, ctrl);
2692 Node *log_addr = new AddPNode(phase->C->top(), buffer, next_index);
2693 phase->register_new_node(log_addr, ctrl);
2694 Node* log_store = new StorePNode(ctrl, raw_mem, log_addr, adr_type, pre_val, MemNode::unordered);
2695 phase->register_new_node(log_store, ctrl);
2696 // update the index
2697 Node* index_update = new StoreXNode(ctrl, log_store, index_adr, adr_type, next_index, MemNode::unordered);
2698 phase->register_new_node(index_update, ctrl);
2699
2700 // Fast-path case
2701 region2->init_req(_fast_path, ctrl);
2702 phi2->init_req(_fast_path, index_update);
2703
2704 ctrl = full;
2705
2706 Node* base = find_bottom_mem(ctrl, phase);
2707
2708 MergeMemNode* mm = MergeMemNode::make(base);
2709 mm->set_memory_at(Compile::AliasIdxRaw, raw_mem);
2710 phase->register_new_node(mm, ctrl);
2711
2712 Node* call = new CallLeafNode(ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type(), CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), "shenandoah_wb_pre", TypeRawPtr::BOTTOM);
2713 call->init_req(TypeFunc::Control, ctrl);
2714 call->init_req(TypeFunc::I_O, phase->C->top());
2715 call->init_req(TypeFunc::Memory, mm);
2716 call->init_req(TypeFunc::FramePtr, phase->C->top());
2717 call->init_req(TypeFunc::ReturnAdr, phase->C->top());
2718 call->init_req(TypeFunc::Parms, pre_val);
2719 call->init_req(TypeFunc::Parms+1, thread);
2720 phase->register_control(call, loop, ctrl);
2721
2722 Node* ctrl_proj = new ProjNode(call, TypeFunc::Control);
2723 phase->register_control(ctrl_proj, loop, call);
2724 Node* mem_proj = new ProjNode(call, TypeFunc::Memory);
2725 phase->register_new_node(mem_proj, call);
2726
2727 // Slow-path case
2728 region2->init_req(_slow_path, ctrl_proj);
2729 phi2->init_req(_slow_path, mem_proj);
2730
2731 phase->register_control(region2, loop, reg2_ctrl);
2732 phase->register_new_node(phi2, region2);
2733
2734 region->init_req(_heap_unstable, region2);
2735 phi->init_req(_heap_unstable, phi2);
2736
2737 phase->register_control(region, loop, heap_stable_ctrl->in(0));
2738 phase->register_new_node(phi, region);
2739
2740 fix_ctrl(barrier, region, fixer, uses, uses_to_ignore, last, phase);
2741 for(uint next = 0; next < uses.size(); next++ ) {
2742 Node *n = uses.at(next);
2743 assert(phase->get_ctrl(n) == init_ctrl, "bad control");
2744 assert(n != init_raw_mem, "should leave input raw mem above the barrier");
2745 phase->set_ctrl(n, region);
2746 follow_barrier_uses(n, init_ctrl, uses, phase);
2747 }
2748 fixer.fix_mem(init_ctrl, region, init_raw_mem, raw_mem_for_ctrl, phi, uses);
2749
2750 phase->igvn().replace_node(barrier, pre_val);
2751 }
2752
2753 for (int i = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count(); i > 0; i--) {
2754 int cnt = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count();
2755 ShenandoahWriteBarrierNode* wb = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barrier(i-1);
2756
2757 uint last = phase->C->unique();
2758 Node* ctrl = phase->get_ctrl(wb);
2759 Node* orig_ctrl = ctrl;
2760
2761 Node* raw_mem = fixer.find_mem(ctrl, wb);
2762 Node* init_raw_mem = raw_mem;
2763 Node* raw_mem_for_ctrl = fixer.find_mem(ctrl, NULL);
2764 int alias = phase->C->get_alias_index(wb->adr_type());
2765 Node* wb_mem = wb->in(Memory);
2766 Node* init_wb_mem = wb_mem;
2767
2768 Node* val = wb->in(ValueIn);
2769 Node* wbproj = wb->find_out_with(Op_ShenandoahWBMemProj);
2770 IdealLoopTree *loop = phase->get_loop(ctrl);
2771
2772 assert(val->Opcode() != Op_ShenandoahWriteBarrier, "No chain of write barriers");
2773
2774 CallStaticJavaNode* unc = wb->pin_and_expand_null_check(phase->igvn());
2775 Node* unc_ctrl = NULL;
2776 if (unc != NULL) {
2777 if (val->in(0) != ctrl) {
2778 unc = NULL;
2779 } else {
2780 unc_ctrl = val->in(0);
2781 }
2782 }
2783
2784 Node* uncasted_val = val;
2785 if (unc != NULL) {
2786 uncasted_val = val->in(1);
2787 }
2788
2789 Node* heap_stable_ctrl = NULL;
2790 Node* null_ctrl = NULL;
2791
2792 assert(val->bottom_type()->make_oopptr(), "need oop");
2793 assert(val->bottom_type()->make_oopptr()->const_oop() == NULL, "expect non-constant");
2794
2795 enum { _heap_stable = 1, _heap_unstable, PATH_LIMIT };
2796 Node* region = new RegionNode(PATH_LIMIT);
2797 Node* val_phi = new PhiNode(region, uncasted_val->bottom_type()->is_oopptr());
2798 Node* mem_phi = PhiNode::make(region, wb_mem, Type::MEMORY, phase->C->alias_type(wb->adr_type())->adr_type());
2799 Node* raw_mem_phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);
2800
2801 enum { _not_cset = 1, _not_equal, _evac_path, _null_path, PATH_LIMIT2 };
2802 Node* region2 = new RegionNode(PATH_LIMIT2);
2803 Node* val_phi2 = new PhiNode(region2, uncasted_val->bottom_type()->is_oopptr());
2804 Node* mem_phi2 = PhiNode::make(region2, wb_mem, Type::MEMORY, phase->C->alias_type(wb->adr_type())->adr_type());
2805 Node* raw_mem_phi2 = PhiNode::make(region2, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);
2806
2807 // Stable path.
2808 test_heap_stable(ctrl, raw_mem, heap_stable_ctrl, phase);
2809 IfNode* heap_stable_iff = heap_stable_ctrl->in(0)->as_If();
2810
2811 // Heap stable case
2812 region->init_req(_heap_stable, heap_stable_ctrl);
2813 val_phi->init_req(_heap_stable, uncasted_val);
2814 mem_phi->init_req(_heap_stable, wb_mem);
2815 raw_mem_phi->init_req(_heap_stable, raw_mem);
2816
2817 Node* reg2_ctrl = NULL;
2818 // Null case
2819 test_null(ctrl, val, null_ctrl, phase);
2820 if (null_ctrl != NULL) {
2821 reg2_ctrl = null_ctrl->in(0);
2822 region2->init_req(_null_path, null_ctrl);
2823 val_phi2->init_req(_null_path, uncasted_val);
2824 mem_phi2->init_req(_null_path, wb_mem);
2825 raw_mem_phi2->init_req(_null_path, raw_mem);
2826 } else {
2827 region2->del_req(_null_path);
2828 val_phi2->del_req(_null_path);
2829 mem_phi2->del_req(_null_path);
2830 raw_mem_phi2->del_req(_null_path);
2831 }
2832
2833 // Test for in-cset.
2834 // Wires !in_cset(obj) to slot 2 of region and phis
2835 Node* not_cset_ctrl = NULL;
2836 in_cset_fast_test(ctrl, not_cset_ctrl, uncasted_val, raw_mem, phase);
2837 if (not_cset_ctrl != NULL) {
2838 if (reg2_ctrl == NULL) reg2_ctrl = not_cset_ctrl->in(0);
2839 region2->init_req(_not_cset, not_cset_ctrl);
2840 val_phi2->init_req(_not_cset, uncasted_val);
2841 mem_phi2->init_req(_not_cset, wb_mem);
2842 raw_mem_phi2->init_req(_not_cset, raw_mem);
2843 }
2844
2845 // Resolve object when orig-value is in cset.
2846 // Make the unconditional resolve for fwdptr, not the read barrier.
2847 Node* new_val = uncasted_val;
2848 if (unc_ctrl != NULL) {
2849 // Clone the null check in this branch to allow implicit null check
2850 new_val = clone_null_check(ctrl, val, unc_ctrl, phase);
2851 fix_null_check(unc, unc_ctrl, ctrl->in(0)->as_If()->proj_out(0), uses, phase);
2852
2853 IfNode* iff = unc_ctrl->in(0)->as_If();
2854 phase->igvn().replace_input_of(iff, 1, phase->igvn().intcon(1));
2855 }
2856 Node* addr = new AddPNode(new_val, uncasted_val, phase->igvn().MakeConX(ShenandoahBrooksPointer::byte_offset()));
2857 phase->register_new_node(addr, ctrl);
2858 assert(val->bottom_type()->isa_oopptr(), "what else?");
2859 const TypePtr* obj_type = val->bottom_type()->is_oopptr();
2860 const TypePtr* adr_type = ShenandoahBarrierNode::brooks_pointer_type(obj_type);
2861 Node* fwd = new LoadPNode(ctrl, wb_mem, addr, adr_type, obj_type, MemNode::unordered);
2862 phase->register_new_node(fwd, ctrl);
2863
2864 // Only branch to WB stub if object is not forwarded; otherwise reply with fwd ptr
2865 Node* cmp = new CmpPNode(fwd, new_val);
2866 phase->register_new_node(cmp, ctrl);
2867 Node* bol = new BoolNode(cmp, BoolTest::eq);
2868 phase->register_new_node(bol, ctrl);
2869
2870 IfNode* iff = new IfNode(ctrl, bol, PROB_UNLIKELY(0.999), COUNT_UNKNOWN);
2871 if (reg2_ctrl == NULL) reg2_ctrl = iff;
2872 phase->register_control(iff, loop, ctrl);
2873 Node* if_not_eq = new IfFalseNode(iff);
2874 phase->register_control(if_not_eq, loop, iff);
2875 Node* if_eq = new IfTrueNode(iff);
2876 phase->register_control(if_eq, loop, iff);
2877
2878 // Wire up not-equal-path in slots 3.
2879 region2->init_req(_not_equal, if_not_eq);
2880 val_phi2->init_req(_not_equal, fwd);
2881 mem_phi2->init_req(_not_equal, wb_mem);
2882 raw_mem_phi2->init_req(_not_equal, raw_mem);
2883
2884 // Call wb-stub and wire up that path in slots 4
2885 Node* result_mem = NULL;
2886 ctrl = if_eq;
2887 call_wb_stub(ctrl, new_val, result_mem,
2888 raw_mem, wb_mem,
2889 alias, phase);
2890 region2->init_req(_evac_path, ctrl);
2891 val_phi2->init_req(_evac_path, new_val);
2892 mem_phi2->init_req(_evac_path, result_mem);
2893 raw_mem_phi2->init_req(_evac_path, result_mem);
2894
2895 phase->register_control(region2, loop, reg2_ctrl);
2896 phase->register_new_node(val_phi2, region2);
2897 phase->register_new_node(mem_phi2, region2);
2898 phase->register_new_node(raw_mem_phi2, region2);
2899
2900 region->init_req(_heap_unstable, region2);
2901 val_phi->init_req(_heap_unstable, val_phi2);
2902 mem_phi->init_req(_heap_unstable, mem_phi2);
2903 raw_mem_phi->init_req(_heap_unstable, raw_mem_phi2);
2904
2905 phase->register_control(region, loop, heap_stable_iff);
2906 Node* out_val = val_phi;
2907 phase->register_new_node(val_phi, region);
2908 phase->register_new_node(mem_phi, region);
2909 phase->register_new_node(raw_mem_phi, region);
2910
2911 fix_ctrl(wb, region, fixer, uses, uses_to_ignore, last, phase);
2912
2913 ctrl = orig_ctrl;
2914
2915 phase->igvn().replace_input_of(wbproj, ShenandoahWBMemProjNode::WriteBarrier, phase->C->top());
2916 phase->igvn().replace_node(wbproj, mem_phi);
2917 if (unc != NULL) {
2918 for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) {
2919 Node* u = val->fast_out(i);
2920 Node* c = phase->ctrl_or_self(u);
2921 if (u != wb && (c != ctrl || is_dominator_same_ctrl(c, wb, u, phase))) {
2922 phase->igvn().rehash_node_delayed(u);
2923 int nb = u->replace_edge(val, out_val);
2924 --i, imax -= nb;
2925 }
2926 }
2927 if (val->outcnt() == 0) {
2928 phase->igvn()._worklist.push(val);
2929 }
2930 }
2931 phase->igvn().replace_node(wb, out_val);
2932
2933 follow_barrier_uses(mem_phi, ctrl, uses, phase);
2934 follow_barrier_uses(out_val, ctrl, uses, phase);
2935
2936 for(uint next = 0; next < uses.size(); next++ ) {
2937 Node *n = uses.at(next);
2938 assert(phase->get_ctrl(n) == ctrl, "bad control");
2939 assert(n != init_raw_mem, "should leave input raw mem above the barrier");
2940 phase->set_ctrl(n, region);
2941 follow_barrier_uses(n, ctrl, uses, phase);
2942 }
2943
2944 // The slow path call produces memory: hook the raw memory phi
2945 // from the expanded write barrier with the rest of the graph
2946 // which may require adding memory phis at every post dominated
2947 // region and at enclosing loop heads. Use the memory state
2948 // collected in memory_nodes to fix the memory graph. Update that
2949 // memory state as we go.
2950 fixer.fix_mem(ctrl, region, init_raw_mem, raw_mem_for_ctrl, raw_mem_phi, uses);
2951 assert(ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count() == cnt - 1, "not replaced");
2952 }
2953
2954 assert(ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count() == 0, "all write barrier nodes should have been replaced");
2955 }
2956
2957 void ShenandoahWriteBarrierNode::move_heap_stable_test_out_of_loop(IfNode* iff, PhaseIdealLoop* phase) {
2958 IdealLoopTree *loop = phase->get_loop(iff);
2959 Node* loop_head = loop->_head;
2960 Node* entry_c = loop_head->in(LoopNode::EntryControl);
2961
2962 Node* bol = iff->in(1);
2963 Node* cmp = bol->in(1);
2964 Node* andi = cmp->in(1);
2965 Node* load = andi->in(1);
2966
2967 assert(is_gc_state_load(load), "broken");
2968 if (!phase->is_dominator(load->in(0), entry_c)) {
2969 Node* mem_ctrl = NULL;
2970 Node* mem = dom_mem(load->in(MemNode::Memory), loop_head, Compile::AliasIdxRaw, mem_ctrl, phase);
2971 load = load->clone();
2972 load->set_req(MemNode::Memory, mem);
2973 load->set_req(0, entry_c);
2974 phase->register_new_node(load, entry_c);
2975 andi = andi->clone();
2976 andi->set_req(1, load);
2977 phase->register_new_node(andi, entry_c);
2978 cmp = cmp->clone();
2979 cmp->set_req(1, andi);
2980 phase->register_new_node(cmp, entry_c);
2981 bol = bol->clone();
2982 bol->set_req(1, cmp);
2983 phase->register_new_node(bol, entry_c);
2984
2985 Node* old_bol =iff->in(1);
2986 phase->igvn().replace_input_of(iff, 1, bol);
2987 }
2988 }
2989
2990 bool ShenandoahWriteBarrierNode::identical_backtoback_ifs(Node *n, PhaseIdealLoop* phase) {
2991 if (!n->is_If() || n->is_CountedLoopEnd()) {
2992 return false;
2993 }
2994 Node* region = n->in(0);
2995
2996 if (!region->is_Region()) {
2997 return false;
2998 }
2999 Node* dom = phase->idom(region);
3000 if (!dom->is_If()) {
3001 return false;
3002 }
3003
3004 if (!is_heap_stable_test(n) || !is_heap_stable_test(dom)) {
3005 return false;
3006 }
3007
3008 IfNode* dom_if = dom->as_If();
3009 Node* proj_true = dom_if->proj_out(1);
3010 Node* proj_false = dom_if->proj_out(0);
3011
3012 for (uint i = 1; i < region->req(); i++) {
3013 if (phase->is_dominator(proj_true, region->in(i))) {
3014 continue;
3015 }
3016 if (phase->is_dominator(proj_false, region->in(i))) {
3017 continue;
3018 }
3019 return false;
3020 }
3021
3022 return true;
3023 }
3024
3025 void ShenandoahWriteBarrierNode::merge_back_to_back_tests(Node* n, PhaseIdealLoop* phase) {
3026 assert(is_heap_stable_test(n), "no other tests");
3027 if (identical_backtoback_ifs(n, phase)) {
3028 Node* n_ctrl = n->in(0);
3029 if (phase->can_split_if(n_ctrl)) {
3030 IfNode* dom_if = phase->idom(n_ctrl)->as_If();
3031 if (is_heap_stable_test(n)) {
3032 Node* gc_state_load = n->in(1)->in(1)->in(1)->in(1);
3033 assert(is_gc_state_load(gc_state_load), "broken");
3034 Node* dom_gc_state_load = dom_if->in(1)->in(1)->in(1)->in(1);
3035 assert(is_gc_state_load(dom_gc_state_load), "broken");
3036 if (gc_state_load != dom_gc_state_load) {
3037 phase->igvn().replace_node(gc_state_load, dom_gc_state_load);
3038 }
3039 }
3040 PhiNode* bolphi = PhiNode::make_blank(n_ctrl, n->in(1));
3041 Node* proj_true = dom_if->proj_out(1);
3042 Node* proj_false = dom_if->proj_out(0);
3043 Node* con_true = phase->igvn().makecon(TypeInt::ONE);
3044 Node* con_false = phase->igvn().makecon(TypeInt::ZERO);
3045
3046 for (uint i = 1; i < n_ctrl->req(); i++) {
3047 if (phase->is_dominator(proj_true, n_ctrl->in(i))) {
3048 bolphi->init_req(i, con_true);
3049 } else {
3050 assert(phase->is_dominator(proj_false, n_ctrl->in(i)), "bad if");
3051 bolphi->init_req(i, con_false);
3052 }
3053 }
3054 phase->register_new_node(bolphi, n_ctrl);
3055 phase->igvn().replace_input_of(n, 1, bolphi);
3056 phase->do_split_if(n);
3057 }
3058 }
3059 }
3060
3061 void ShenandoahWriteBarrierNode::optimize_after_expansion(VectorSet &visited, Node_Stack &stack, Node_List &old_new, PhaseIdealLoop* phase) {
3062 Node_List heap_stable_tests;
3063 Node_List gc_state_loads;
3064
3065 stack.push(phase->C->start(), 0);
3066 do {
3067 Node* n = stack.node();
3068 uint i = stack.index();
3069
3070 if (i < n->outcnt()) {
3071 Node* u = n->raw_out(i);
3072 stack.set_index(i+1);
3073 if (!visited.test_set(u->_idx)) {
3074 stack.push(u, 0);
3075 }
3076 } else {
3077 stack.pop();
3078 if (ShenandoahCommonGCStateLoads && ShenandoahWriteBarrierNode::is_gc_state_load(n)) {
3079 gc_state_loads.push(n);
3080 }
3081 if (n->is_If() && ShenandoahWriteBarrierNode::is_heap_stable_test(n)) {
3082 heap_stable_tests.push(n);
3083 }
3084 }
3085 } while (stack.size() > 0);
3086
3087 bool progress;
3088 do {
3089 progress = false;
3090 for (uint i = 0; i < gc_state_loads.size(); i++) {
3091 Node* n = gc_state_loads.at(i);
3092 if (n->outcnt() != 0) {
3093 progress |= ShenandoahWriteBarrierNode::try_common_gc_state_load(n, phase);
3094 }
3095 }
3096 } while (progress);
3097
3098 for (uint i = 0; i < heap_stable_tests.size(); i++) {
3099 Node* n = heap_stable_tests.at(i);
3100 assert(is_heap_stable_test(n), "only evacuation test");
3101 merge_back_to_back_tests(n, phase);
3102 }
3103
3104 if (!phase->C->major_progress()) {
3105 VectorSet seen(Thread::current()->resource_area());
3106 for (uint i = 0; i < heap_stable_tests.size(); i++) {
3107 Node* n = heap_stable_tests.at(i);
3108 IdealLoopTree* loop = phase->get_loop(n);
3109 if (loop != phase->ltree_root() &&
3110 loop->_child == NULL &&
3111 !loop->_irreducible) {
3112 LoopNode* head = loop->_head->as_Loop();
3113 if ((!head->is_CountedLoop() || head->as_CountedLoop()->is_main_loop() || head->as_CountedLoop()->is_normal_loop()) &&
3114 !seen.test_set(head->_idx) &&
3115 loop->policy_unswitching(phase, true)) {
3116 IfNode* iff = phase->find_unswitching_candidate(loop, true);
3117 if (iff != NULL && is_heap_stable_test(iff)) {
3118 if (head->is_strip_mined()) {
3119 head->verify_strip_mined(0);
3120 OuterStripMinedLoopNode* outer = head->as_CountedLoop()->outer_loop();
3121 OuterStripMinedLoopEndNode* le = head->outer_loop_end();
3122 Node* new_outer = new LoopNode(outer->in(LoopNode::EntryControl), outer->in(LoopNode::LoopBackControl));
3123 phase->register_control(new_outer, phase->get_loop(outer), outer->in(LoopNode::EntryControl));
3124 Node* new_le = new IfNode(le->in(0), le->in(1), le->_prob, le->_fcnt);
3125 phase->register_control(new_le, phase->get_loop(le), le->in(0));
3126 phase->lazy_replace(outer, new_outer);
3127 phase->lazy_replace(le, new_le);
3128 head->clear_strip_mined();
3129 }
3130 phase->do_unswitching(loop, old_new, true);
3131 }
3132 }
3133 }
3134 }
3135 }
3136 }
3137
3138 #ifdef ASSERT
3139 void ShenandoahBarrierNode::verify_raw_mem(RootNode* root) {
3140 const bool trace = false;
3141 ResourceMark rm;
3142 Unique_Node_List nodes;
3143 Unique_Node_List controls;
3144 Unique_Node_List memories;
3145
3146 nodes.push(root);
3147 for (uint next = 0; next < nodes.size(); next++) {
3148 Node *n = nodes.at(next);
3149 if (ShenandoahBarrierSetC2::is_shenandoah_wb_call(n)) {
3150 controls.push(n);
3151 if (trace) { tty->print("XXXXXX verifying"); n->dump(); }
3152 for (uint next2 = 0; next2 < controls.size(); next2++) {
3153 Node *m = controls.at(next2);
3154 if (!m->is_Loop() || controls.member(m->in(LoopNode::EntryControl)) || 1) {
3155 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
3156 Node* u = m->fast_out(i);
3157 if (u->is_CFG() && !u->is_Root() &&
3158 !(u->Opcode() == Op_CProj && u->in(0)->Opcode() == Op_NeverBranch && u->as_Proj()->_con == 1) &&
3159 !(u->is_Region() && u->unique_ctrl_out()->Opcode() == Op_Halt)) {
3160 if (trace) { tty->print("XXXXXX pushing control"); u->dump(); }
3161 controls.push(u);
3162 }
3163 }
3164 }
3165 }
3166 memories.push(n->as_Call()->proj_out(TypeFunc::Memory));
3167 for (uint next2 = 0; next2 < memories.size(); next2++) {
3168 Node *m = memories.at(next2);
3169 assert(m->bottom_type() == Type::MEMORY, "");
3170 if (!m->is_Phi() || !m->in(0)->is_Loop() || controls.member(m->in(0)->in(LoopNode::EntryControl)) || 1) {
3171 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
3172 Node* u = m->fast_out(i);
3173 if (u->bottom_type() == Type::MEMORY && (u->is_Mem() || u->is_ClearArray())) {
3174 if (trace) { tty->print("XXXXXX pushing memory"); u->dump(); }
3175 memories.push(u);
3176 } else if (u->is_LoadStore()) {
3177 if (trace) { tty->print("XXXXXX pushing memory"); u->find_out_with(Op_SCMemProj)->dump(); }
3178 memories.push(u->find_out_with(Op_SCMemProj));
3179 } else if (u->is_MergeMem() && u->as_MergeMem()->memory_at(Compile::AliasIdxRaw) == m) {
3180 if (trace) { tty->print("XXXXXX pushing memory"); u->dump(); }
3181 memories.push(u);
3182 } else if (u->is_Phi()) {
3183 assert(u->bottom_type() == Type::MEMORY, "");
3184 if (u->adr_type() == TypeRawPtr::BOTTOM || u->adr_type() == TypePtr::BOTTOM) {
3185 assert(controls.member(u->in(0)), "");
3186 if (trace) { tty->print("XXXXXX pushing memory"); u->dump(); }
3187 memories.push(u);
3188 }
3189 } else if (u->is_SafePoint() || u->is_MemBar()) {
3190 for (DUIterator_Fast jmax, j = u->fast_outs(jmax); j < jmax; j++) {
3191 Node* uu = u->fast_out(j);
3192 if (uu->bottom_type() == Type::MEMORY) {
3193 if (trace) { tty->print("XXXXXX pushing memory"); uu->dump(); }
3194 memories.push(uu);
3195 }
3196 }
3197 }
3198 }
3199 }
3200 }
3201 for (uint next2 = 0; next2 < controls.size(); next2++) {
3202 Node *m = controls.at(next2);
3203 if (m->is_Region()) {
3204 bool all_in = true;
3205 for (uint i = 1; i < m->req(); i++) {
3206 if (!controls.member(m->in(i))) {
3207 all_in = false;
3208 break;
3209 }
3210 }
3211 if (trace) { tty->print("XXX verifying %s", all_in ? "all in" : ""); m->dump(); }
3212 bool found_phi = false;
3213 for (DUIterator_Fast jmax, j = m->fast_outs(jmax); j < jmax && !found_phi; j++) {
3214 Node* u = m->fast_out(j);
3215 if (u->is_Phi() && memories.member(u)) {
3216 found_phi = true;
3217 for (uint i = 1; i < u->req() && found_phi; i++) {
3218 Node* k = u->in(i);
3219 if (memories.member(k) != controls.member(m->in(i))) {
3220 found_phi = false;
3221 }
3222 }
3223 }
3224 }
3225 assert(found_phi || all_in, "");
3226 }
3227 }
3228 controls.clear();
3229 memories.clear();
3230 }
3231 for( uint i = 0; i < n->len(); ++i ) {
3232 Node *m = n->in(i);
3233 if (m != NULL) {
3234 nodes.push(m);
3235 }
3236 }
3237 }
3238 }
3239 #endif
3240
3241 const Type* ShenandoahEnqueueBarrierNode::bottom_type() const {
3242 if (in(1) == NULL || in(1)->is_top()) {
3243 return Type::TOP;
3244 }
3245 const Type* t = in(1)->bottom_type();
3246 if (t == TypePtr::NULL_PTR) {
3247 return t;
3248 }
3249 return t->is_oopptr()->cast_to_nonconst();
3250 }
3251
3252 const Type* ShenandoahEnqueueBarrierNode::Value(PhaseGVN* phase) const {
3253 if (in(1) == NULL) {
3254 return Type::TOP;
3255 }
3256 const Type* t = phase->type(in(1));
3257 if (t == Type::TOP) {
3258 return Type::TOP;
3259 }
3260 if (t == TypePtr::NULL_PTR) {
3261 return t;
3262 }
3263 return t->is_oopptr()->cast_to_nonconst();
3264 }
3265
3266 int ShenandoahEnqueueBarrierNode::needed(Node* n) {
3267 if (n == NULL ||
3268 n->is_Allocate() ||
3269 n->Opcode() == Op_ShenandoahEnqueueBarrier ||
3270 n->bottom_type() == TypePtr::NULL_PTR ||
3271 (n->bottom_type()->make_oopptr() != NULL && n->bottom_type()->make_oopptr()->const_oop() != NULL)) {
3272 return NotNeeded;
3273 }
3274 if (n->is_Phi() ||
3275 n->is_CMove()) {
3276 return MaybeNeeded;
3277 }
3278 return Needed;
3279 }
3280
3281 Node* ShenandoahEnqueueBarrierNode::next(Node* n) {
3282 for (;;) {
3283 if (n == NULL) {
3284 return n;
3285 } else if (n->bottom_type() == TypePtr::NULL_PTR) {
3286 return n;
3287 } else if (n->bottom_type()->make_oopptr() != NULL && n->bottom_type()->make_oopptr()->const_oop() != NULL) {
3288 return n;
3289 } else if (n->is_ConstraintCast() ||
3290 n->Opcode() == Op_DecodeN ||
3291 n->Opcode() == Op_EncodeP) {
3292 n = n->in(1);
3293 } else if (n->is_Proj()) {
3294 n = n->in(0);
3295 } else {
3296 return n;
3297 }
3298 }
3299 ShouldNotReachHere();
3300 return NULL;
3301 }
3302
3303 Node* ShenandoahEnqueueBarrierNode::Identity(PhaseGVN* phase) {
3304 PhaseIterGVN* igvn = phase->is_IterGVN();
3305
3306 Node* n = next(in(1));
3307
3308 int cont = needed(n);
3309
3310 if (cont == NotNeeded) {
3311 return in(1);
3312 } else if (cont == MaybeNeeded) {
3313 if (igvn == NULL) {
3314 phase->record_for_igvn(this);
3315 return this;
3316 } else {
3317 ResourceMark rm;
3318 Unique_Node_List wq;
3319 uint wq_i = 0;
3320
3321 for (;;) {
3322 if (n->is_Phi()) {
3323 for (uint i = 1; i < n->req(); i++) {
3324 Node* m = n->in(i);
3325 if (m != NULL) {
3326 wq.push(m);
3327 }
3328 }
3329 } else {
3330 assert(n->is_CMove(), "nothing else here");
3331 Node* m = n->in(CMoveNode::IfFalse);
3332 wq.push(m);
3333 m = n->in(CMoveNode::IfTrue);
3334 wq.push(m);
3335 }
3336 Node* orig_n = NULL;
3337 do {
3338 if (wq_i >= wq.size()) {
3339 return in(1);
3340 }
3341 n = wq.at(wq_i);
3342 wq_i++;
3343 orig_n = n;
3344 n = next(n);
3345 cont = needed(n);
3346 if (cont == Needed) {
3347 return this;
3348 }
3349 } while (cont != MaybeNeeded || (orig_n != n && wq.member(n)));
3350 }
3351 }
3352 }
3353
3354 return this;
3355 }
3356
3357 #ifdef ASSERT
3358 static bool has_never_branch(Node* root) {
3359 for (uint i = 1; i < root->req(); i++) {
3360 Node* in = root->in(i);
3361 if (in != NULL && in->Opcode() == Op_Halt && in->in(0)->is_Proj() && in->in(0)->in(0)->Opcode() == Op_NeverBranch) {
3362 return true;
3363 }
3364 }
3365 return false;
3366 }
3367 #endif
3368
3369 void MemoryGraphFixer::collect_memory_nodes() {
3370 Node_Stack stack(0);
3371 VectorSet visited(Thread::current()->resource_area());
3372 Node_List regions;
3373
3374 // Walk the raw memory graph and create a mapping from CFG node to
3375 // memory node. Exclude phis for now.
3376 stack.push(_phase->C->root(), 1);
3377 do {
3378 Node* n = stack.node();
3379 int opc = n->Opcode();
3380 uint i = stack.index();
3381 if (i < n->req()) {
3382 Node* mem = NULL;
3383 if (opc == Op_Root) {
3384 Node* in = n->in(i);
3385 int in_opc = in->Opcode();
3386 if (in_opc == Op_Return || in_opc == Op_Rethrow) {
3387 mem = in->in(TypeFunc::Memory);
3388 } else if (in_opc == Op_Halt) {
3389 if (!in->in(0)->is_Region()) {
3390 Node* proj = in->in(0);
3391 assert(proj->is_Proj(), "");
3392 Node* in = proj->in(0);
3393 assert(in->is_CallStaticJava() || in->Opcode() == Op_NeverBranch || in->Opcode() == Op_Catch || proj->is_IfProj(), "");
3394 if (in->is_CallStaticJava()) {
3395 mem = in->in(TypeFunc::Memory);
3396 } else if (in->Opcode() == Op_Catch) {
3397 Node* call = in->in(0)->in(0);
3398 assert(call->is_Call(), "");
3399 mem = call->in(TypeFunc::Memory);
3400 }
3401 }
3402 } else {
3403 #ifdef ASSERT
3404 n->dump();
3405 in->dump();
3406 #endif
3407 ShouldNotReachHere();
3408 }
3409 } else {
3410 assert(n->is_Phi() && n->bottom_type() == Type::MEMORY, "");
3411 assert(n->adr_type() == TypePtr::BOTTOM || _phase->C->get_alias_index(n->adr_type()) == _alias, "");
3412 mem = n->in(i);
3413 }
3414 i++;
3415 stack.set_index(i);
3416 if (mem == NULL) {
3417 continue;
3418 }
3419 for (;;) {
3420 if (visited.test_set(mem->_idx) || mem->is_Start()) {
3421 break;
3422 }
3423 if (mem->is_Phi()) {
3424 stack.push(mem, 2);
3425 mem = mem->in(1);
3426 } else if (mem->is_Proj()) {
3427 stack.push(mem, mem->req());
3428 mem = mem->in(0);
3429 } else if (mem->is_SafePoint() || mem->is_MemBar()) {
3430 mem = mem->in(TypeFunc::Memory);
3431 } else if (mem->is_MergeMem()) {
3432 MergeMemNode* mm = mem->as_MergeMem();
3433 mem = mm->memory_at(_alias);
3434 } else if (mem->is_Store() || mem->is_LoadStore() || mem->is_ClearArray()) {
3435 assert(_alias == Compile::AliasIdxRaw, "");
3436 stack.push(mem, mem->req());
3437 mem = mem->in(MemNode::Memory);
3438 } else if (mem->Opcode() == Op_ShenandoahWriteBarrier) {
3439 assert(_alias != Compile::AliasIdxRaw, "");
3440 mem = mem->in(ShenandoahBarrierNode::Memory);
3441 } else if (mem->Opcode() == Op_ShenandoahWBMemProj) {
3442 stack.push(mem, mem->req());
3443 mem = mem->in(ShenandoahWBMemProjNode::WriteBarrier);
3444 } else {
3445 #ifdef ASSERT
3446 mem->dump();
3447 #endif
3448 ShouldNotReachHere();
3449 }
3450 }
3451 } else {
3452 if (n->is_Phi()) {
3453 // Nothing
3454 } else if (!n->is_Root()) {
3455 Node* c = get_ctrl(n);
3456 _memory_nodes.map(c->_idx, n);
3457 }
3458 stack.pop();
3459 }
3460 } while(stack.is_nonempty());
3461
3462 // Iterate over CFG nodes in rpo and propagate memory state to
3463 // compute memory state at regions, creating new phis if needed.
3464 Node_List rpo_list;
3465 visited.Clear();
3466 _phase->rpo(_phase->C->root(), stack, visited, rpo_list);
3467 Node* root = rpo_list.pop();
3468 assert(root == _phase->C->root(), "");
3469
3470 const bool trace = false;
3471 #ifdef ASSERT
3472 if (trace) {
3473 for (int i = rpo_list.size() - 1; i >= 0; i--) {
3474 Node* c = rpo_list.at(i);
3475 if (_memory_nodes[c->_idx] != NULL) {
3476 tty->print("X %d", c->_idx); _memory_nodes[c->_idx]->dump();
3477 }
3478 }
3479 }
3480 #endif
3481 uint last = _phase->C->unique();
3482
3483 #ifdef ASSERT
3484 uint8_t max_depth = 0;
3485 for (LoopTreeIterator iter(_phase->ltree_root()); !iter.done(); iter.next()) {
3486 IdealLoopTree* lpt = iter.current();
3487 max_depth = MAX2(max_depth, lpt->_nest);
3488 }
3489 #endif
3490
3491 bool progress = true;
3492 int iteration = 0;
3493 Node_List dead_phis;
3494 while (progress) {
3495 progress = false;
3496 iteration++;
3497 assert(iteration <= 2+max_depth || _phase->C->has_irreducible_loop(), "");
3498 if (trace) { tty->print_cr("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"); }
3499 IdealLoopTree* last_updated_ilt = NULL;
3500 for (int i = rpo_list.size() - 1; i >= 0; i--) {
3501 Node* c = rpo_list.at(i);
3502
3503 Node* prev_mem = _memory_nodes[c->_idx];
3504 if (c->is_Region() && (_include_lsm || !c->is_OuterStripMinedLoop())) {
3505 Node* prev_region = regions[c->_idx];
3506 Node* unique = NULL;
3507 for (uint j = 1; j < c->req() && unique != NodeSentinel; j++) {
3508 Node* m = _memory_nodes[c->in(j)->_idx];
3509 assert(m != NULL || (c->is_Loop() && j == LoopNode::LoopBackControl && iteration == 1) || _phase->C->has_irreducible_loop() || has_never_branch(_phase->C->root()), "expect memory state");
3510 if (m != NULL) {
3511 if (m == prev_region && ((c->is_Loop() && j == LoopNode::LoopBackControl) || (prev_region->is_Phi() && prev_region->in(0) == c))) {
3512 assert(c->is_Loop() && j == LoopNode::LoopBackControl || _phase->C->has_irreducible_loop(), "");
3513 // continue
3514 } else if (unique == NULL) {
3515 unique = m;
3516 } else if (m == unique) {
3517 // continue
3518 } else {
3519 unique = NodeSentinel;
3520 }
3521 }
3522 }
3523 assert(unique != NULL, "empty phi???");
3524 if (unique != NodeSentinel) {
3525 if (prev_region != NULL && prev_region->is_Phi() && prev_region->in(0) == c) {
3526 dead_phis.push(prev_region);
3527 }
3528 regions.map(c->_idx, unique);
3529 } else {
3530 Node* phi = NULL;
3531 if (prev_region != NULL && prev_region->is_Phi() && prev_region->in(0) == c && prev_region->_idx >= last) {
3532 phi = prev_region;
3533 for (uint k = 1; k < c->req(); k++) {
3534 Node* m = _memory_nodes[c->in(k)->_idx];
3535 assert(m != NULL, "expect memory state");
3536 phi->set_req(k, m);
3537 }
3538 } else {
3539 for (DUIterator_Fast jmax, j = c->fast_outs(jmax); j < jmax && phi == NULL; j++) {
3540 Node* u = c->fast_out(j);
3541 if (u->is_Phi() && u->bottom_type() == Type::MEMORY &&
3542 (u->adr_type() == TypePtr::BOTTOM || _phase->C->get_alias_index(u->adr_type()) == _alias)) {
3543 phi = u;
3544 for (uint k = 1; k < c->req() && phi != NULL; k++) {
3545 Node* m = _memory_nodes[c->in(k)->_idx];
3546 assert(m != NULL, "expect memory state");
3547 if (u->in(k) != m) {
3548 phi = NULL;
3549 }
3550 }
3551 }
3552 }
3553 if (phi == NULL) {
3554 phi = new PhiNode(c, Type::MEMORY, _phase->C->get_adr_type(_alias));
3555 for (uint k = 1; k < c->req(); k++) {
3556 Node* m = _memory_nodes[c->in(k)->_idx];
3557 assert(m != NULL, "expect memory state");
3558 phi->init_req(k, m);
3559 }
3560 }
3561 }
3562 assert(phi != NULL, "");
3563 regions.map(c->_idx, phi);
3564 }
3565 Node* current_region = regions[c->_idx];
3566 if (current_region != prev_region) {
3567 progress = true;
3568 if (prev_region == prev_mem) {
3569 _memory_nodes.map(c->_idx, current_region);
3570 }
3571 }
3572 } else if (prev_mem == NULL || prev_mem->is_Phi() || ctrl_or_self(prev_mem) != c) {
3573 Node* m = _memory_nodes[_phase->idom(c)->_idx];
3574 assert(m != NULL, "expect memory state");
3575 if (m != prev_mem) {
3576 _memory_nodes.map(c->_idx, m);
3577 progress = true;
3578 }
3579 }
3580 #ifdef ASSERT
3581 if (trace) { tty->print("X %d", c->_idx); _memory_nodes[c->_idx]->dump(); }
3582 #endif
3583 }
3584 }
3585
3586 // Replace existing phi with computed memory state for that region
3587 // if different (could be a new phi or a dominating memory node if
3588 // that phi was found to be useless).
3589 while (dead_phis.size() > 0) {
3590 Node* n = dead_phis.pop();
3591 n->replace_by(_phase->C->top());
3592 n->destruct();
3593 }
3594 for (int i = rpo_list.size() - 1; i >= 0; i--) {
3595 Node* c = rpo_list.at(i);
3596 if (c->is_Region() && (_include_lsm || !c->is_OuterStripMinedLoop())) {
3597 Node* n = regions[c->_idx];
3598 if (n->is_Phi() && n->_idx >= last && n->in(0) == c) {
3599 _phase->register_new_node(n, c);
3600 }
3601 }
3602 }
3603 for (int i = rpo_list.size() - 1; i >= 0; i--) {
3604 Node* c = rpo_list.at(i);
3605 if (c->is_Region() && (_include_lsm || !c->is_OuterStripMinedLoop())) {
3606 Node* n = regions[c->_idx];
3607 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) {
3608 Node* u = c->fast_out(i);
3609 if (u->is_Phi() && u->bottom_type() == Type::MEMORY &&
3610 u != n) {
3611 if (u->adr_type() == TypePtr::BOTTOM) {
3612 fix_memory_uses(u, n, n, c);
3613 } else if (_phase->C->get_alias_index(u->adr_type()) == _alias) {
3614 _phase->lazy_replace(u, n);
3615 --i; --imax;
3616 }
3617 }
3618 }
3619 }
3620 }
3621 }
3622
3623 Node* MemoryGraphFixer::get_ctrl(Node* n) const {
3624 Node* c = _phase->get_ctrl(n);
3625 if (n->is_Proj() && n->in(0) != NULL && n->in(0)->is_Call()) {
3626 assert(c == n->in(0), "");
3627 CallNode* call = c->as_Call();
3628 CallProjections projs;
3629 call->extract_projections(&projs, true, false);
3630 if (projs.catchall_memproj != NULL) {
3631 if (projs.fallthrough_memproj == n) {
3632 c = projs.fallthrough_catchproj;
3633 } else {
3634 assert(projs.catchall_memproj == n, "");
3635 c = projs.catchall_catchproj;
3636 }
3637 }
3638 }
3639 return c;
3640 }
3641
3642 Node* MemoryGraphFixer::ctrl_or_self(Node* n) const {
3643 if (_phase->has_ctrl(n))
3644 return get_ctrl(n);
3645 else {
3646 assert (n->is_CFG(), "must be a CFG node");
3647 return n;
3648 }
3649 }
3650
3651 bool MemoryGraphFixer::mem_is_valid(Node* m, Node* c) const {
3652 return m != NULL && get_ctrl(m) == c;
3653 }
3654
3655 Node* MemoryGraphFixer::find_mem(Node* ctrl, Node* n) const {
3656 assert(n == NULL || _phase->ctrl_or_self(n) == ctrl, "");
3657 Node* mem = _memory_nodes[ctrl->_idx];
3658 Node* c = ctrl;
3659 while (!mem_is_valid(mem, c) &&
3660 (!c->is_CatchProj() || mem == NULL || c->in(0)->in(0)->in(0) != get_ctrl(mem))) {
3661 c = _phase->idom(c);
3662 mem = _memory_nodes[c->_idx];
3663 }
3664 if (n != NULL && mem_is_valid(mem, c)) {
3665 while (!ShenandoahWriteBarrierNode::is_dominator_same_ctrl(c, mem, n, _phase) && _phase->ctrl_or_self(mem) == ctrl) {
3666 mem = next_mem(mem, _alias);
3667 }
3668 if (mem->is_MergeMem()) {
3669 mem = mem->as_MergeMem()->memory_at(_alias);
3670 }
3671 if (!mem_is_valid(mem, c)) {
3672 do {
3673 c = _phase->idom(c);
3674 mem = _memory_nodes[c->_idx];
3675 } while (!mem_is_valid(mem, c) &&
3676 (!c->is_CatchProj() || mem == NULL || c->in(0)->in(0)->in(0) != get_ctrl(mem)));
3677 }
3678 }
3679 assert(mem->bottom_type() == Type::MEMORY, "");
3680 return mem;
3681 }
3682
3683 bool MemoryGraphFixer::has_mem_phi(Node* region) const {
3684 for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) {
3685 Node* use = region->fast_out(i);
3686 if (use->is_Phi() && use->bottom_type() == Type::MEMORY &&
3687 (_phase->C->get_alias_index(use->adr_type()) == _alias)) {
3688 return true;
3689 }
3690 }
3691 return false;
3692 }
3693
3694 void MemoryGraphFixer::fix_mem(Node* ctrl, Node* new_ctrl, Node* mem, Node* mem_for_ctrl, Node* new_mem, Unique_Node_List& uses) {
3695 assert(_phase->ctrl_or_self(new_mem) == new_ctrl, "");
3696 const bool trace = false;
3697 DEBUG_ONLY(if (trace) { tty->print("ZZZ control is"); ctrl->dump(); });
3698 DEBUG_ONLY(if (trace) { tty->print("ZZZ mem is"); mem->dump(); });
3699 GrowableArray<Node*> phis;
3700 if (mem_for_ctrl != mem) {
3701 Node* old = mem_for_ctrl;
3702 Node* prev = NULL;
3703 while (old != mem) {
3704 prev = old;
3705 if (old->is_Store() || old->is_ClearArray() || old->is_LoadStore()) {
3706 assert(_alias == Compile::AliasIdxRaw, "");
3707 old = old->in(MemNode::Memory);
3708 } else if (old->Opcode() == Op_SCMemProj) {
3709 assert(_alias == Compile::AliasIdxRaw, "");
3710 old = old->in(0);
3711 } else if (old->Opcode() == Op_ShenandoahWBMemProj) {
3712 assert(_alias != Compile::AliasIdxRaw, "");
3713 old = old->in(ShenandoahWBMemProjNode::WriteBarrier);
3714 } else if (old->Opcode() == Op_ShenandoahWriteBarrier) {
3715 assert(_alias != Compile::AliasIdxRaw, "");
3716 old = old->in(ShenandoahBarrierNode::Memory);
3717 } else {
3718 ShouldNotReachHere();
3719 }
3720 }
3721 assert(prev != NULL, "");
3722 if (new_ctrl != ctrl) {
3723 _memory_nodes.map(ctrl->_idx, mem);
3724 _memory_nodes.map(new_ctrl->_idx, mem_for_ctrl);
3725 }
3726 uint input = prev->Opcode() == Op_ShenandoahWriteBarrier ? (uint)ShenandoahBarrierNode::Memory : (uint)MemNode::Memory;
3727 _phase->igvn().replace_input_of(prev, input, new_mem);
3728 } else {
3729 uses.clear();
3730 _memory_nodes.map(new_ctrl->_idx, new_mem);
3731 uses.push(new_ctrl);
3732 for(uint next = 0; next < uses.size(); next++ ) {
3733 Node *n = uses.at(next);
3734 assert(n->is_CFG(), "");
3735 DEBUG_ONLY(if (trace) { tty->print("ZZZ ctrl"); n->dump(); });
3736 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
3737 Node* u = n->fast_out(i);
3738 if (!u->is_Root() && u->is_CFG() && u != n) {
3739 Node* m = _memory_nodes[u->_idx];
3740 if (u->is_Region() && (!u->is_OuterStripMinedLoop() || _include_lsm) &&
3741 !has_mem_phi(u) &&
3742 u->unique_ctrl_out()->Opcode() != Op_Halt) {
3743 DEBUG_ONLY(if (trace) { tty->print("ZZZ region"); u->dump(); });
3744 DEBUG_ONLY(if (trace && m != NULL) { tty->print("ZZZ mem"); m->dump(); });
3745
3746 if (!mem_is_valid(m, u) || !m->is_Phi()) {
3747 bool push = true;
3748 bool create_phi = true;
3749 if (_phase->is_dominator(new_ctrl, u)) {
3750 create_phi = false;
3751 } else if (!_phase->C->has_irreducible_loop()) {
3752 IdealLoopTree* loop = _phase->get_loop(ctrl);
3753 bool do_check = true;
3754 IdealLoopTree* l = loop;
3755 create_phi = false;
3756 while (l != _phase->ltree_root()) {
3757 if (_phase->is_dominator(l->_head, u) && _phase->is_dominator(_phase->idom(u), l->_head)) {
3758 create_phi = true;
3759 do_check = false;
3760 break;
3761 }
3762 l = l->_parent;
3763 }
3764
3765 if (do_check) {
3766 assert(!create_phi, "");
3767 IdealLoopTree* u_loop = _phase->get_loop(u);
3768 if (u_loop != _phase->ltree_root() && u_loop->is_member(loop)) {
3769 Node* c = ctrl;
3770 while (!_phase->is_dominator(c, u_loop->tail())) {
3771 c = _phase->idom(c);
3772 }
3773 if (!_phase->is_dominator(c, u)) {
3774 do_check = false;
3775 }
3776 }
3777 }
3778
3779 if (do_check && _phase->is_dominator(_phase->idom(u), new_ctrl)) {
3780 create_phi = true;
3781 }
3782 }
3783 if (create_phi) {
3784 Node* phi = new PhiNode(u, Type::MEMORY, _phase->C->get_adr_type(_alias));
3785 _phase->register_new_node(phi, u);
3786 phis.push(phi);
3787 DEBUG_ONLY(if (trace) { tty->print("ZZZ new phi"); phi->dump(); });
3788 if (!mem_is_valid(m, u)) {
3789 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting mem"); phi->dump(); });
3790 _memory_nodes.map(u->_idx, phi);
3791 } else {
3792 DEBUG_ONLY(if (trace) { tty->print("ZZZ NOT setting mem"); m->dump(); });
3793 for (;;) {
3794 assert(m->is_Mem() || m->is_LoadStore() || m->is_Proj() || m->Opcode() == Op_ShenandoahWriteBarrier || m->Opcode() == Op_ShenandoahWBMemProj, "");
3795 Node* next = NULL;
3796 if (m->is_Proj()) {
3797 next = m->in(0);
3798 } else if (m->Opcode() == Op_ShenandoahWBMemProj) {
3799 next = m->in(ShenandoahWBMemProjNode::WriteBarrier);
3800 } else if (m->is_Mem() || m->is_LoadStore()) {
3801 assert(_alias == Compile::AliasIdxRaw, "");
3802 next = m->in(MemNode::Memory);
3803 } else {
3804 assert(_alias != Compile::AliasIdxRaw, "");
3805 assert (m->Opcode() == Op_ShenandoahWriteBarrier, "");
3806 next = m->in(ShenandoahBarrierNode::Memory);
3807 }
3808 if (_phase->get_ctrl(next) != u) {
3809 break;
3810 }
3811 if (next->is_MergeMem()) {
3812 assert(_phase->get_ctrl(next->as_MergeMem()->memory_at(_alias)) != u, "");
3813 break;
3814 }
3815 if (next->is_Phi()) {
3816 assert(next->adr_type() == TypePtr::BOTTOM && next->in(0) == u, "");
3817 break;
3818 }
3819 m = next;
3820 }
3821
3822 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting to phi"); m->dump(); });
3823 assert(m->is_Mem() || m->is_LoadStore() || m->Opcode() == Op_ShenandoahWriteBarrier, "");
3824 uint input = (m->is_Mem() || m->is_LoadStore()) ? (uint)MemNode::Memory : (uint)ShenandoahBarrierNode::Memory;
3825 _phase->igvn().replace_input_of(m, input, phi);
3826 push = false;
3827 }
3828 } else {
3829 DEBUG_ONLY(if (trace) { tty->print("ZZZ skipping region"); u->dump(); });
3830 }
3831 if (push) {
3832 uses.push(u);
3833 }
3834 }
3835 } else if (!mem_is_valid(m, u) &&
3836 !(u->Opcode() == Op_CProj && u->in(0)->Opcode() == Op_NeverBranch && u->as_Proj()->_con == 1)) {
3837 uses.push(u);
3838 }
3839 }
3840 }
3841 }
3842 for (int i = 0; i < phis.length(); i++) {
3843 Node* n = phis.at(i);
3844 Node* r = n->in(0);
3845 DEBUG_ONLY(if (trace) { tty->print("ZZZ fixing new phi"); n->dump(); });
3846 for (uint j = 1; j < n->req(); j++) {
3847 Node* m = find_mem(r->in(j), NULL);
3848 _phase->igvn().replace_input_of(n, j, m);
3849 DEBUG_ONLY(if (trace) { tty->print("ZZZ fixing new phi: %d", j); m->dump(); });
3850 }
3851 }
3852 }
3853 uint last = _phase->C->unique();
3854 MergeMemNode* mm = NULL;
3855 int alias = _alias;
3856 DEBUG_ONLY(if (trace) { tty->print("ZZZ raw mem is"); mem->dump(); });
3857 for (DUIterator i = mem->outs(); mem->has_out(i); i++) {
3858 Node* u = mem->out(i);
3859 if (u->_idx < last) {
3860 if (u->is_Mem()) {
3861 if (_phase->C->get_alias_index(u->adr_type()) == alias) {
3862 Node* m = find_mem(_phase->get_ctrl(u), u);
3863 if (m != mem) {
3864 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting memory of use"); u->dump(); });
3865 _phase->igvn().replace_input_of(u, MemNode::Memory, m);
3866 --i;
3867 }
3868 }
3869 } else if (u->is_MergeMem()) {
3870 MergeMemNode* u_mm = u->as_MergeMem();
3871 if (u_mm->memory_at(alias) == mem) {
3872 MergeMemNode* newmm = NULL;
3873 for (DUIterator_Fast jmax, j = u->fast_outs(jmax); j < jmax; j++) {
3874 Node* uu = u->fast_out(j);
3875 assert(!uu->is_MergeMem(), "chain of MergeMems?");
3876 if (uu->is_Phi()) {
3877 assert(uu->adr_type() == TypePtr::BOTTOM, "");
3878 Node* region = uu->in(0);
3879 int nb = 0;
3880 for (uint k = 1; k < uu->req(); k++) {
3881 if (uu->in(k) == u) {
3882 Node* m = find_mem(region->in(k), NULL);
3883 if (m != mem) {
3884 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting memory of phi %d", k); uu->dump(); });
3885 newmm = clone_merge_mem(u, mem, m, _phase->ctrl_or_self(m), i);
3886 if (newmm != u) {
3887 _phase->igvn().replace_input_of(uu, k, newmm);
3888 nb++;
3889 --jmax;
3890 }
3891 }
3892 }
3893 }
3894 if (nb > 0) {
3895 --j;
3896 }
3897 } else {
3898 Node* m = find_mem(_phase->ctrl_or_self(uu), uu);
3899 if (m != mem) {
3900 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting memory of use"); uu->dump(); });
3901 newmm = clone_merge_mem(u, mem, m, _phase->ctrl_or_self(m), i);
3902 if (newmm != u) {
3903 _phase->igvn().replace_input_of(uu, uu->find_edge(u), newmm);
3904 --j, --jmax;
3905 }
3906 }
3907 }
3908 }
3909 }
3910 } else if (u->is_Phi()) {
3911 assert(u->bottom_type() == Type::MEMORY, "what else?");
3912 if (_phase->C->get_alias_index(u->adr_type()) == alias || u->adr_type() == TypePtr::BOTTOM) {
3913 Node* region = u->in(0);
3914 bool replaced = false;
3915 for (uint j = 1; j < u->req(); j++) {
3916 if (u->in(j) == mem) {
3917 Node* m = find_mem(region->in(j), NULL);
3918 Node* nnew = m;
3919 if (m != mem) {
3920 if (u->adr_type() == TypePtr::BOTTOM) {
3921 mm = allocate_merge_mem(mem, m, _phase->ctrl_or_self(m));
3922 nnew = mm;
3923 }
3924 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting memory of phi %d", j); u->dump(); });
3925 _phase->igvn().replace_input_of(u, j, nnew);
3926 replaced = true;
3927 }
3928 }
3929 }
3930 if (replaced) {
3931 --i;
3932 }
3933 }
3934 } else if ((u->adr_type() == TypePtr::BOTTOM && u->Opcode() != Op_StrInflatedCopy) ||
3935 u->adr_type() == NULL) {
3936 assert(u->adr_type() != NULL ||
3937 u->Opcode() == Op_Rethrow ||
3938 u->Opcode() == Op_Return ||
3939 u->Opcode() == Op_SafePoint ||
3940 (u->is_CallStaticJava() && u->as_CallStaticJava()->uncommon_trap_request() != 0) ||
3941 (u->is_CallStaticJava() && u->as_CallStaticJava()->_entry_point == OptoRuntime::rethrow_stub()) ||
3942 u->Opcode() == Op_CallLeaf, "");
3943 Node* m = find_mem(_phase->ctrl_or_self(u), u);
3944 if (m != mem) {
3945 mm = allocate_merge_mem(mem, m, _phase->get_ctrl(m));
3946 _phase->igvn().replace_input_of(u, u->find_edge(mem), mm);
3947 --i;
3948 }
3949 } else if (_phase->C->get_alias_index(u->adr_type()) == alias) {
3950 Node* m = find_mem(_phase->ctrl_or_self(u), u);
3951 if (m != mem) {
3952 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting memory of use"); u->dump(); });
3953 _phase->igvn().replace_input_of(u, u->find_edge(mem), m);
3954 --i;
3955 }
3956 } else if (u->adr_type() != TypePtr::BOTTOM &&
3957 _memory_nodes[_phase->ctrl_or_self(u)->_idx] == u) {
3958 Node* m = find_mem(_phase->ctrl_or_self(u), u);
3959 assert(m != mem, "");
3960 // u is on the wrong slice...
3961 assert(u->is_ClearArray(), "");
3962 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting memory of use"); u->dump(); });
3963 _phase->igvn().replace_input_of(u, u->find_edge(mem), m);
3964 --i;
3965 }
3966 }
3967 }
3968 #ifdef ASSERT
3969 assert(new_mem->outcnt() > 0, "");
3970 for (int i = 0; i < phis.length(); i++) {
3971 Node* n = phis.at(i);
3972 assert(n->outcnt() > 0, "new phi must have uses now");
3973 }
3974 #endif
3975 }
3976
3977 MergeMemNode* MemoryGraphFixer::allocate_merge_mem(Node* mem, Node* rep_proj, Node* rep_ctrl) const {
3978 MergeMemNode* mm = MergeMemNode::make(mem);
3979 mm->set_memory_at(_alias, rep_proj);
3980 _phase->register_new_node(mm, rep_ctrl);
3981 return mm;
3982 }
3983
3984 MergeMemNode* MemoryGraphFixer::clone_merge_mem(Node* u, Node* mem, Node* rep_proj, Node* rep_ctrl, DUIterator& i) const {
3985 MergeMemNode* newmm = NULL;
3986 MergeMemNode* u_mm = u->as_MergeMem();
3987 Node* c = _phase->get_ctrl(u);
3988 if (_phase->is_dominator(c, rep_ctrl)) {
3989 c = rep_ctrl;
3990 } else {
3991 assert(_phase->is_dominator(rep_ctrl, c), "one must dominate the other");
3992 }
3993 if (u->outcnt() == 1) {
3994 if (u->req() > (uint)_alias && u->in(_alias) == mem) {
3995 _phase->igvn().replace_input_of(u, _alias, rep_proj);
3996 --i;
3997 } else {
3998 _phase->igvn().rehash_node_delayed(u);
3999 u_mm->set_memory_at(_alias, rep_proj);
4000 }
4001 newmm = u_mm;
4002 _phase->set_ctrl_and_loop(u, c);
4003 } else {
4004 // can't simply clone u and then change one of its input because
4005 // it adds and then removes an edge which messes with the
4006 // DUIterator
4007 newmm = MergeMemNode::make(u_mm->base_memory());
4008 for (uint j = 0; j < u->req(); j++) {
4009 if (j < newmm->req()) {
4010 if (j == (uint)_alias) {
4011 newmm->set_req(j, rep_proj);
4012 } else if (newmm->in(j) != u->in(j)) {
4013 newmm->set_req(j, u->in(j));
4014 }
4015 } else if (j == (uint)_alias) {
4016 newmm->add_req(rep_proj);
4017 } else {
4018 newmm->add_req(u->in(j));
4019 }
4020 }
4021 if ((uint)_alias >= u->req()) {
4022 newmm->set_memory_at(_alias, rep_proj);
4023 }
4024 _phase->register_new_node(newmm, c);
4025 }
4026 return newmm;
4027 }
4028
4029 bool MemoryGraphFixer::should_process_phi(Node* phi) const {
4030 if (phi->adr_type() == TypePtr::BOTTOM) {
4031 Node* region = phi->in(0);
4032 for (DUIterator_Fast jmax, j = region->fast_outs(jmax); j < jmax; j++) {
4033 Node* uu = region->fast_out(j);
4034 if (uu->is_Phi() && uu != phi && uu->bottom_type() == Type::MEMORY && _phase->C->get_alias_index(uu->adr_type()) == _alias) {
4035 return false;
4036 }
4037 }
4038 return true;
4039 }
4040 return _phase->C->get_alias_index(phi->adr_type()) == _alias;
4041 }
4042
4043 void MemoryGraphFixer::fix_memory_uses(Node* mem, Node* replacement, Node* rep_proj, Node* rep_ctrl) const {
4044 uint last = _phase-> C->unique();
4045 MergeMemNode* mm = NULL;
4046 assert(mem->bottom_type() == Type::MEMORY, "");
4047 for (DUIterator i = mem->outs(); mem->has_out(i); i++) {
4048 Node* u = mem->out(i);
4049 if (u != replacement && u->_idx < last) {
4050 if (u->is_ShenandoahBarrier() && _alias != Compile::AliasIdxRaw) {
4051 if (_phase->C->get_alias_index(u->adr_type()) == _alias && ShenandoahWriteBarrierNode::is_dominator(rep_ctrl, _phase->ctrl_or_self(u), replacement, u, _phase)) {
4052 _phase->igvn().replace_input_of(u, u->find_edge(mem), rep_proj);
4053 assert(u->find_edge(mem) == -1, "only one edge");
4054 --i;
4055 }
4056 } else if (u->is_Mem()) {
4057 if (_phase->C->get_alias_index(u->adr_type()) == _alias && ShenandoahWriteBarrierNode::is_dominator(rep_ctrl, _phase->ctrl_or_self(u), replacement, u, _phase)) {
4058 assert(_alias == Compile::AliasIdxRaw , "only raw memory can lead to a memory operation");
4059 _phase->igvn().replace_input_of(u, u->find_edge(mem), rep_proj);
4060 assert(u->find_edge(mem) == -1, "only one edge");
4061 --i;
4062 }
4063 } else if (u->is_MergeMem()) {
4064 MergeMemNode* u_mm = u->as_MergeMem();
4065 if (u_mm->memory_at(_alias) == mem) {
4066 MergeMemNode* newmm = NULL;
4067 for (DUIterator_Fast jmax, j = u->fast_outs(jmax); j < jmax; j++) {
4068 Node* uu = u->fast_out(j);
4069 assert(!uu->is_MergeMem(), "chain of MergeMems?");
4070 if (uu->is_Phi()) {
4071 if (should_process_phi(uu)) {
4072 Node* region = uu->in(0);
4073 int nb = 0;
4074 for (uint k = 1; k < uu->req(); k++) {
4075 if (uu->in(k) == u && _phase->is_dominator(rep_ctrl, region->in(k))) {
4076 if (newmm == NULL) {
4077 newmm = clone_merge_mem(u, mem, rep_proj, rep_ctrl, i);
4078 }
4079 if (newmm != u) {
4080 _phase->igvn().replace_input_of(uu, k, newmm);
4081 nb++;
4082 --jmax;
4083 }
4084 }
4085 }
4086 if (nb > 0) {
4087 --j;
4088 }
4089 }
4090 } else {
4091 if (rep_ctrl != uu && ShenandoahWriteBarrierNode::is_dominator(rep_ctrl, _phase->ctrl_or_self(uu), replacement, uu, _phase)) {
4092 if (newmm == NULL) {
4093 newmm = clone_merge_mem(u, mem, rep_proj, rep_ctrl, i);
4094 }
4095 if (newmm != u) {
4096 _phase->igvn().replace_input_of(uu, uu->find_edge(u), newmm);
4097 --j, --jmax;
4098 }
4099 }
4100 }
4101 }
4102 }
4103 } else if (u->is_Phi()) {
4104 assert(u->bottom_type() == Type::MEMORY, "what else?");
4105 Node* region = u->in(0);
4106 if (should_process_phi(u)) {
4107 bool replaced = false;
4108 for (uint j = 1; j < u->req(); j++) {
4109 if (u->in(j) == mem && _phase->is_dominator(rep_ctrl, region->in(j))) {
4110 Node* nnew = rep_proj;
4111 if (u->adr_type() == TypePtr::BOTTOM) {
4112 if (mm == NULL) {
4113 mm = allocate_merge_mem(mem, rep_proj, rep_ctrl);
4114 }
4115 nnew = mm;
4116 }
4117 _phase->igvn().replace_input_of(u, j, nnew);
4118 replaced = true;
4119 }
4120 }
4121 if (replaced) {
4122 --i;
4123 }
4124
4125 }
4126 } else if ((u->adr_type() == TypePtr::BOTTOM && u->Opcode() != Op_StrInflatedCopy) ||
4127 u->adr_type() == NULL) {
4128 assert(u->adr_type() != NULL ||
4129 u->Opcode() == Op_Rethrow ||
4130 u->Opcode() == Op_Return ||
4131 u->Opcode() == Op_SafePoint ||
4132 (u->is_CallStaticJava() && u->as_CallStaticJava()->uncommon_trap_request() != 0) ||
4133 (u->is_CallStaticJava() && u->as_CallStaticJava()->_entry_point == OptoRuntime::rethrow_stub()) ||
4134 u->Opcode() == Op_CallLeaf, "");
4135 if (ShenandoahWriteBarrierNode::is_dominator(rep_ctrl, _phase->ctrl_or_self(u), replacement, u, _phase)) {
4136 if (mm == NULL) {
4137 mm = allocate_merge_mem(mem, rep_proj, rep_ctrl);
4138 }
4139 _phase->igvn().replace_input_of(u, u->find_edge(mem), mm);
4140 --i;
4141 }
4142 } else if (_phase->C->get_alias_index(u->adr_type()) == _alias) {
4143 if (ShenandoahWriteBarrierNode::is_dominator(rep_ctrl, _phase->ctrl_or_self(u), replacement, u, _phase)) {
4144 _phase->igvn().replace_input_of(u, u->find_edge(mem), rep_proj);
4145 --i;
4146 }
4147 }
4148 }
4149 }
4150 }
4151
4152 void MemoryGraphFixer::remove(Node* n) {
4153 assert(n->Opcode() == Op_ShenandoahWBMemProj, "");
4154 Node* c = _phase->get_ctrl(n);
4155 Node* mem = find_mem(c, NULL);
4156 if (mem == n) {
4157 _memory_nodes.map(c->_idx, mem->in(ShenandoahWBMemProjNode::WriteBarrier)->in(ShenandoahBarrierNode::Memory));
4158 }
4159 }
4160