24 #include "precompiled.hpp"
25
26 #include "gc/shenandoah/c2/shenandoahSupport.hpp"
27 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
28 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
29 #include "gc/shenandoah/shenandoahBrooksPointer.hpp"
30 #include "gc/shenandoah/shenandoahHeap.hpp"
31 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
32 #include "gc/shenandoah/shenandoahRuntime.hpp"
33 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
34 #include "opto/arraycopynode.hpp"
35 #include "opto/block.hpp"
36 #include "opto/callnode.hpp"
37 #include "opto/castnode.hpp"
38 #include "opto/movenode.hpp"
39 #include "opto/phaseX.hpp"
40 #include "opto/rootnode.hpp"
41 #include "opto/runtime.hpp"
42 #include "opto/subnode.hpp"
43
44 Node* ShenandoahBarrierNode::skip_through_barrier(Node* n) {
45 if (n == NULL) {
46 return NULL;
47 }
48 if (n->Opcode() == Op_ShenandoahEnqueueBarrier) {
49 n = n->in(1);
50 }
51
52 if (n->is_ShenandoahBarrier()) {
53 return n->in(ValueIn);
54 } else if (n->is_Phi() &&
55 n->req() == 3 &&
56 n->in(1) != NULL &&
57 n->in(1)->is_ShenandoahBarrier() &&
58 n->in(2) != NULL &&
59 n->in(2)->bottom_type() == TypePtr::NULL_PTR &&
60 n->in(0) != NULL &&
61 n->in(0)->in(1) != NULL &&
62 n->in(0)->in(1)->is_IfProj() &&
63 n->in(0)->in(2) != NULL &&
64 n->in(0)->in(2)->is_IfProj() &&
65 n->in(0)->in(1)->in(0) != NULL &&
66 n->in(0)->in(1)->in(0) == n->in(0)->in(2)->in(0) &&
67 n->in(1)->in(ValueIn)->Opcode() == Op_CastPP) {
68 Node* iff = n->in(0)->in(1)->in(0);
69 Node* res = n->in(1)->in(ValueIn)->in(1);
70 if (iff->is_If() &&
71 iff->in(1) != NULL &&
72 iff->in(1)->is_Bool() &&
73 iff->in(1)->as_Bool()->_test._test == BoolTest::ne &&
74 iff->in(1)->in(1) != NULL &&
75 iff->in(1)->in(1)->Opcode() == Op_CmpP &&
76 iff->in(1)->in(1)->in(1) != NULL &&
77 iff->in(1)->in(1)->in(1) == res &&
78 iff->in(1)->in(1)->in(2) != NULL &&
79 iff->in(1)->in(1)->in(2)->bottom_type() == TypePtr::NULL_PTR) {
80 return res;
81 }
82 }
83 return n;
84 }
85
86 bool ShenandoahBarrierNode::needs_barrier(PhaseGVN* phase, ShenandoahBarrierNode* orig, Node* n, Node* rb_mem, bool allow_fromspace) {
87 Unique_Node_List visited;
88 return needs_barrier_impl(phase, orig, n, rb_mem, allow_fromspace, visited);
89 }
90
91 bool ShenandoahBarrierNode::needs_barrier_impl(PhaseGVN* phase, ShenandoahBarrierNode* orig, Node* n, Node* rb_mem, bool allow_fromspace, Unique_Node_List &visited) {
92 if (visited.member(n)) {
93 return false; // Been there.
94 }
95 visited.push(n);
96
97 if (n->is_Allocate()) {
98 return false;
99 }
100
101 if (n->is_Call()) {
102 return true;
103 }
104
105 const Type* type = phase->type(n);
106 if (type == Type::TOP) {
107 return false;
108 }
109 if (type->make_ptr()->higher_equal(TypePtr::NULL_PTR)) {
110 return false;
111 }
112 if (type->make_oopptr() && type->make_oopptr()->const_oop() != NULL) {
113 return false;
114 }
115
116 if (ShenandoahOptimizeStableFinals) {
117 const TypeAryPtr* ary = type->isa_aryptr();
118 if (ary && ary->is_stable() && allow_fromspace) {
119 return false;
120 }
121 }
122
123 if (n->is_CheckCastPP() || n->is_ConstraintCast() || n->Opcode() == Op_ShenandoahEnqueueBarrier) {
124 return needs_barrier_impl(phase, orig, n->in(1), rb_mem, allow_fromspace, visited);
125 }
126 if (n->is_Parm()) {
127 return true;
128 }
129 if (n->is_Proj()) {
130 return needs_barrier_impl(phase, orig, n->in(0), rb_mem, allow_fromspace, visited);
131 }
132
133 if (n->Opcode() == Op_ShenandoahWBMemProj) {
134 return needs_barrier_impl(phase, orig, n->in(ShenandoahWBMemProjNode::WriteBarrier), rb_mem, allow_fromspace, visited);
135 }
136 if (n->is_Phi()) {
137 bool need_barrier = false;
138 for (uint i = 1; i < n->req() && ! need_barrier; i++) {
139 Node* input = n->in(i);
140 if (input == NULL) {
141 need_barrier = true; // Phi not complete yet?
142 } else if (needs_barrier_impl(phase, orig, input, rb_mem, allow_fromspace, visited)) {
143 need_barrier = true;
144 }
145 }
146 return need_barrier;
147 }
148 if (n->is_CMove()) {
149 return needs_barrier_impl(phase, orig, n->in(CMoveNode::IfFalse), rb_mem, allow_fromspace, visited) ||
150 needs_barrier_impl(phase, orig, n->in(CMoveNode::IfTrue ), rb_mem, allow_fromspace, visited);
151 }
152 if (n->Opcode() == Op_CreateEx) {
153 return true;
154 }
155 if (n->Opcode() == Op_ShenandoahWriteBarrier) {
156 return false;
157 }
158 if (n->Opcode() == Op_ShenandoahReadBarrier) {
159 if (rb_mem == n->in(Memory)) {
160 return false;
161 } else {
162 return true;
163 }
164 }
165
166 if (n->Opcode() == Op_LoadP ||
167 n->Opcode() == Op_LoadN ||
168 n->Opcode() == Op_GetAndSetP ||
169 n->Opcode() == Op_CompareAndExchangeP ||
170 n->Opcode() == Op_ShenandoahCompareAndExchangeP ||
171 n->Opcode() == Op_GetAndSetN ||
172 n->Opcode() == Op_CompareAndExchangeN ||
173 n->Opcode() == Op_ShenandoahCompareAndExchangeN) {
174 return true;
175 }
176 if (n->Opcode() == Op_DecodeN ||
177 n->Opcode() == Op_EncodeP) {
178 return needs_barrier_impl(phase, orig, n->in(1), rb_mem, allow_fromspace, visited);
179 }
180
181 #ifdef ASSERT
182 tty->print("need barrier on?: "); n->dump();
183 ShouldNotReachHere();
184 #endif
185 return true;
186 }
187
188 bool ShenandoahReadBarrierNode::dominates_memory_rb_impl(PhaseGVN* phase,
189 Node* b1,
190 Node* b2,
191 Node* current,
192 bool linear) {
193 ResourceMark rm;
194 VectorSet visited(Thread::current()->resource_area());
195 Node_Stack phis(0);
196
197 for(int i = 0; i < 10; i++) {
198 if (current == NULL) {
199 return false;
200 } else if (visited.test_set(current->_idx) || current->is_top() || current == b1) {
201 current = NULL;
202 while (phis.is_nonempty() && current == NULL) {
203 uint idx = phis.index();
204 Node* phi = phis.node();
205 if (idx >= phi->req()) {
206 phis.pop();
207 } else {
208 current = phi->in(idx);
209 phis.set_index(idx+1);
210 }
211 }
212 if (current == NULL) {
213 return true;
214 }
215 } else if (current == phase->C->immutable_memory()) {
216 return false;
217 } else if (current->isa_Phi()) {
218 if (!linear) {
219 return false;
220 }
221 phis.push(current, 2);
222 current = current->in(1);
223 } else if (current->Opcode() == Op_ShenandoahWriteBarrier) {
224 const Type* in_type = current->bottom_type();
225 const Type* this_type = b2->bottom_type();
226 if (is_independent(in_type, this_type)) {
227 current = current->in(Memory);
228 } else {
229 return false;
230 }
231 } else if (current->Opcode() == Op_ShenandoahWBMemProj) {
232 current = current->in(ShenandoahWBMemProjNode::WriteBarrier);
233 } else if (current->is_Proj()) {
234 current = current->in(0);
235 } else if (current->is_Call()) {
236 return false; // TODO: Maybe improve by looking at the call's memory effects?
237 } else if (current->is_MemBar()) {
238 return false; // TODO: Do we need to stop at *any* membar?
239 } else if (current->is_MergeMem()) {
240 const TypePtr* adr_type = brooks_pointer_type(phase->type(b2));
241 uint alias_idx = phase->C->get_alias_index(adr_type);
242 current = current->as_MergeMem()->memory_at(alias_idx);
243 } else {
244 #ifdef ASSERT
245 current->dump();
246 #endif
247 ShouldNotReachHere();
248 return false;
249 }
250 }
251 return false;
252 }
253
254 bool ShenandoahReadBarrierNode::is_independent(Node* mem) {
255 if (mem->is_Phi() || mem->is_Proj() || mem->is_MergeMem()) {
256 return true;
257 } else if (mem->Opcode() == Op_ShenandoahWBMemProj) {
258 return true;
259 } else if (mem->Opcode() == Op_ShenandoahWriteBarrier) {
260 const Type* mem_type = mem->bottom_type();
261 const Type* this_type = bottom_type();
262 if (is_independent(mem_type, this_type)) {
263 return true;
264 } else {
265 return false;
266 }
267 } else if (mem->is_Call() || mem->is_MemBar()) {
268 return false;
269 }
270 #ifdef ASSERT
271 mem->dump();
272 #endif
273 ShouldNotReachHere();
274 return true;
275 }
276
277 bool ShenandoahReadBarrierNode::dominates_memory_rb(PhaseGVN* phase, Node* b1, Node* b2, bool linear) {
278 return dominates_memory_rb_impl(phase, b1->in(Memory), b2, b2->in(Memory), linear);
279 }
280
281 bool ShenandoahReadBarrierNode::is_independent(const Type* in_type, const Type* this_type) {
282 assert(in_type->isa_oopptr(), "expect oop ptr");
283 assert(this_type->isa_oopptr(), "expect oop ptr");
284
285 ciKlass* in_kls = in_type->is_oopptr()->klass();
286 ciKlass* this_kls = this_type->is_oopptr()->klass();
287 if (in_kls != NULL && this_kls != NULL &&
288 in_kls->is_loaded() && this_kls->is_loaded() &&
289 (!in_kls->is_subclass_of(this_kls)) &&
290 (!this_kls->is_subclass_of(in_kls))) {
291 return true;
292 }
293 return false;
294 }
295
296 Node* ShenandoahReadBarrierNode::Ideal(PhaseGVN *phase, bool can_reshape) {
297 if (! can_reshape) {
298 return NULL;
299 }
300
301 if (in(Memory) == phase->C->immutable_memory()) return NULL;
302
303 // If memory input is a MergeMem, take the appropriate slice out of it.
304 Node* mem_in = in(Memory);
305 if (mem_in->isa_MergeMem()) {
306 const TypePtr* adr_type = brooks_pointer_type(bottom_type());
307 uint alias_idx = phase->C->get_alias_index(adr_type);
308 mem_in = mem_in->as_MergeMem()->memory_at(alias_idx);
309 set_req(Memory, mem_in);
310 return this;
311 }
312
313 Node* input = in(Memory);
314 if (input->Opcode() == Op_ShenandoahWBMemProj) {
315 ResourceMark rm;
316 VectorSet seen(Thread::current()->resource_area());
317 Node* n = in(Memory);
318 while (n->Opcode() == Op_ShenandoahWBMemProj &&
319 n->in(ShenandoahWBMemProjNode::WriteBarrier) != NULL &&
320 n->in(ShenandoahWBMemProjNode::WriteBarrier)->Opcode() == Op_ShenandoahWriteBarrier &&
321 n->in(ShenandoahWBMemProjNode::WriteBarrier)->in(Memory) != NULL) {
322 if (seen.test_set(n->_idx)) {
323 return NULL; // loop
324 }
325 n = n->in(ShenandoahWBMemProjNode::WriteBarrier)->in(Memory);
326 }
327
328 Node* wb = input->in(ShenandoahWBMemProjNode::WriteBarrier);
329 const Type* in_type = phase->type(wb);
330 // is_top() test not sufficient here: we can come here after CCP
331 // in a dead branch of the graph that has not yet been removed.
332 if (in_type == Type::TOP) return NULL; // Dead path.
333 assert(wb->Opcode() == Op_ShenandoahWriteBarrier, "expect write barrier");
334 if (is_independent(in_type, _type)) {
335 phase->igvn_rehash_node_delayed(wb);
336 set_req(Memory, wb->in(Memory));
337 if (can_reshape && input->outcnt() == 0) {
338 phase->is_IterGVN()->_worklist.push(input);
339 }
340 return this;
341 }
342 }
343 return NULL;
344 }
345
346 ShenandoahWriteBarrierNode::ShenandoahWriteBarrierNode(Compile* C, Node* ctrl, Node* mem, Node* obj)
347 : ShenandoahBarrierNode(ctrl, mem, obj, false) {
348 assert(UseShenandoahGC && ShenandoahWriteBarrier, "should be enabled");
349 ShenandoahBarrierSetC2::bsc2()->state()->add_shenandoah_barrier(this);
350 }
351
352 Node* ShenandoahWriteBarrierNode::Identity(PhaseGVN* phase) {
353 assert(in(0) != NULL, "should have control");
354 PhaseIterGVN* igvn = phase->is_IterGVN();
355 Node* mem_in = in(Memory);
356 Node* mem_proj = NULL;
357
358 if (igvn != NULL) {
359 mem_proj = find_out_with(Op_ShenandoahWBMemProj);
360 if (mem_in == mem_proj) {
361 return this;
362 }
363 }
364
365 Node* replacement = Identity_impl(phase);
366 if (igvn != NULL) {
367 if (replacement != NULL && replacement != this && mem_proj != NULL) {
368 igvn->replace_node(mem_proj, mem_in);
369 }
370 }
371 return replacement;
372 }
373
374 Node* ShenandoahWriteBarrierNode::Ideal(PhaseGVN *phase, bool can_reshape) {
375 assert(in(0) != NULL, "should have control");
376 if (!can_reshape) {
377 return NULL;
378 }
379
380 Node* mem_in = in(Memory);
381
382 if (mem_in->isa_MergeMem()) {
383 const TypePtr* adr_type = brooks_pointer_type(bottom_type());
384 uint alias_idx = phase->C->get_alias_index(adr_type);
385 mem_in = mem_in->as_MergeMem()->memory_at(alias_idx);
386 set_req(Memory, mem_in);
387 return this;
388 }
389
390 Node* val = in(ValueIn);
391 if (val->is_ShenandoahBarrier()) {
392 set_req(ValueIn, val->in(ValueIn));
393 return this;
394 }
395
396 return NULL;
397 }
398
399 bool ShenandoahWriteBarrierNode::expand(Compile* C, PhaseIterGVN& igvn) {
400 if (UseShenandoahGC) {
401 if (ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count() > 0 || (!ShenandoahWriteBarrier && ShenandoahStoreValEnqueueBarrier)) {
402 bool attempt_more_loopopts = ShenandoahLoopOptsAfterExpansion;
403 C->clear_major_progress();
404 PhaseIdealLoop ideal_loop(igvn, LoopOptsShenandoahExpand);
405 if (C->failing()) return false;
406 PhaseIdealLoop::verify(igvn);
407 DEBUG_ONLY(ShenandoahBarrierNode::verify_raw_mem(C->root());)
408 if (attempt_more_loopopts) {
409 C->set_major_progress();
410 if (!C->optimize_loops(igvn, LoopOptsShenandoahPostExpand)) {
411 return false;
412 }
413 C->clear_major_progress();
414 }
415 }
416 }
417 return true;
418 }
419
420 bool ShenandoahWriteBarrierNode::is_heap_state_test(Node* iff, int mask) {
421 if (!UseShenandoahGC) {
422 return false;
423 }
424 assert(iff->is_If(), "bad input");
425 if (iff->Opcode() != Op_If) {
426 return false;
427 }
428 Node* bol = iff->in(1);
429 if (!bol->is_Bool() || bol->as_Bool()->_test._test != BoolTest::ne) {
430 return false;
431 }
432 Node* cmp = bol->in(1);
433 if (cmp->Opcode() != Op_CmpI) {
434 return false;
435 }
436 Node* in1 = cmp->in(1);
437 Node* in2 = cmp->in(2);
438 if (in2->find_int_con(-1) != 0) {
439 return false;
440 }
441 if (in1->Opcode() != Op_AndI) {
442 return false;
443 }
444 in2 = in1->in(2);
445 if (in2->find_int_con(-1) != mask) {
446 return false;
447 }
448 in1 = in1->in(1);
449
450 return is_gc_state_load(in1);
451 }
452
453 bool ShenandoahWriteBarrierNode::is_heap_stable_test(Node* iff) {
454 return is_heap_state_test(iff, ShenandoahHeap::HAS_FORWARDED);
455 }
456
457 bool ShenandoahWriteBarrierNode::is_gc_state_load(Node *n) {
458 if (!UseShenandoahGC) {
459 return false;
460 }
461 if (n->Opcode() != Op_LoadB && n->Opcode() != Op_LoadUB) {
462 return false;
463 }
464 Node* addp = n->in(MemNode::Address);
465 if (!addp->is_AddP()) {
466 return false;
467 }
468 Node* base = addp->in(AddPNode::Address);
469 Node* off = addp->in(AddPNode::Offset);
470 if (base->Opcode() != Op_ThreadLocal) {
471 return false;
472 }
473 if (off->find_intptr_t_con(-1) != in_bytes(ShenandoahThreadLocalData::gc_state_offset())) {
474 return false;
475 }
476 return true;
477 }
478
479 bool ShenandoahWriteBarrierNode::has_safepoint_between(Node* start, Node* stop, PhaseIdealLoop *phase) {
480 assert(phase->is_dominator(stop, start), "bad inputs");
481 ResourceMark rm;
482 Unique_Node_List wq;
483 wq.push(start);
484 for (uint next = 0; next < wq.size(); next++) {
485 Node *m = wq.at(next);
486 if (m == stop) {
487 continue;
488 }
489 if (m->is_SafePoint() && !m->is_CallLeaf()) {
490 return true;
491 }
492 if (m->is_Region()) {
493 for (uint i = 1; i < m->req(); i++) {
494 wq.push(m->in(i));
495 }
496 } else {
497 wq.push(m->in(0));
498 }
499 }
500 return false;
501 }
502
503 bool ShenandoahWriteBarrierNode::try_common_gc_state_load(Node *n, PhaseIdealLoop *phase) {
504 assert(is_gc_state_load(n), "inconsistent");
505 Node* addp = n->in(MemNode::Address);
506 Node* dominator = NULL;
507 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
508 Node* u = addp->fast_out(i);
509 assert(is_gc_state_load(u), "inconsistent");
510 if (u != n && phase->is_dominator(u->in(0), n->in(0))) {
511 if (dominator == NULL) {
512 dominator = u;
513 } else {
514 if (phase->dom_depth(u->in(0)) < phase->dom_depth(dominator->in(0))) {
515 dominator = u;
516 }
517 }
518 }
519 }
520 if (dominator == NULL || has_safepoint_between(n->in(0), dominator->in(0), phase)) {
521 return false;
522 }
523 phase->igvn().replace_node(n, dominator);
524
525 return true;
526 }
527
528 bool ShenandoahBarrierNode::dominates_memory_impl(PhaseGVN* phase,
529 Node* b1,
530 Node* b2,
531 Node* current,
532 bool linear) {
533 ResourceMark rm;
534 VectorSet visited(Thread::current()->resource_area());
535 Node_Stack phis(0);
536
537 for(int i = 0; i < 10; i++) {
538 if (current == NULL) {
539 return false;
540 } else if (visited.test_set(current->_idx) || current->is_top() || current == b1) {
541 current = NULL;
542 while (phis.is_nonempty() && current == NULL) {
543 uint idx = phis.index();
544 Node* phi = phis.node();
545 if (idx >= phi->req()) {
546 phis.pop();
547 } else {
548 current = phi->in(idx);
549 phis.set_index(idx+1);
550 }
551 }
552 if (current == NULL) {
553 return true;
554 }
555 } else if (current == b2) {
556 return false;
557 } else if (current == phase->C->immutable_memory()) {
558 return false;
559 } else if (current->isa_Phi()) {
560 if (!linear) {
561 return false;
562 }
563 phis.push(current, 2);
564 current = current->in(1);
565 } else if (current->Opcode() == Op_ShenandoahWriteBarrier) {
566 current = current->in(Memory);
567 } else if (current->Opcode() == Op_ShenandoahWBMemProj) {
568 current = current->in(ShenandoahWBMemProjNode::WriteBarrier);
569 } else if (current->is_Proj()) {
570 current = current->in(0);
571 } else if (current->is_Call()) {
572 current = current->in(TypeFunc::Memory);
573 } else if (current->is_MemBar()) {
574 current = current->in(TypeFunc::Memory);
575 } else if (current->is_MergeMem()) {
576 const TypePtr* adr_type = brooks_pointer_type(phase->type(b2));
577 uint alias_idx = phase->C->get_alias_index(adr_type);
578 current = current->as_MergeMem()->memory_at(alias_idx);
579 } else {
580 #ifdef ASSERT
581 current->dump();
582 #endif
583 ShouldNotReachHere();
584 return false;
585 }
586 }
587 return false;
588 }
589
590 /**
591 * Determines if b1 dominates b2 through memory inputs. It returns true if:
592 * - b1 can be reached by following each branch in b2's memory input (through phis, etc)
593 * - or we get back to b2 (i.e. through a loop) without seeing b1
594 * In all other cases, (in particular, if we reach immutable_memory without having seen b1)
595 * we return false.
596 */
597 bool ShenandoahBarrierNode::dominates_memory(PhaseGVN* phase, Node* b1, Node* b2, bool linear) {
598 return dominates_memory_impl(phase, b1, b2, b2->in(Memory), linear);
599 }
600
601 Node* ShenandoahBarrierNode::Identity_impl(PhaseGVN* phase) {
602 Node* n = in(ValueIn);
603
604 Node* rb_mem = Opcode() == Op_ShenandoahReadBarrier ? in(Memory) : NULL;
605 if (! needs_barrier(phase, this, n, rb_mem, _allow_fromspace)) {
606 return n;
607 }
608
609 // Try to find a write barrier sibling with identical inputs that we can fold into.
610 for (DUIterator i = n->outs(); n->has_out(i); i++) {
611 Node* sibling = n->out(i);
612 if (sibling == this) {
613 continue;
614 }
615 if (sibling->Opcode() != Op_ShenandoahWriteBarrier) {
616 continue;
617 }
618
619 assert(sibling->in(ValueIn) == in(ValueIn), "sanity");
620 assert(sibling->Opcode() == Op_ShenandoahWriteBarrier, "sanity");
621
622 if (dominates_memory(phase, sibling, this, phase->is_IterGVN() == NULL)) {
623 return sibling;
624 }
625 }
626 return this;
627 }
628
629 #ifndef PRODUCT
630 void ShenandoahBarrierNode::dump_spec(outputStream *st) const {
631 const TypePtr* adr = adr_type();
632 if (adr == NULL) {
633 return;
634 }
635 st->print(" @");
636 adr->dump_on(st);
637 st->print(" (");
638 Compile::current()->alias_type(adr)->adr_type()->dump_on(st);
639 st->print(") ");
640 }
641 #endif
642
643 Node* ShenandoahReadBarrierNode::Identity(PhaseGVN* phase) {
644 Node* id = Identity_impl(phase);
645
646 if (id == this && phase->is_IterGVN()) {
647 Node* n = in(ValueIn);
648 // No success in super call. Try to combine identical read barriers.
649 for (DUIterator i = n->outs(); n->has_out(i); i++) {
650 Node* sibling = n->out(i);
651 if (sibling == this || sibling->Opcode() != Op_ShenandoahReadBarrier) {
652 continue;
653 }
654 assert(sibling->in(ValueIn) == in(ValueIn), "sanity");
655 if (phase->is_IterGVN()->hash_find(sibling) &&
656 sibling->bottom_type() == bottom_type() &&
657 sibling->in(Control) == in(Control) &&
658 dominates_memory_rb(phase, sibling, this, phase->is_IterGVN() == NULL)) {
659 return sibling;
660 }
661 }
662 }
663 return id;
664 }
665
666 const Type* ShenandoahBarrierNode::Value(PhaseGVN* phase) const {
667 // Either input is TOP ==> the result is TOP
668 const Type *t1 = phase->type(in(Memory));
669 if (t1 == Type::TOP) return Type::TOP;
670 const Type *t2 = phase->type(in(ValueIn));
671 if( t2 == Type::TOP ) return Type::TOP;
672
673 if (t2 == TypePtr::NULL_PTR) {
674 return _type;
675 }
676
677 const Type* type = t2->is_oopptr()->cast_to_nonconst();
678 return type;
679 }
680
681 uint ShenandoahBarrierNode::hash() const {
682 return TypeNode::hash() + _allow_fromspace;
683 }
684
685 bool ShenandoahBarrierNode::cmp(const Node& n) const {
686 return _allow_fromspace == ((ShenandoahBarrierNode&) n)._allow_fromspace
687 && TypeNode::cmp(n);
688 }
689
690 uint ShenandoahBarrierNode::size_of() const {
691 return sizeof(*this);
692 }
693
694 Node* ShenandoahWBMemProjNode::Identity(PhaseGVN* phase) {
695 Node* wb = in(WriteBarrier);
696 if (wb->is_top()) return phase->C->top(); // Dead path.
697
698 assert(wb->Opcode() == Op_ShenandoahWriteBarrier, "expect write barrier");
699 PhaseIterGVN* igvn = phase->is_IterGVN();
700 // We can't do the below unless the graph is fully constructed.
701 if (igvn == NULL) {
702 return this;
703 }
704
705 // If the mem projection has no barrier users, it's not needed anymore.
706 if (wb->outcnt() == 1) {
707 return wb->in(ShenandoahBarrierNode::Memory);
708 }
709
710 return this;
711 }
712
713 #ifdef ASSERT
714 bool ShenandoahBarrierNode::verify_helper(Node* in, Node_Stack& phis, VectorSet& visited, verify_type t, bool trace, Unique_Node_List& barriers_used) {
715 assert(phis.size() == 0, "");
716
717 while (true) {
718 if (in->bottom_type() == TypePtr::NULL_PTR) {
719 if (trace) {tty->print_cr("NULL");}
720 } else if (!in->bottom_type()->make_ptr()->make_oopptr()) {
721 if (trace) {tty->print_cr("Non oop");}
722 } else if (t == ShenandoahLoad && ShenandoahOptimizeStableFinals &&
723 in->bottom_type()->make_ptr()->isa_aryptr() &&
724 in->bottom_type()->make_ptr()->is_aryptr()->is_stable()) {
725 if (trace) {tty->print_cr("Stable array load");}
726 } else {
727 if (in->is_ConstraintCast()) {
728 in = in->in(1);
729 continue;
730 } else if (in->is_AddP()) {
731 assert(!in->in(AddPNode::Address)->is_top(), "no raw memory access");
732 in = in->in(AddPNode::Address);
733 continue;
734 } else if (in->is_Con()) {
735 if (trace) {tty->print("Found constant"); in->dump();}
736 } else if (in->is_ShenandoahBarrier()) {
737 if (t == ShenandoahOopStore) {
738 if (in->Opcode() != Op_ShenandoahWriteBarrier) {
739 return false;
740 }
741 uint i = 0;
742 for (; i < phis.size(); i++) {
743 Node* n = phis.node_at(i);
744 if (n->Opcode() == Op_ShenandoahEnqueueBarrier) {
745 break;
746 }
747 }
748 if (i == phis.size()) {
749 return false;
750 }
751 } else if (t == ShenandoahStore && in->Opcode() != Op_ShenandoahWriteBarrier) {
752 return false;
753 }
754 barriers_used.push(in);
755 if (trace) {tty->print("Found barrier"); in->dump();}
756 } else if (in->Opcode() == Op_ShenandoahEnqueueBarrier) {
757 if (t != ShenandoahOopStore) {
758 in = in->in(1);
759 continue;
760 }
761 if (trace) {tty->print("Found enqueue barrier"); in->dump();}
762 phis.push(in, in->req());
763 in = in->in(1);
764 continue;
765 } else if (in->is_Proj() && in->in(0)->is_Allocate()) {
766 if (trace) {tty->print("Found alloc"); in->in(0)->dump();}
767 } else if (in->is_Phi()) {
768 if (!visited.test_set(in->_idx)) {
769 if (trace) {tty->print("Pushed phi:"); in->dump();}
770 phis.push(in, 2);
771 in = in->in(1);
772 continue;
773 }
774 if (trace) {tty->print("Already seen phi:"); in->dump();}
775 } else if (in->Opcode() == Op_CMoveP || in->Opcode() == Op_CMoveN) {
776 if (!visited.test_set(in->_idx)) {
777 if (trace) {tty->print("Pushed cmovep:"); in->dump();}
778 phis.push(in, CMoveNode::IfTrue);
779 in = in->in(CMoveNode::IfFalse);
780 continue;
781 }
782 if (trace) {tty->print("Already seen cmovep:"); in->dump();}
783 } else if (in->Opcode() == Op_EncodeP || in->Opcode() == Op_DecodeN) {
784 in = in->in(1);
785 continue;
786 } else {
792 uint idx = phis.index();
793 Node* phi = phis.node();
794 if (idx >= phi->req()) {
795 if (trace) {tty->print("Popped phi:"); phi->dump();}
796 phis.pop();
797 continue;
798 }
799 if (trace) {tty->print("Next entry(%d) for phi:", idx); phi->dump();}
800 in = phi->in(idx);
801 phis.set_index(idx+1);
802 cont = true;
803 break;
804 }
805 if (!cont) {
806 break;
807 }
808 }
809 return true;
810 }
811
812 void ShenandoahBarrierNode::report_verify_failure(const char *msg, Node *n1, Node *n2) {
813 if (n1 != NULL) {
814 n1->dump(+10);
815 }
816 if (n2 != NULL) {
817 n2->dump(+10);
818 }
819 fatal("%s", msg);
820 }
821
822 void ShenandoahBarrierNode::verify(RootNode* root) {
823 ResourceMark rm;
824 Unique_Node_List wq;
825 GrowableArray<Node*> barriers;
826 Unique_Node_List barriers_used;
827 Node_Stack phis(0);
828 VectorSet visited(Thread::current()->resource_area());
829 const bool trace = false;
830 const bool verify_no_useless_barrier = false;
831
832 wq.push(root);
833 for (uint next = 0; next < wq.size(); next++) {
834 Node *n = wq.at(next);
835 if (n->is_Load()) {
836 const bool trace = false;
837 if (trace) {tty->print("Verifying"); n->dump();}
838 if (n->Opcode() == Op_LoadRange || n->Opcode() == Op_LoadKlass || n->Opcode() == Op_LoadNKlass) {
839 if (trace) {tty->print_cr("Load range/klass");}
840 } else {
841 const TypePtr* adr_type = n->as_Load()->adr_type();
842
854 assert(k->is_instance_klass(), "");
855 ciInstanceKlass* ik = (ciInstanceKlass*)k;
856 int offset = adr_type->offset();
857
858 if ((ik->debug_final_field_at(offset) && ShenandoahOptimizeInstanceFinals) ||
859 (ik->debug_stable_field_at(offset) && ShenandoahOptimizeStableFinals)) {
860 if (trace) {tty->print_cr("Final/stable");}
861 verify = false;
862 } else if (k == ciEnv::current()->Class_klass() &&
863 tinst->const_oop() != NULL &&
864 tinst->offset() >= (ik->size_helper() * wordSize)) {
865 ciInstanceKlass* k = tinst->const_oop()->as_instance()->java_lang_Class_klass()->as_instance_klass();
866 ciField* field = k->get_field_by_offset(tinst->offset(), true);
867 if ((ShenandoahOptimizeStaticFinals && field->is_final()) ||
868 (ShenandoahOptimizeStableFinals && field->is_stable())) {
869 verify = false;
870 }
871 }
872 }
873
874 if (verify && !ShenandoahBarrierNode::verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahLoad, trace, barriers_used)) {
875 report_verify_failure("Shenandoah verification: Load should have barriers", n);
876 }
877 }
878 }
879 } else if (n->is_Store()) {
880 const bool trace = false;
881
882 if (trace) {tty->print("Verifying"); n->dump();}
883 if (n->in(MemNode::ValueIn)->bottom_type()->make_oopptr()) {
884 Node* adr = n->in(MemNode::Address);
885 bool verify = true;
886
887 if (adr->is_AddP() && adr->in(AddPNode::Base)->is_top()) {
888 adr = adr->in(AddPNode::Address);
889 if (adr->is_AddP()) {
890 assert(adr->in(AddPNode::Base)->is_top(), "");
891 adr = adr->in(AddPNode::Address);
892 if (adr->Opcode() == Op_LoadP &&
893 adr->in(MemNode::Address)->in(AddPNode::Base)->is_top() &&
894 adr->in(MemNode::Address)->in(AddPNode::Address)->Opcode() == Op_ThreadLocal &&
895 adr->in(MemNode::Address)->in(AddPNode::Offset)->find_intptr_t_con(-1) == in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset())) {
896 if (trace) {tty->print_cr("SATB prebarrier");}
897 verify = false;
898 }
899 }
900 }
901
902 if (verify && !ShenandoahBarrierNode::verify_helper(n->in(MemNode::ValueIn), phis, visited, ShenandoahStoreValEnqueueBarrier ? ShenandoahOopStore : ShenandoahValue, trace, barriers_used)) {
903 report_verify_failure("Shenandoah verification: Store should have barriers", n);
904 }
905 }
906 if (!ShenandoahBarrierNode::verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahStore, trace, barriers_used)) {
907 report_verify_failure("Shenandoah verification: Store (address) should have barriers", n);
908 }
909 } else if (n->Opcode() == Op_CmpP) {
910 const bool trace = false;
911
912 Node* in1 = n->in(1);
913 Node* in2 = n->in(2);
914 if (in1->bottom_type()->isa_oopptr()) {
915 if (trace) {tty->print("Verifying"); n->dump();}
916
917 bool mark_inputs = false;
918 if (in1->bottom_type() == TypePtr::NULL_PTR || in2->bottom_type() == TypePtr::NULL_PTR ||
919 (in1->is_Con() || in2->is_Con())) {
920 if (trace) {tty->print_cr("Comparison against a constant");}
921 mark_inputs = true;
922 } else if ((in1->is_CheckCastPP() && in1->in(1)->is_Proj() && in1->in(1)->in(0)->is_Allocate()) ||
923 (in2->is_CheckCastPP() && in2->in(1)->is_Proj() && in2->in(1)->in(0)->is_Allocate())) {
924 if (trace) {tty->print_cr("Comparison with newly alloc'ed object");}
925 mark_inputs = true;
926 } else {
927 assert(in2->bottom_type()->isa_oopptr(), "");
928
929 if (!ShenandoahBarrierNode::verify_helper(in1, phis, visited, ShenandoahStore, trace, barriers_used) ||
930 !ShenandoahBarrierNode::verify_helper(in2, phis, visited, ShenandoahStore, trace, barriers_used)) {
931 report_verify_failure("Shenandoah verification: Cmp should have barriers", n);
932 }
933 }
934 if (verify_no_useless_barrier &&
935 mark_inputs &&
936 (!ShenandoahBarrierNode::verify_helper(in1, phis, visited, ShenandoahValue, trace, barriers_used) ||
937 !ShenandoahBarrierNode::verify_helper(in2, phis, visited, ShenandoahValue, trace, barriers_used))) {
938 phis.clear();
939 visited.Reset();
940 }
941 }
942 } else if (n->is_LoadStore()) {
943 if (n->in(MemNode::ValueIn)->bottom_type()->make_ptr() &&
944 !ShenandoahBarrierNode::verify_helper(n->in(MemNode::ValueIn), phis, visited, ShenandoahStoreValEnqueueBarrier ? ShenandoahOopStore : ShenandoahValue, trace, barriers_used)) {
945 report_verify_failure("Shenandoah verification: LoadStore (value) should have barriers", n);
946 }
947
948 if (n->in(MemNode::Address)->bottom_type()->make_oopptr() && !ShenandoahBarrierNode::verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahStore, trace, barriers_used)) {
949 report_verify_failure("Shenandoah verification: LoadStore (address) should have barriers", n);
950 }
951 } else if (n->Opcode() == Op_CallLeafNoFP || n->Opcode() == Op_CallLeaf) {
952 CallNode* call = n->as_Call();
953
954 static struct {
955 const char* name;
956 struct {
957 int pos;
958 verify_type t;
959 } args[6];
960 } calls[] = {
961 "aescrypt_encryptBlock",
962 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad },
963 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
964 "aescrypt_decryptBlock",
965 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad },
966 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
967 "multiplyToLen",
968 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad }, { TypeFunc::Parms+4, ShenandoahStore },
1024 "sha512_implCompressMB",
1025 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
1026 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
1027 "encodeBlock",
1028 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahStore }, { -1, ShenandoahNone },
1029 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
1030 };
1031
1032 if (call->is_call_to_arraycopystub()) {
1033 Node* dest = NULL;
1034 const TypeTuple* args = n->as_Call()->_tf->domain();
1035 for (uint i = TypeFunc::Parms, j = 0; i < args->cnt(); i++) {
1036 if (args->field_at(i)->isa_ptr()) {
1037 j++;
1038 if (j == 2) {
1039 dest = n->in(i);
1040 break;
1041 }
1042 }
1043 }
1044 if (!ShenandoahBarrierNode::verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahLoad, trace, barriers_used) ||
1045 !ShenandoahBarrierNode::verify_helper(dest, phis, visited, ShenandoahStore, trace, barriers_used)) {
1046 report_verify_failure("Shenandoah verification: ArrayCopy should have barriers", n);
1047 }
1048 } else if (strlen(call->_name) > 5 &&
1049 !strcmp(call->_name + strlen(call->_name) - 5, "_fill")) {
1050 if (!ShenandoahBarrierNode::verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahStore, trace, barriers_used)) {
1051 report_verify_failure("Shenandoah verification: _fill should have barriers", n);
1052 }
1053 } else if (!strcmp(call->_name, "shenandoah_wb_pre")) {
1054 // skip
1055 } else {
1056 const int calls_len = sizeof(calls) / sizeof(calls[0]);
1057 int i = 0;
1058 for (; i < calls_len; i++) {
1059 if (!strcmp(calls[i].name, call->_name)) {
1060 break;
1061 }
1062 }
1063 if (i != calls_len) {
1064 const uint args_len = sizeof(calls[0].args) / sizeof(calls[0].args[0]);
1065 for (uint j = 0; j < args_len; j++) {
1066 int pos = calls[i].args[j].pos;
1067 if (pos == -1) {
1068 break;
1069 }
1070 if (!ShenandoahBarrierNode::verify_helper(call->in(pos), phis, visited, calls[i].args[j].t, trace, barriers_used)) {
1071 report_verify_failure("Shenandoah verification: intrinsic calls should have barriers", n);
1072 }
1073 }
1074 for (uint j = TypeFunc::Parms; j < call->req(); j++) {
1075 if (call->in(j)->bottom_type()->make_ptr() &&
1076 call->in(j)->bottom_type()->make_ptr()->isa_oopptr()) {
1077 uint k = 0;
1078 for (; k < args_len && calls[i].args[k].pos != (int)j; k++);
1079 if (k == args_len) {
1080 fatal("arg %d for call %s not covered", j, call->_name);
1081 }
1082 }
1083 }
1084 } else {
1085 for (uint j = TypeFunc::Parms; j < call->req(); j++) {
1086 if (call->in(j)->bottom_type()->make_ptr() &&
1087 call->in(j)->bottom_type()->make_ptr()->isa_oopptr()) {
1088 fatal("%s not covered", call->_name);
1089 }
1090 }
1091 }
1092 }
1093 } else if (n->is_ShenandoahBarrier()) {
1094 assert(!barriers.contains(n), "");
1095 assert(n->Opcode() != Op_ShenandoahWriteBarrier || n->find_out_with(Op_ShenandoahWBMemProj) != NULL, "bad shenandoah write barrier");
1096 assert(n->Opcode() != Op_ShenandoahWriteBarrier || n->outcnt() > 1, "bad shenandoah write barrier");
1097 barriers.push(n);
1098 } else if (n->Opcode() == Op_ShenandoahEnqueueBarrier) {
1099 // skip
1100 } else if (n->Opcode() == Op_ShenandoahWBMemProj) {
1101 assert(n->in(0) == NULL && n->in(ShenandoahWBMemProjNode::WriteBarrier)->Opcode() == Op_ShenandoahWriteBarrier, "strange ShenandoahWBMemProj");
1102 } else if (n->is_AddP()
1103 || n->is_Phi()
1104 || n->is_ConstraintCast()
1105 || n->Opcode() == Op_Return
1106 || n->Opcode() == Op_CMoveP
1107 || n->Opcode() == Op_CMoveN
1108 || n->Opcode() == Op_Rethrow
1109 || n->is_MemBar()
1110 || n->Opcode() == Op_Conv2B
1111 || n->Opcode() == Op_SafePoint
1112 || n->is_CallJava()
1113 || n->Opcode() == Op_Unlock
1114 || n->Opcode() == Op_EncodeP
1115 || n->Opcode() == Op_DecodeN) {
1116 // nothing to do
1117 } else {
1118 static struct {
1119 int opcode;
1120 struct {
1121 int pos;
1148 { { 1, ShenandoahLoad }, { -1, ShenandoahNone} },
1149 Op_StrIndexOfChar,
1150 { { 2, ShenandoahLoad }, { -1, ShenandoahNone } },
1151 };
1152
1153 const int others_len = sizeof(others) / sizeof(others[0]);
1154 int i = 0;
1155 for (; i < others_len; i++) {
1156 if (others[i].opcode == n->Opcode()) {
1157 break;
1158 }
1159 }
1160 uint stop = n->is_Call() ? n->as_Call()->tf()->domain()->cnt() : n->req();
1161 if (i != others_len) {
1162 const uint inputs_len = sizeof(others[0].inputs) / sizeof(others[0].inputs[0]);
1163 for (uint j = 0; j < inputs_len; j++) {
1164 int pos = others[i].inputs[j].pos;
1165 if (pos == -1) {
1166 break;
1167 }
1168 if (!ShenandoahBarrierNode::verify_helper(n->in(pos), phis, visited, others[i].inputs[j].t, trace, barriers_used)) {
1169 report_verify_failure("Shenandoah verification: intrinsic calls should have barriers", n);
1170 }
1171 }
1172 for (uint j = 1; j < stop; j++) {
1173 if (n->in(j) != NULL && n->in(j)->bottom_type()->make_ptr() &&
1174 n->in(j)->bottom_type()->make_ptr()->make_oopptr()) {
1175 uint k = 0;
1176 for (; k < inputs_len && others[i].inputs[k].pos != (int)j; k++);
1177 if (k == inputs_len) {
1178 fatal("arg %d for node %s not covered", j, n->Name());
1179 }
1180 }
1181 }
1182 } else {
1183 for (uint j = 1; j < stop; j++) {
1184 if (n->in(j) != NULL && n->in(j)->bottom_type()->make_ptr() &&
1185 n->in(j)->bottom_type()->make_ptr()->make_oopptr()) {
1186 fatal("%s not covered", n->Name());
1187 }
1188 }
1189 }
1190 }
1191
1192 if (n->is_SafePoint()) {
1193 SafePointNode* sfpt = n->as_SafePoint();
1194 if (verify_no_useless_barrier && sfpt->jvms() != NULL) {
1195 for (uint i = sfpt->jvms()->scloff(); i < sfpt->jvms()->endoff(); i++) {
1196 if (!ShenandoahBarrierNode::verify_helper(sfpt->in(i), phis, visited, ShenandoahLoad, trace, barriers_used)) {
1197 phis.clear();
1198 visited.Reset();
1199 }
1200 }
1201 }
1202 }
1203 for( uint i = 0; i < n->len(); ++i ) {
1204 Node *m = n->in(i);
1205 if (m == NULL) continue;
1206
1207 // In most cases, inputs should be known to be non null. If it's
1208 // not the case, it could be a missing cast_not_null() in an
1209 // intrinsic or support might be needed in AddPNode::Ideal() to
1210 // avoid a NULL+offset input.
1211 if (!(n->is_Phi() ||
1212 (n->is_SafePoint() && (!n->is_CallRuntime() || !strcmp(n->as_Call()->_name, "shenandoah_wb_pre") || !strcmp(n->as_Call()->_name, "unsafe_arraycopy"))) ||
1213 n->Opcode() == Op_CmpP ||
1214 n->Opcode() == Op_CmpN ||
1215 (n->Opcode() == Op_StoreP && i == StoreNode::ValueIn) ||
1216 (n->Opcode() == Op_StoreN && i == StoreNode::ValueIn) ||
1217 n->is_ConstraintCast() ||
1218 n->Opcode() == Op_Return ||
1219 n->Opcode() == Op_Conv2B ||
1220 n->is_AddP() ||
1221 n->Opcode() == Op_CMoveP ||
1222 n->Opcode() == Op_CMoveN ||
1223 n->Opcode() == Op_Rethrow ||
1224 n->is_MemBar() ||
1225 n->is_Mem() ||
1226 n->Opcode() == Op_AryEq ||
1227 n->Opcode() == Op_SCMemProj ||
1228 n->Opcode() == Op_EncodeP ||
1229 n->Opcode() == Op_DecodeN ||
1230 n->Opcode() == Op_ShenandoahWriteBarrier ||
1231 n->Opcode() == Op_ShenandoahWBMemProj ||
1232 n->Opcode() == Op_ShenandoahEnqueueBarrier)) {
1233 if (m->bottom_type()->make_oopptr() && m->bottom_type()->make_oopptr()->meet(TypePtr::NULL_PTR) == m->bottom_type()) {
1234 report_verify_failure("Shenandoah verification: null input", n, m);
1235 }
1236 }
1237
1238 wq.push(m);
1239 }
1240 }
1241
1242 if (verify_no_useless_barrier) {
1243 for (int i = 0; i < barriers.length(); i++) {
1244 Node* n = barriers.at(i);
1245 if (!barriers_used.member(n)) {
1246 tty->print("XXX useless barrier"); n->dump(-2);
1247 ShouldNotReachHere();
1248 }
1249 }
1250 }
1251 }
1252 #endif
1253
1254 bool ShenandoahBarrierNode::is_dominator_same_ctrl(Node*c, Node* d, Node* n, PhaseIdealLoop* phase) {
1255 // That both nodes have the same control is not sufficient to prove
1256 // domination, verify that there's no path from d to n
1257 ResourceMark rm;
1258 Unique_Node_List wq;
1259 wq.push(d);
1260 for (uint next = 0; next < wq.size(); next++) {
1261 Node *m = wq.at(next);
1262 if (m == n) {
1263 return false;
1264 }
1265 if (m->is_Phi() && m->in(0)->is_Loop()) {
1266 assert(phase->ctrl_or_self(m->in(LoopNode::EntryControl)) != c, "following loop entry should lead to new control");
1267 } else {
1268 for (uint i = 0; i < m->req(); i++) {
1269 if (m->in(i) != NULL && phase->ctrl_or_self(m->in(i)) == c) {
1270 wq.push(m->in(i));
1271 }
1272 }
1273 }
1274 }
1275 return true;
1276 }
1277
1278 bool ShenandoahBarrierNode::is_dominator(Node *d_c, Node *n_c, Node* d, Node* n, PhaseIdealLoop* phase) {
1279 if (d_c != n_c) {
1280 return phase->is_dominator(d_c, n_c);
1281 }
1282 return is_dominator_same_ctrl(d_c, d, n, phase);
1283 }
1284
1285 Node* next_mem(Node* mem, int alias) {
1286 Node* res = NULL;
1287 if (mem->is_Proj()) {
1288 res = mem->in(0);
1289 } else if (mem->is_SafePoint() || mem->is_MemBar()) {
1290 res = mem->in(TypeFunc::Memory);
1291 } else if (mem->is_Phi()) {
1292 res = mem->in(1);
1293 } else if (mem->is_ShenandoahBarrier()) {
1294 res = mem->in(ShenandoahBarrierNode::Memory);
1295 } else if (mem->is_MergeMem()) {
1296 res = mem->as_MergeMem()->memory_at(alias);
1297 } else if (mem->is_Store() || mem->is_LoadStore() || mem->is_ClearArray()) {
1298 assert(alias = Compile::AliasIdxRaw, "following raw memory can't lead to a barrier");
1299 res = mem->in(MemNode::Memory);
1300 } else if (mem->Opcode() == Op_ShenandoahWBMemProj) {
1301 res = mem->in(ShenandoahWBMemProjNode::WriteBarrier);
1302 } else {
1303 #ifdef ASSERT
1304 mem->dump();
1305 #endif
1306 ShouldNotReachHere();
1307 }
1308 return res;
1309 }
1310
1311 Node* ShenandoahBarrierNode::no_branches(Node* c, Node* dom, bool allow_one_proj, PhaseIdealLoop* phase) {
1312 Node* iffproj = NULL;
1313 while (c != dom) {
1314 Node* next = phase->idom(c);
1315 assert(next->unique_ctrl_out() == c || c->is_Proj() || c->is_Region(), "multiple control flow out but no proj or region?");
1316 if (c->is_Region()) {
1317 ResourceMark rm;
1318 Unique_Node_List wq;
1319 wq.push(c);
1320 for (uint i = 0; i < wq.size(); i++) {
1321 Node *n = wq.at(i);
1322 if (n == next) {
1323 continue;
1324 }
1325 if (n->is_Region()) {
1326 for (uint j = 1; j < n->req(); j++) {
1327 wq.push(n->in(j));
1328 }
1329 } else {
1330 wq.push(n->in(0));
1331 }
1356 iffproj = c;
1357 } else {
1358 return NodeSentinel;
1359 }
1360 }
1361 } else if (c->Opcode() == Op_JumpProj) {
1362 return NodeSentinel; // unsupported
1363 } else if (c->Opcode() == Op_CatchProj) {
1364 return NodeSentinel; // unsupported
1365 } else if (c->Opcode() == Op_CProj && next->Opcode() == Op_NeverBranch) {
1366 return NodeSentinel; // unsupported
1367 } else {
1368 assert(next->unique_ctrl_out() == c, "unsupported branch pattern");
1369 }
1370 }
1371 c = next;
1372 }
1373 return iffproj;
1374 }
1375
1376 bool ShenandoahBarrierNode::build_loop_late_post(PhaseIdealLoop* phase, Node* n) {
1377 if (n->Opcode() == Op_ShenandoahReadBarrier ||
1378 n->Opcode() == Op_ShenandoahWriteBarrier ||
1379 n->Opcode() == Op_ShenandoahWBMemProj) {
1380
1381 phase->build_loop_late_post_work(n, false);
1382
1383 if (n->Opcode() == Op_ShenandoahWriteBarrier) {
1384 // The write barrier and its memory proj must have the same
1385 // control otherwise some loop opts could put nodes (Phis) between
1386 // them
1387 Node* proj = n->find_out_with(Op_ShenandoahWBMemProj);
1388 if (proj != NULL) {
1389 phase->set_ctrl_and_loop(proj, phase->get_ctrl(n));
1390 }
1391 }
1392 return true;
1393 }
1394 return false;
1395 }
1396
1397 bool ShenandoahBarrierNode::sink_node(PhaseIdealLoop* phase, Node* ctrl, Node* n_ctrl) {
1398 ctrl = phase->find_non_split_ctrl(ctrl);
1399 assert(phase->dom_depth(n_ctrl) <= phase->dom_depth(ctrl), "n is later than its clone");
1400 set_req(0, ctrl);
1401 phase->register_new_node(this, ctrl);
1402 return true;
1403 }
1404
1405 #ifdef ASSERT
1406 void ShenandoahWriteBarrierNode::memory_dominates_all_paths_helper(Node* c, Node* rep_ctrl, Unique_Node_List& controls, PhaseIdealLoop* phase) {
1407 const bool trace = false;
1408 if (trace) { tty->print("X control is"); c->dump(); }
1409
1410 uint start = controls.size();
1411 controls.push(c);
1412 for (uint i = start; i < controls.size(); i++) {
1413 Node *n = controls.at(i);
1414
1415 if (trace) { tty->print("X from"); n->dump(); }
1416
1417 if (n == rep_ctrl) {
1418 continue;
1419 }
1420
1421 if (n->is_Proj()) {
1422 Node* n_dom = n->in(0);
1423 IdealLoopTree* n_dom_loop = phase->get_loop(n_dom);
1424 if (n->is_IfProj() && n_dom->outcnt() == 2) {
1425 n_dom_loop = phase->get_loop(n_dom->as_If()->proj_out(n->as_Proj()->_con == 0 ? 1 : 0));
1426 }
1427 if (n_dom_loop != phase->ltree_root()) {
1428 Node* tail = n_dom_loop->tail();
1429 if (tail->is_Region()) {
1430 for (uint j = 1; j < tail->req(); j++) {
1431 if (phase->is_dominator(n_dom, tail->in(j)) && !phase->is_dominator(n, tail->in(j))) {
1432 assert(phase->is_dominator(rep_ctrl, tail->in(j)), "why are we here?");
1433 // entering loop from below, mark backedge
1434 if (trace) { tty->print("X pushing backedge"); tail->in(j)->dump(); }
1435 controls.push(tail->in(j));
1436 //assert(n->in(0) == n_dom, "strange flow control");
1437 }
1438 }
1439 } else if (phase->get_loop(n) != n_dom_loop && phase->is_dominator(n_dom, tail)) {
1440 // entering loop from below, mark backedge
1441 if (trace) { tty->print("X pushing backedge"); tail->dump(); }
1442 controls.push(tail);
1443 //assert(n->in(0) == n_dom, "strange flow control");
1444 }
1445 }
1446 }
1447
1448 if (n->is_Loop()) {
1449 Node* c = n->in(LoopNode::EntryControl);
1450 if (trace) { tty->print("X pushing"); c->dump(); }
1451 controls.push(c);
1452 } else if (n->is_Region()) {
1453 for (uint i = 1; i < n->req(); i++) {
1454 Node* c = n->in(i);
1455 if (trace) { tty->print("X pushing"); c->dump(); }
1456 controls.push(c);
1457 }
1458 } else {
1459 Node* c = n->in(0);
1460 if (trace) { tty->print("X pushing"); c->dump(); }
1461 controls.push(c);
1462 }
1463 }
1464 }
1465
1466 bool ShenandoahWriteBarrierNode::memory_dominates_all_paths(Node* mem, Node* rep_ctrl, int alias, PhaseIdealLoop* phase) {
1467 const bool trace = false;
1468 if (trace) {
1469 tty->print("XXX mem is"); mem->dump();
1470 tty->print("XXX rep ctrl is"); rep_ctrl->dump();
1471 tty->print_cr("XXX alias is %d", alias);
1472 }
1473 ResourceMark rm;
1474 Unique_Node_List wq;
1475 Unique_Node_List controls;
1476 wq.push(mem);
1477 for (uint next = 0; next < wq.size(); next++) {
1478 Node *nn = wq.at(next);
1479 if (trace) { tty->print("XX from mem"); nn->dump(); }
1480 assert(nn->bottom_type() == Type::MEMORY, "memory only");
1481
1482 if (nn->is_Phi()) {
1483 Node* r = nn->in(0);
1484 for (DUIterator_Fast jmax, j = r->fast_outs(jmax); j < jmax; j++) {
1485 Node* u = r->fast_out(j);
1486 if (u->is_Phi() && u->bottom_type() == Type::MEMORY && u != nn &&
1487 (u->adr_type() == TypePtr::BOTTOM || phase->C->get_alias_index(u->adr_type()) == alias)) {
1488 if (trace) { tty->print("XX Next mem (other phi)"); u->dump(); }
1489 wq.push(u);
1490 }
1491 }
1492 }
1493
1494 for (DUIterator_Fast imax, i = nn->fast_outs(imax); i < imax; i++) {
1495 Node* use = nn->fast_out(i);
1496
1497 if (trace) { tty->print("XX use %p", use->adr_type()); use->dump(); }
1498 if (use->is_CFG() && use->in(TypeFunc::Memory) == nn) {
1499 Node* c = use->in(0);
1500 if (phase->is_dominator(rep_ctrl, c)) {
1501 memory_dominates_all_paths_helper(c, rep_ctrl, controls, phase);
1502 } else if (use->is_CallStaticJava() && use->as_CallStaticJava()->uncommon_trap_request() != 0 && c->is_Region()) {
1503 Node* region = c;
1504 if (trace) { tty->print("XX unc region"); region->dump(); }
1505 for (uint j = 1; j < region->req(); j++) {
1506 if (phase->is_dominator(rep_ctrl, region->in(j))) {
1507 if (trace) { tty->print("XX unc follows"); region->in(j)->dump(); }
1508 memory_dominates_all_paths_helper(region->in(j), rep_ctrl, controls, phase);
1509 }
1510 }
1511 }
1512 //continue;
1513 } else if (use->is_Phi()) {
1514 assert(use->bottom_type() == Type::MEMORY, "bad phi");
1515 if ((use->adr_type() == TypePtr::BOTTOM) ||
1516 phase->C->get_alias_index(use->adr_type()) == alias) {
1517 for (uint j = 1; j < use->req(); j++) {
1518 if (use->in(j) == nn) {
1519 Node* c = use->in(0)->in(j);
1520 if (phase->is_dominator(rep_ctrl, c)) {
1521 memory_dominates_all_paths_helper(c, rep_ctrl, controls, phase);
1522 }
1523 }
1524 }
1525 }
1526 // continue;
1527 }
1528
1529 if (use->is_MergeMem()) {
1530 if (use->as_MergeMem()->memory_at(alias) == nn) {
1531 if (trace) { tty->print("XX Next mem"); use->dump(); }
1532 // follow the memory edges
1533 wq.push(use);
1534 }
1535 } else if (use->is_Phi()) {
1536 assert(use->bottom_type() == Type::MEMORY, "bad phi");
1537 if ((use->adr_type() == TypePtr::BOTTOM) ||
1538 phase->C->get_alias_index(use->adr_type()) == alias) {
1539 if (trace) { tty->print("XX Next mem"); use->dump(); }
1540 // follow the memory edges
1541 wq.push(use);
1542 }
1543 } else if (use->bottom_type() == Type::MEMORY &&
1544 (use->adr_type() == TypePtr::BOTTOM || phase->C->get_alias_index(use->adr_type()) == alias)) {
1545 if (trace) { tty->print("XX Next mem"); use->dump(); }
1546 // follow the memory edges
1547 wq.push(use);
1548 } else if ((use->is_SafePoint() || use->is_MemBar()) &&
1549 (use->adr_type() == TypePtr::BOTTOM || phase->C->get_alias_index(use->adr_type()) == alias)) {
1550 for (DUIterator_Fast jmax, j = use->fast_outs(jmax); j < jmax; j++) {
1551 Node* u = use->fast_out(j);
1552 if (u->bottom_type() == Type::MEMORY) {
1553 if (trace) { tty->print("XX Next mem"); u->dump(); }
1554 // follow the memory edges
1555 wq.push(u);
1556 }
1557 }
1558 } else if (use->Opcode() == Op_ShenandoahWriteBarrier && phase->C->get_alias_index(use->adr_type()) == alias) {
1559 Node* m = use->find_out_with(Op_ShenandoahWBMemProj);
1560 if (m != NULL) {
1561 if (trace) { tty->print("XX Next mem"); m->dump(); }
1562 // follow the memory edges
1563 wq.push(m);
1564 }
1565 }
1566 }
1567 }
1568
1569 if (controls.size() == 0) {
1570 return false;
1571 }
1572
1573 for (uint i = 0; i < controls.size(); i++) {
1574 Node *n = controls.at(i);
1575
1576 if (trace) { tty->print("X checking"); n->dump(); }
1577
1578 if (n->unique_ctrl_out() != NULL) {
1579 continue;
1580 }
1581
1582 if (n->Opcode() == Op_NeverBranch) {
1583 Node* taken = n->as_Multi()->proj_out(0);
1584 if (!controls.member(taken)) {
1585 if (trace) { tty->print("X not seen"); taken->dump(); }
1586 return false;
1587 }
1588 continue;
1589 }
1590
1591 for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) {
1592 Node* u = n->fast_out(j);
1593
1594 if (u->is_CFG()) {
1595 if (!controls.member(u)) {
1596 if (u->is_Proj() && u->as_Proj()->is_uncommon_trap_proj(Deoptimization::Reason_none)) {
1597 if (trace) { tty->print("X not seen but unc"); u->dump(); }
1598 } else {
1599 Node* c = u;
1600 do {
1601 c = c->unique_ctrl_out();
1602 } while (c != NULL && c->is_Region());
1603 if (c != NULL && c->Opcode() == Op_Halt) {
1604 if (trace) { tty->print("X not seen but halt"); c->dump(); }
1605 } else {
1606 if (trace) { tty->print("X not seen"); u->dump(); }
1607 return false;
1608 }
1609 }
1610 } else {
1611 if (trace) { tty->print("X seen"); u->dump(); }
1612 }
1613 }
1614 }
1615 }
1616 return true;
1617 }
1618 #endif
1619
1620 Node* ShenandoahBarrierNode::dom_mem(Node* mem, Node*& mem_ctrl, Node* n, Node* rep_ctrl, int alias, PhaseIdealLoop* phase) {
1621 ResourceMark rm;
1622 VectorSet wq(Thread::current()->resource_area());
1623 wq.set(mem->_idx);
1624 mem_ctrl = phase->get_ctrl(mem);
1625 while (!is_dominator(mem_ctrl, rep_ctrl, mem, n, phase)) {
1626 mem = next_mem(mem, alias);
1627 if (wq.test_set(mem->_idx)) {
1628 return NULL; // hit an unexpected loop
1629 }
1630 mem_ctrl = phase->ctrl_or_self(mem);
1631 }
1632 if (mem->is_MergeMem()) {
1633 mem = mem->as_MergeMem()->memory_at(alias);
1634 mem_ctrl = phase->ctrl_or_self(mem);
1635 }
1636 return mem;
1637 }
1638
1639 Node* ShenandoahBarrierNode::dom_mem(Node* mem, Node* ctrl, int alias, Node*& mem_ctrl, PhaseIdealLoop* phase) {
1640 ResourceMark rm;
1641 VectorSet wq(Thread::current()->resource_area());
1642 wq.set(mem->_idx);
1643 mem_ctrl = phase->ctrl_or_self(mem);
1644 while (!phase->is_dominator(mem_ctrl, ctrl) || mem_ctrl == ctrl) {
1645 mem = next_mem(mem, alias);
1646 if (wq.test_set(mem->_idx)) {
1647 return NULL;
1648 }
1649 mem_ctrl = phase->ctrl_or_self(mem);
1650 }
1651 if (mem->is_MergeMem()) {
1652 mem = mem->as_MergeMem()->memory_at(alias);
1653 mem_ctrl = phase->ctrl_or_self(mem);
1654 }
1655 return mem;
1656 }
1657
1658 static void disconnect_barrier_mem(Node* wb, PhaseIterGVN& igvn) {
1659 Node* mem_in = wb->in(ShenandoahBarrierNode::Memory);
1660 Node* proj = wb->find_out_with(Op_ShenandoahWBMemProj);
1661
1662 for (DUIterator_Last imin, i = proj->last_outs(imin); i >= imin; ) {
1663 Node* u = proj->last_out(i);
1664 igvn.rehash_node_delayed(u);
1665 int nb = u->replace_edge(proj, mem_in);
1666 assert(nb > 0, "no replacement?");
1667 i -= nb;
1668 }
1669 }
1670
1671 Node* ShenandoahWriteBarrierNode::move_above_predicates(LoopNode* cl, Node* val_ctrl, PhaseIdealLoop* phase) {
1672 Node* entry = cl->skip_strip_mined(-1)->in(LoopNode::EntryControl);
1673 Node* above_pred = phase->skip_all_loop_predicates(entry);
1674 Node* ctrl = entry;
1675 while (ctrl != above_pred) {
1676 Node* next = ctrl->in(0);
1677 if (!phase->is_dominator(val_ctrl, next)) {
1678 break;
1679 }
1680 ctrl = next;
1681 }
1682 return ctrl;
1683 }
1684
1685 static MemoryGraphFixer* find_fixer(GrowableArray<MemoryGraphFixer*>& memory_graph_fixers, int alias) {
1686 for (int i = 0; i < memory_graph_fixers.length(); i++) {
1687 if (memory_graph_fixers.at(i)->alias() == alias) {
1688 return memory_graph_fixers.at(i);
1689 }
1690 }
1691 return NULL;
1692 }
1693
1694 static MemoryGraphFixer* create_fixer(GrowableArray<MemoryGraphFixer*>& memory_graph_fixers, int alias, PhaseIdealLoop* phase, bool include_lsm) {
1695 assert(find_fixer(memory_graph_fixers, alias) == NULL, "none should exist yet");
1696 MemoryGraphFixer* fixer = new MemoryGraphFixer(alias, include_lsm, phase);
1697 memory_graph_fixers.push(fixer);
1698 return fixer;
1699 }
1700
1701 void ShenandoahWriteBarrierNode::try_move_before_loop_helper(LoopNode* cl, Node* val_ctrl, GrowableArray<MemoryGraphFixer*>& memory_graph_fixers, PhaseIdealLoop* phase, bool include_lsm, Unique_Node_List& uses) {
1702 assert(cl->is_Loop(), "bad control");
1703 Node* ctrl = move_above_predicates(cl, val_ctrl, phase);
1704 Node* mem_ctrl = NULL;
1705 int alias = phase->C->get_alias_index(adr_type());
1706
1707 MemoryGraphFixer* fixer = find_fixer(memory_graph_fixers, alias);
1708 if (fixer == NULL) {
1709 fixer = create_fixer(memory_graph_fixers, alias, phase, include_lsm);
1710 }
1711
1712 Node* proj = find_out_with(Op_ShenandoahWBMemProj);
1713
1714 fixer->remove(proj);
1715 Node* mem = fixer->find_mem(ctrl, NULL);
1716
1717 assert(!ShenandoahVerifyOptoBarriers || memory_dominates_all_paths(mem, ctrl, alias, phase), "can't fix the memory graph");
1718
1719 phase->set_ctrl_and_loop(this, ctrl);
1720 phase->igvn().replace_input_of(this, Control, ctrl);
1721
1722 disconnect_barrier_mem(this, phase->igvn());
1723
1724 phase->igvn().replace_input_of(this, Memory, mem);
1725 phase->set_ctrl_and_loop(proj, ctrl);
1726
1727 fixer->fix_mem(ctrl, ctrl, mem, mem, proj, uses);
1728 assert(proj->outcnt() > 0, "disconnected write barrier");
1729 }
1730
1731 LoopNode* ShenandoahWriteBarrierNode::try_move_before_pre_loop(Node* c, Node* val_ctrl, PhaseIdealLoop* phase) {
1732 // A write barrier between a pre and main loop can get in the way of
1733 // vectorization. Move it above the pre loop if possible
1734 CountedLoopNode* cl = NULL;
1735 if (c->is_IfFalse() &&
1736 c->in(0)->is_CountedLoopEnd()) {
1737 cl = c->in(0)->as_CountedLoopEnd()->loopnode();
1738 } else if (c->is_IfProj() &&
1739 c->in(0)->is_If() &&
1740 c->in(0)->in(0)->is_IfFalse() &&
1741 c->in(0)->in(0)->in(0)->is_CountedLoopEnd()) {
1742 cl = c->in(0)->in(0)->in(0)->as_CountedLoopEnd()->loopnode();
1743 }
1744 if (cl != NULL &&
1745 cl->is_pre_loop() &&
1746 val_ctrl != cl &&
1747 phase->is_dominator(val_ctrl, cl)) {
1748 return cl;
1749 }
1750 return NULL;
1751 }
1752
1753 void ShenandoahWriteBarrierNode::try_move_before_loop(GrowableArray<MemoryGraphFixer*>& memory_graph_fixers, PhaseIdealLoop* phase, bool include_lsm, Unique_Node_List& uses) {
1754 Node *n_ctrl = phase->get_ctrl(this);
1755 IdealLoopTree *n_loop = phase->get_loop(n_ctrl);
1756 Node* val = in(ValueIn);
1757 Node* val_ctrl = phase->get_ctrl(val);
1758 if (n_loop != phase->ltree_root() && !n_loop->_irreducible) {
1759 IdealLoopTree *val_loop = phase->get_loop(val_ctrl);
1760 Node* mem = in(Memory);
1761 IdealLoopTree *mem_loop = phase->get_loop(phase->get_ctrl(mem));
1762 if (!n_loop->is_member(val_loop) &&
1763 n_loop->is_member(mem_loop)) {
1764 Node* n_loop_head = n_loop->_head;
1765
1766 if (n_loop_head->is_Loop()) {
1767 LoopNode* loop = n_loop_head->as_Loop();
1768 if (n_loop_head->is_CountedLoop() && n_loop_head->as_CountedLoop()->is_main_loop()) {
1769 LoopNode* res = try_move_before_pre_loop(n_loop_head->in(LoopNode::EntryControl), val_ctrl, phase);
1770 if (res != NULL) {
1771 loop = res;
1772 }
1773 }
1774
1775 try_move_before_loop_helper(loop, val_ctrl, memory_graph_fixers, phase, include_lsm, uses);
1776 }
1777 }
1778 }
1779 LoopNode* ctrl = try_move_before_pre_loop(in(0), val_ctrl, phase);
1780 if (ctrl != NULL) {
1781 try_move_before_loop_helper(ctrl, val_ctrl, memory_graph_fixers, phase, include_lsm, uses);
1782 }
1783 }
1784
1785 Node* ShenandoahWriteBarrierNode::would_subsume(ShenandoahBarrierNode* other, PhaseIdealLoop* phase) {
1786 Node* val = in(ValueIn);
1787 Node* val_ctrl = phase->get_ctrl(val);
1788 Node* other_mem = other->in(Memory);
1789 Node* other_ctrl = phase->get_ctrl(other);
1790 Node* this_ctrl = phase->get_ctrl(this);
1791 IdealLoopTree* this_loop = phase->get_loop(this_ctrl);
1792 IdealLoopTree* other_loop = phase->get_loop(other_ctrl);
1793
1794 Node* ctrl = phase->dom_lca(other_ctrl, this_ctrl);
1795
1796 if (ctrl->is_Proj() &&
1797 ctrl->in(0)->is_Call() &&
1798 ctrl->unique_ctrl_out() != NULL &&
1799 ctrl->unique_ctrl_out()->Opcode() == Op_Catch &&
1800 !phase->is_dominator(val_ctrl, ctrl->in(0)->in(0))) {
1801 return NULL;
1802 }
1803
1804 IdealLoopTree* loop = phase->get_loop(ctrl);
1805
1806 // We don't want to move a write barrier in a loop
1807 // If the LCA is in a inner loop, try a control out of loop if possible
1808 while (!loop->is_member(this_loop) && (other->Opcode() != Op_ShenandoahWriteBarrier || !loop->is_member(other_loop))) {
1809 ctrl = phase->idom(ctrl);
1810 if (ctrl->is_MultiBranch()) {
1811 ctrl = ctrl->in(0);
1812 }
1813 if (ctrl != val_ctrl && phase->is_dominator(ctrl, val_ctrl)) {
1814 return NULL;
1815 }
1816 loop = phase->get_loop(ctrl);
1817 }
1818
1819 if (ShenandoahDontIncreaseWBFreq) {
1820 Node* this_iffproj = no_branches(this_ctrl, ctrl, true, phase);
1821 if (other->Opcode() == Op_ShenandoahWriteBarrier) {
1822 Node* other_iffproj = no_branches(other_ctrl, ctrl, true, phase);
1823 if (other_iffproj == NULL || this_iffproj == NULL) {
1824 return ctrl;
1825 } else if (other_iffproj != NodeSentinel && this_iffproj != NodeSentinel &&
1826 other_iffproj->in(0) == this_iffproj->in(0)) {
1827 return ctrl;
1828 }
1829 } else if (this_iffproj == NULL) {
1830 return ctrl;
1831 }
1832 return NULL;
1833 }
1834
1835 return ctrl;
1836 }
1837
1838 void ShenandoahWriteBarrierNode::optimize_before_expansion(PhaseIdealLoop* phase, GrowableArray<MemoryGraphFixer*> memory_graph_fixers, bool include_lsm) {
1839 bool progress = false;
1840 Unique_Node_List uses;
1841 do {
1842 progress = false;
1843 for (int i = 0; i < ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count(); i++) {
1844 ShenandoahWriteBarrierNode* wb = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barrier(i);
1845
1846 wb->try_move_before_loop(memory_graph_fixers, phase, include_lsm, uses);
1847
1848 Node* val = wb->in(ValueIn);
1849
1850 for (DUIterator_Fast jmax, j = val->fast_outs(jmax); j < jmax; j++) {
1851 Node* u = val->fast_out(j);
1852 if (u != wb && u->is_ShenandoahBarrier()) {
1853 Node* rep_ctrl = wb->would_subsume(u->as_ShenandoahBarrier(), phase);
1854
1855 if (rep_ctrl != NULL) {
1856 Node* other = u;
1857 Node* val_ctrl = phase->get_ctrl(val);
1858 if (rep_ctrl->is_Proj() &&
1859 rep_ctrl->in(0)->is_Call() &&
1860 rep_ctrl->unique_ctrl_out() != NULL &&
1861 rep_ctrl->unique_ctrl_out()->Opcode() == Op_Catch) {
1862 rep_ctrl = rep_ctrl->in(0)->in(0);
1863
1864 assert(phase->is_dominator(val_ctrl, rep_ctrl), "bad control");
1865 } else {
1866 LoopNode* c = ShenandoahWriteBarrierNode::try_move_before_pre_loop(rep_ctrl, val_ctrl, phase);
1867 if (c != NULL) {
1868 rep_ctrl = ShenandoahWriteBarrierNode::move_above_predicates(c, val_ctrl, phase);
1869 } else {
1870 while (rep_ctrl->is_IfProj()) {
1871 CallStaticJavaNode* unc = rep_ctrl->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none);
1872 if (unc != NULL) {
1873 int req = unc->uncommon_trap_request();
1874 Deoptimization::DeoptReason trap_reason = Deoptimization::trap_request_reason(req);
1875 if ((trap_reason == Deoptimization::Reason_loop_limit_check ||
1876 trap_reason == Deoptimization::Reason_predicate ||
1877 trap_reason == Deoptimization::Reason_profile_predicate) &&
1878 phase->is_dominator(val_ctrl, rep_ctrl->in(0)->in(0))) {
1879 rep_ctrl = rep_ctrl->in(0)->in(0);
1880 continue;
1881 }
1882 }
1883 break;
1884 }
1885 }
1886 }
1887
1888 Node* wb_ctrl = phase->get_ctrl(wb);
1889 Node* other_ctrl = phase->get_ctrl(other);
1890 int alias = phase->C->get_alias_index(wb->adr_type());
1891 MemoryGraphFixer* fixer = find_fixer(memory_graph_fixers, alias);;
1892 if (!is_dominator(wb_ctrl, other_ctrl, wb, other, phase)) {
1893 if (fixer == NULL) {
1894 fixer = create_fixer(memory_graph_fixers, alias, phase, include_lsm);
1895 }
1896 Node* mem = fixer->find_mem(rep_ctrl, phase->get_ctrl(other) == rep_ctrl ? other : NULL);
1897
1898 if (mem->has_out_with(Op_Lock) || mem->has_out_with(Op_Unlock)) {
1899 continue;
1900 }
1901
1902 Node* wb_proj = wb->find_out_with(Op_ShenandoahWBMemProj);
1903 fixer->remove(wb_proj);
1904 Node* mem_for_ctrl = fixer->find_mem(rep_ctrl, NULL);
1905
1906 if (wb->in(Memory) != mem) {
1907 disconnect_barrier_mem(wb, phase->igvn());
1908 phase->igvn().replace_input_of(wb, Memory, mem);
1909 }
1910 if (rep_ctrl != wb_ctrl) {
1911 phase->set_ctrl_and_loop(wb, rep_ctrl);
1912 phase->igvn().replace_input_of(wb, Control, rep_ctrl);
1913 phase->set_ctrl_and_loop(wb_proj, rep_ctrl);
1914 progress = true;
1915 }
1916
1917 fixer->fix_mem(rep_ctrl, rep_ctrl, mem, mem_for_ctrl, wb_proj, uses);
1918
1919 assert(!ShenandoahVerifyOptoBarriers || ShenandoahWriteBarrierNode::memory_dominates_all_paths(mem, rep_ctrl, alias, phase), "can't fix the memory graph");
1920 }
1921
1922 if (other->Opcode() == Op_ShenandoahWriteBarrier) {
1923 Node* other_proj = other->find_out_with(Op_ShenandoahWBMemProj);
1924 if (fixer != NULL) {
1925 fixer->remove(other_proj);
1926 }
1927 phase->igvn().replace_node(other_proj, other->in(Memory));
1928 }
1929 phase->igvn().replace_node(other, wb);
1930 --j; --jmax;
1931 }
1932 }
1933 }
1934 }
1935 } while(progress);
1936 }
1937
1938 // Some code duplication with PhaseIdealLoop::split_if_with_blocks_pre()
1939 Node* ShenandoahWriteBarrierNode::try_split_thru_phi(PhaseIdealLoop* phase) {
1940 Node *ctrl = phase->get_ctrl(this);
1941 if (ctrl == NULL) {
1942 return this;
1943 }
1944 Node *blk = phase->has_local_phi_input(this);
1945 if (blk == NULL) {
1946 return this;
1947 }
1948
1949 if (in(0) != blk) {
1950 return this;
1951 }
1952
1953 int policy = blk->req() >> 2;
1954
1955 if (blk->is_CountedLoop()) {
1956 IdealLoopTree *lp = phase->get_loop(blk);
1957 if (lp && lp->_rce_candidate) {
1958 return this;
1959 }
1960 }
1961
1962 if (phase->C->live_nodes() > 35000) {
1963 return this;
1964 }
1965
1966 uint unique = phase->C->unique();
1967 Node *phi = phase->split_thru_phi(this, blk, policy);
1968 if (phi == NULL) {
1969 return this;
1970 }
1971
1972 Node* mem_phi = new PhiNode(blk, Type::MEMORY, phase->C->alias_type(adr_type())->adr_type());
1973 for (uint i = 1; i < blk->req(); i++) {
1974 Node* n = phi->in(i);
1975 if (n->Opcode() == Op_ShenandoahWriteBarrier &&
1976 n->_idx >= unique) {
1977 Node* proj = new ShenandoahWBMemProjNode(n);
1978 phase->register_new_node(proj, phase->get_ctrl(n));
1979 mem_phi->init_req(i, proj);
1980 } else {
1981 Node* mem = in(ShenandoahBarrierNode::Memory);
1982 if (mem->is_Phi() && mem->in(0) == blk) {
1983 mem = mem->in(i);
1984 }
1985 mem_phi->init_req(i, mem);
1986 }
1987 }
1988 phase->register_new_node(mem_phi, blk);
1989
1990
1991 Node* proj = find_out_with(Op_ShenandoahWBMemProj);
1992 phase->igvn().replace_node(proj, mem_phi);
1993 phase->igvn().replace_node(this, phi);
1994
1995 return phi;
1996 }
1997
1998 void ShenandoahReadBarrierNode::try_move(PhaseIdealLoop* phase) {
1999 Node *n_ctrl = phase->get_ctrl(this);
2000 if (n_ctrl == NULL) {
2001 return;
2002 }
2003 Node* mem = in(MemNode::Memory);
2004 int alias = phase->C->get_alias_index(adr_type());
2005 const bool trace = false;
2006
2007 #ifdef ASSERT
2008 if (trace) { tty->print("Trying to move mem of"); dump(); }
2009 #endif
2010
2011 Node* new_mem = mem;
2012
2013 ResourceMark rm;
2014 VectorSet seen(Thread::current()->resource_area());
2015 Node_List phis;
2016
2017 for (;;) {
2018 #ifdef ASSERT
2019 if (trace) { tty->print("Looking for dominator from"); mem->dump(); }
2020 #endif
2021 if (mem->is_Proj() && mem->in(0)->is_Start()) {
2022 if (new_mem != in(MemNode::Memory)) {
2023 #ifdef ASSERT
2024 if (trace) { tty->print("XXX Setting mem to"); new_mem->dump(); tty->print(" for "); dump(); }
2025 #endif
2026 phase->igvn().replace_input_of(this, MemNode::Memory, new_mem);
2027 }
2028 return;
2029 }
2030
2031 Node* candidate = mem;
2032 do {
2033 if (!is_independent(mem)) {
2034 if (trace) { tty->print_cr("Not independent"); }
2035 if (new_mem != in(MemNode::Memory)) {
2036 #ifdef ASSERT
2037 if (trace) { tty->print("XXX Setting mem to"); new_mem->dump(); tty->print(" for "); dump(); }
2038 #endif
2039 phase->igvn().replace_input_of(this, MemNode::Memory, new_mem);
2040 }
2041 return;
2042 }
2043 if (seen.test_set(mem->_idx)) {
2044 if (trace) { tty->print_cr("Already seen"); }
2045 ShouldNotReachHere();
2046 // Strange graph
2047 if (new_mem != in(MemNode::Memory)) {
2048 #ifdef ASSERT
2049 if (trace) { tty->print("XXX Setting mem to"); new_mem->dump(); tty->print(" for "); dump(); }
2050 #endif
2051 phase->igvn().replace_input_of(this, MemNode::Memory, new_mem);
2052 }
2053 return;
2054 }
2055 if (mem->is_Phi()) {
2056 phis.push(mem);
2057 }
2058 mem = next_mem(mem, alias);
2059 if (mem->bottom_type() == Type::MEMORY) {
2060 candidate = mem;
2061 }
2062 assert(is_dominator(phase->ctrl_or_self(mem), n_ctrl, mem, this, phase) == phase->is_dominator(phase->ctrl_or_self(mem), n_ctrl), "strange dominator");
2063 #ifdef ASSERT
2064 if (trace) { tty->print("Next mem is"); mem->dump(); }
2065 #endif
2066 } while (mem->bottom_type() != Type::MEMORY || !phase->is_dominator(phase->ctrl_or_self(mem), n_ctrl));
2067
2068 assert(mem->bottom_type() == Type::MEMORY, "bad mem");
2069
2070 bool not_dom = false;
2071 for (uint i = 0; i < phis.size() && !not_dom; i++) {
2072 Node* nn = phis.at(i);
2073
2074 #ifdef ASSERT
2075 if (trace) { tty->print("Looking from phi"); nn->dump(); }
2076 #endif
2077 assert(nn->is_Phi(), "phis only");
2078 for (uint j = 2; j < nn->req() && !not_dom; j++) {
2079 Node* m = nn->in(j);
2080 #ifdef ASSERT
2081 if (trace) { tty->print("Input %d is", j); m->dump(); }
2082 #endif
2083 while (m != mem && !seen.test_set(m->_idx)) {
2084 if (is_dominator(phase->ctrl_or_self(m), phase->ctrl_or_self(mem), m, mem, phase)) {
2085 not_dom = true;
2086 // Scheduling anomaly
2087 #ifdef ASSERT
2088 if (trace) { tty->print("Giving up"); m->dump(); }
2089 #endif
2090 break;
2091 }
2092 if (!is_independent(m)) {
2093 if (trace) { tty->print_cr("Not independent"); }
2094 if (new_mem != in(MemNode::Memory)) {
2095 #ifdef ASSERT
2096 if (trace) { tty->print("XXX Setting mem to"); new_mem->dump(); tty->print(" for "); dump(); }
2097 #endif
2098 phase->igvn().replace_input_of(this, MemNode::Memory, new_mem);
2099 }
2100 return;
2101 }
2102 if (m->is_Phi()) {
2103 phis.push(m);
2104 }
2105 m = next_mem(m, alias);
2106 #ifdef ASSERT
2107 if (trace) { tty->print("Next mem is"); m->dump(); }
2108 #endif
2109 }
2110 }
2111 }
2112 if (!not_dom) {
2113 new_mem = mem;
2114 phis.clear();
2115 } else {
2116 seen.Clear();
2117 }
2118 }
2119 }
2120
2121 CallStaticJavaNode* ShenandoahWriteBarrierNode::pin_and_expand_null_check(PhaseIterGVN& igvn) {
2122 Node* val = in(ValueIn);
2123
2124 const Type* val_t = igvn.type(val);
2125
2126 if (val_t->meet(TypePtr::NULL_PTR) != val_t &&
2127 val->Opcode() == Op_CastPP &&
2128 val->in(0) != NULL &&
2129 val->in(0)->Opcode() == Op_IfTrue &&
2130 val->in(0)->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none) &&
2131 val->in(0)->in(0)->is_If() &&
2132 val->in(0)->in(0)->in(1)->Opcode() == Op_Bool &&
2133 val->in(0)->in(0)->in(1)->as_Bool()->_test._test == BoolTest::ne &&
2134 val->in(0)->in(0)->in(1)->in(1)->Opcode() == Op_CmpP &&
2135 val->in(0)->in(0)->in(1)->in(1)->in(1) == val->in(1) &&
2136 val->in(0)->in(0)->in(1)->in(1)->in(2)->bottom_type() == TypePtr::NULL_PTR) {
2137 assert(val->in(0)->in(0)->in(1)->in(1)->in(1) == val->in(1), "");
2138 CallStaticJavaNode* unc = val->in(0)->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none);
2139 return unc;
2140 }
2141 return NULL;
2142 }
2143
2144 void ShenandoahWriteBarrierNode::pin_and_expand_move_barrier(PhaseIdealLoop* phase, GrowableArray<MemoryGraphFixer*>& memory_graph_fixers, Unique_Node_List& uses) {
2145 Node* unc = pin_and_expand_null_check(phase->igvn());
2146 Node* val = in(ValueIn);
2147
2148 if (unc != NULL) {
2149 Node* ctrl = phase->get_ctrl(this);
2150 Node* unc_ctrl = val->in(0);
2151
2152 // Don't move write barrier in a loop
2153 IdealLoopTree* loop = phase->get_loop(ctrl);
2154 IdealLoopTree* unc_loop = phase->get_loop(unc_ctrl);
2155
2156 if (!unc_loop->is_member(loop)) {
2157 return;
2158 }
2159
2160 Node* branch = no_branches(ctrl, unc_ctrl, false, phase);
2161 assert(branch == NULL || branch == NodeSentinel, "was not looking for a branch");
2162 if (branch == NodeSentinel) {
2163 return;
2164 }
2165
2166 RegionNode* r = new RegionNode(3);
2167 IfNode* iff = unc_ctrl->in(0)->as_If();
2168
2169 Node* ctrl_use = unc_ctrl->unique_ctrl_out();
2170 Node* unc_ctrl_clone = unc_ctrl->clone();
2171 phase->register_control(unc_ctrl_clone, loop, iff);
2172 Node* c = unc_ctrl_clone;
2173 Node* new_cast = clone_null_check(c, val, unc_ctrl_clone, phase);
2174 r->init_req(1, new_cast->in(0)->in(0)->as_If()->proj_out(0));
2175
2176 phase->igvn().replace_input_of(unc_ctrl, 0, c->in(0));
2177 phase->set_idom(unc_ctrl, c->in(0), phase->dom_depth(unc_ctrl));
2178 phase->lazy_replace(c, unc_ctrl);
2179 c = NULL;;
2180 phase->igvn().replace_input_of(val, 0, unc_ctrl_clone);
2181 phase->set_ctrl(val, unc_ctrl_clone);
2182
2183 IfNode* new_iff = new_cast->in(0)->in(0)->as_If();
2184 fix_null_check(unc, unc_ctrl_clone, r, uses, phase);
2185 Node* iff_proj = iff->proj_out(0);
2186 r->init_req(2, iff_proj);
2187 phase->register_control(r, phase->ltree_root(), iff);
2188
2189 Node* new_bol = new_iff->in(1)->clone();
2190 Node* new_cmp = new_bol->in(1)->clone();
2191 assert(new_cmp->Opcode() == Op_CmpP, "broken");
2192 assert(new_cmp->in(1) == val->in(1), "broken");
2193 new_bol->set_req(1, new_cmp);
2194 new_cmp->set_req(1, this);
2195 phase->register_new_node(new_bol, new_iff->in(0));
2196 phase->register_new_node(new_cmp, new_iff->in(0));
2197 phase->igvn().replace_input_of(new_iff, 1, new_bol);
2198 phase->igvn().replace_input_of(new_cast, 1, this);
2199
2200 for (DUIterator_Fast imax, i = this->fast_outs(imax); i < imax; i++) {
2201 Node* u = this->fast_out(i);
2202 if (u == new_cast || u->Opcode() == Op_ShenandoahWBMemProj || u == new_cmp) {
2203 continue;
2204 }
2205 phase->igvn().rehash_node_delayed(u);
2206 int nb = u->replace_edge(this, new_cast);
2207 assert(nb > 0, "no update?");
2208 --i; imax -= nb;
2209 }
2210
2211 for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) {
2212 Node* u = val->fast_out(i);
2213 if (u == this) {
2214 continue;
2215 }
2216 phase->igvn().rehash_node_delayed(u);
2217 int nb = u->replace_edge(val, new_cast);
2218 assert(nb > 0, "no update?");
2219 --i; imax -= nb;
2220 }
2221
2222 Node* new_ctrl = unc_ctrl_clone;
2223
2224 int alias = phase->C->get_alias_index(adr_type());
2225 MemoryGraphFixer* fixer = find_fixer(memory_graph_fixers, alias);
2226 if (fixer == NULL) {
2227 fixer = create_fixer(memory_graph_fixers, alias, phase, true);
2228 }
2229
2230 Node* proj = find_out_with(Op_ShenandoahWBMemProj);
2231 fixer->remove(proj);
2232 Node* mem = fixer->find_mem(new_ctrl, NULL);
2233
2234 if (in(Memory) != mem) {
2235 disconnect_barrier_mem(this, phase->igvn());
2236 phase->igvn().replace_input_of(this, Memory, mem);
2237 }
2238
2239 phase->set_ctrl_and_loop(this, new_ctrl);
2240 phase->igvn().replace_input_of(this, Control, new_ctrl);
2241 phase->set_ctrl_and_loop(proj, new_ctrl);
2242
2243 fixer->fix_mem(new_ctrl, new_ctrl, mem, mem, proj, uses);
2244 }
2245 }
2246
2247 void ShenandoahWriteBarrierNode::pin_and_expand_helper(PhaseIdealLoop* phase) {
2248 Node* val = in(ValueIn);
2249 CallStaticJavaNode* unc = pin_and_expand_null_check(phase->igvn());
2250 Node* rep = this;
2251 Node* ctrl = phase->get_ctrl(this);
2252 if (unc != NULL && val->in(0) == ctrl) {
2253 Node* unc_ctrl = val->in(0);
2254 IfNode* other_iff = unc_ctrl->unique_ctrl_out()->as_If();
2255 ProjNode* other_unc_ctrl = other_iff->proj_out(1);
2256 Node* cast = NULL;
2257 for (DUIterator_Fast imax, i = other_unc_ctrl->fast_outs(imax); i < imax && cast == NULL; i++) {
2258 Node* u = other_unc_ctrl->fast_out(i);
2259 if (u->Opcode() == Op_CastPP && u->in(1) == this) {
2260 cast = u;
2261 }
2262 }
2263 assert(other_unc_ctrl->is_uncommon_trap_if_pattern(Deoptimization::Reason_none) == unc, "broken");
2264 rep = cast;
2265 }
2266
2267 // Replace all uses of barrier's input that are dominated by ctrl
2268 // with the value returned by the barrier: no need to keep both
2269 // live.
2270 for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) {
2271 Node* u = val->fast_out(i);
2272 if (u != this) {
2273 if (u->is_Phi()) {
2274 int nb = 0;
2275 for (uint j = 1; j < u->req(); j++) {
2276 if (u->in(j) == val) {
2277 Node* c = u->in(0)->in(j);
2278 if (phase->is_dominator(ctrl, c)) {
2279 phase->igvn().replace_input_of(u, j, rep);
2280 nb++;
2281 }
2282 }
2283 }
2284 if (nb > 0) {
2285 imax -= nb;
2286 --i;
2287 }
2288 } else {
2289 Node* c = phase->ctrl_or_self(u);
2290 if (is_dominator(ctrl, c, this, u, phase)) {
2291 phase->igvn().rehash_node_delayed(u);
2292 int nb = u->replace_edge(val, rep);
2293 assert(nb > 0, "no update?");
2294 --i, imax -= nb;
2295 }
2296 }
2297 }
2298 }
2299 }
2300
2301 Node* ShenandoahWriteBarrierNode::find_bottom_mem(Node* ctrl, PhaseIdealLoop* phase) {
2302 Node* mem = NULL;
2303 Node* c = ctrl;
2304 do {
2305 if (c->is_Region()) {
2306 Node* phi_bottom = NULL;
2307 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax && mem == NULL; i++) {
2308 Node* u = c->fast_out(i);
2309 if (u->is_Phi() && u->bottom_type() == Type::MEMORY) {
2310 if (u->adr_type() == TypePtr::BOTTOM) {
2311 mem = u;
2312 }
2313 }
2314 }
2315 } else {
2316 if (c->is_Call() && c->as_Call()->adr_type() != NULL) {
2317 CallProjections projs;
2318 c->as_Call()->extract_projections(&projs, true, false);
2319 if (projs.fallthrough_memproj != NULL) {
2320 if (projs.fallthrough_memproj->adr_type() == TypePtr::BOTTOM) {
2321 if (projs.catchall_memproj == NULL) {
2322 mem = projs.fallthrough_memproj;
2323 } else {
2324 if (phase->is_dominator(projs.fallthrough_catchproj, ctrl)) {
2325 mem = projs.fallthrough_memproj;
2326 } else {
2327 assert(phase->is_dominator(projs.catchall_catchproj, ctrl), "one proj must dominate barrier");
2328 mem = projs.catchall_memproj;
2329 }
2330 }
2331 }
2332 } else {
2333 Node* proj = c->as_Call()->proj_out(TypeFunc::Memory);
2334 if (proj != NULL &&
2335 proj->adr_type() == TypePtr::BOTTOM) {
2336 mem = proj;
2337 }
2338 }
2339 } else {
2340 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) {
2341 Node* u = c->fast_out(i);
2342 if (u->is_Proj() &&
2343 u->bottom_type() == Type::MEMORY &&
2344 u->adr_type() == TypePtr::BOTTOM) {
2345 assert(c->is_SafePoint() || c->is_MemBar() || c->is_Start(), "");
2346 assert(mem == NULL, "only one proj");
2347 mem = u;
2348 }
2349 }
2350 assert(!c->is_Call() || c->as_Call()->adr_type() != NULL || mem == NULL, "no mem projection expected");
2351 }
2352 }
2353 c = phase->idom(c);
2354 } while (mem == NULL);
2355 return mem;
2356 }
2357
2358 void ShenandoahWriteBarrierNode::follow_barrier_uses(Node* n, Node* ctrl, Unique_Node_List& uses, PhaseIdealLoop* phase) {
2359 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
2360 Node* u = n->fast_out(i);
2361 if (!u->is_CFG() && phase->get_ctrl(u) == ctrl && (!u->is_Phi() || !u->in(0)->is_Loop() || u->in(LoopNode::LoopBackControl) != n)) {
2362 uses.push(u);
2363 }
2364 }
2365 }
2366
2367 static void hide_strip_mined_loop(OuterStripMinedLoopNode* outer, CountedLoopNode* inner, PhaseIdealLoop* phase) {
2368 OuterStripMinedLoopEndNode* le = inner->outer_loop_end();
2369 Node* new_outer = new LoopNode(outer->in(LoopNode::EntryControl), outer->in(LoopNode::LoopBackControl));
2370 phase->register_control(new_outer, phase->get_loop(outer), outer->in(LoopNode::EntryControl));
2371 Node* new_le = new IfNode(le->in(0), le->in(1), le->_prob, le->_fcnt);
2372 phase->register_control(new_le, phase->get_loop(le), le->in(0));
2373 phase->lazy_replace(outer, new_outer);
2374 phase->lazy_replace(le, new_le);
2375 inner->clear_strip_mined();
2376 }
2377
2378 void ShenandoahWriteBarrierNode::test_heap_stable(Node*& ctrl, Node* raw_mem, Node*& heap_stable_ctrl,
2379 PhaseIdealLoop* phase) {
2380 IdealLoopTree* loop = phase->get_loop(ctrl);
2381 Node* thread = new ThreadLocalNode();
2382 phase->register_new_node(thread, ctrl);
2383 Node* offset = phase->igvn().MakeConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
2384 phase->set_ctrl(offset, phase->C->root());
2385 Node* gc_state_addr = new AddPNode(phase->C->top(), thread, offset);
2386 phase->register_new_node(gc_state_addr, ctrl);
2387 uint gc_state_idx = Compile::AliasIdxRaw;
2388 const TypePtr* gc_state_adr_type = NULL; // debug-mode-only argument
2389 debug_only(gc_state_adr_type = phase->C->get_adr_type(gc_state_idx));
2390
2391 Node* gc_state = new LoadBNode(ctrl, raw_mem, gc_state_addr, gc_state_adr_type, TypeInt::BYTE, MemNode::unordered);
2392 phase->register_new_node(gc_state, ctrl);
2393 Node* heap_stable_and = new AndINode(gc_state, phase->igvn().intcon(ShenandoahHeap::HAS_FORWARDED));
2394 phase->register_new_node(heap_stable_and, ctrl);
2395 Node* heap_stable_cmp = new CmpINode(heap_stable_and, phase->igvn().zerocon(T_INT));
2396 phase->register_new_node(heap_stable_cmp, ctrl);
2397 Node* heap_stable_test = new BoolNode(heap_stable_cmp, BoolTest::ne);
2398 phase->register_new_node(heap_stable_test, ctrl);
2399 IfNode* heap_stable_iff = new IfNode(ctrl, heap_stable_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN);
2400 phase->register_control(heap_stable_iff, loop, ctrl);
2401
2402 heap_stable_ctrl = new IfFalseNode(heap_stable_iff);
2403 phase->register_control(heap_stable_ctrl, loop, heap_stable_iff);
2404 ctrl = new IfTrueNode(heap_stable_iff);
2405 phase->register_control(ctrl, loop, heap_stable_iff);
2406
2407 assert(is_heap_stable_test(heap_stable_iff), "Should match the shape");
2408 }
2409
2410 void ShenandoahWriteBarrierNode::test_null(Node*& ctrl, Node* val, Node*& null_ctrl, PhaseIdealLoop* phase) {
2411 const Type* val_t = phase->igvn().type(val);
2412 if (val_t->meet(TypePtr::NULL_PTR) == val_t) {
2413 IdealLoopTree* loop = phase->get_loop(ctrl);
2414 Node* null_cmp = new CmpPNode(val, phase->igvn().zerocon(T_OBJECT));
2415 phase->register_new_node(null_cmp, ctrl);
2416 Node* null_test = new BoolNode(null_cmp, BoolTest::ne);
2417 phase->register_new_node(null_test, ctrl);
2418 IfNode* null_iff = new IfNode(ctrl, null_test, PROB_LIKELY(0.999), COUNT_UNKNOWN);
2419 phase->register_control(null_iff, loop, ctrl);
2420 ctrl = new IfTrueNode(null_iff);
2421 phase->register_control(ctrl, loop, null_iff);
2422 null_ctrl = new IfFalseNode(null_iff);
2423 phase->register_control(null_ctrl, loop, null_iff);
2424 }
2425 }
2426
2427 Node* ShenandoahWriteBarrierNode::clone_null_check(Node*& c, Node* val, Node* unc_ctrl, PhaseIdealLoop* phase) {
2428 IdealLoopTree *loop = phase->get_loop(c);
2429 Node* iff = unc_ctrl->in(0);
2430 assert(iff->is_If(), "broken");
2431 Node* new_iff = iff->clone();
2432 new_iff->set_req(0, c);
2433 phase->register_control(new_iff, loop, c);
2434 Node* iffalse = new IfFalseNode(new_iff->as_If());
2435 phase->register_control(iffalse, loop, new_iff);
2436 Node* iftrue = new IfTrueNode(new_iff->as_If());
2437 phase->register_control(iftrue, loop, new_iff);
2438 c = iftrue;
2439 const Type *t = phase->igvn().type(val);
2440 assert(val->Opcode() == Op_CastPP, "expect cast to non null here");
2441 Node* uncasted_val = val->in(1);
2442 val = new CastPPNode(uncasted_val, t);
2443 val->init_req(0, c);
2444 phase->register_new_node(val, c);
2445 return val;
2446 }
2447
2448 void ShenandoahWriteBarrierNode::fix_null_check(Node* unc, Node* unc_ctrl, Node* new_unc_ctrl,
2449 Unique_Node_List& uses, PhaseIdealLoop* phase) {
2450 IfNode* iff = unc_ctrl->in(0)->as_If();
2451 Node* proj = iff->proj_out(0);
2452 assert(proj != unc_ctrl, "bad projection");
2453 Node* use = proj->unique_ctrl_out();
2454
2455 assert(use == unc || use->is_Region(), "what else?");
2456
2457 uses.clear();
2458 if (use == unc) {
2459 phase->set_idom(use, new_unc_ctrl, phase->dom_depth(use));
2460 for (uint i = 1; i < unc->req(); i++) {
2461 Node* n = unc->in(i);
2462 if (phase->has_ctrl(n) && phase->get_ctrl(n) == proj) {
2463 uses.push(n);
2464 }
2465 }
2466 } else {
2467 assert(use->is_Region(), "what else?");
2468 uint idx = 1;
2477 for(uint next = 0; next < uses.size(); next++ ) {
2478 Node *n = uses.at(next);
2479 assert(phase->get_ctrl(n) == proj, "bad control");
2480 phase->set_ctrl_and_loop(n, new_unc_ctrl);
2481 if (n->in(0) == proj) {
2482 phase->igvn().replace_input_of(n, 0, new_unc_ctrl);
2483 }
2484 for (uint i = 0; i < n->req(); i++) {
2485 Node* m = n->in(i);
2486 if (m != NULL && phase->has_ctrl(m) && phase->get_ctrl(m) == proj) {
2487 uses.push(m);
2488 }
2489 }
2490 }
2491
2492 phase->igvn().rehash_node_delayed(use);
2493 int nb = use->replace_edge(proj, new_unc_ctrl);
2494 assert(nb == 1, "only use expected");
2495 }
2496
2497 void ShenandoahWriteBarrierNode::in_cset_fast_test(Node*& ctrl, Node*& not_cset_ctrl, Node* val, Node* raw_mem, PhaseIdealLoop* phase) {
2498 IdealLoopTree *loop = phase->get_loop(ctrl);
2499 Node* raw_rbtrue = new CastP2XNode(ctrl, val);
2500 phase->register_new_node(raw_rbtrue, ctrl);
2501 Node* cset_offset = new URShiftXNode(raw_rbtrue, phase->igvn().intcon(ShenandoahHeapRegion::region_size_bytes_shift_jint()));
2502 phase->register_new_node(cset_offset, ctrl);
2503 Node* in_cset_fast_test_base_addr = phase->igvn().makecon(TypeRawPtr::make(ShenandoahHeap::in_cset_fast_test_addr()));
2504 phase->set_ctrl(in_cset_fast_test_base_addr, phase->C->root());
2505 Node* in_cset_fast_test_adr = new AddPNode(phase->C->top(), in_cset_fast_test_base_addr, cset_offset);
2506 phase->register_new_node(in_cset_fast_test_adr, ctrl);
2507 uint in_cset_fast_test_idx = Compile::AliasIdxRaw;
2508 const TypePtr* in_cset_fast_test_adr_type = NULL; // debug-mode-only argument
2509 debug_only(in_cset_fast_test_adr_type = phase->C->get_adr_type(in_cset_fast_test_idx));
2510 Node* in_cset_fast_test_load = new LoadBNode(ctrl, raw_mem, in_cset_fast_test_adr, in_cset_fast_test_adr_type, TypeInt::BYTE, MemNode::unordered);
2511 phase->register_new_node(in_cset_fast_test_load, ctrl);
2512 Node* in_cset_fast_test_cmp = new CmpINode(in_cset_fast_test_load, phase->igvn().zerocon(T_INT));
2513 phase->register_new_node(in_cset_fast_test_cmp, ctrl);
2514 Node* in_cset_fast_test_test = new BoolNode(in_cset_fast_test_cmp, BoolTest::eq);
2515 phase->register_new_node(in_cset_fast_test_test, ctrl);
2516 IfNode* in_cset_fast_test_iff = new IfNode(ctrl, in_cset_fast_test_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN);
2517 phase->register_control(in_cset_fast_test_iff, loop, ctrl);
2518
2519 not_cset_ctrl = new IfTrueNode(in_cset_fast_test_iff);
2520 phase->register_control(not_cset_ctrl, loop, in_cset_fast_test_iff);
2521
2522 ctrl = new IfFalseNode(in_cset_fast_test_iff);
2523 phase->register_control(ctrl, loop, in_cset_fast_test_iff);
2524 }
2525
2526 void ShenandoahWriteBarrierNode::call_wb_stub(Node*& ctrl, Node*& val, Node*& result_mem,
2527 Node* raw_mem, Node* wb_mem,
2528 int alias,
2529 PhaseIdealLoop* phase) {
2530 IdealLoopTree*loop = phase->get_loop(ctrl);
2531 const TypePtr* obj_type = phase->igvn().type(val)->is_oopptr()->cast_to_nonconst();
2532
2533 // The slow path stub consumes and produces raw memory in addition
2534 // to the existing memory edges
2535 Node* base = find_bottom_mem(ctrl, phase);
2536
2537 MergeMemNode* mm = MergeMemNode::make(base);
2538 mm->set_memory_at(alias, wb_mem);
2539 mm->set_memory_at(Compile::AliasIdxRaw, raw_mem);
2540 phase->register_new_node(mm, ctrl);
2541
2542 Node* call = new CallLeafNode(ShenandoahBarrierSetC2::shenandoah_write_barrier_Type(), CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_JRT), "shenandoah_write_barrier", TypeRawPtr::BOTTOM);
2543 call->init_req(TypeFunc::Control, ctrl);
2544 call->init_req(TypeFunc::I_O, phase->C->top());
2545 call->init_req(TypeFunc::Memory, mm);
2546 call->init_req(TypeFunc::FramePtr, phase->C->top());
2547 call->init_req(TypeFunc::ReturnAdr, phase->C->top());
2548 call->init_req(TypeFunc::Parms, val);
2549 phase->register_control(call, loop, ctrl);
2550 ctrl = new ProjNode(call, TypeFunc::Control);
2551 phase->register_control(ctrl, loop, call);
2552 result_mem = new ProjNode(call, TypeFunc::Memory);
2553 phase->register_new_node(result_mem, call);
2554 val = new ProjNode(call, TypeFunc::Parms);
2555 phase->register_new_node(val, call);
2556 val = new CheckCastPPNode(ctrl, val, obj_type);
2557 phase->register_new_node(val, ctrl);
2558 }
2559
2560 void ShenandoahWriteBarrierNode::fix_ctrl(Node* barrier, Node* region, const MemoryGraphFixer& fixer, Unique_Node_List& uses, Unique_Node_List& uses_to_ignore, uint last, PhaseIdealLoop* phase) {
2561 Node* ctrl = phase->get_ctrl(barrier);
2562 Node* init_raw_mem = fixer.find_mem(ctrl, barrier);
2563
2564 // Update the control of all nodes that should be after the
2565 // barrier control flow
2566 uses.clear();
2567 // Every node that is control dependent on the barrier's input
2568 // control will be after the expanded barrier. The raw memory (if
2569 // its memory is control dependent on the barrier's input control)
2570 // must stay above the barrier.
2571 uses_to_ignore.clear();
2572 if (phase->has_ctrl(init_raw_mem) && phase->get_ctrl(init_raw_mem) == ctrl && !init_raw_mem->is_Phi()) {
2573 uses_to_ignore.push(init_raw_mem);
2574 }
2575 for (uint next = 0; next < uses_to_ignore.size(); next++) {
2576 Node *n = uses_to_ignore.at(next);
2577 for (uint i = 0; i < n->req(); i++) {
2578 Node* in = n->in(i);
2579 if (in != NULL && phase->has_ctrl(in) && phase->get_ctrl(in) == ctrl) {
2580 uses_to_ignore.push(in);
2593 if (c != ctrl ||
2594 is_dominator_same_ctrl(old_c, barrier, u, phase) ||
2595 ShenandoahBarrierSetC2::is_shenandoah_state_load(u)) {
2596 phase->igvn().rehash_node_delayed(u);
2597 int nb = u->replace_edge(ctrl, region);
2598 if (u->is_CFG()) {
2599 if (phase->idom(u) == ctrl) {
2600 phase->set_idom(u, region, phase->dom_depth(region));
2601 }
2602 } else if (phase->get_ctrl(u) == ctrl) {
2603 assert(u != init_raw_mem, "should leave input raw mem above the barrier");
2604 uses.push(u);
2605 }
2606 assert(nb == 1, "more than 1 ctrl input?");
2607 --i, imax -= nb;
2608 }
2609 }
2610 }
2611 }
2612
2613 void ShenandoahWriteBarrierNode::pin_and_expand(PhaseIdealLoop* phase) {
2614 Node_List enqueue_barriers;
2615 if (ShenandoahStoreValEnqueueBarrier) {
2616 Unique_Node_List wq;
2617 wq.push(phase->C->root());
2618 for (uint i = 0; i < wq.size(); i++) {
2619 Node* n = wq.at(i);
2620 if (n->Opcode() == Op_ShenandoahEnqueueBarrier) {
2621 enqueue_barriers.push(n);
2622 }
2623 for (uint i = 0; i < n->req(); i++) {
2624 Node* in = n->in(i);
2625 if (in != NULL) {
2626 wq.push(in);
2627 }
2628 }
2629 }
2630 }
2631
2632 const bool trace = false;
2633
2634 // Collect raw memory state at CFG points in the entire graph and
2635 // record it in memory_nodes. Optimize the raw memory graph in the
2636 // process. Optimizing the memory graph also makes the memory graph
2637 // simpler.
2638 GrowableArray<MemoryGraphFixer*> memory_graph_fixers;
2639
2640 // Let's try to common write barriers again
2641 optimize_before_expansion(phase, memory_graph_fixers, true);
2642
2643 Unique_Node_List uses;
2644 for (int i = 0; i < ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count(); i++) {
2645 ShenandoahWriteBarrierNode* wb = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barrier(i);
2646 Node* ctrl = phase->get_ctrl(wb);
2647
2648 Node* val = wb->in(ValueIn);
2649 if (ctrl->is_Proj() && ctrl->in(0)->is_CallJava()) {
2650 assert(is_dominator(phase->get_ctrl(val), ctrl->in(0)->in(0), val, ctrl->in(0), phase), "can't move");
2651 phase->set_ctrl(wb, ctrl->in(0)->in(0));
2652 } else if (ctrl->is_CallRuntime()) {
2653 assert(is_dominator(phase->get_ctrl(val), ctrl->in(0), val, ctrl, phase), "can't move");
2654 phase->set_ctrl(wb, ctrl->in(0));
2655 }
2656
2657 assert(wb->Opcode() == Op_ShenandoahWriteBarrier, "only for write barriers");
2658 // Look for a null check that dominates this barrier and move the
2659 // barrier right after the null check to enable implicit null
2660 // checks
2661 wb->pin_and_expand_move_barrier(phase, memory_graph_fixers, uses);
2662
2663 wb->pin_and_expand_helper(phase);
2664 }
2665
2666 for (uint i = 0; i < enqueue_barriers.size(); i++) {
2667 Node* barrier = enqueue_barriers.at(i);
2668 Node* ctrl = phase->get_ctrl(barrier);
2669 IdealLoopTree* loop = phase->get_loop(ctrl);
2670 if (loop->_head->is_OuterStripMinedLoop()) {
2671 // Expanding a barrier here will break loop strip mining
2672 // verification. Transform the loop so the loop nest doesn't
2673 // appear as strip mined.
2674 OuterStripMinedLoopNode* outer = loop->_head->as_OuterStripMinedLoop();
2675 hide_strip_mined_loop(outer, outer->unique_ctrl_out()->as_CountedLoop(), phase);
2676 }
2677 }
2678
2679 for (int i = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count(); i > 0; i--) {
2680 int cnt = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count();
2681 ShenandoahWriteBarrierNode* wb = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barrier(i-1);
2682 Node* ctrl = phase->get_ctrl(wb);
2683 IdealLoopTree* loop = phase->get_loop(ctrl);
2684 if (loop->_head->is_OuterStripMinedLoop()) {
2685 // Expanding a barrier here will break loop strip mining
2686 // verification. Transform the loop so the loop nest doesn't
2687 // appear as strip mined.
2688 OuterStripMinedLoopNode* outer = loop->_head->as_OuterStripMinedLoop();
2689 hide_strip_mined_loop(outer, outer->unique_ctrl_out()->as_CountedLoop(), phase);
2690 }
2691 }
2692
2693 MemoryGraphFixer fixer(Compile::AliasIdxRaw, true, phase);
2694 Unique_Node_List uses_to_ignore;
2695 for (uint i = 0; i < enqueue_barriers.size(); i++) {
2696 Node* barrier = enqueue_barriers.at(i);
2697 Node* pre_val = barrier->in(1);
2698
2699 if (phase->igvn().type(pre_val)->higher_equal(TypePtr::NULL_PTR)) {
2700 ShouldNotReachHere();
2701 continue;
2702 }
2703
2704 Node* ctrl = phase->get_ctrl(barrier);
2705
2706 if (ctrl->is_Proj() && ctrl->in(0)->is_CallJava()) {
2707 assert(is_dominator(phase->get_ctrl(pre_val), ctrl->in(0)->in(0), pre_val, ctrl->in(0), phase), "can't move");
2708 ctrl = ctrl->in(0)->in(0);
2709 phase->set_ctrl(barrier, ctrl);
2710 } else if (ctrl->is_CallRuntime()) {
2711 assert(is_dominator(phase->get_ctrl(pre_val), ctrl->in(0), pre_val, ctrl, phase), "can't move");
2712 ctrl = ctrl->in(0);
2713 phase->set_ctrl(barrier, ctrl);
2714 }
2715
2716 Node* init_ctrl = ctrl;
2753 phase->register_new_node(thread, ctrl);
2754 Node* buffer_adr = new AddPNode(phase->C->top(), thread, phase->igvn().MakeConX(buffer_offset));
2755 phase->register_new_node(buffer_adr, ctrl);
2756 Node* index_adr = new AddPNode(phase->C->top(), thread, phase->igvn().MakeConX(index_offset));
2757 phase->register_new_node(index_adr, ctrl);
2758
2759 BasicType index_bt = TypeX_X->basic_type();
2760 assert(sizeof(size_t) == type2aelembytes(index_bt), "Loading G1 SATBMarkQueue::_index with wrong size.");
2761 const TypePtr* adr_type = TypeRawPtr::BOTTOM;
2762 Node* index = new LoadXNode(ctrl, raw_mem, index_adr, adr_type, TypeX_X, MemNode::unordered);
2763 phase->register_new_node(index, ctrl);
2764 Node* index_cmp = new CmpXNode(index, phase->igvn().MakeConX(0));
2765 phase->register_new_node(index_cmp, ctrl);
2766 Node* index_test = new BoolNode(index_cmp, BoolTest::ne);
2767 phase->register_new_node(index_test, ctrl);
2768 IfNode* queue_full_iff = new IfNode(ctrl, index_test, PROB_LIKELY(0.999), COUNT_UNKNOWN);
2769 if (reg2_ctrl == NULL) reg2_ctrl = queue_full_iff;
2770 phase->register_control(queue_full_iff, loop, ctrl);
2771 Node* not_full = new IfTrueNode(queue_full_iff);
2772 phase->register_control(not_full, loop, queue_full_iff);
2773 Node* full = new IfFalseNode(queue_full_iff);
2774 phase->register_control(full, loop, queue_full_iff);
2775
2776 ctrl = not_full;
2777
2778 Node* next_index = new SubXNode(index, phase->igvn().MakeConX(sizeof(intptr_t)));
2779 phase->register_new_node(next_index, ctrl);
2780
2781 Node* buffer = new LoadPNode(ctrl, raw_mem, buffer_adr, adr_type, TypeRawPtr::NOTNULL, MemNode::unordered);
2782 phase->register_new_node(buffer, ctrl);
2783 Node *log_addr = new AddPNode(phase->C->top(), buffer, next_index);
2784 phase->register_new_node(log_addr, ctrl);
2785 Node* log_store = new StorePNode(ctrl, raw_mem, log_addr, adr_type, pre_val, MemNode::unordered);
2786 phase->register_new_node(log_store, ctrl);
2787 // update the index
2788 Node* index_update = new StoreXNode(ctrl, log_store, index_adr, adr_type, next_index, MemNode::unordered);
2789 phase->register_new_node(index_update, ctrl);
2790
2791 // Fast-path case
2792 region2->init_req(_fast_path, ctrl);
2793 phi2->init_req(_fast_path, index_update);
2794
2795 ctrl = full;
2796
2797 Node* base = find_bottom_mem(ctrl, phase);
2798
2799 MergeMemNode* mm = MergeMemNode::make(base);
2800 mm->set_memory_at(Compile::AliasIdxRaw, raw_mem);
2801 phase->register_new_node(mm, ctrl);
2802
2803 Node* call = new CallLeafNode(ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type(), CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), "shenandoah_wb_pre", TypeRawPtr::BOTTOM);
2804 call->init_req(TypeFunc::Control, ctrl);
2805 call->init_req(TypeFunc::I_O, phase->C->top());
2806 call->init_req(TypeFunc::Memory, mm);
2807 call->init_req(TypeFunc::FramePtr, phase->C->top());
2808 call->init_req(TypeFunc::ReturnAdr, phase->C->top());
2809 call->init_req(TypeFunc::Parms, pre_val);
2810 call->init_req(TypeFunc::Parms+1, thread);
2811 phase->register_control(call, loop, ctrl);
2812
2813 Node* ctrl_proj = new ProjNode(call, TypeFunc::Control);
2814 phase->register_control(ctrl_proj, loop, call);
2815 Node* mem_proj = new ProjNode(call, TypeFunc::Memory);
2816 phase->register_new_node(mem_proj, call);
2817
2818 // Slow-path case
2819 region2->init_req(_slow_path, ctrl_proj);
2820 phi2->init_req(_slow_path, mem_proj);
2821
2822 phase->register_control(region2, loop, reg2_ctrl);
2823 phase->register_new_node(phi2, region2);
2824
2825 region->init_req(_heap_unstable, region2);
2826 phi->init_req(_heap_unstable, phi2);
2827
2828 phase->register_control(region, loop, heap_stable_ctrl->in(0));
2829 phase->register_new_node(phi, region);
2830
2831 fix_ctrl(barrier, region, fixer, uses, uses_to_ignore, last, phase);
2832 for(uint next = 0; next < uses.size(); next++ ) {
2833 Node *n = uses.at(next);
2834 assert(phase->get_ctrl(n) == init_ctrl, "bad control");
2835 assert(n != init_raw_mem, "should leave input raw mem above the barrier");
2836 phase->set_ctrl(n, region);
2837 follow_barrier_uses(n, init_ctrl, uses, phase);
2838 }
2839 fixer.fix_mem(init_ctrl, region, init_raw_mem, raw_mem_for_ctrl, phi, uses);
2840
2841 phase->igvn().replace_node(barrier, pre_val);
2842 }
2843
2844 for (int i = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count(); i > 0; i--) {
2845 int cnt = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count();
2846 ShenandoahWriteBarrierNode* wb = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barrier(i-1);
2847
2848 uint last = phase->C->unique();
2849 Node* ctrl = phase->get_ctrl(wb);
2850 Node* orig_ctrl = ctrl;
2851
2852 Node* raw_mem = fixer.find_mem(ctrl, wb);
2853 Node* init_raw_mem = raw_mem;
2854 Node* raw_mem_for_ctrl = fixer.find_mem(ctrl, NULL);
2855 int alias = phase->C->get_alias_index(wb->adr_type());
2856 Node* wb_mem = wb->in(Memory);
2857 Node* init_wb_mem = wb_mem;
2858
2859 Node* val = wb->in(ValueIn);
2860 Node* wbproj = wb->find_out_with(Op_ShenandoahWBMemProj);
2861 IdealLoopTree *loop = phase->get_loop(ctrl);
2862
2863 assert(val->Opcode() != Op_ShenandoahWriteBarrier, "No chain of write barriers");
2864
2865 CallStaticJavaNode* unc = wb->pin_and_expand_null_check(phase->igvn());
2866 Node* unc_ctrl = NULL;
2867 if (unc != NULL) {
2868 if (val->in(0) != ctrl) {
2869 unc = NULL;
2870 } else {
2871 unc_ctrl = val->in(0);
2872 }
2873 }
2874
2875 Node* uncasted_val = val;
2876 if (unc != NULL) {
2877 uncasted_val = val->in(1);
2878 }
2879
2880 Node* heap_stable_ctrl = NULL;
2881 Node* null_ctrl = NULL;
2882
2883 assert(val->bottom_type()->make_oopptr(), "need oop");
2884 assert(val->bottom_type()->make_oopptr()->const_oop() == NULL, "expect non-constant");
2885
2886 enum { _heap_stable = 1, _heap_unstable, PATH_LIMIT };
2887 Node* region = new RegionNode(PATH_LIMIT);
2888 Node* val_phi = new PhiNode(region, uncasted_val->bottom_type()->is_oopptr());
2889 Node* mem_phi = PhiNode::make(region, wb_mem, Type::MEMORY, phase->C->alias_type(wb->adr_type())->adr_type());
2890 Node* raw_mem_phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);
2891
2892 enum { _not_cset = 1, _not_equal, _evac_path, _null_path, PATH_LIMIT2 };
2893 Node* region2 = new RegionNode(PATH_LIMIT2);
2894 Node* val_phi2 = new PhiNode(region2, uncasted_val->bottom_type()->is_oopptr());
2895 Node* mem_phi2 = PhiNode::make(region2, wb_mem, Type::MEMORY, phase->C->alias_type(wb->adr_type())->adr_type());
2896 Node* raw_mem_phi2 = PhiNode::make(region2, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);
2897
2898 // Stable path.
2899 test_heap_stable(ctrl, raw_mem, heap_stable_ctrl, phase);
2900 IfNode* heap_stable_iff = heap_stable_ctrl->in(0)->as_If();
2901
2902 // Heap stable case
2903 region->init_req(_heap_stable, heap_stable_ctrl);
2904 val_phi->init_req(_heap_stable, uncasted_val);
2905 mem_phi->init_req(_heap_stable, wb_mem);
2906 raw_mem_phi->init_req(_heap_stable, raw_mem);
2907
2908 Node* reg2_ctrl = NULL;
2909 // Null case
2910 test_null(ctrl, val, null_ctrl, phase);
2911 if (null_ctrl != NULL) {
2912 reg2_ctrl = null_ctrl->in(0);
2913 region2->init_req(_null_path, null_ctrl);
2914 val_phi2->init_req(_null_path, uncasted_val);
2915 mem_phi2->init_req(_null_path, wb_mem);
2916 raw_mem_phi2->init_req(_null_path, raw_mem);
2917 } else {
2918 region2->del_req(_null_path);
2919 val_phi2->del_req(_null_path);
2920 mem_phi2->del_req(_null_path);
2921 raw_mem_phi2->del_req(_null_path);
2922 }
2923
2924 // Test for in-cset.
2925 // Wires !in_cset(obj) to slot 2 of region and phis
2926 Node* not_cset_ctrl = NULL;
2927 in_cset_fast_test(ctrl, not_cset_ctrl, uncasted_val, raw_mem, phase);
2928 if (not_cset_ctrl != NULL) {
2929 if (reg2_ctrl == NULL) reg2_ctrl = not_cset_ctrl->in(0);
2930 region2->init_req(_not_cset, not_cset_ctrl);
2931 val_phi2->init_req(_not_cset, uncasted_val);
2932 mem_phi2->init_req(_not_cset, wb_mem);
2933 raw_mem_phi2->init_req(_not_cset, raw_mem);
2934 }
2935
2936 // Resolve object when orig-value is in cset.
2937 // Make the unconditional resolve for fwdptr, not the read barrier.
2938 Node* new_val = uncasted_val;
2939 if (unc_ctrl != NULL) {
2940 // Clone the null check in this branch to allow implicit null check
2941 new_val = clone_null_check(ctrl, val, unc_ctrl, phase);
2942 fix_null_check(unc, unc_ctrl, ctrl->in(0)->as_If()->proj_out(0), uses, phase);
2943
2944 IfNode* iff = unc_ctrl->in(0)->as_If();
2945 phase->igvn().replace_input_of(iff, 1, phase->igvn().intcon(1));
2946 }
2947 Node* addr = new AddPNode(new_val, uncasted_val, phase->igvn().MakeConX(ShenandoahBrooksPointer::byte_offset()));
2948 phase->register_new_node(addr, ctrl);
2949 assert(val->bottom_type()->isa_oopptr(), "what else?");
2950 const TypePtr* obj_type = val->bottom_type()->is_oopptr();
2951 const TypePtr* adr_type = ShenandoahBarrierNode::brooks_pointer_type(obj_type);
2952 Node* fwd = new LoadPNode(ctrl, wb_mem, addr, adr_type, obj_type, MemNode::unordered);
2953 phase->register_new_node(fwd, ctrl);
2954
2955 // Only branch to WB stub if object is not forwarded; otherwise reply with fwd ptr
2956 Node* cmp = new CmpPNode(fwd, new_val);
2957 phase->register_new_node(cmp, ctrl);
2958 Node* bol = new BoolNode(cmp, BoolTest::eq);
2959 phase->register_new_node(bol, ctrl);
2960
2961 IfNode* iff = new IfNode(ctrl, bol, PROB_UNLIKELY(0.999), COUNT_UNKNOWN);
2962 if (reg2_ctrl == NULL) reg2_ctrl = iff;
2963 phase->register_control(iff, loop, ctrl);
2964 Node* if_not_eq = new IfFalseNode(iff);
2965 phase->register_control(if_not_eq, loop, iff);
2966 Node* if_eq = new IfTrueNode(iff);
2967 phase->register_control(if_eq, loop, iff);
2968
2969 // Wire up not-equal-path in slots 3.
2970 region2->init_req(_not_equal, if_not_eq);
2971 val_phi2->init_req(_not_equal, fwd);
2972 mem_phi2->init_req(_not_equal, wb_mem);
2973 raw_mem_phi2->init_req(_not_equal, raw_mem);
2974
2975 // Call wb-stub and wire up that path in slots 4
2976 Node* result_mem = NULL;
2977 ctrl = if_eq;
2978 call_wb_stub(ctrl, new_val, result_mem,
2979 raw_mem, wb_mem,
2980 alias, phase);
2981 region2->init_req(_evac_path, ctrl);
2982 val_phi2->init_req(_evac_path, new_val);
2983 mem_phi2->init_req(_evac_path, result_mem);
2984 raw_mem_phi2->init_req(_evac_path, result_mem);
2985
2986 phase->register_control(region2, loop, reg2_ctrl);
2987 phase->register_new_node(val_phi2, region2);
2988 phase->register_new_node(mem_phi2, region2);
2989 phase->register_new_node(raw_mem_phi2, region2);
2990
2991 region->init_req(_heap_unstable, region2);
2992 val_phi->init_req(_heap_unstable, val_phi2);
2993 mem_phi->init_req(_heap_unstable, mem_phi2);
2994 raw_mem_phi->init_req(_heap_unstable, raw_mem_phi2);
2995
2996 phase->register_control(region, loop, heap_stable_iff);
2997 Node* out_val = val_phi;
2998 phase->register_new_node(val_phi, region);
2999 phase->register_new_node(mem_phi, region);
3000 phase->register_new_node(raw_mem_phi, region);
3001
3002 fix_ctrl(wb, region, fixer, uses, uses_to_ignore, last, phase);
3003
3004 ctrl = orig_ctrl;
3005
3006 phase->igvn().replace_input_of(wbproj, ShenandoahWBMemProjNode::WriteBarrier, phase->C->top());
3007 phase->igvn().replace_node(wbproj, mem_phi);
3008 if (unc != NULL) {
3009 for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) {
3010 Node* u = val->fast_out(i);
3011 Node* c = phase->ctrl_or_self(u);
3012 if (u != wb && (c != ctrl || is_dominator_same_ctrl(c, wb, u, phase))) {
3013 phase->igvn().rehash_node_delayed(u);
3014 int nb = u->replace_edge(val, out_val);
3015 --i, imax -= nb;
3016 }
3017 }
3018 if (val->outcnt() == 0) {
3019 phase->igvn()._worklist.push(val);
3020 }
3021 }
3022 phase->igvn().replace_node(wb, out_val);
3023
3024 follow_barrier_uses(mem_phi, ctrl, uses, phase);
3025 follow_barrier_uses(out_val, ctrl, uses, phase);
3026
3027 for(uint next = 0; next < uses.size(); next++ ) {
3028 Node *n = uses.at(next);
3029 assert(phase->get_ctrl(n) == ctrl, "bad control");
3030 assert(n != init_raw_mem, "should leave input raw mem above the barrier");
3031 phase->set_ctrl(n, region);
3032 follow_barrier_uses(n, ctrl, uses, phase);
3033 }
3034
3035 // The slow path call produces memory: hook the raw memory phi
3036 // from the expanded write barrier with the rest of the graph
3037 // which may require adding memory phis at every post dominated
3038 // region and at enclosing loop heads. Use the memory state
3039 // collected in memory_nodes to fix the memory graph. Update that
3040 // memory state as we go.
3041 fixer.fix_mem(ctrl, region, init_raw_mem, raw_mem_for_ctrl, raw_mem_phi, uses);
3042 assert(ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count() == cnt - 1, "not replaced");
3043 }
3044
3045 assert(ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count() == 0, "all write barrier nodes should have been replaced");
3046 }
3047
3048 void ShenandoahWriteBarrierNode::move_heap_stable_test_out_of_loop(IfNode* iff, PhaseIdealLoop* phase) {
3049 IdealLoopTree *loop = phase->get_loop(iff);
3050 Node* loop_head = loop->_head;
3051 Node* entry_c = loop_head->in(LoopNode::EntryControl);
3052
3053 Node* bol = iff->in(1);
3054 Node* cmp = bol->in(1);
3055 Node* andi = cmp->in(1);
3056 Node* load = andi->in(1);
3057
3058 assert(is_gc_state_load(load), "broken");
3059 if (!phase->is_dominator(load->in(0), entry_c)) {
3060 Node* mem_ctrl = NULL;
3061 Node* mem = dom_mem(load->in(MemNode::Memory), loop_head, Compile::AliasIdxRaw, mem_ctrl, phase);
3062 load = load->clone();
3063 load->set_req(MemNode::Memory, mem);
3064 load->set_req(0, entry_c);
3065 phase->register_new_node(load, entry_c);
3066 andi = andi->clone();
3067 andi->set_req(1, load);
3068 phase->register_new_node(andi, entry_c);
3069 cmp = cmp->clone();
3070 cmp->set_req(1, andi);
3071 phase->register_new_node(cmp, entry_c);
3072 bol = bol->clone();
3073 bol->set_req(1, cmp);
3074 phase->register_new_node(bol, entry_c);
3075
3076 Node* old_bol =iff->in(1);
3077 phase->igvn().replace_input_of(iff, 1, bol);
3078 }
3079 }
3080
3081 bool ShenandoahWriteBarrierNode::identical_backtoback_ifs(Node *n, PhaseIdealLoop* phase) {
3082 if (!n->is_If() || n->is_CountedLoopEnd()) {
3083 return false;
3084 }
3085 Node* region = n->in(0);
3086
3087 if (!region->is_Region()) {
3088 return false;
3089 }
3090 Node* dom = phase->idom(region);
3091 if (!dom->is_If()) {
3092 return false;
3093 }
3094
3095 if (!is_heap_stable_test(n) || !is_heap_stable_test(dom)) {
3096 return false;
3097 }
3098
3099 IfNode* dom_if = dom->as_If();
3100 Node* proj_true = dom_if->proj_out(1);
3101 Node* proj_false = dom_if->proj_out(0);
3102
3103 for (uint i = 1; i < region->req(); i++) {
3104 if (phase->is_dominator(proj_true, region->in(i))) {
3105 continue;
3106 }
3107 if (phase->is_dominator(proj_false, region->in(i))) {
3108 continue;
3109 }
3110 return false;
3111 }
3112
3113 return true;
3114 }
3115
3116 void ShenandoahWriteBarrierNode::merge_back_to_back_tests(Node* n, PhaseIdealLoop* phase) {
3117 assert(is_heap_stable_test(n), "no other tests");
3118 if (identical_backtoback_ifs(n, phase)) {
3119 Node* n_ctrl = n->in(0);
3120 if (phase->can_split_if(n_ctrl)) {
3121 IfNode* dom_if = phase->idom(n_ctrl)->as_If();
3122 if (is_heap_stable_test(n)) {
3123 Node* gc_state_load = n->in(1)->in(1)->in(1)->in(1);
3124 assert(is_gc_state_load(gc_state_load), "broken");
3125 Node* dom_gc_state_load = dom_if->in(1)->in(1)->in(1)->in(1);
3126 assert(is_gc_state_load(dom_gc_state_load), "broken");
3127 if (gc_state_load != dom_gc_state_load) {
3128 phase->igvn().replace_node(gc_state_load, dom_gc_state_load);
3129 }
3130 }
3131 PhiNode* bolphi = PhiNode::make_blank(n_ctrl, n->in(1));
3132 Node* proj_true = dom_if->proj_out(1);
3133 Node* proj_false = dom_if->proj_out(0);
3134 Node* con_true = phase->igvn().makecon(TypeInt::ONE);
3135 Node* con_false = phase->igvn().makecon(TypeInt::ZERO);
3136
3137 for (uint i = 1; i < n_ctrl->req(); i++) {
3138 if (phase->is_dominator(proj_true, n_ctrl->in(i))) {
3139 bolphi->init_req(i, con_true);
3140 } else {
3141 assert(phase->is_dominator(proj_false, n_ctrl->in(i)), "bad if");
3142 bolphi->init_req(i, con_false);
3143 }
3144 }
3145 phase->register_new_node(bolphi, n_ctrl);
3146 phase->igvn().replace_input_of(n, 1, bolphi);
3147 phase->do_split_if(n);
3148 }
3149 }
3150 }
3151
3152 IfNode* ShenandoahWriteBarrierNode::find_unswitching_candidate(const IdealLoopTree *loop, PhaseIdealLoop* phase) {
3153 // Find first invariant test that doesn't exit the loop
3154 LoopNode *head = loop->_head->as_Loop();
3155 IfNode* unswitch_iff = NULL;
3156 Node* n = head->in(LoopNode::LoopBackControl);
3157 int loop_has_sfpts = -1;
3158 while (n != head) {
3159 Node* n_dom = phase->idom(n);
3160 if (n->is_Region()) {
3161 if (n_dom->is_If()) {
3162 IfNode* iff = n_dom->as_If();
3163 if (iff->in(1)->is_Bool()) {
3164 BoolNode* bol = iff->in(1)->as_Bool();
3165 if (bol->in(1)->is_Cmp()) {
3166 // If condition is invariant and not a loop exit,
3167 // then found reason to unswitch.
3168 if (is_heap_stable_test(iff) &&
3169 (loop_has_sfpts == -1 || loop_has_sfpts == 0)) {
3170 assert(!loop->is_loop_exit(iff), "both branches should be in the loop");
3171 if (loop_has_sfpts == -1) {
3172 for(uint i = 0; i < loop->_body.size(); i++) {
3177 }
3178 }
3179 if (loop_has_sfpts == -1) {
3180 loop_has_sfpts = 0;
3181 }
3182 }
3183 if (!loop_has_sfpts) {
3184 unswitch_iff = iff;
3185 }
3186 }
3187 }
3188 }
3189 }
3190 }
3191 n = n_dom;
3192 }
3193 return unswitch_iff;
3194 }
3195
3196
3197 void ShenandoahWriteBarrierNode::optimize_after_expansion(VectorSet &visited, Node_Stack &stack, Node_List &old_new, PhaseIdealLoop* phase) {
3198 Node_List heap_stable_tests;
3199 Node_List gc_state_loads;
3200
3201 stack.push(phase->C->start(), 0);
3202 do {
3203 Node* n = stack.node();
3204 uint i = stack.index();
3205
3206 if (i < n->outcnt()) {
3207 Node* u = n->raw_out(i);
3208 stack.set_index(i+1);
3209 if (!visited.test_set(u->_idx)) {
3210 stack.push(u, 0);
3211 }
3212 } else {
3213 stack.pop();
3214 if (ShenandoahCommonGCStateLoads && is_gc_state_load(n)) {
3215 gc_state_loads.push(n);
3216 }
3217 if (n->is_If() && is_heap_stable_test(n)) {
3218 heap_stable_tests.push(n);
3219 }
3220 }
3257 move_heap_stable_test_out_of_loop(iff, phase);
3258 if (loop->policy_unswitching(phase)) {
3259 if (head->is_strip_mined()) {
3260 OuterStripMinedLoopNode* outer = head->as_CountedLoop()->outer_loop();
3261 hide_strip_mined_loop(outer, head->as_CountedLoop(), phase);
3262 }
3263 phase->do_unswitching(loop, old_new);
3264 } else {
3265 // Not proceeding with unswitching. Move load back in
3266 // the loop.
3267 phase->igvn().replace_input_of(iff, 1, bol);
3268 }
3269 }
3270 }
3271 }
3272 }
3273 }
3274 }
3275
3276 #ifdef ASSERT
3277 void ShenandoahBarrierNode::verify_raw_mem(RootNode* root) {
3278 const bool trace = false;
3279 ResourceMark rm;
3280 Unique_Node_List nodes;
3281 Unique_Node_List controls;
3282 Unique_Node_List memories;
3283
3284 nodes.push(root);
3285 for (uint next = 0; next < nodes.size(); next++) {
3286 Node *n = nodes.at(next);
3287 if (ShenandoahBarrierSetC2::is_shenandoah_wb_call(n)) {
3288 controls.push(n);
3289 if (trace) { tty->print("XXXXXX verifying"); n->dump(); }
3290 for (uint next2 = 0; next2 < controls.size(); next2++) {
3291 Node *m = controls.at(next2);
3292 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
3293 Node* u = m->fast_out(i);
3294 if (u->is_CFG() && !u->is_Root() &&
3295 !(u->Opcode() == Op_CProj && u->in(0)->Opcode() == Op_NeverBranch && u->as_Proj()->_con == 1) &&
3296 !(u->is_Region() && u->unique_ctrl_out()->Opcode() == Op_Halt)) {
3297 if (trace) { tty->print("XXXXXX pushing control"); u->dump(); }
3355 }
3356 }
3357 }
3358 }
3359 assert(found_phi || all_in, "");
3360 }
3361 }
3362 controls.clear();
3363 memories.clear();
3364 }
3365 for( uint i = 0; i < n->len(); ++i ) {
3366 Node *m = n->in(i);
3367 if (m != NULL) {
3368 nodes.push(m);
3369 }
3370 }
3371 }
3372 }
3373 #endif
3374
3375 const Type* ShenandoahEnqueueBarrierNode::bottom_type() const {
3376 if (in(1) == NULL || in(1)->is_top()) {
3377 return Type::TOP;
3378 }
3379 const Type* t = in(1)->bottom_type();
3380 if (t == TypePtr::NULL_PTR) {
3381 return t;
3382 }
3383 return t->is_oopptr()->cast_to_nonconst();
3384 }
3385
3386 const Type* ShenandoahEnqueueBarrierNode::Value(PhaseGVN* phase) const {
3387 if (in(1) == NULL) {
3388 return Type::TOP;
3389 }
3390 const Type* t = phase->type(in(1));
3391 if (t == Type::TOP) {
3392 return Type::TOP;
3393 }
3394 if (t == TypePtr::NULL_PTR) {
3514 uint i = stack.index();
3515 if (i < n->req()) {
3516 Node* mem = NULL;
3517 if (opc == Op_Root) {
3518 Node* in = n->in(i);
3519 int in_opc = in->Opcode();
3520 if (in_opc == Op_Return || in_opc == Op_Rethrow) {
3521 mem = in->in(TypeFunc::Memory);
3522 } else if (in_opc == Op_Halt) {
3523 if (!in->in(0)->is_Region()) {
3524 Node* proj = in->in(0);
3525 assert(proj->is_Proj(), "");
3526 Node* in = proj->in(0);
3527 assert(in->is_CallStaticJava() || in->Opcode() == Op_NeverBranch || in->Opcode() == Op_Catch || proj->is_IfProj(), "");
3528 if (in->is_CallStaticJava()) {
3529 mem = in->in(TypeFunc::Memory);
3530 } else if (in->Opcode() == Op_Catch) {
3531 Node* call = in->in(0)->in(0);
3532 assert(call->is_Call(), "");
3533 mem = call->in(TypeFunc::Memory);
3534 }
3535 }
3536 } else {
3537 #ifdef ASSERT
3538 n->dump();
3539 in->dump();
3540 #endif
3541 ShouldNotReachHere();
3542 }
3543 } else {
3544 assert(n->is_Phi() && n->bottom_type() == Type::MEMORY, "");
3545 assert(n->adr_type() == TypePtr::BOTTOM || _phase->C->get_alias_index(n->adr_type()) == _alias, "");
3546 mem = n->in(i);
3547 }
3548 i++;
3549 stack.set_index(i);
3550 if (mem == NULL) {
3551 continue;
3552 }
3553 for (;;) {
3554 if (visited.test_set(mem->_idx) || mem->is_Start()) {
3555 break;
3556 }
3557 if (mem->is_Phi()) {
3558 stack.push(mem, 2);
3559 mem = mem->in(1);
3560 } else if (mem->is_Proj()) {
3561 stack.push(mem, mem->req());
3562 mem = mem->in(0);
3563 } else if (mem->is_SafePoint() || mem->is_MemBar()) {
3564 mem = mem->in(TypeFunc::Memory);
3565 } else if (mem->is_MergeMem()) {
3566 MergeMemNode* mm = mem->as_MergeMem();
3567 mem = mm->memory_at(_alias);
3568 } else if (mem->is_Store() || mem->is_LoadStore() || mem->is_ClearArray()) {
3569 assert(_alias == Compile::AliasIdxRaw, "");
3570 stack.push(mem, mem->req());
3571 mem = mem->in(MemNode::Memory);
3572 } else if (mem->Opcode() == Op_ShenandoahWriteBarrier) {
3573 assert(_alias != Compile::AliasIdxRaw, "");
3574 mem = mem->in(ShenandoahBarrierNode::Memory);
3575 } else if (mem->Opcode() == Op_ShenandoahWBMemProj) {
3576 stack.push(mem, mem->req());
3577 mem = mem->in(ShenandoahWBMemProjNode::WriteBarrier);
3578 } else {
3579 #ifdef ASSERT
3580 mem->dump();
3581 #endif
3582 ShouldNotReachHere();
3583 }
3584 }
3585 } else {
3586 if (n->is_Phi()) {
3587 // Nothing
3588 } else if (!n->is_Root()) {
3589 Node* c = get_ctrl(n);
3590 _memory_nodes.map(c->_idx, n);
3591 }
3592 stack.pop();
3593 }
3594 } while(stack.is_nonempty());
3595
3596 // Iterate over CFG nodes in rpo and propagate memory state to
3597 // compute memory state at regions, creating new phis if needed.
3611 }
3612 }
3613 }
3614 #endif
3615 uint last = _phase->C->unique();
3616
3617 #ifdef ASSERT
3618 uint8_t max_depth = 0;
3619 for (LoopTreeIterator iter(_phase->ltree_root()); !iter.done(); iter.next()) {
3620 IdealLoopTree* lpt = iter.current();
3621 max_depth = MAX2(max_depth, lpt->_nest);
3622 }
3623 #endif
3624
3625 bool progress = true;
3626 int iteration = 0;
3627 Node_List dead_phis;
3628 while (progress) {
3629 progress = false;
3630 iteration++;
3631 assert(iteration <= 2+max_depth || _phase->C->has_irreducible_loop(), "");
3632 if (trace) { tty->print_cr("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"); }
3633 IdealLoopTree* last_updated_ilt = NULL;
3634 for (int i = rpo_list.size() - 1; i >= 0; i--) {
3635 Node* c = rpo_list.at(i);
3636
3637 Node* prev_mem = _memory_nodes[c->_idx];
3638 if (c->is_Region() && (_include_lsm || !c->is_OuterStripMinedLoop())) {
3639 Node* prev_region = regions[c->_idx];
3640 Node* unique = NULL;
3641 for (uint j = 1; j < c->req() && unique != NodeSentinel; j++) {
3642 Node* m = _memory_nodes[c->in(j)->_idx];
3643 assert(m != NULL || (c->is_Loop() && j == LoopNode::LoopBackControl && iteration == 1) || _phase->C->has_irreducible_loop() || has_never_branch(_phase->C->root()), "expect memory state");
3644 if (m != NULL) {
3645 if (m == prev_region && ((c->is_Loop() && j == LoopNode::LoopBackControl) || (prev_region->is_Phi() && prev_region->in(0) == c))) {
3646 assert(c->is_Loop() && j == LoopNode::LoopBackControl || _phase->C->has_irreducible_loop(), "");
3647 // continue
3648 } else if (unique == NULL) {
3649 unique = m;
3650 } else if (m == unique) {
3651 // continue
3779 else {
3780 assert (n->is_CFG(), "must be a CFG node");
3781 return n;
3782 }
3783 }
3784
3785 bool MemoryGraphFixer::mem_is_valid(Node* m, Node* c) const {
3786 return m != NULL && get_ctrl(m) == c;
3787 }
3788
3789 Node* MemoryGraphFixer::find_mem(Node* ctrl, Node* n) const {
3790 assert(n == NULL || _phase->ctrl_or_self(n) == ctrl, "");
3791 Node* mem = _memory_nodes[ctrl->_idx];
3792 Node* c = ctrl;
3793 while (!mem_is_valid(mem, c) &&
3794 (!c->is_CatchProj() || mem == NULL || c->in(0)->in(0)->in(0) != get_ctrl(mem))) {
3795 c = _phase->idom(c);
3796 mem = _memory_nodes[c->_idx];
3797 }
3798 if (n != NULL && mem_is_valid(mem, c)) {
3799 while (!ShenandoahWriteBarrierNode::is_dominator_same_ctrl(c, mem, n, _phase) && _phase->ctrl_or_self(mem) == ctrl) {
3800 mem = next_mem(mem, _alias);
3801 }
3802 if (mem->is_MergeMem()) {
3803 mem = mem->as_MergeMem()->memory_at(_alias);
3804 }
3805 if (!mem_is_valid(mem, c)) {
3806 do {
3807 c = _phase->idom(c);
3808 mem = _memory_nodes[c->_idx];
3809 } while (!mem_is_valid(mem, c) &&
3810 (!c->is_CatchProj() || mem == NULL || c->in(0)->in(0)->in(0) != get_ctrl(mem)));
3811 }
3812 }
3813 assert(mem->bottom_type() == Type::MEMORY, "");
3814 return mem;
3815 }
3816
3817 bool MemoryGraphFixer::has_mem_phi(Node* region) const {
3818 for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) {
3819 Node* use = region->fast_out(i);
3825 return false;
3826 }
3827
3828 void MemoryGraphFixer::fix_mem(Node* ctrl, Node* new_ctrl, Node* mem, Node* mem_for_ctrl, Node* new_mem, Unique_Node_List& uses) {
3829 assert(_phase->ctrl_or_self(new_mem) == new_ctrl, "");
3830 const bool trace = false;
3831 DEBUG_ONLY(if (trace) { tty->print("ZZZ control is"); ctrl->dump(); });
3832 DEBUG_ONLY(if (trace) { tty->print("ZZZ mem is"); mem->dump(); });
3833 GrowableArray<Node*> phis;
3834 if (mem_for_ctrl != mem) {
3835 Node* old = mem_for_ctrl;
3836 Node* prev = NULL;
3837 while (old != mem) {
3838 prev = old;
3839 if (old->is_Store() || old->is_ClearArray() || old->is_LoadStore()) {
3840 assert(_alias == Compile::AliasIdxRaw, "");
3841 old = old->in(MemNode::Memory);
3842 } else if (old->Opcode() == Op_SCMemProj) {
3843 assert(_alias == Compile::AliasIdxRaw, "");
3844 old = old->in(0);
3845 } else if (old->Opcode() == Op_ShenandoahWBMemProj) {
3846 assert(_alias != Compile::AliasIdxRaw, "");
3847 old = old->in(ShenandoahWBMemProjNode::WriteBarrier);
3848 } else if (old->Opcode() == Op_ShenandoahWriteBarrier) {
3849 assert(_alias != Compile::AliasIdxRaw, "");
3850 old = old->in(ShenandoahBarrierNode::Memory);
3851 } else {
3852 ShouldNotReachHere();
3853 }
3854 }
3855 assert(prev != NULL, "");
3856 if (new_ctrl != ctrl) {
3857 _memory_nodes.map(ctrl->_idx, mem);
3858 _memory_nodes.map(new_ctrl->_idx, mem_for_ctrl);
3859 }
3860 uint input = prev->Opcode() == Op_ShenandoahWriteBarrier ? (uint)ShenandoahBarrierNode::Memory : (uint)MemNode::Memory;
3861 _phase->igvn().replace_input_of(prev, input, new_mem);
3862 } else {
3863 uses.clear();
3864 _memory_nodes.map(new_ctrl->_idx, new_mem);
3865 uses.push(new_ctrl);
3866 for(uint next = 0; next < uses.size(); next++ ) {
3867 Node *n = uses.at(next);
3868 assert(n->is_CFG(), "");
3869 DEBUG_ONLY(if (trace) { tty->print("ZZZ ctrl"); n->dump(); });
3870 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
3871 Node* u = n->fast_out(i);
3872 if (!u->is_Root() && u->is_CFG() && u != n) {
3873 Node* m = _memory_nodes[u->_idx];
3874 if (u->is_Region() && (!u->is_OuterStripMinedLoop() || _include_lsm) &&
3875 !has_mem_phi(u) &&
3876 u->unique_ctrl_out()->Opcode() != Op_Halt) {
3877 DEBUG_ONLY(if (trace) { tty->print("ZZZ region"); u->dump(); });
3878 DEBUG_ONLY(if (trace && m != NULL) { tty->print("ZZZ mem"); m->dump(); });
3879
3880 if (!mem_is_valid(m, u) || !m->is_Phi()) {
3908 do_check = false;
3909 }
3910 }
3911 }
3912
3913 if (do_check && _phase->is_dominator(_phase->idom(u), new_ctrl)) {
3914 create_phi = true;
3915 }
3916 }
3917 if (create_phi) {
3918 Node* phi = new PhiNode(u, Type::MEMORY, _phase->C->get_adr_type(_alias));
3919 _phase->register_new_node(phi, u);
3920 phis.push(phi);
3921 DEBUG_ONLY(if (trace) { tty->print("ZZZ new phi"); phi->dump(); });
3922 if (!mem_is_valid(m, u)) {
3923 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting mem"); phi->dump(); });
3924 _memory_nodes.map(u->_idx, phi);
3925 } else {
3926 DEBUG_ONLY(if (trace) { tty->print("ZZZ NOT setting mem"); m->dump(); });
3927 for (;;) {
3928 assert(m->is_Mem() || m->is_LoadStore() || m->is_Proj() || m->Opcode() == Op_ShenandoahWriteBarrier || m->Opcode() == Op_ShenandoahWBMemProj, "");
3929 Node* next = NULL;
3930 if (m->is_Proj()) {
3931 next = m->in(0);
3932 } else if (m->Opcode() == Op_ShenandoahWBMemProj) {
3933 next = m->in(ShenandoahWBMemProjNode::WriteBarrier);
3934 } else if (m->is_Mem() || m->is_LoadStore()) {
3935 assert(_alias == Compile::AliasIdxRaw, "");
3936 next = m->in(MemNode::Memory);
3937 } else {
3938 assert(_alias != Compile::AliasIdxRaw, "");
3939 assert (m->Opcode() == Op_ShenandoahWriteBarrier, "");
3940 next = m->in(ShenandoahBarrierNode::Memory);
3941 }
3942 if (_phase->get_ctrl(next) != u) {
3943 break;
3944 }
3945 if (next->is_MergeMem()) {
3946 assert(_phase->get_ctrl(next->as_MergeMem()->memory_at(_alias)) != u, "");
3947 break;
3948 }
3949 if (next->is_Phi()) {
3950 assert(next->adr_type() == TypePtr::BOTTOM && next->in(0) == u, "");
3951 break;
3952 }
3953 m = next;
3954 }
3955
3956 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting to phi"); m->dump(); });
3957 assert(m->is_Mem() || m->is_LoadStore() || m->Opcode() == Op_ShenandoahWriteBarrier, "");
3958 uint input = (m->is_Mem() || m->is_LoadStore()) ? (uint)MemNode::Memory : (uint)ShenandoahBarrierNode::Memory;
3959 _phase->igvn().replace_input_of(m, input, phi);
3960 push = false;
3961 }
3962 } else {
3963 DEBUG_ONLY(if (trace) { tty->print("ZZZ skipping region"); u->dump(); });
3964 }
3965 if (push) {
3966 uses.push(u);
3967 }
3968 }
3969 } else if (!mem_is_valid(m, u) &&
3970 !(u->Opcode() == Op_CProj && u->in(0)->Opcode() == Op_NeverBranch && u->as_Proj()->_con == 1)) {
3971 uses.push(u);
3972 }
3973 }
3974 }
3975 }
3976 for (int i = 0; i < phis.length(); i++) {
3977 Node* n = phis.at(i);
3978 Node* r = n->in(0);
4164 if (phi->adr_type() == TypePtr::BOTTOM) {
4165 Node* region = phi->in(0);
4166 for (DUIterator_Fast jmax, j = region->fast_outs(jmax); j < jmax; j++) {
4167 Node* uu = region->fast_out(j);
4168 if (uu->is_Phi() && uu != phi && uu->bottom_type() == Type::MEMORY && _phase->C->get_alias_index(uu->adr_type()) == _alias) {
4169 return false;
4170 }
4171 }
4172 return true;
4173 }
4174 return _phase->C->get_alias_index(phi->adr_type()) == _alias;
4175 }
4176
4177 void MemoryGraphFixer::fix_memory_uses(Node* mem, Node* replacement, Node* rep_proj, Node* rep_ctrl) const {
4178 uint last = _phase-> C->unique();
4179 MergeMemNode* mm = NULL;
4180 assert(mem->bottom_type() == Type::MEMORY, "");
4181 for (DUIterator i = mem->outs(); mem->has_out(i); i++) {
4182 Node* u = mem->out(i);
4183 if (u != replacement && u->_idx < last) {
4184 if (u->is_ShenandoahBarrier() && _alias != Compile::AliasIdxRaw) {
4185 if (_phase->C->get_alias_index(u->adr_type()) == _alias && ShenandoahWriteBarrierNode::is_dominator(rep_ctrl, _phase->ctrl_or_self(u), replacement, u, _phase)) {
4186 _phase->igvn().replace_input_of(u, u->find_edge(mem), rep_proj);
4187 assert(u->find_edge(mem) == -1, "only one edge");
4188 --i;
4189 }
4190 } else if (u->is_Mem()) {
4191 if (_phase->C->get_alias_index(u->adr_type()) == _alias && ShenandoahWriteBarrierNode::is_dominator(rep_ctrl, _phase->ctrl_or_self(u), replacement, u, _phase)) {
4192 assert(_alias == Compile::AliasIdxRaw , "only raw memory can lead to a memory operation");
4193 _phase->igvn().replace_input_of(u, u->find_edge(mem), rep_proj);
4194 assert(u->find_edge(mem) == -1, "only one edge");
4195 --i;
4196 }
4197 } else if (u->is_MergeMem()) {
4198 MergeMemNode* u_mm = u->as_MergeMem();
4199 if (u_mm->memory_at(_alias) == mem) {
4200 MergeMemNode* newmm = NULL;
4201 for (DUIterator_Fast jmax, j = u->fast_outs(jmax); j < jmax; j++) {
4202 Node* uu = u->fast_out(j);
4203 assert(!uu->is_MergeMem(), "chain of MergeMems?");
4204 if (uu->is_Phi()) {
4205 if (should_process_phi(uu)) {
4206 Node* region = uu->in(0);
4207 int nb = 0;
4208 for (uint k = 1; k < uu->req(); k++) {
4209 if (uu->in(k) == u && _phase->is_dominator(rep_ctrl, region->in(k))) {
4210 if (newmm == NULL) {
4211 newmm = clone_merge_mem(u, mem, rep_proj, rep_ctrl, i);
4212 }
4213 if (newmm != u) {
4214 _phase->igvn().replace_input_of(uu, k, newmm);
4215 nb++;
4216 --jmax;
4217 }
4218 }
4219 }
4220 if (nb > 0) {
4221 --j;
4222 }
4223 }
4224 } else {
4225 if (rep_ctrl != uu && ShenandoahWriteBarrierNode::is_dominator(rep_ctrl, _phase->ctrl_or_self(uu), replacement, uu, _phase)) {
4226 if (newmm == NULL) {
4227 newmm = clone_merge_mem(u, mem, rep_proj, rep_ctrl, i);
4228 }
4229 if (newmm != u) {
4230 _phase->igvn().replace_input_of(uu, uu->find_edge(u), newmm);
4231 --j, --jmax;
4232 }
4233 }
4234 }
4235 }
4236 }
4237 } else if (u->is_Phi()) {
4238 assert(u->bottom_type() == Type::MEMORY, "what else?");
4239 Node* region = u->in(0);
4240 if (should_process_phi(u)) {
4241 bool replaced = false;
4242 for (uint j = 1; j < u->req(); j++) {
4243 if (u->in(j) == mem && _phase->is_dominator(rep_ctrl, region->in(j))) {
4244 Node* nnew = rep_proj;
4245 if (u->adr_type() == TypePtr::BOTTOM) {
4246 if (mm == NULL) {
4247 mm = allocate_merge_mem(mem, rep_proj, rep_ctrl);
4248 }
4249 nnew = mm;
4250 }
4251 _phase->igvn().replace_input_of(u, j, nnew);
4252 replaced = true;
4253 }
4254 }
4255 if (replaced) {
4256 --i;
4257 }
4258
4259 }
4260 } else if ((u->adr_type() == TypePtr::BOTTOM && u->Opcode() != Op_StrInflatedCopy) ||
4261 u->adr_type() == NULL) {
4262 assert(u->adr_type() != NULL ||
4263 u->Opcode() == Op_Rethrow ||
4264 u->Opcode() == Op_Return ||
4265 u->Opcode() == Op_SafePoint ||
4266 (u->is_CallStaticJava() && u->as_CallStaticJava()->uncommon_trap_request() != 0) ||
4267 (u->is_CallStaticJava() && u->as_CallStaticJava()->_entry_point == OptoRuntime::rethrow_stub()) ||
4268 u->Opcode() == Op_CallLeaf, "");
4269 if (ShenandoahWriteBarrierNode::is_dominator(rep_ctrl, _phase->ctrl_or_self(u), replacement, u, _phase)) {
4270 if (mm == NULL) {
4271 mm = allocate_merge_mem(mem, rep_proj, rep_ctrl);
4272 }
4273 _phase->igvn().replace_input_of(u, u->find_edge(mem), mm);
4274 --i;
4275 }
4276 } else if (_phase->C->get_alias_index(u->adr_type()) == _alias) {
4277 if (ShenandoahWriteBarrierNode::is_dominator(rep_ctrl, _phase->ctrl_or_self(u), replacement, u, _phase)) {
4278 _phase->igvn().replace_input_of(u, u->find_edge(mem), rep_proj);
4279 --i;
4280 }
4281 }
4282 }
4283 }
4284 }
4285
4286 void MemoryGraphFixer::remove(Node* n) {
4287 assert(n->Opcode() == Op_ShenandoahWBMemProj, "");
4288 Node* c = _phase->get_ctrl(n);
4289 Node* mem = find_mem(c, NULL);
4290 if (mem == n) {
4291 _memory_nodes.map(c->_idx, mem->in(ShenandoahWBMemProjNode::WriteBarrier)->in(ShenandoahBarrierNode::Memory));
4292 }
4293 }
|
24 #include "precompiled.hpp"
25
26 #include "gc/shenandoah/c2/shenandoahSupport.hpp"
27 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
28 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
29 #include "gc/shenandoah/shenandoahBrooksPointer.hpp"
30 #include "gc/shenandoah/shenandoahHeap.hpp"
31 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
32 #include "gc/shenandoah/shenandoahRuntime.hpp"
33 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
34 #include "opto/arraycopynode.hpp"
35 #include "opto/block.hpp"
36 #include "opto/callnode.hpp"
37 #include "opto/castnode.hpp"
38 #include "opto/movenode.hpp"
39 #include "opto/phaseX.hpp"
40 #include "opto/rootnode.hpp"
41 #include "opto/runtime.hpp"
42 #include "opto/subnode.hpp"
43
44 bool ShenandoahBarrierC2Support::expand(Compile* C, PhaseIterGVN& igvn) {
45 ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state();
46 if ((state->enqueue_barriers_count() +
47 state->load_reference_barriers_count()) > 0) {
48 bool attempt_more_loopopts = ShenandoahLoopOptsAfterExpansion;
49 C->clear_major_progress();
50 PhaseIdealLoop ideal_loop(igvn, LoopOptsShenandoahExpand);
51 if (C->failing()) return false;
52 PhaseIdealLoop::verify(igvn);
53 DEBUG_ONLY(verify_raw_mem(C->root());)
54 if (attempt_more_loopopts) {
55 C->set_major_progress();
56 if (!C->optimize_loops(igvn, LoopOptsShenandoahPostExpand)) {
57 return false;
58 }
59 C->clear_major_progress();
60 }
61 }
62 return true;
63 }
64
65 bool ShenandoahBarrierC2Support::is_heap_state_test(Node* iff, int mask) {
66 if (!UseShenandoahGC) {
67 return false;
68 }
69 assert(iff->is_If(), "bad input");
70 if (iff->Opcode() != Op_If) {
71 return false;
72 }
73 Node* bol = iff->in(1);
74 if (!bol->is_Bool() || bol->as_Bool()->_test._test != BoolTest::ne) {
75 return false;
76 }
77 Node* cmp = bol->in(1);
78 if (cmp->Opcode() != Op_CmpI) {
79 return false;
80 }
81 Node* in1 = cmp->in(1);
82 Node* in2 = cmp->in(2);
83 if (in2->find_int_con(-1) != 0) {
84 return false;
85 }
86 if (in1->Opcode() != Op_AndI) {
87 return false;
88 }
89 in2 = in1->in(2);
90 if (in2->find_int_con(-1) != mask) {
91 return false;
92 }
93 in1 = in1->in(1);
94
95 return is_gc_state_load(in1);
96 }
97
98 bool ShenandoahBarrierC2Support::is_heap_stable_test(Node* iff) {
99 return is_heap_state_test(iff, ShenandoahHeap::HAS_FORWARDED);
100 }
101
102 bool ShenandoahBarrierC2Support::is_gc_state_load(Node *n) {
103 if (!UseShenandoahGC) {
104 return false;
105 }
106 if (n->Opcode() != Op_LoadB && n->Opcode() != Op_LoadUB) {
107 return false;
108 }
109 Node* addp = n->in(MemNode::Address);
110 if (!addp->is_AddP()) {
111 return false;
112 }
113 Node* base = addp->in(AddPNode::Address);
114 Node* off = addp->in(AddPNode::Offset);
115 if (base->Opcode() != Op_ThreadLocal) {
116 return false;
117 }
118 if (off->find_intptr_t_con(-1) != in_bytes(ShenandoahThreadLocalData::gc_state_offset())) {
119 return false;
120 }
121 return true;
122 }
123
124 bool ShenandoahBarrierC2Support::has_safepoint_between(Node* start, Node* stop, PhaseIdealLoop *phase) {
125 assert(phase->is_dominator(stop, start), "bad inputs");
126 ResourceMark rm;
127 Unique_Node_List wq;
128 wq.push(start);
129 for (uint next = 0; next < wq.size(); next++) {
130 Node *m = wq.at(next);
131 if (m == stop) {
132 continue;
133 }
134 if (m->is_SafePoint() && !m->is_CallLeaf()) {
135 return true;
136 }
137 if (m->is_Region()) {
138 for (uint i = 1; i < m->req(); i++) {
139 wq.push(m->in(i));
140 }
141 } else {
142 wq.push(m->in(0));
143 }
144 }
145 return false;
146 }
147
148 bool ShenandoahBarrierC2Support::try_common_gc_state_load(Node *n, PhaseIdealLoop *phase) {
149 assert(is_gc_state_load(n), "inconsistent");
150 Node* addp = n->in(MemNode::Address);
151 Node* dominator = NULL;
152 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
153 Node* u = addp->fast_out(i);
154 assert(is_gc_state_load(u), "inconsistent");
155 if (u != n && phase->is_dominator(u->in(0), n->in(0))) {
156 if (dominator == NULL) {
157 dominator = u;
158 } else {
159 if (phase->dom_depth(u->in(0)) < phase->dom_depth(dominator->in(0))) {
160 dominator = u;
161 }
162 }
163 }
164 }
165 if (dominator == NULL || has_safepoint_between(n->in(0), dominator->in(0), phase)) {
166 return false;
167 }
168 phase->igvn().replace_node(n, dominator);
169
170 return true;
171 }
172
173 #ifdef ASSERT
174 bool ShenandoahBarrierC2Support::verify_helper(Node* in, Node_Stack& phis, VectorSet& visited, verify_type t, bool trace, Unique_Node_List& barriers_used) {
175 assert(phis.size() == 0, "");
176
177 while (true) {
178 if (in->bottom_type() == TypePtr::NULL_PTR) {
179 if (trace) {tty->print_cr("NULL");}
180 } else if (!in->bottom_type()->make_ptr()->make_oopptr()) {
181 if (trace) {tty->print_cr("Non oop");}
182 } else if (t == ShenandoahLoad && ShenandoahOptimizeStableFinals &&
183 in->bottom_type()->make_ptr()->isa_aryptr() &&
184 in->bottom_type()->make_ptr()->is_aryptr()->is_stable()) {
185 if (trace) {tty->print_cr("Stable array load");}
186 } else {
187 if (in->is_ConstraintCast()) {
188 in = in->in(1);
189 continue;
190 } else if (in->is_AddP()) {
191 assert(!in->in(AddPNode::Address)->is_top(), "no raw memory access");
192 in = in->in(AddPNode::Address);
193 continue;
194 } else if (in->is_Con()) {
195 if (trace) {
196 tty->print("Found constant");
197 in->dump();
198 }
199 } else if (in->Opcode() == Op_Parm) {
200 if (trace) {
201 tty->print("Found argument");
202 }
203 } else if (in->Opcode() == Op_CreateEx) {
204 if (trace) {
205 tty->print("Found create-exception");
206 }
207 } else if (in->Opcode() == Op_LoadP && in->adr_type() == TypeRawPtr::BOTTOM) {
208 if (trace) {
209 tty->print("Found raw LoadP (OSR argument?)");
210 }
211 } else if (in->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
212 if (t == ShenandoahOopStore) {
213 uint i = 0;
214 for (; i < phis.size(); i++) {
215 Node* n = phis.node_at(i);
216 if (n->Opcode() == Op_ShenandoahEnqueueBarrier) {
217 break;
218 }
219 }
220 if (i == phis.size()) {
221 return false;
222 }
223 }
224 barriers_used.push(in);
225 if (trace) {tty->print("Found barrier"); in->dump();}
226 } else if (in->Opcode() == Op_ShenandoahEnqueueBarrier) {
227 if (t != ShenandoahOopStore) {
228 in = in->in(1);
229 continue;
230 }
231 if (trace) {tty->print("Found enqueue barrier"); in->dump();}
232 phis.push(in, in->req());
233 in = in->in(1);
234 continue;
235 } else if (in->is_Proj() && in->in(0)->is_Allocate()) {
236 if (trace) {
237 tty->print("Found alloc");
238 in->in(0)->dump();
239 }
240 } else if (in->is_Proj() && (in->in(0)->Opcode() == Op_CallStaticJava || in->in(0)->Opcode() == Op_CallDynamicJava)) {
241 if (trace) {
242 tty->print("Found Java call");
243 }
244 } else if (in->is_Phi()) {
245 if (!visited.test_set(in->_idx)) {
246 if (trace) {tty->print("Pushed phi:"); in->dump();}
247 phis.push(in, 2);
248 in = in->in(1);
249 continue;
250 }
251 if (trace) {tty->print("Already seen phi:"); in->dump();}
252 } else if (in->Opcode() == Op_CMoveP || in->Opcode() == Op_CMoveN) {
253 if (!visited.test_set(in->_idx)) {
254 if (trace) {tty->print("Pushed cmovep:"); in->dump();}
255 phis.push(in, CMoveNode::IfTrue);
256 in = in->in(CMoveNode::IfFalse);
257 continue;
258 }
259 if (trace) {tty->print("Already seen cmovep:"); in->dump();}
260 } else if (in->Opcode() == Op_EncodeP || in->Opcode() == Op_DecodeN) {
261 in = in->in(1);
262 continue;
263 } else {
269 uint idx = phis.index();
270 Node* phi = phis.node();
271 if (idx >= phi->req()) {
272 if (trace) {tty->print("Popped phi:"); phi->dump();}
273 phis.pop();
274 continue;
275 }
276 if (trace) {tty->print("Next entry(%d) for phi:", idx); phi->dump();}
277 in = phi->in(idx);
278 phis.set_index(idx+1);
279 cont = true;
280 break;
281 }
282 if (!cont) {
283 break;
284 }
285 }
286 return true;
287 }
288
289 void ShenandoahBarrierC2Support::report_verify_failure(const char* msg, Node* n1, Node* n2) {
290 if (n1 != NULL) {
291 n1->dump(+10);
292 }
293 if (n2 != NULL) {
294 n2->dump(+10);
295 }
296 fatal("%s", msg);
297 }
298
299 void ShenandoahBarrierC2Support::verify(RootNode* root) {
300 ResourceMark rm;
301 Unique_Node_List wq;
302 GrowableArray<Node*> barriers;
303 Unique_Node_List barriers_used;
304 Node_Stack phis(0);
305 VectorSet visited(Thread::current()->resource_area());
306 const bool trace = false;
307 const bool verify_no_useless_barrier = false;
308
309 wq.push(root);
310 for (uint next = 0; next < wq.size(); next++) {
311 Node *n = wq.at(next);
312 if (n->is_Load()) {
313 const bool trace = false;
314 if (trace) {tty->print("Verifying"); n->dump();}
315 if (n->Opcode() == Op_LoadRange || n->Opcode() == Op_LoadKlass || n->Opcode() == Op_LoadNKlass) {
316 if (trace) {tty->print_cr("Load range/klass");}
317 } else {
318 const TypePtr* adr_type = n->as_Load()->adr_type();
319
331 assert(k->is_instance_klass(), "");
332 ciInstanceKlass* ik = (ciInstanceKlass*)k;
333 int offset = adr_type->offset();
334
335 if ((ik->debug_final_field_at(offset) && ShenandoahOptimizeInstanceFinals) ||
336 (ik->debug_stable_field_at(offset) && ShenandoahOptimizeStableFinals)) {
337 if (trace) {tty->print_cr("Final/stable");}
338 verify = false;
339 } else if (k == ciEnv::current()->Class_klass() &&
340 tinst->const_oop() != NULL &&
341 tinst->offset() >= (ik->size_helper() * wordSize)) {
342 ciInstanceKlass* k = tinst->const_oop()->as_instance()->java_lang_Class_klass()->as_instance_klass();
343 ciField* field = k->get_field_by_offset(tinst->offset(), true);
344 if ((ShenandoahOptimizeStaticFinals && field->is_final()) ||
345 (ShenandoahOptimizeStableFinals && field->is_stable())) {
346 verify = false;
347 }
348 }
349 }
350
351 if (verify && !verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahLoad, trace, barriers_used)) {
352 report_verify_failure("Shenandoah verification: Load should have barriers", n);
353 }
354 }
355 }
356 } else if (n->is_Store()) {
357 const bool trace = false;
358
359 if (trace) {tty->print("Verifying"); n->dump();}
360 if (n->in(MemNode::ValueIn)->bottom_type()->make_oopptr()) {
361 Node* adr = n->in(MemNode::Address);
362 bool verify = true;
363
364 if (adr->is_AddP() && adr->in(AddPNode::Base)->is_top()) {
365 adr = adr->in(AddPNode::Address);
366 if (adr->is_AddP()) {
367 assert(adr->in(AddPNode::Base)->is_top(), "");
368 adr = adr->in(AddPNode::Address);
369 if (adr->Opcode() == Op_LoadP &&
370 adr->in(MemNode::Address)->in(AddPNode::Base)->is_top() &&
371 adr->in(MemNode::Address)->in(AddPNode::Address)->Opcode() == Op_ThreadLocal &&
372 adr->in(MemNode::Address)->in(AddPNode::Offset)->find_intptr_t_con(-1) == in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset())) {
373 if (trace) {tty->print_cr("SATB prebarrier");}
374 verify = false;
375 }
376 }
377 }
378
379 if (verify && !verify_helper(n->in(MemNode::ValueIn), phis, visited, ShenandoahStoreValEnqueueBarrier ? ShenandoahOopStore : ShenandoahValue, trace, barriers_used)) {
380 report_verify_failure("Shenandoah verification: Store should have barriers", n);
381 }
382 }
383 if (!verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahStore, trace, barriers_used)) {
384 report_verify_failure("Shenandoah verification: Store (address) should have barriers", n);
385 }
386 } else if (n->Opcode() == Op_CmpP) {
387 const bool trace = false;
388
389 Node* in1 = n->in(1);
390 Node* in2 = n->in(2);
391 if (in1->bottom_type()->isa_oopptr()) {
392 if (trace) {tty->print("Verifying"); n->dump();}
393
394 bool mark_inputs = false;
395 if (in1->bottom_type() == TypePtr::NULL_PTR || in2->bottom_type() == TypePtr::NULL_PTR ||
396 (in1->is_Con() || in2->is_Con())) {
397 if (trace) {tty->print_cr("Comparison against a constant");}
398 mark_inputs = true;
399 } else if ((in1->is_CheckCastPP() && in1->in(1)->is_Proj() && in1->in(1)->in(0)->is_Allocate()) ||
400 (in2->is_CheckCastPP() && in2->in(1)->is_Proj() && in2->in(1)->in(0)->is_Allocate())) {
401 if (trace) {tty->print_cr("Comparison with newly alloc'ed object");}
402 mark_inputs = true;
403 } else {
404 assert(in2->bottom_type()->isa_oopptr(), "");
405
406 if (!verify_helper(in1, phis, visited, ShenandoahStore, trace, barriers_used) ||
407 !verify_helper(in2, phis, visited, ShenandoahStore, trace, barriers_used)) {
408 report_verify_failure("Shenandoah verification: Cmp should have barriers", n);
409 }
410 }
411 if (verify_no_useless_barrier &&
412 mark_inputs &&
413 (!verify_helper(in1, phis, visited, ShenandoahValue, trace, barriers_used) ||
414 !verify_helper(in2, phis, visited, ShenandoahValue, trace, barriers_used))) {
415 phis.clear();
416 visited.Reset();
417 }
418 }
419 } else if (n->is_LoadStore()) {
420 if (n->in(MemNode::ValueIn)->bottom_type()->make_ptr() &&
421 !verify_helper(n->in(MemNode::ValueIn), phis, visited, ShenandoahStoreValEnqueueBarrier ? ShenandoahOopStore : ShenandoahValue, trace, barriers_used)) {
422 report_verify_failure("Shenandoah verification: LoadStore (value) should have barriers", n);
423 }
424
425 if (n->in(MemNode::Address)->bottom_type()->make_oopptr() && !verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahStore, trace, barriers_used)) {
426 report_verify_failure("Shenandoah verification: LoadStore (address) should have barriers", n);
427 }
428 } else if (n->Opcode() == Op_CallLeafNoFP || n->Opcode() == Op_CallLeaf) {
429 CallNode* call = n->as_Call();
430
431 static struct {
432 const char* name;
433 struct {
434 int pos;
435 verify_type t;
436 } args[6];
437 } calls[] = {
438 "aescrypt_encryptBlock",
439 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad },
440 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
441 "aescrypt_decryptBlock",
442 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad },
443 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
444 "multiplyToLen",
445 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad }, { TypeFunc::Parms+4, ShenandoahStore },
501 "sha512_implCompressMB",
502 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
503 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
504 "encodeBlock",
505 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahStore }, { -1, ShenandoahNone },
506 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
507 };
508
509 if (call->is_call_to_arraycopystub()) {
510 Node* dest = NULL;
511 const TypeTuple* args = n->as_Call()->_tf->domain();
512 for (uint i = TypeFunc::Parms, j = 0; i < args->cnt(); i++) {
513 if (args->field_at(i)->isa_ptr()) {
514 j++;
515 if (j == 2) {
516 dest = n->in(i);
517 break;
518 }
519 }
520 }
521 if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahLoad, trace, barriers_used) ||
522 !verify_helper(dest, phis, visited, ShenandoahStore, trace, barriers_used)) {
523 report_verify_failure("Shenandoah verification: ArrayCopy should have barriers", n);
524 }
525 } else if (strlen(call->_name) > 5 &&
526 !strcmp(call->_name + strlen(call->_name) - 5, "_fill")) {
527 if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahStore, trace, barriers_used)) {
528 report_verify_failure("Shenandoah verification: _fill should have barriers", n);
529 }
530 } else if (!strcmp(call->_name, "shenandoah_wb_pre")) {
531 // skip
532 } else {
533 const int calls_len = sizeof(calls) / sizeof(calls[0]);
534 int i = 0;
535 for (; i < calls_len; i++) {
536 if (!strcmp(calls[i].name, call->_name)) {
537 break;
538 }
539 }
540 if (i != calls_len) {
541 const uint args_len = sizeof(calls[0].args) / sizeof(calls[0].args[0]);
542 for (uint j = 0; j < args_len; j++) {
543 int pos = calls[i].args[j].pos;
544 if (pos == -1) {
545 break;
546 }
547 if (!verify_helper(call->in(pos), phis, visited, calls[i].args[j].t, trace, barriers_used)) {
548 report_verify_failure("Shenandoah verification: intrinsic calls should have barriers", n);
549 }
550 }
551 for (uint j = TypeFunc::Parms; j < call->req(); j++) {
552 if (call->in(j)->bottom_type()->make_ptr() &&
553 call->in(j)->bottom_type()->make_ptr()->isa_oopptr()) {
554 uint k = 0;
555 for (; k < args_len && calls[i].args[k].pos != (int)j; k++);
556 if (k == args_len) {
557 fatal("arg %d for call %s not covered", j, call->_name);
558 }
559 }
560 }
561 } else {
562 for (uint j = TypeFunc::Parms; j < call->req(); j++) {
563 if (call->in(j)->bottom_type()->make_ptr() &&
564 call->in(j)->bottom_type()->make_ptr()->isa_oopptr()) {
565 fatal("%s not covered", call->_name);
566 }
567 }
568 }
569 }
570 } else if (n->Opcode() == Op_ShenandoahEnqueueBarrier || n->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
571 // skip
572 } else if (n->is_AddP()
573 || n->is_Phi()
574 || n->is_ConstraintCast()
575 || n->Opcode() == Op_Return
576 || n->Opcode() == Op_CMoveP
577 || n->Opcode() == Op_CMoveN
578 || n->Opcode() == Op_Rethrow
579 || n->is_MemBar()
580 || n->Opcode() == Op_Conv2B
581 || n->Opcode() == Op_SafePoint
582 || n->is_CallJava()
583 || n->Opcode() == Op_Unlock
584 || n->Opcode() == Op_EncodeP
585 || n->Opcode() == Op_DecodeN) {
586 // nothing to do
587 } else {
588 static struct {
589 int opcode;
590 struct {
591 int pos;
618 { { 1, ShenandoahLoad }, { -1, ShenandoahNone} },
619 Op_StrIndexOfChar,
620 { { 2, ShenandoahLoad }, { -1, ShenandoahNone } },
621 };
622
623 const int others_len = sizeof(others) / sizeof(others[0]);
624 int i = 0;
625 for (; i < others_len; i++) {
626 if (others[i].opcode == n->Opcode()) {
627 break;
628 }
629 }
630 uint stop = n->is_Call() ? n->as_Call()->tf()->domain()->cnt() : n->req();
631 if (i != others_len) {
632 const uint inputs_len = sizeof(others[0].inputs) / sizeof(others[0].inputs[0]);
633 for (uint j = 0; j < inputs_len; j++) {
634 int pos = others[i].inputs[j].pos;
635 if (pos == -1) {
636 break;
637 }
638 if (!verify_helper(n->in(pos), phis, visited, others[i].inputs[j].t, trace, barriers_used)) {
639 report_verify_failure("Shenandoah verification: intrinsic calls should have barriers", n);
640 }
641 }
642 for (uint j = 1; j < stop; j++) {
643 if (n->in(j) != NULL && n->in(j)->bottom_type()->make_ptr() &&
644 n->in(j)->bottom_type()->make_ptr()->make_oopptr()) {
645 uint k = 0;
646 for (; k < inputs_len && others[i].inputs[k].pos != (int)j; k++);
647 if (k == inputs_len) {
648 fatal("arg %d for node %s not covered", j, n->Name());
649 }
650 }
651 }
652 } else {
653 for (uint j = 1; j < stop; j++) {
654 if (n->in(j) != NULL && n->in(j)->bottom_type()->make_ptr() &&
655 n->in(j)->bottom_type()->make_ptr()->make_oopptr()) {
656 fatal("%s not covered", n->Name());
657 }
658 }
659 }
660 }
661
662 if (n->is_SafePoint()) {
663 SafePointNode* sfpt = n->as_SafePoint();
664 if (verify_no_useless_barrier && sfpt->jvms() != NULL) {
665 for (uint i = sfpt->jvms()->scloff(); i < sfpt->jvms()->endoff(); i++) {
666 if (!verify_helper(sfpt->in(i), phis, visited, ShenandoahLoad, trace, barriers_used)) {
667 phis.clear();
668 visited.Reset();
669 }
670 }
671 }
672 }
673 for( uint i = 0; i < n->len(); ++i ) {
674 Node *m = n->in(i);
675 if (m == NULL) continue;
676
677 // In most cases, inputs should be known to be non null. If it's
678 // not the case, it could be a missing cast_not_null() in an
679 // intrinsic or support might be needed in AddPNode::Ideal() to
680 // avoid a NULL+offset input.
681 if (!(n->is_Phi() ||
682 (n->is_SafePoint() && (!n->is_CallRuntime() || !strcmp(n->as_Call()->_name, "shenandoah_wb_pre") || !strcmp(n->as_Call()->_name, "unsafe_arraycopy"))) ||
683 n->Opcode() == Op_CmpP ||
684 n->Opcode() == Op_CmpN ||
685 (n->Opcode() == Op_StoreP && i == StoreNode::ValueIn) ||
686 (n->Opcode() == Op_StoreN && i == StoreNode::ValueIn) ||
687 n->is_ConstraintCast() ||
688 n->Opcode() == Op_Return ||
689 n->Opcode() == Op_Conv2B ||
690 n->is_AddP() ||
691 n->Opcode() == Op_CMoveP ||
692 n->Opcode() == Op_CMoveN ||
693 n->Opcode() == Op_Rethrow ||
694 n->is_MemBar() ||
695 n->is_Mem() ||
696 n->Opcode() == Op_AryEq ||
697 n->Opcode() == Op_SCMemProj ||
698 n->Opcode() == Op_EncodeP ||
699 n->Opcode() == Op_DecodeN ||
700 n->Opcode() == Op_ShenandoahEnqueueBarrier ||
701 n->Opcode() == Op_ShenandoahLoadReferenceBarrier)) {
702 if (m->bottom_type()->make_oopptr() && m->bottom_type()->make_oopptr()->meet(TypePtr::NULL_PTR) == m->bottom_type()) {
703 report_verify_failure("Shenandoah verification: null input", n, m);
704 }
705 }
706
707 wq.push(m);
708 }
709 }
710
711 if (verify_no_useless_barrier) {
712 for (int i = 0; i < barriers.length(); i++) {
713 Node* n = barriers.at(i);
714 if (!barriers_used.member(n)) {
715 tty->print("XXX useless barrier"); n->dump(-2);
716 ShouldNotReachHere();
717 }
718 }
719 }
720 }
721 #endif
722
723 bool ShenandoahBarrierC2Support::is_dominator_same_ctrl(Node* c, Node* d, Node* n, PhaseIdealLoop* phase) {
724 // That both nodes have the same control is not sufficient to prove
725 // domination, verify that there's no path from d to n
726 ResourceMark rm;
727 Unique_Node_List wq;
728 wq.push(d);
729 for (uint next = 0; next < wq.size(); next++) {
730 Node *m = wq.at(next);
731 if (m == n) {
732 return false;
733 }
734 if (m->is_Phi() && m->in(0)->is_Loop()) {
735 assert(phase->ctrl_or_self(m->in(LoopNode::EntryControl)) != c, "following loop entry should lead to new control");
736 } else {
737 for (uint i = 0; i < m->req(); i++) {
738 if (m->in(i) != NULL && phase->ctrl_or_self(m->in(i)) == c) {
739 wq.push(m->in(i));
740 }
741 }
742 }
743 }
744 return true;
745 }
746
747 bool ShenandoahBarrierC2Support::is_dominator(Node* d_c, Node* n_c, Node* d, Node* n, PhaseIdealLoop* phase) {
748 if (d_c != n_c) {
749 return phase->is_dominator(d_c, n_c);
750 }
751 return is_dominator_same_ctrl(d_c, d, n, phase);
752 }
753
754 Node* next_mem(Node* mem, int alias) {
755 Node* res = NULL;
756 if (mem->is_Proj()) {
757 res = mem->in(0);
758 } else if (mem->is_SafePoint() || mem->is_MemBar()) {
759 res = mem->in(TypeFunc::Memory);
760 } else if (mem->is_Phi()) {
761 res = mem->in(1);
762 } else if (mem->is_MergeMem()) {
763 res = mem->as_MergeMem()->memory_at(alias);
764 } else if (mem->is_Store() || mem->is_LoadStore() || mem->is_ClearArray()) {
765 assert(alias = Compile::AliasIdxRaw, "following raw memory can't lead to a barrier");
766 res = mem->in(MemNode::Memory);
767 } else {
768 #ifdef ASSERT
769 mem->dump();
770 #endif
771 ShouldNotReachHere();
772 }
773 return res;
774 }
775
776 Node* ShenandoahBarrierC2Support::no_branches(Node* c, Node* dom, bool allow_one_proj, PhaseIdealLoop* phase) {
777 Node* iffproj = NULL;
778 while (c != dom) {
779 Node* next = phase->idom(c);
780 assert(next->unique_ctrl_out() == c || c->is_Proj() || c->is_Region(), "multiple control flow out but no proj or region?");
781 if (c->is_Region()) {
782 ResourceMark rm;
783 Unique_Node_List wq;
784 wq.push(c);
785 for (uint i = 0; i < wq.size(); i++) {
786 Node *n = wq.at(i);
787 if (n == next) {
788 continue;
789 }
790 if (n->is_Region()) {
791 for (uint j = 1; j < n->req(); j++) {
792 wq.push(n->in(j));
793 }
794 } else {
795 wq.push(n->in(0));
796 }
821 iffproj = c;
822 } else {
823 return NodeSentinel;
824 }
825 }
826 } else if (c->Opcode() == Op_JumpProj) {
827 return NodeSentinel; // unsupported
828 } else if (c->Opcode() == Op_CatchProj) {
829 return NodeSentinel; // unsupported
830 } else if (c->Opcode() == Op_CProj && next->Opcode() == Op_NeverBranch) {
831 return NodeSentinel; // unsupported
832 } else {
833 assert(next->unique_ctrl_out() == c, "unsupported branch pattern");
834 }
835 }
836 c = next;
837 }
838 return iffproj;
839 }
840
841 Node* ShenandoahBarrierC2Support::dom_mem(Node* mem, Node* ctrl, int alias, Node*& mem_ctrl, PhaseIdealLoop* phase) {
842 ResourceMark rm;
843 VectorSet wq(Thread::current()->resource_area());
844 wq.set(mem->_idx);
845 mem_ctrl = phase->ctrl_or_self(mem);
846 while (!phase->is_dominator(mem_ctrl, ctrl) || mem_ctrl == ctrl) {
847 mem = next_mem(mem, alias);
848 if (wq.test_set(mem->_idx)) {
849 return NULL;
850 }
851 mem_ctrl = phase->ctrl_or_self(mem);
852 }
853 if (mem->is_MergeMem()) {
854 mem = mem->as_MergeMem()->memory_at(alias);
855 mem_ctrl = phase->ctrl_or_self(mem);
856 }
857 return mem;
858 }
859
860 Node* ShenandoahBarrierC2Support::find_bottom_mem(Node* ctrl, PhaseIdealLoop* phase) {
861 Node* mem = NULL;
862 Node* c = ctrl;
863 do {
864 if (c->is_Region()) {
865 Node* phi_bottom = NULL;
866 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax && mem == NULL; i++) {
867 Node* u = c->fast_out(i);
868 if (u->is_Phi() && u->bottom_type() == Type::MEMORY) {
869 if (u->adr_type() == TypePtr::BOTTOM) {
870 mem = u;
871 }
872 }
873 }
874 } else {
875 if (c->is_Call() && c->as_Call()->adr_type() != NULL) {
876 CallProjections projs;
877 c->as_Call()->extract_projections(&projs, true, false);
878 if (projs.fallthrough_memproj != NULL) {
879 if (projs.fallthrough_memproj->adr_type() == TypePtr::BOTTOM) {
880 if (projs.catchall_memproj == NULL) {
881 mem = projs.fallthrough_memproj;
882 } else {
883 if (phase->is_dominator(projs.fallthrough_catchproj, ctrl)) {
884 mem = projs.fallthrough_memproj;
885 } else {
886 assert(phase->is_dominator(projs.catchall_catchproj, ctrl), "one proj must dominate barrier");
887 mem = projs.catchall_memproj;
888 }
889 }
890 }
891 } else {
892 Node* proj = c->as_Call()->proj_out(TypeFunc::Memory);
893 if (proj != NULL &&
894 proj->adr_type() == TypePtr::BOTTOM) {
895 mem = proj;
896 }
897 }
898 } else {
899 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) {
900 Node* u = c->fast_out(i);
901 if (u->is_Proj() &&
902 u->bottom_type() == Type::MEMORY &&
903 u->adr_type() == TypePtr::BOTTOM) {
904 assert(c->is_SafePoint() || c->is_MemBar() || c->is_Start(), "");
905 assert(mem == NULL, "only one proj");
906 mem = u;
907 }
908 }
909 assert(!c->is_Call() || c->as_Call()->adr_type() != NULL || mem == NULL, "no mem projection expected");
910 }
911 }
912 c = phase->idom(c);
913 } while (mem == NULL);
914 return mem;
915 }
916
917 void ShenandoahBarrierC2Support::follow_barrier_uses(Node* n, Node* ctrl, Unique_Node_List& uses, PhaseIdealLoop* phase) {
918 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
919 Node* u = n->fast_out(i);
920 if (!u->is_CFG() && phase->get_ctrl(u) == ctrl && (!u->is_Phi() || !u->in(0)->is_Loop() || u->in(LoopNode::LoopBackControl) != n)) {
921 uses.push(u);
922 }
923 }
924 }
925
926 static void hide_strip_mined_loop(OuterStripMinedLoopNode* outer, CountedLoopNode* inner, PhaseIdealLoop* phase) {
927 OuterStripMinedLoopEndNode* le = inner->outer_loop_end();
928 Node* new_outer = new LoopNode(outer->in(LoopNode::EntryControl), outer->in(LoopNode::LoopBackControl));
929 phase->register_control(new_outer, phase->get_loop(outer), outer->in(LoopNode::EntryControl));
930 Node* new_le = new IfNode(le->in(0), le->in(1), le->_prob, le->_fcnt);
931 phase->register_control(new_le, phase->get_loop(le), le->in(0));
932 phase->lazy_replace(outer, new_outer);
933 phase->lazy_replace(le, new_le);
934 inner->clear_strip_mined();
935 }
936
937 void ShenandoahBarrierC2Support::test_heap_stable(Node*& ctrl, Node* raw_mem, Node*& heap_stable_ctrl,
938 PhaseIdealLoop* phase) {
939 IdealLoopTree* loop = phase->get_loop(ctrl);
940 Node* thread = new ThreadLocalNode();
941 phase->register_new_node(thread, ctrl);
942 Node* offset = phase->igvn().MakeConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
943 phase->set_ctrl(offset, phase->C->root());
944 Node* gc_state_addr = new AddPNode(phase->C->top(), thread, offset);
945 phase->register_new_node(gc_state_addr, ctrl);
946 uint gc_state_idx = Compile::AliasIdxRaw;
947 const TypePtr* gc_state_adr_type = NULL; // debug-mode-only argument
948 debug_only(gc_state_adr_type = phase->C->get_adr_type(gc_state_idx));
949
950 Node* gc_state = new LoadBNode(ctrl, raw_mem, gc_state_addr, gc_state_adr_type, TypeInt::BYTE, MemNode::unordered);
951 phase->register_new_node(gc_state, ctrl);
952 Node* heap_stable_and = new AndINode(gc_state, phase->igvn().intcon(ShenandoahHeap::HAS_FORWARDED));
953 phase->register_new_node(heap_stable_and, ctrl);
954 Node* heap_stable_cmp = new CmpINode(heap_stable_and, phase->igvn().zerocon(T_INT));
955 phase->register_new_node(heap_stable_cmp, ctrl);
956 Node* heap_stable_test = new BoolNode(heap_stable_cmp, BoolTest::ne);
957 phase->register_new_node(heap_stable_test, ctrl);
958 IfNode* heap_stable_iff = new IfNode(ctrl, heap_stable_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN);
959 phase->register_control(heap_stable_iff, loop, ctrl);
960
961 heap_stable_ctrl = new IfFalseNode(heap_stable_iff);
962 phase->register_control(heap_stable_ctrl, loop, heap_stable_iff);
963 ctrl = new IfTrueNode(heap_stable_iff);
964 phase->register_control(ctrl, loop, heap_stable_iff);
965
966 assert(is_heap_stable_test(heap_stable_iff), "Should match the shape");
967 }
968
969 void ShenandoahBarrierC2Support::test_null(Node*& ctrl, Node* val, Node*& null_ctrl, PhaseIdealLoop* phase) {
970 const Type* val_t = phase->igvn().type(val);
971 if (val_t->meet(TypePtr::NULL_PTR) == val_t) {
972 IdealLoopTree* loop = phase->get_loop(ctrl);
973 Node* null_cmp = new CmpPNode(val, phase->igvn().zerocon(T_OBJECT));
974 phase->register_new_node(null_cmp, ctrl);
975 Node* null_test = new BoolNode(null_cmp, BoolTest::ne);
976 phase->register_new_node(null_test, ctrl);
977 IfNode* null_iff = new IfNode(ctrl, null_test, PROB_LIKELY(0.999), COUNT_UNKNOWN);
978 phase->register_control(null_iff, loop, ctrl);
979 ctrl = new IfTrueNode(null_iff);
980 phase->register_control(ctrl, loop, null_iff);
981 null_ctrl = new IfFalseNode(null_iff);
982 phase->register_control(null_ctrl, loop, null_iff);
983 }
984 }
985
986 Node* ShenandoahBarrierC2Support::clone_null_check(Node*& c, Node* val, Node* unc_ctrl, PhaseIdealLoop* phase) {
987 IdealLoopTree *loop = phase->get_loop(c);
988 Node* iff = unc_ctrl->in(0);
989 assert(iff->is_If(), "broken");
990 Node* new_iff = iff->clone();
991 new_iff->set_req(0, c);
992 phase->register_control(new_iff, loop, c);
993 Node* iffalse = new IfFalseNode(new_iff->as_If());
994 phase->register_control(iffalse, loop, new_iff);
995 Node* iftrue = new IfTrueNode(new_iff->as_If());
996 phase->register_control(iftrue, loop, new_iff);
997 c = iftrue;
998 const Type *t = phase->igvn().type(val);
999 assert(val->Opcode() == Op_CastPP, "expect cast to non null here");
1000 Node* uncasted_val = val->in(1);
1001 val = new CastPPNode(uncasted_val, t);
1002 val->init_req(0, c);
1003 phase->register_new_node(val, c);
1004 return val;
1005 }
1006
1007 void ShenandoahBarrierC2Support::fix_null_check(Node* unc, Node* unc_ctrl, Node* new_unc_ctrl,
1008 Unique_Node_List& uses, PhaseIdealLoop* phase) {
1009 IfNode* iff = unc_ctrl->in(0)->as_If();
1010 Node* proj = iff->proj_out(0);
1011 assert(proj != unc_ctrl, "bad projection");
1012 Node* use = proj->unique_ctrl_out();
1013
1014 assert(use == unc || use->is_Region(), "what else?");
1015
1016 uses.clear();
1017 if (use == unc) {
1018 phase->set_idom(use, new_unc_ctrl, phase->dom_depth(use));
1019 for (uint i = 1; i < unc->req(); i++) {
1020 Node* n = unc->in(i);
1021 if (phase->has_ctrl(n) && phase->get_ctrl(n) == proj) {
1022 uses.push(n);
1023 }
1024 }
1025 } else {
1026 assert(use->is_Region(), "what else?");
1027 uint idx = 1;
1036 for(uint next = 0; next < uses.size(); next++ ) {
1037 Node *n = uses.at(next);
1038 assert(phase->get_ctrl(n) == proj, "bad control");
1039 phase->set_ctrl_and_loop(n, new_unc_ctrl);
1040 if (n->in(0) == proj) {
1041 phase->igvn().replace_input_of(n, 0, new_unc_ctrl);
1042 }
1043 for (uint i = 0; i < n->req(); i++) {
1044 Node* m = n->in(i);
1045 if (m != NULL && phase->has_ctrl(m) && phase->get_ctrl(m) == proj) {
1046 uses.push(m);
1047 }
1048 }
1049 }
1050
1051 phase->igvn().rehash_node_delayed(use);
1052 int nb = use->replace_edge(proj, new_unc_ctrl);
1053 assert(nb == 1, "only use expected");
1054 }
1055
1056 void ShenandoahBarrierC2Support::in_cset_fast_test(Node*& ctrl, Node*& not_cset_ctrl, Node* val, Node* raw_mem, PhaseIdealLoop* phase) {
1057 IdealLoopTree *loop = phase->get_loop(ctrl);
1058 Node* raw_rbtrue = new CastP2XNode(ctrl, val);
1059 phase->register_new_node(raw_rbtrue, ctrl);
1060 Node* cset_offset = new URShiftXNode(raw_rbtrue, phase->igvn().intcon(ShenandoahHeapRegion::region_size_bytes_shift_jint()));
1061 phase->register_new_node(cset_offset, ctrl);
1062 Node* in_cset_fast_test_base_addr = phase->igvn().makecon(TypeRawPtr::make(ShenandoahHeap::in_cset_fast_test_addr()));
1063 phase->set_ctrl(in_cset_fast_test_base_addr, phase->C->root());
1064 Node* in_cset_fast_test_adr = new AddPNode(phase->C->top(), in_cset_fast_test_base_addr, cset_offset);
1065 phase->register_new_node(in_cset_fast_test_adr, ctrl);
1066 uint in_cset_fast_test_idx = Compile::AliasIdxRaw;
1067 const TypePtr* in_cset_fast_test_adr_type = NULL; // debug-mode-only argument
1068 debug_only(in_cset_fast_test_adr_type = phase->C->get_adr_type(in_cset_fast_test_idx));
1069 Node* in_cset_fast_test_load = new LoadBNode(ctrl, raw_mem, in_cset_fast_test_adr, in_cset_fast_test_adr_type, TypeInt::BYTE, MemNode::unordered);
1070 phase->register_new_node(in_cset_fast_test_load, ctrl);
1071 Node* in_cset_fast_test_cmp = new CmpINode(in_cset_fast_test_load, phase->igvn().zerocon(T_INT));
1072 phase->register_new_node(in_cset_fast_test_cmp, ctrl);
1073 Node* in_cset_fast_test_test = new BoolNode(in_cset_fast_test_cmp, BoolTest::eq);
1074 phase->register_new_node(in_cset_fast_test_test, ctrl);
1075 IfNode* in_cset_fast_test_iff = new IfNode(ctrl, in_cset_fast_test_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN);
1076 phase->register_control(in_cset_fast_test_iff, loop, ctrl);
1077
1078 not_cset_ctrl = new IfTrueNode(in_cset_fast_test_iff);
1079 phase->register_control(not_cset_ctrl, loop, in_cset_fast_test_iff);
1080
1081 ctrl = new IfFalseNode(in_cset_fast_test_iff);
1082 phase->register_control(ctrl, loop, in_cset_fast_test_iff);
1083 }
1084
1085 void ShenandoahBarrierC2Support::call_lrb_stub(Node*& ctrl, Node*& val, Node*& result_mem, Node* raw_mem, PhaseIdealLoop* phase) {
1086 IdealLoopTree*loop = phase->get_loop(ctrl);
1087 const TypePtr* obj_type = phase->igvn().type(val)->is_oopptr()->cast_to_nonconst();
1088
1089 // The slow path stub consumes and produces raw memory in addition
1090 // to the existing memory edges
1091 Node* base = find_bottom_mem(ctrl, phase);
1092 MergeMemNode* mm = MergeMemNode::make(base);
1093 mm->set_memory_at(Compile::AliasIdxRaw, raw_mem);
1094 phase->register_new_node(mm, ctrl);
1095
1096 Node* call = new CallLeafNode(ShenandoahBarrierSetC2::shenandoah_write_barrier_Type(), CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_JRT), "shenandoah_write_barrier", TypeRawPtr::BOTTOM);
1097 call->init_req(TypeFunc::Control, ctrl);
1098 call->init_req(TypeFunc::I_O, phase->C->top());
1099 call->init_req(TypeFunc::Memory, mm);
1100 call->init_req(TypeFunc::FramePtr, phase->C->top());
1101 call->init_req(TypeFunc::ReturnAdr, phase->C->top());
1102 call->init_req(TypeFunc::Parms, val);
1103 phase->register_control(call, loop, ctrl);
1104 ctrl = new ProjNode(call, TypeFunc::Control);
1105 phase->register_control(ctrl, loop, call);
1106 result_mem = new ProjNode(call, TypeFunc::Memory);
1107 phase->register_new_node(result_mem, call);
1108 val = new ProjNode(call, TypeFunc::Parms);
1109 phase->register_new_node(val, call);
1110 val = new CheckCastPPNode(ctrl, val, obj_type);
1111 phase->register_new_node(val, ctrl);
1112 }
1113
1114 void ShenandoahBarrierC2Support::fix_ctrl(Node* barrier, Node* region, const MemoryGraphFixer& fixer, Unique_Node_List& uses, Unique_Node_List& uses_to_ignore, uint last, PhaseIdealLoop* phase) {
1115 Node* ctrl = phase->get_ctrl(barrier);
1116 Node* init_raw_mem = fixer.find_mem(ctrl, barrier);
1117
1118 // Update the control of all nodes that should be after the
1119 // barrier control flow
1120 uses.clear();
1121 // Every node that is control dependent on the barrier's input
1122 // control will be after the expanded barrier. The raw memory (if
1123 // its memory is control dependent on the barrier's input control)
1124 // must stay above the barrier.
1125 uses_to_ignore.clear();
1126 if (phase->has_ctrl(init_raw_mem) && phase->get_ctrl(init_raw_mem) == ctrl && !init_raw_mem->is_Phi()) {
1127 uses_to_ignore.push(init_raw_mem);
1128 }
1129 for (uint next = 0; next < uses_to_ignore.size(); next++) {
1130 Node *n = uses_to_ignore.at(next);
1131 for (uint i = 0; i < n->req(); i++) {
1132 Node* in = n->in(i);
1133 if (in != NULL && phase->has_ctrl(in) && phase->get_ctrl(in) == ctrl) {
1134 uses_to_ignore.push(in);
1147 if (c != ctrl ||
1148 is_dominator_same_ctrl(old_c, barrier, u, phase) ||
1149 ShenandoahBarrierSetC2::is_shenandoah_state_load(u)) {
1150 phase->igvn().rehash_node_delayed(u);
1151 int nb = u->replace_edge(ctrl, region);
1152 if (u->is_CFG()) {
1153 if (phase->idom(u) == ctrl) {
1154 phase->set_idom(u, region, phase->dom_depth(region));
1155 }
1156 } else if (phase->get_ctrl(u) == ctrl) {
1157 assert(u != init_raw_mem, "should leave input raw mem above the barrier");
1158 uses.push(u);
1159 }
1160 assert(nb == 1, "more than 1 ctrl input?");
1161 --i, imax -= nb;
1162 }
1163 }
1164 }
1165 }
1166
1167 static Node* create_phis_on_call_return(Node* ctrl, Node* c, Node* n, Node* n_clone, const CallProjections& projs, PhaseIdealLoop* phase) {
1168 Node* region = NULL;
1169 while (c != ctrl) {
1170 if (c->is_Region()) {
1171 region = c;
1172 }
1173 c = phase->idom(c);
1174 }
1175 assert(region != NULL, "");
1176 Node* phi = new PhiNode(region, n->bottom_type());
1177 for (uint j = 1; j < region->req(); j++) {
1178 Node* in = region->in(j);
1179 if (phase->is_dominator(projs.fallthrough_catchproj, in)) {
1180 phi->init_req(j, n);
1181 } else if (phase->is_dominator(projs.catchall_catchproj, in)) {
1182 phi->init_req(j, n_clone);
1183 } else {
1184 phi->init_req(j, create_phis_on_call_return(ctrl, in, n, n_clone, projs, phase));
1185 }
1186 }
1187 phase->register_new_node(phi, region);
1188 return phi;
1189 }
1190
1191 void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) {
1192 ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state();
1193
1194 // Collect raw memory state at CFG points in the entire graph and
1195 // record it in memory_nodes. Optimize the raw memory graph in the
1196 // process. Optimizing the memory graph also makes the memory graph
1197 // simpler.
1198 GrowableArray<MemoryGraphFixer*> memory_graph_fixers;
1199
1200 Unique_Node_List uses;
1201 for (int i = 0; i < state->enqueue_barriers_count(); i++) {
1202 Node* barrier = state->enqueue_barrier(i);
1203 Node* ctrl = phase->get_ctrl(barrier);
1204 IdealLoopTree* loop = phase->get_loop(ctrl);
1205 if (loop->_head->is_OuterStripMinedLoop()) {
1206 // Expanding a barrier here will break loop strip mining
1207 // verification. Transform the loop so the loop nest doesn't
1208 // appear as strip mined.
1209 OuterStripMinedLoopNode* outer = loop->_head->as_OuterStripMinedLoop();
1210 hide_strip_mined_loop(outer, outer->unique_ctrl_out()->as_CountedLoop(), phase);
1211 }
1212 }
1213
1214 Node_Stack stack(0);
1215 Node_List clones;
1216 for (int i = state->load_reference_barriers_count() - 1; i >= 0; i--) {
1217 ShenandoahLoadReferenceBarrierNode* lrb = state->load_reference_barrier(i);
1218 if (lrb->get_barrier_strength() == ShenandoahLoadReferenceBarrierNode::NONE) {
1219 continue;
1220 }
1221
1222 Node* ctrl = phase->get_ctrl(lrb);
1223 Node* val = lrb->in(ShenandoahLoadReferenceBarrierNode::ValueIn);
1224
1225 CallStaticJavaNode* unc = NULL;
1226 Node* unc_ctrl = NULL;
1227 Node* uncasted_val = val;
1228
1229 for (DUIterator_Fast imax, i = lrb->fast_outs(imax); i < imax; i++) {
1230 Node* u = lrb->fast_out(i);
1231 if (u->Opcode() == Op_CastPP &&
1232 u->in(0) != NULL &&
1233 phase->is_dominator(u->in(0), ctrl)) {
1234 const Type* u_t = phase->igvn().type(u);
1235
1236 if (u_t->meet(TypePtr::NULL_PTR) != u_t &&
1237 u->in(0)->Opcode() == Op_IfTrue &&
1238 u->in(0)->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none) &&
1239 u->in(0)->in(0)->is_If() &&
1240 u->in(0)->in(0)->in(1)->Opcode() == Op_Bool &&
1241 u->in(0)->in(0)->in(1)->as_Bool()->_test._test == BoolTest::ne &&
1242 u->in(0)->in(0)->in(1)->in(1)->Opcode() == Op_CmpP &&
1243 u->in(0)->in(0)->in(1)->in(1)->in(1) == val &&
1244 u->in(0)->in(0)->in(1)->in(1)->in(2)->bottom_type() == TypePtr::NULL_PTR) {
1245 IdealLoopTree* loop = phase->get_loop(ctrl);
1246 IdealLoopTree* unc_loop = phase->get_loop(u->in(0));
1247
1248 if (!unc_loop->is_member(loop)) {
1249 continue;
1250 }
1251
1252 Node* branch = no_branches(ctrl, u->in(0), false, phase);
1253 assert(branch == NULL || branch == NodeSentinel, "was not looking for a branch");
1254 if (branch == NodeSentinel) {
1255 continue;
1256 }
1257
1258 phase->igvn().replace_input_of(u, 1, val);
1259 phase->igvn().replace_input_of(lrb, ShenandoahLoadReferenceBarrierNode::ValueIn, u);
1260 phase->set_ctrl(u, u->in(0));
1261 phase->set_ctrl(lrb, u->in(0));
1262 unc = u->in(0)->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none);
1263 unc_ctrl = u->in(0);
1264 val = u;
1265
1266 for (DUIterator_Fast jmax, j = val->fast_outs(jmax); j < jmax; j++) {
1267 Node* u = val->fast_out(j);
1268 if (u == lrb) continue;
1269 phase->igvn().rehash_node_delayed(u);
1270 int nb = u->replace_edge(val, lrb);
1271 --j; jmax -= nb;
1272 }
1273
1274 RegionNode* r = new RegionNode(3);
1275 IfNode* iff = unc_ctrl->in(0)->as_If();
1276
1277 Node* ctrl_use = unc_ctrl->unique_ctrl_out();
1278 Node* unc_ctrl_clone = unc_ctrl->clone();
1279 phase->register_control(unc_ctrl_clone, loop, iff);
1280 Node* c = unc_ctrl_clone;
1281 Node* new_cast = clone_null_check(c, val, unc_ctrl_clone, phase);
1282 r->init_req(1, new_cast->in(0)->in(0)->as_If()->proj_out(0));
1283
1284 phase->igvn().replace_input_of(unc_ctrl, 0, c->in(0));
1285 phase->set_idom(unc_ctrl, c->in(0), phase->dom_depth(unc_ctrl));
1286 phase->lazy_replace(c, unc_ctrl);
1287 c = NULL;;
1288 phase->igvn().replace_input_of(val, 0, unc_ctrl_clone);
1289 phase->set_ctrl(val, unc_ctrl_clone);
1290
1291 IfNode* new_iff = new_cast->in(0)->in(0)->as_If();
1292 fix_null_check(unc, unc_ctrl_clone, r, uses, phase);
1293 Node* iff_proj = iff->proj_out(0);
1294 r->init_req(2, iff_proj);
1295 phase->register_control(r, phase->ltree_root(), iff);
1296
1297 Node* new_bol = new_iff->in(1)->clone();
1298 Node* new_cmp = new_bol->in(1)->clone();
1299 assert(new_cmp->Opcode() == Op_CmpP, "broken");
1300 assert(new_cmp->in(1) == val->in(1), "broken");
1301 new_bol->set_req(1, new_cmp);
1302 new_cmp->set_req(1, lrb);
1303 phase->register_new_node(new_bol, new_iff->in(0));
1304 phase->register_new_node(new_cmp, new_iff->in(0));
1305 phase->igvn().replace_input_of(new_iff, 1, new_bol);
1306 phase->igvn().replace_input_of(new_cast, 1, lrb);
1307
1308 for (DUIterator_Fast imax, i = lrb->fast_outs(imax); i < imax; i++) {
1309 Node* u = lrb->fast_out(i);
1310 if (u == new_cast || u == new_cmp) {
1311 continue;
1312 }
1313 phase->igvn().rehash_node_delayed(u);
1314 int nb = u->replace_edge(lrb, new_cast);
1315 assert(nb > 0, "no update?");
1316 --i; imax -= nb;
1317 }
1318
1319 for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) {
1320 Node* u = val->fast_out(i);
1321 if (u == lrb) {
1322 continue;
1323 }
1324 phase->igvn().rehash_node_delayed(u);
1325 int nb = u->replace_edge(val, new_cast);
1326 assert(nb > 0, "no update?");
1327 --i; imax -= nb;
1328 }
1329
1330 ctrl = unc_ctrl_clone;
1331 phase->set_ctrl_and_loop(lrb, ctrl);
1332 break;
1333 }
1334 }
1335 }
1336 if (ctrl->is_Proj() && ctrl->in(0)->is_CallJava()) {
1337 CallNode* call = ctrl->in(0)->as_CallJava();
1338 CallProjections projs;
1339 call->extract_projections(&projs, false, false);
1340
1341 Node* lrb_clone = lrb->clone();
1342 phase->register_new_node(lrb_clone, projs.catchall_catchproj);
1343 phase->set_ctrl(lrb, projs.fallthrough_catchproj);
1344
1345 stack.push(lrb, 0);
1346 clones.push(lrb_clone);
1347
1348 do {
1349 assert(stack.size() == clones.size(), "");
1350 Node* n = stack.node();
1351 #ifdef ASSERT
1352 if (n->is_Load()) {
1353 Node* mem = n->in(MemNode::Memory);
1354 for (DUIterator_Fast jmax, j = mem->fast_outs(jmax); j < jmax; j++) {
1355 Node* u = mem->fast_out(j);
1356 assert(!u->is_Store() || !u->is_LoadStore() || phase->get_ctrl(u) != ctrl, "anti dependent store?");
1357 }
1358 }
1359 #endif
1360 uint idx = stack.index();
1361 Node* n_clone = clones.at(clones.size()-1);
1362 if (idx < n->outcnt()) {
1363 Node* u = n->raw_out(idx);
1364 Node* c = phase->ctrl_or_self(u);
1365 if (c == ctrl) {
1366 stack.set_index(idx+1);
1367 assert(!u->is_CFG(), "");
1368 stack.push(u, 0);
1369 Node* u_clone = u->clone();
1370 int nb = u_clone->replace_edge(n, n_clone);
1371 assert(nb > 0, "should have replaced some uses");
1372 phase->register_new_node(u_clone, projs.catchall_catchproj);
1373 clones.push(u_clone);
1374 phase->set_ctrl(u, projs.fallthrough_catchproj);
1375 } else {
1376 bool replaced = false;
1377 if (u->is_Phi()) {
1378 for (uint k = 1; k < u->req(); k++) {
1379 if (u->in(k) == n) {
1380 if (phase->is_dominator(projs.catchall_catchproj, u->in(0)->in(k))) {
1381 phase->igvn().replace_input_of(u, k, n_clone);
1382 replaced = true;
1383 } else if (!phase->is_dominator(projs.fallthrough_catchproj, u->in(0)->in(k))) {
1384 phase->igvn().replace_input_of(u, k, create_phis_on_call_return(ctrl, u->in(0)->in(k), n, n_clone, projs, phase));
1385 replaced = true;
1386 }
1387 }
1388 }
1389 } else {
1390 if (phase->is_dominator(projs.catchall_catchproj, c)) {
1391 phase->igvn().rehash_node_delayed(u);
1392 int nb = u->replace_edge(n, n_clone);
1393 assert(nb > 0, "should have replaced some uses");
1394 replaced = true;
1395 } else if (!phase->is_dominator(projs.fallthrough_catchproj, c)) {
1396 phase->igvn().rehash_node_delayed(u);
1397 int nb = u->replace_edge(n, create_phis_on_call_return(ctrl, c, n, n_clone, projs, phase));
1398 assert(nb > 0, "should have replaced some uses");
1399 replaced = true;
1400 }
1401 }
1402 if (!replaced) {
1403 stack.set_index(idx+1);
1404 }
1405 }
1406 } else {
1407 // assert(n_clone->outcnt() > 0, "");
1408 // assert(n->outcnt() > 0, "");
1409 stack.pop();
1410 clones.pop();
1411 }
1412 } while (stack.size() > 0);
1413 assert(stack.size() == 0 && clones.size() == 0, "");
1414 ctrl = projs.fallthrough_catchproj;
1415 }
1416 }
1417
1418 // Expand load-reference-barriers
1419 MemoryGraphFixer fixer(Compile::AliasIdxRaw, true, phase);
1420 Unique_Node_List uses_to_ignore;
1421 for (int i = state->load_reference_barriers_count() - 1; i >= 0; i--) {
1422 ShenandoahLoadReferenceBarrierNode* lrb = state->load_reference_barrier(i);
1423 if (lrb->get_barrier_strength() == ShenandoahLoadReferenceBarrierNode::NONE) {
1424 phase->igvn().replace_node(lrb, lrb->in(ShenandoahLoadReferenceBarrierNode::ValueIn));
1425 continue;
1426 }
1427 uint last = phase->C->unique();
1428 Node* ctrl = phase->get_ctrl(lrb);
1429 Node* val = lrb->in(ShenandoahLoadReferenceBarrierNode::ValueIn);
1430
1431
1432 Node* orig_ctrl = ctrl;
1433
1434 Node* raw_mem = fixer.find_mem(ctrl, lrb);
1435 Node* init_raw_mem = raw_mem;
1436 Node* raw_mem_for_ctrl = fixer.find_mem(ctrl, NULL);
1437 // int alias = phase->C->get_alias_index(lrb->adr_type());
1438
1439 IdealLoopTree *loop = phase->get_loop(ctrl);
1440 CallStaticJavaNode* unc = lrb->pin_and_expand_null_check(phase->igvn());
1441 Node* unc_ctrl = NULL;
1442 if (unc != NULL) {
1443 if (val->in(ShenandoahLoadReferenceBarrierNode::Control) != ctrl) {
1444 unc = NULL;
1445 } else {
1446 unc_ctrl = val->in(ShenandoahLoadReferenceBarrierNode::Control);
1447 }
1448 }
1449
1450 Node* uncasted_val = val;
1451 if (unc != NULL) {
1452 uncasted_val = val->in(1);
1453 }
1454
1455 Node* heap_stable_ctrl = NULL;
1456 Node* null_ctrl = NULL;
1457
1458 assert(val->bottom_type()->make_oopptr(), "need oop");
1459 assert(val->bottom_type()->make_oopptr()->const_oop() == NULL, "expect non-constant");
1460
1461 enum { _heap_stable = 1, _not_cset, _not_equal, _evac_path, _null_path, PATH_LIMIT };
1462 Node* region = new RegionNode(PATH_LIMIT);
1463 Node* val_phi = new PhiNode(region, uncasted_val->bottom_type()->is_oopptr());
1464 Node* raw_mem_phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);
1465
1466 // Stable path.
1467 test_heap_stable(ctrl, raw_mem, heap_stable_ctrl, phase);
1468 IfNode* heap_stable_iff = heap_stable_ctrl->in(0)->as_If();
1469
1470 // Heap stable case
1471 region->init_req(_heap_stable, heap_stable_ctrl);
1472 val_phi->init_req(_heap_stable, uncasted_val);
1473 raw_mem_phi->init_req(_heap_stable, raw_mem);
1474
1475 Node* reg2_ctrl = NULL;
1476 // Null case
1477 test_null(ctrl, val, null_ctrl, phase);
1478 if (null_ctrl != NULL) {
1479 reg2_ctrl = null_ctrl->in(0);
1480 region->init_req(_null_path, null_ctrl);
1481 val_phi->init_req(_null_path, uncasted_val);
1482 raw_mem_phi->init_req(_null_path, raw_mem);
1483 } else {
1484 region->del_req(_null_path);
1485 val_phi->del_req(_null_path);
1486 raw_mem_phi->del_req(_null_path);
1487 }
1488
1489 // Test for in-cset.
1490 // Wires !in_cset(obj) to slot 2 of region and phis
1491 Node* not_cset_ctrl = NULL;
1492 in_cset_fast_test(ctrl, not_cset_ctrl, uncasted_val, raw_mem, phase);
1493 if (not_cset_ctrl != NULL) {
1494 if (reg2_ctrl == NULL) reg2_ctrl = not_cset_ctrl->in(0);
1495 region->init_req(_not_cset, not_cset_ctrl);
1496 val_phi->init_req(_not_cset, uncasted_val);
1497 raw_mem_phi->init_req(_not_cset, raw_mem);
1498 }
1499
1500 // Resolve object when orig-value is in cset.
1501 // Make the unconditional resolve for fwdptr.
1502 Node* new_val = uncasted_val;
1503 if (unc_ctrl != NULL) {
1504 // Clone the null check in this branch to allow implicit null check
1505 new_val = clone_null_check(ctrl, val, unc_ctrl, phase);
1506 fix_null_check(unc, unc_ctrl, ctrl->in(0)->as_If()->proj_out(0), uses, phase);
1507
1508 IfNode* iff = unc_ctrl->in(0)->as_If();
1509 phase->igvn().replace_input_of(iff, 1, phase->igvn().intcon(1));
1510 }
1511 Node* addr = new AddPNode(new_val, uncasted_val, phase->igvn().MakeConX(ShenandoahBrooksPointer::byte_offset()));
1512 phase->register_new_node(addr, ctrl);
1513 assert(val->bottom_type()->isa_oopptr(), "what else?");
1514 const TypePtr* obj_type = val->bottom_type()->is_oopptr();
1515 const TypePtr* adr_type = TypeRawPtr::BOTTOM;
1516 Node* fwd = new LoadPNode(ctrl, raw_mem, addr, adr_type, obj_type, MemNode::unordered);
1517 phase->register_new_node(fwd, ctrl);
1518
1519 // Only branch to LRB stub if object is not forwarded; otherwise reply with fwd ptr
1520 Node* cmp = new CmpPNode(fwd, new_val);
1521 phase->register_new_node(cmp, ctrl);
1522 Node* bol = new BoolNode(cmp, BoolTest::eq);
1523 phase->register_new_node(bol, ctrl);
1524
1525 IfNode* iff = new IfNode(ctrl, bol, PROB_UNLIKELY(0.999), COUNT_UNKNOWN);
1526 if (reg2_ctrl == NULL) reg2_ctrl = iff;
1527 phase->register_control(iff, loop, ctrl);
1528 Node* if_not_eq = new IfFalseNode(iff);
1529 phase->register_control(if_not_eq, loop, iff);
1530 Node* if_eq = new IfTrueNode(iff);
1531 phase->register_control(if_eq, loop, iff);
1532
1533 // Wire up not-equal-path in slots 3.
1534 region->init_req(_not_equal, if_not_eq);
1535 val_phi->init_req(_not_equal, fwd);
1536 raw_mem_phi->init_req(_not_equal, raw_mem);
1537
1538 // Call wb-stub and wire up that path in slots 4
1539 Node* result_mem = NULL;
1540 ctrl = if_eq;
1541 call_lrb_stub(ctrl, fwd, result_mem, raw_mem, phase);
1542 region->init_req(_evac_path, ctrl);
1543 val_phi->init_req(_evac_path, fwd);
1544 raw_mem_phi->init_req(_evac_path, result_mem);
1545
1546 phase->register_control(region, loop, heap_stable_iff);
1547 Node* out_val = val_phi;
1548 phase->register_new_node(val_phi, region);
1549 phase->register_new_node(raw_mem_phi, region);
1550
1551 fix_ctrl(lrb, region, fixer, uses, uses_to_ignore, last, phase);
1552
1553 ctrl = orig_ctrl;
1554
1555 if (unc != NULL) {
1556 for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) {
1557 Node* u = val->fast_out(i);
1558 Node* c = phase->ctrl_or_self(u);
1559 if (u != lrb && (c != ctrl || is_dominator_same_ctrl(c, lrb, u, phase))) {
1560 phase->igvn().rehash_node_delayed(u);
1561 int nb = u->replace_edge(val, out_val);
1562 --i, imax -= nb;
1563 }
1564 }
1565 if (val->outcnt() == 0) {
1566 phase->igvn()._worklist.push(val);
1567 }
1568 }
1569 phase->igvn().replace_node(lrb, out_val);
1570
1571 follow_barrier_uses(out_val, ctrl, uses, phase);
1572
1573 for(uint next = 0; next < uses.size(); next++ ) {
1574 Node *n = uses.at(next);
1575 assert(phase->get_ctrl(n) == ctrl, "bad control");
1576 assert(n != init_raw_mem, "should leave input raw mem above the barrier");
1577 phase->set_ctrl(n, region);
1578 follow_barrier_uses(n, ctrl, uses, phase);
1579 }
1580
1581 // The slow path call produces memory: hook the raw memory phi
1582 // from the expanded load reference barrier with the rest of the graph
1583 // which may require adding memory phis at every post dominated
1584 // region and at enclosing loop heads. Use the memory state
1585 // collected in memory_nodes to fix the memory graph. Update that
1586 // memory state as we go.
1587 fixer.fix_mem(ctrl, region, init_raw_mem, raw_mem_for_ctrl, raw_mem_phi, uses);
1588 }
1589 // Done expanding load-reference-barriers.
1590 assert(ShenandoahBarrierSetC2::bsc2()->state()->load_reference_barriers_count() == 0, "all load reference barrier nodes should have been replaced");
1591
1592 for (int i = state->enqueue_barriers_count() - 1; i >= 0; i--) {
1593 Node* barrier = state->enqueue_barrier(i);
1594 Node* pre_val = barrier->in(1);
1595
1596 if (phase->igvn().type(pre_val)->higher_equal(TypePtr::NULL_PTR)) {
1597 ShouldNotReachHere();
1598 continue;
1599 }
1600
1601 Node* ctrl = phase->get_ctrl(barrier);
1602
1603 if (ctrl->is_Proj() && ctrl->in(0)->is_CallJava()) {
1604 assert(is_dominator(phase->get_ctrl(pre_val), ctrl->in(0)->in(0), pre_val, ctrl->in(0), phase), "can't move");
1605 ctrl = ctrl->in(0)->in(0);
1606 phase->set_ctrl(barrier, ctrl);
1607 } else if (ctrl->is_CallRuntime()) {
1608 assert(is_dominator(phase->get_ctrl(pre_val), ctrl->in(0), pre_val, ctrl, phase), "can't move");
1609 ctrl = ctrl->in(0);
1610 phase->set_ctrl(barrier, ctrl);
1611 }
1612
1613 Node* init_ctrl = ctrl;
1650 phase->register_new_node(thread, ctrl);
1651 Node* buffer_adr = new AddPNode(phase->C->top(), thread, phase->igvn().MakeConX(buffer_offset));
1652 phase->register_new_node(buffer_adr, ctrl);
1653 Node* index_adr = new AddPNode(phase->C->top(), thread, phase->igvn().MakeConX(index_offset));
1654 phase->register_new_node(index_adr, ctrl);
1655
1656 BasicType index_bt = TypeX_X->basic_type();
1657 assert(sizeof(size_t) == type2aelembytes(index_bt), "Loading G1 SATBMarkQueue::_index with wrong size.");
1658 const TypePtr* adr_type = TypeRawPtr::BOTTOM;
1659 Node* index = new LoadXNode(ctrl, raw_mem, index_adr, adr_type, TypeX_X, MemNode::unordered);
1660 phase->register_new_node(index, ctrl);
1661 Node* index_cmp = new CmpXNode(index, phase->igvn().MakeConX(0));
1662 phase->register_new_node(index_cmp, ctrl);
1663 Node* index_test = new BoolNode(index_cmp, BoolTest::ne);
1664 phase->register_new_node(index_test, ctrl);
1665 IfNode* queue_full_iff = new IfNode(ctrl, index_test, PROB_LIKELY(0.999), COUNT_UNKNOWN);
1666 if (reg2_ctrl == NULL) reg2_ctrl = queue_full_iff;
1667 phase->register_control(queue_full_iff, loop, ctrl);
1668 Node* not_full = new IfTrueNode(queue_full_iff);
1669 phase->register_control(not_full, loop, queue_full_iff);
1670 Node* full = new IfFalseNode(queue_full_iff);
1671 phase->register_control(full, loop, queue_full_iff);
1672
1673 ctrl = not_full;
1674
1675 Node* next_index = new SubXNode(index, phase->igvn().MakeConX(sizeof(intptr_t)));
1676 phase->register_new_node(next_index, ctrl);
1677
1678 Node* buffer = new LoadPNode(ctrl, raw_mem, buffer_adr, adr_type, TypeRawPtr::NOTNULL, MemNode::unordered);
1679 phase->register_new_node(buffer, ctrl);
1680 Node *log_addr = new AddPNode(phase->C->top(), buffer, next_index);
1681 phase->register_new_node(log_addr, ctrl);
1682 Node* log_store = new StorePNode(ctrl, raw_mem, log_addr, adr_type, pre_val, MemNode::unordered);
1683 phase->register_new_node(log_store, ctrl);
1684 // update the index
1685 Node* index_update = new StoreXNode(ctrl, log_store, index_adr, adr_type, next_index, MemNode::unordered);
1686 phase->register_new_node(index_update, ctrl);
1687
1688 // Fast-path case
1689 region2->init_req(_fast_path, ctrl);
1690 phi2->init_req(_fast_path, index_update);
1691
1692 ctrl = full;
1693
1694 Node* base = find_bottom_mem(ctrl, phase);
1695
1696 MergeMemNode* mm = MergeMemNode::make(base);
1697 mm->set_memory_at(Compile::AliasIdxRaw, raw_mem);
1698 phase->register_new_node(mm, ctrl);
1699
1700 Node* call = new CallLeafNode(ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type(), CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), "shenandoah_wb_pre", TypeRawPtr::BOTTOM);
1701 call->init_req(TypeFunc::Control, ctrl);
1702 call->init_req(TypeFunc::I_O, phase->C->top());
1703 call->init_req(TypeFunc::Memory, mm);
1704 call->init_req(TypeFunc::FramePtr, phase->C->top());
1705 call->init_req(TypeFunc::ReturnAdr, phase->C->top());
1706 call->init_req(TypeFunc::Parms, pre_val);
1707 call->init_req(TypeFunc::Parms+1, thread);
1708 phase->register_control(call, loop, ctrl);
1709
1710 Node* ctrl_proj = new ProjNode(call, TypeFunc::Control);
1711 phase->register_control(ctrl_proj, loop, call);
1712 Node* mem_proj = new ProjNode(call, TypeFunc::Memory);
1713 phase->register_new_node(mem_proj, call);
1714
1715 // Slow-path case
1716 region2->init_req(_slow_path, ctrl_proj);
1717 phi2->init_req(_slow_path, mem_proj);
1718
1719 phase->register_control(region2, loop, reg2_ctrl);
1720 phase->register_new_node(phi2, region2);
1721
1722 region->init_req(_heap_unstable, region2);
1723 phi->init_req(_heap_unstable, phi2);
1724
1725 phase->register_control(region, loop, heap_stable_ctrl->in(0));
1726 phase->register_new_node(phi, region);
1727
1728 fix_ctrl(barrier, region, fixer, uses, uses_to_ignore, last, phase);
1729 for(uint next = 0; next < uses.size(); next++ ) {
1730 Node *n = uses.at(next);
1731 assert(phase->get_ctrl(n) == init_ctrl, "bad control");
1732 assert(n != init_raw_mem, "should leave input raw mem above the barrier");
1733 phase->set_ctrl(n, region);
1734 follow_barrier_uses(n, init_ctrl, uses, phase);
1735 }
1736 fixer.fix_mem(init_ctrl, region, init_raw_mem, raw_mem_for_ctrl, phi, uses);
1737
1738 phase->igvn().replace_node(barrier, pre_val);
1739 }
1740 assert(state->enqueue_barriers_count() == 0, "all enqueue barrier nodes should have been replaced");
1741
1742 }
1743
1744 void ShenandoahBarrierC2Support::move_heap_stable_test_out_of_loop(IfNode* iff, PhaseIdealLoop* phase) {
1745 IdealLoopTree *loop = phase->get_loop(iff);
1746 Node* loop_head = loop->_head;
1747 Node* entry_c = loop_head->in(LoopNode::EntryControl);
1748
1749 Node* bol = iff->in(1);
1750 Node* cmp = bol->in(1);
1751 Node* andi = cmp->in(1);
1752 Node* load = andi->in(1);
1753
1754 assert(is_gc_state_load(load), "broken");
1755 if (!phase->is_dominator(load->in(0), entry_c)) {
1756 Node* mem_ctrl = NULL;
1757 Node* mem = dom_mem(load->in(MemNode::Memory), loop_head, Compile::AliasIdxRaw, mem_ctrl, phase);
1758 load = load->clone();
1759 load->set_req(MemNode::Memory, mem);
1760 load->set_req(0, entry_c);
1761 phase->register_new_node(load, entry_c);
1762 andi = andi->clone();
1763 andi->set_req(1, load);
1764 phase->register_new_node(andi, entry_c);
1765 cmp = cmp->clone();
1766 cmp->set_req(1, andi);
1767 phase->register_new_node(cmp, entry_c);
1768 bol = bol->clone();
1769 bol->set_req(1, cmp);
1770 phase->register_new_node(bol, entry_c);
1771
1772 Node* old_bol =iff->in(1);
1773 phase->igvn().replace_input_of(iff, 1, bol);
1774 }
1775 }
1776
1777 bool ShenandoahBarrierC2Support::identical_backtoback_ifs(Node* n, PhaseIdealLoop* phase) {
1778 if (!n->is_If() || n->is_CountedLoopEnd()) {
1779 return false;
1780 }
1781 Node* region = n->in(0);
1782
1783 if (!region->is_Region()) {
1784 return false;
1785 }
1786 Node* dom = phase->idom(region);
1787 if (!dom->is_If()) {
1788 return false;
1789 }
1790
1791 if (!is_heap_stable_test(n) || !is_heap_stable_test(dom)) {
1792 return false;
1793 }
1794
1795 IfNode* dom_if = dom->as_If();
1796 Node* proj_true = dom_if->proj_out(1);
1797 Node* proj_false = dom_if->proj_out(0);
1798
1799 for (uint i = 1; i < region->req(); i++) {
1800 if (phase->is_dominator(proj_true, region->in(i))) {
1801 continue;
1802 }
1803 if (phase->is_dominator(proj_false, region->in(i))) {
1804 continue;
1805 }
1806 return false;
1807 }
1808
1809 return true;
1810 }
1811
1812 void ShenandoahBarrierC2Support::merge_back_to_back_tests(Node* n, PhaseIdealLoop* phase) {
1813 assert(is_heap_stable_test(n), "no other tests");
1814 if (identical_backtoback_ifs(n, phase)) {
1815 Node* n_ctrl = n->in(0);
1816 if (phase->can_split_if(n_ctrl)) {
1817 IfNode* dom_if = phase->idom(n_ctrl)->as_If();
1818 if (is_heap_stable_test(n)) {
1819 Node* gc_state_load = n->in(1)->in(1)->in(1)->in(1);
1820 assert(is_gc_state_load(gc_state_load), "broken");
1821 Node* dom_gc_state_load = dom_if->in(1)->in(1)->in(1)->in(1);
1822 assert(is_gc_state_load(dom_gc_state_load), "broken");
1823 if (gc_state_load != dom_gc_state_load) {
1824 phase->igvn().replace_node(gc_state_load, dom_gc_state_load);
1825 }
1826 }
1827 PhiNode* bolphi = PhiNode::make_blank(n_ctrl, n->in(1));
1828 Node* proj_true = dom_if->proj_out(1);
1829 Node* proj_false = dom_if->proj_out(0);
1830 Node* con_true = phase->igvn().makecon(TypeInt::ONE);
1831 Node* con_false = phase->igvn().makecon(TypeInt::ZERO);
1832
1833 for (uint i = 1; i < n_ctrl->req(); i++) {
1834 if (phase->is_dominator(proj_true, n_ctrl->in(i))) {
1835 bolphi->init_req(i, con_true);
1836 } else {
1837 assert(phase->is_dominator(proj_false, n_ctrl->in(i)), "bad if");
1838 bolphi->init_req(i, con_false);
1839 }
1840 }
1841 phase->register_new_node(bolphi, n_ctrl);
1842 phase->igvn().replace_input_of(n, 1, bolphi);
1843 phase->do_split_if(n);
1844 }
1845 }
1846 }
1847
1848 IfNode* ShenandoahBarrierC2Support::find_unswitching_candidate(const IdealLoopTree* loop, PhaseIdealLoop* phase) {
1849 // Find first invariant test that doesn't exit the loop
1850 LoopNode *head = loop->_head->as_Loop();
1851 IfNode* unswitch_iff = NULL;
1852 Node* n = head->in(LoopNode::LoopBackControl);
1853 int loop_has_sfpts = -1;
1854 while (n != head) {
1855 Node* n_dom = phase->idom(n);
1856 if (n->is_Region()) {
1857 if (n_dom->is_If()) {
1858 IfNode* iff = n_dom->as_If();
1859 if (iff->in(1)->is_Bool()) {
1860 BoolNode* bol = iff->in(1)->as_Bool();
1861 if (bol->in(1)->is_Cmp()) {
1862 // If condition is invariant and not a loop exit,
1863 // then found reason to unswitch.
1864 if (is_heap_stable_test(iff) &&
1865 (loop_has_sfpts == -1 || loop_has_sfpts == 0)) {
1866 assert(!loop->is_loop_exit(iff), "both branches should be in the loop");
1867 if (loop_has_sfpts == -1) {
1868 for(uint i = 0; i < loop->_body.size(); i++) {
1873 }
1874 }
1875 if (loop_has_sfpts == -1) {
1876 loop_has_sfpts = 0;
1877 }
1878 }
1879 if (!loop_has_sfpts) {
1880 unswitch_iff = iff;
1881 }
1882 }
1883 }
1884 }
1885 }
1886 }
1887 n = n_dom;
1888 }
1889 return unswitch_iff;
1890 }
1891
1892
1893 void ShenandoahBarrierC2Support::optimize_after_expansion(VectorSet &visited, Node_Stack &stack, Node_List &old_new, PhaseIdealLoop* phase) {
1894 Node_List heap_stable_tests;
1895 Node_List gc_state_loads;
1896 stack.push(phase->C->start(), 0);
1897 do {
1898 Node* n = stack.node();
1899 uint i = stack.index();
1900
1901 if (i < n->outcnt()) {
1902 Node* u = n->raw_out(i);
1903 stack.set_index(i+1);
1904 if (!visited.test_set(u->_idx)) {
1905 stack.push(u, 0);
1906 }
1907 } else {
1908 stack.pop();
1909 if (ShenandoahCommonGCStateLoads && is_gc_state_load(n)) {
1910 gc_state_loads.push(n);
1911 }
1912 if (n->is_If() && is_heap_stable_test(n)) {
1913 heap_stable_tests.push(n);
1914 }
1915 }
1952 move_heap_stable_test_out_of_loop(iff, phase);
1953 if (loop->policy_unswitching(phase)) {
1954 if (head->is_strip_mined()) {
1955 OuterStripMinedLoopNode* outer = head->as_CountedLoop()->outer_loop();
1956 hide_strip_mined_loop(outer, head->as_CountedLoop(), phase);
1957 }
1958 phase->do_unswitching(loop, old_new);
1959 } else {
1960 // Not proceeding with unswitching. Move load back in
1961 // the loop.
1962 phase->igvn().replace_input_of(iff, 1, bol);
1963 }
1964 }
1965 }
1966 }
1967 }
1968 }
1969 }
1970
1971 #ifdef ASSERT
1972 void ShenandoahBarrierC2Support::verify_raw_mem(RootNode* root) {
1973 const bool trace = false;
1974 ResourceMark rm;
1975 Unique_Node_List nodes;
1976 Unique_Node_List controls;
1977 Unique_Node_List memories;
1978
1979 nodes.push(root);
1980 for (uint next = 0; next < nodes.size(); next++) {
1981 Node *n = nodes.at(next);
1982 if (ShenandoahBarrierSetC2::is_shenandoah_wb_call(n)) {
1983 controls.push(n);
1984 if (trace) { tty->print("XXXXXX verifying"); n->dump(); }
1985 for (uint next2 = 0; next2 < controls.size(); next2++) {
1986 Node *m = controls.at(next2);
1987 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
1988 Node* u = m->fast_out(i);
1989 if (u->is_CFG() && !u->is_Root() &&
1990 !(u->Opcode() == Op_CProj && u->in(0)->Opcode() == Op_NeverBranch && u->as_Proj()->_con == 1) &&
1991 !(u->is_Region() && u->unique_ctrl_out()->Opcode() == Op_Halt)) {
1992 if (trace) { tty->print("XXXXXX pushing control"); u->dump(); }
2050 }
2051 }
2052 }
2053 }
2054 assert(found_phi || all_in, "");
2055 }
2056 }
2057 controls.clear();
2058 memories.clear();
2059 }
2060 for( uint i = 0; i < n->len(); ++i ) {
2061 Node *m = n->in(i);
2062 if (m != NULL) {
2063 nodes.push(m);
2064 }
2065 }
2066 }
2067 }
2068 #endif
2069
2070 ShenandoahEnqueueBarrierNode::ShenandoahEnqueueBarrierNode(Node* val) : Node(NULL, val) {
2071 ShenandoahBarrierSetC2::bsc2()->state()->add_enqueue_barrier(this);
2072 }
2073
2074 const Type* ShenandoahEnqueueBarrierNode::bottom_type() const {
2075 if (in(1) == NULL || in(1)->is_top()) {
2076 return Type::TOP;
2077 }
2078 const Type* t = in(1)->bottom_type();
2079 if (t == TypePtr::NULL_PTR) {
2080 return t;
2081 }
2082 return t->is_oopptr()->cast_to_nonconst();
2083 }
2084
2085 const Type* ShenandoahEnqueueBarrierNode::Value(PhaseGVN* phase) const {
2086 if (in(1) == NULL) {
2087 return Type::TOP;
2088 }
2089 const Type* t = phase->type(in(1));
2090 if (t == Type::TOP) {
2091 return Type::TOP;
2092 }
2093 if (t == TypePtr::NULL_PTR) {
2213 uint i = stack.index();
2214 if (i < n->req()) {
2215 Node* mem = NULL;
2216 if (opc == Op_Root) {
2217 Node* in = n->in(i);
2218 int in_opc = in->Opcode();
2219 if (in_opc == Op_Return || in_opc == Op_Rethrow) {
2220 mem = in->in(TypeFunc::Memory);
2221 } else if (in_opc == Op_Halt) {
2222 if (!in->in(0)->is_Region()) {
2223 Node* proj = in->in(0);
2224 assert(proj->is_Proj(), "");
2225 Node* in = proj->in(0);
2226 assert(in->is_CallStaticJava() || in->Opcode() == Op_NeverBranch || in->Opcode() == Op_Catch || proj->is_IfProj(), "");
2227 if (in->is_CallStaticJava()) {
2228 mem = in->in(TypeFunc::Memory);
2229 } else if (in->Opcode() == Op_Catch) {
2230 Node* call = in->in(0)->in(0);
2231 assert(call->is_Call(), "");
2232 mem = call->in(TypeFunc::Memory);
2233 } else if (in->Opcode() == Op_NeverBranch) {
2234 ResourceMark rm;
2235 Unique_Node_List wq;
2236 wq.push(in);
2237 wq.push(in->as_Multi()->proj_out(0));
2238 for (uint j = 1; j < wq.size(); j++) {
2239 Node* c = wq.at(j);
2240 assert(!c->is_Root(), "shouldn't leave loop");
2241 if (c->is_SafePoint()) {
2242 assert(mem == NULL, "only one safepoint");
2243 mem = c->in(TypeFunc::Memory);
2244 }
2245 for (DUIterator_Fast kmax, k = c->fast_outs(kmax); k < kmax; k++) {
2246 Node* u = c->fast_out(k);
2247 if (u->is_CFG()) {
2248 wq.push(u);
2249 }
2250 }
2251 }
2252 assert(mem != NULL, "should have found safepoint");
2253 }
2254 }
2255 } else {
2256 #ifdef ASSERT
2257 n->dump();
2258 in->dump();
2259 #endif
2260 ShouldNotReachHere();
2261 }
2262 } else {
2263 assert(n->is_Phi() && n->bottom_type() == Type::MEMORY, "");
2264 assert(n->adr_type() == TypePtr::BOTTOM || _phase->C->get_alias_index(n->adr_type()) == _alias, "");
2265 mem = n->in(i);
2266 }
2267 i++;
2268 stack.set_index(i);
2269 if (mem == NULL) {
2270 continue;
2271 }
2272 for (;;) {
2273 if (visited.test_set(mem->_idx) || mem->is_Start()) {
2274 break;
2275 }
2276 if (mem->is_Phi()) {
2277 stack.push(mem, 2);
2278 mem = mem->in(1);
2279 } else if (mem->is_Proj()) {
2280 stack.push(mem, mem->req());
2281 mem = mem->in(0);
2282 } else if (mem->is_SafePoint() || mem->is_MemBar()) {
2283 mem = mem->in(TypeFunc::Memory);
2284 } else if (mem->is_MergeMem()) {
2285 MergeMemNode* mm = mem->as_MergeMem();
2286 mem = mm->memory_at(_alias);
2287 } else if (mem->is_Store() || mem->is_LoadStore() || mem->is_ClearArray()) {
2288 assert(_alias == Compile::AliasIdxRaw, "");
2289 stack.push(mem, mem->req());
2290 mem = mem->in(MemNode::Memory);
2291 } else {
2292 #ifdef ASSERT
2293 mem->dump();
2294 #endif
2295 ShouldNotReachHere();
2296 }
2297 }
2298 } else {
2299 if (n->is_Phi()) {
2300 // Nothing
2301 } else if (!n->is_Root()) {
2302 Node* c = get_ctrl(n);
2303 _memory_nodes.map(c->_idx, n);
2304 }
2305 stack.pop();
2306 }
2307 } while(stack.is_nonempty());
2308
2309 // Iterate over CFG nodes in rpo and propagate memory state to
2310 // compute memory state at regions, creating new phis if needed.
2324 }
2325 }
2326 }
2327 #endif
2328 uint last = _phase->C->unique();
2329
2330 #ifdef ASSERT
2331 uint8_t max_depth = 0;
2332 for (LoopTreeIterator iter(_phase->ltree_root()); !iter.done(); iter.next()) {
2333 IdealLoopTree* lpt = iter.current();
2334 max_depth = MAX2(max_depth, lpt->_nest);
2335 }
2336 #endif
2337
2338 bool progress = true;
2339 int iteration = 0;
2340 Node_List dead_phis;
2341 while (progress) {
2342 progress = false;
2343 iteration++;
2344 assert(iteration <= 2+max_depth || _phase->C->has_irreducible_loop() || has_never_branch(_phase->C->root()), "");
2345 if (trace) { tty->print_cr("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"); }
2346 IdealLoopTree* last_updated_ilt = NULL;
2347 for (int i = rpo_list.size() - 1; i >= 0; i--) {
2348 Node* c = rpo_list.at(i);
2349
2350 Node* prev_mem = _memory_nodes[c->_idx];
2351 if (c->is_Region() && (_include_lsm || !c->is_OuterStripMinedLoop())) {
2352 Node* prev_region = regions[c->_idx];
2353 Node* unique = NULL;
2354 for (uint j = 1; j < c->req() && unique != NodeSentinel; j++) {
2355 Node* m = _memory_nodes[c->in(j)->_idx];
2356 assert(m != NULL || (c->is_Loop() && j == LoopNode::LoopBackControl && iteration == 1) || _phase->C->has_irreducible_loop() || has_never_branch(_phase->C->root()), "expect memory state");
2357 if (m != NULL) {
2358 if (m == prev_region && ((c->is_Loop() && j == LoopNode::LoopBackControl) || (prev_region->is_Phi() && prev_region->in(0) == c))) {
2359 assert(c->is_Loop() && j == LoopNode::LoopBackControl || _phase->C->has_irreducible_loop(), "");
2360 // continue
2361 } else if (unique == NULL) {
2362 unique = m;
2363 } else if (m == unique) {
2364 // continue
2492 else {
2493 assert (n->is_CFG(), "must be a CFG node");
2494 return n;
2495 }
2496 }
2497
2498 bool MemoryGraphFixer::mem_is_valid(Node* m, Node* c) const {
2499 return m != NULL && get_ctrl(m) == c;
2500 }
2501
2502 Node* MemoryGraphFixer::find_mem(Node* ctrl, Node* n) const {
2503 assert(n == NULL || _phase->ctrl_or_self(n) == ctrl, "");
2504 Node* mem = _memory_nodes[ctrl->_idx];
2505 Node* c = ctrl;
2506 while (!mem_is_valid(mem, c) &&
2507 (!c->is_CatchProj() || mem == NULL || c->in(0)->in(0)->in(0) != get_ctrl(mem))) {
2508 c = _phase->idom(c);
2509 mem = _memory_nodes[c->_idx];
2510 }
2511 if (n != NULL && mem_is_valid(mem, c)) {
2512 while (!ShenandoahBarrierC2Support::is_dominator_same_ctrl(c, mem, n, _phase) && _phase->ctrl_or_self(mem) == ctrl) {
2513 mem = next_mem(mem, _alias);
2514 }
2515 if (mem->is_MergeMem()) {
2516 mem = mem->as_MergeMem()->memory_at(_alias);
2517 }
2518 if (!mem_is_valid(mem, c)) {
2519 do {
2520 c = _phase->idom(c);
2521 mem = _memory_nodes[c->_idx];
2522 } while (!mem_is_valid(mem, c) &&
2523 (!c->is_CatchProj() || mem == NULL || c->in(0)->in(0)->in(0) != get_ctrl(mem)));
2524 }
2525 }
2526 assert(mem->bottom_type() == Type::MEMORY, "");
2527 return mem;
2528 }
2529
2530 bool MemoryGraphFixer::has_mem_phi(Node* region) const {
2531 for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) {
2532 Node* use = region->fast_out(i);
2538 return false;
2539 }
2540
2541 void MemoryGraphFixer::fix_mem(Node* ctrl, Node* new_ctrl, Node* mem, Node* mem_for_ctrl, Node* new_mem, Unique_Node_List& uses) {
2542 assert(_phase->ctrl_or_self(new_mem) == new_ctrl, "");
2543 const bool trace = false;
2544 DEBUG_ONLY(if (trace) { tty->print("ZZZ control is"); ctrl->dump(); });
2545 DEBUG_ONLY(if (trace) { tty->print("ZZZ mem is"); mem->dump(); });
2546 GrowableArray<Node*> phis;
2547 if (mem_for_ctrl != mem) {
2548 Node* old = mem_for_ctrl;
2549 Node* prev = NULL;
2550 while (old != mem) {
2551 prev = old;
2552 if (old->is_Store() || old->is_ClearArray() || old->is_LoadStore()) {
2553 assert(_alias == Compile::AliasIdxRaw, "");
2554 old = old->in(MemNode::Memory);
2555 } else if (old->Opcode() == Op_SCMemProj) {
2556 assert(_alias == Compile::AliasIdxRaw, "");
2557 old = old->in(0);
2558 } else {
2559 ShouldNotReachHere();
2560 }
2561 }
2562 assert(prev != NULL, "");
2563 if (new_ctrl != ctrl) {
2564 _memory_nodes.map(ctrl->_idx, mem);
2565 _memory_nodes.map(new_ctrl->_idx, mem_for_ctrl);
2566 }
2567 uint input = (uint)MemNode::Memory;
2568 _phase->igvn().replace_input_of(prev, input, new_mem);
2569 } else {
2570 uses.clear();
2571 _memory_nodes.map(new_ctrl->_idx, new_mem);
2572 uses.push(new_ctrl);
2573 for(uint next = 0; next < uses.size(); next++ ) {
2574 Node *n = uses.at(next);
2575 assert(n->is_CFG(), "");
2576 DEBUG_ONLY(if (trace) { tty->print("ZZZ ctrl"); n->dump(); });
2577 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
2578 Node* u = n->fast_out(i);
2579 if (!u->is_Root() && u->is_CFG() && u != n) {
2580 Node* m = _memory_nodes[u->_idx];
2581 if (u->is_Region() && (!u->is_OuterStripMinedLoop() || _include_lsm) &&
2582 !has_mem_phi(u) &&
2583 u->unique_ctrl_out()->Opcode() != Op_Halt) {
2584 DEBUG_ONLY(if (trace) { tty->print("ZZZ region"); u->dump(); });
2585 DEBUG_ONLY(if (trace && m != NULL) { tty->print("ZZZ mem"); m->dump(); });
2586
2587 if (!mem_is_valid(m, u) || !m->is_Phi()) {
2615 do_check = false;
2616 }
2617 }
2618 }
2619
2620 if (do_check && _phase->is_dominator(_phase->idom(u), new_ctrl)) {
2621 create_phi = true;
2622 }
2623 }
2624 if (create_phi) {
2625 Node* phi = new PhiNode(u, Type::MEMORY, _phase->C->get_adr_type(_alias));
2626 _phase->register_new_node(phi, u);
2627 phis.push(phi);
2628 DEBUG_ONLY(if (trace) { tty->print("ZZZ new phi"); phi->dump(); });
2629 if (!mem_is_valid(m, u)) {
2630 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting mem"); phi->dump(); });
2631 _memory_nodes.map(u->_idx, phi);
2632 } else {
2633 DEBUG_ONLY(if (trace) { tty->print("ZZZ NOT setting mem"); m->dump(); });
2634 for (;;) {
2635 assert(m->is_Mem() || m->is_LoadStore() || m->is_Proj(), "");
2636 Node* next = NULL;
2637 if (m->is_Proj()) {
2638 next = m->in(0);
2639 } else {
2640 assert(m->is_Mem() || m->is_LoadStore(), "");
2641 assert(_alias == Compile::AliasIdxRaw, "");
2642 next = m->in(MemNode::Memory);
2643 }
2644 if (_phase->get_ctrl(next) != u) {
2645 break;
2646 }
2647 if (next->is_MergeMem()) {
2648 assert(_phase->get_ctrl(next->as_MergeMem()->memory_at(_alias)) != u, "");
2649 break;
2650 }
2651 if (next->is_Phi()) {
2652 assert(next->adr_type() == TypePtr::BOTTOM && next->in(0) == u, "");
2653 break;
2654 }
2655 m = next;
2656 }
2657
2658 DEBUG_ONLY(if (trace) { tty->print("ZZZ setting to phi"); m->dump(); });
2659 assert(m->is_Mem() || m->is_LoadStore(), "");
2660 uint input = (uint)MemNode::Memory;
2661 _phase->igvn().replace_input_of(m, input, phi);
2662 push = false;
2663 }
2664 } else {
2665 DEBUG_ONLY(if (trace) { tty->print("ZZZ skipping region"); u->dump(); });
2666 }
2667 if (push) {
2668 uses.push(u);
2669 }
2670 }
2671 } else if (!mem_is_valid(m, u) &&
2672 !(u->Opcode() == Op_CProj && u->in(0)->Opcode() == Op_NeverBranch && u->as_Proj()->_con == 1)) {
2673 uses.push(u);
2674 }
2675 }
2676 }
2677 }
2678 for (int i = 0; i < phis.length(); i++) {
2679 Node* n = phis.at(i);
2680 Node* r = n->in(0);
2866 if (phi->adr_type() == TypePtr::BOTTOM) {
2867 Node* region = phi->in(0);
2868 for (DUIterator_Fast jmax, j = region->fast_outs(jmax); j < jmax; j++) {
2869 Node* uu = region->fast_out(j);
2870 if (uu->is_Phi() && uu != phi && uu->bottom_type() == Type::MEMORY && _phase->C->get_alias_index(uu->adr_type()) == _alias) {
2871 return false;
2872 }
2873 }
2874 return true;
2875 }
2876 return _phase->C->get_alias_index(phi->adr_type()) == _alias;
2877 }
2878
2879 void MemoryGraphFixer::fix_memory_uses(Node* mem, Node* replacement, Node* rep_proj, Node* rep_ctrl) const {
2880 uint last = _phase-> C->unique();
2881 MergeMemNode* mm = NULL;
2882 assert(mem->bottom_type() == Type::MEMORY, "");
2883 for (DUIterator i = mem->outs(); mem->has_out(i); i++) {
2884 Node* u = mem->out(i);
2885 if (u != replacement && u->_idx < last) {
2886 if (u->is_MergeMem()) {
2887 MergeMemNode* u_mm = u->as_MergeMem();
2888 if (u_mm->memory_at(_alias) == mem) {
2889 MergeMemNode* newmm = NULL;
2890 for (DUIterator_Fast jmax, j = u->fast_outs(jmax); j < jmax; j++) {
2891 Node* uu = u->fast_out(j);
2892 assert(!uu->is_MergeMem(), "chain of MergeMems?");
2893 if (uu->is_Phi()) {
2894 if (should_process_phi(uu)) {
2895 Node* region = uu->in(0);
2896 int nb = 0;
2897 for (uint k = 1; k < uu->req(); k++) {
2898 if (uu->in(k) == u && _phase->is_dominator(rep_ctrl, region->in(k))) {
2899 if (newmm == NULL) {
2900 newmm = clone_merge_mem(u, mem, rep_proj, rep_ctrl, i);
2901 }
2902 if (newmm != u) {
2903 _phase->igvn().replace_input_of(uu, k, newmm);
2904 nb++;
2905 --jmax;
2906 }
2907 }
2908 }
2909 if (nb > 0) {
2910 --j;
2911 }
2912 }
2913 } else {
2914 if (rep_ctrl != uu && ShenandoahBarrierC2Support::is_dominator(rep_ctrl, _phase->ctrl_or_self(uu), replacement, uu, _phase)) {
2915 if (newmm == NULL) {
2916 newmm = clone_merge_mem(u, mem, rep_proj, rep_ctrl, i);
2917 }
2918 if (newmm != u) {
2919 _phase->igvn().replace_input_of(uu, uu->find_edge(u), newmm);
2920 --j, --jmax;
2921 }
2922 }
2923 }
2924 }
2925 }
2926 } else if (u->is_Phi()) {
2927 assert(u->bottom_type() == Type::MEMORY, "what else?");
2928 Node* region = u->in(0);
2929 if (should_process_phi(u)) {
2930 bool replaced = false;
2931 for (uint j = 1; j < u->req(); j++) {
2932 if (u->in(j) == mem && _phase->is_dominator(rep_ctrl, region->in(j))) {
2933 Node* nnew = rep_proj;
2934 if (u->adr_type() == TypePtr::BOTTOM) {
2935 if (mm == NULL) {
2936 mm = allocate_merge_mem(mem, rep_proj, rep_ctrl);
2937 }
2938 nnew = mm;
2939 }
2940 _phase->igvn().replace_input_of(u, j, nnew);
2941 replaced = true;
2942 }
2943 }
2944 if (replaced) {
2945 --i;
2946 }
2947
2948 }
2949 } else if ((u->adr_type() == TypePtr::BOTTOM && u->Opcode() != Op_StrInflatedCopy) ||
2950 u->adr_type() == NULL) {
2951 assert(u->adr_type() != NULL ||
2952 u->Opcode() == Op_Rethrow ||
2953 u->Opcode() == Op_Return ||
2954 u->Opcode() == Op_SafePoint ||
2955 u->Opcode() == Op_StoreLConditional ||
2956 (u->is_CallStaticJava() && u->as_CallStaticJava()->uncommon_trap_request() != 0) ||
2957 (u->is_CallStaticJava() && u->as_CallStaticJava()->_entry_point == OptoRuntime::rethrow_stub()) ||
2958 u->Opcode() == Op_CallLeaf, "");
2959 if (ShenandoahBarrierC2Support::is_dominator(rep_ctrl, _phase->ctrl_or_self(u), replacement, u, _phase)) {
2960 if (mm == NULL) {
2961 mm = allocate_merge_mem(mem, rep_proj, rep_ctrl);
2962 }
2963 _phase->igvn().replace_input_of(u, u->find_edge(mem), mm);
2964 --i;
2965 }
2966 } else if (_phase->C->get_alias_index(u->adr_type()) == _alias) {
2967 if (ShenandoahBarrierC2Support::is_dominator(rep_ctrl, _phase->ctrl_or_self(u), replacement, u, _phase)) {
2968 _phase->igvn().replace_input_of(u, u->find_edge(mem), rep_proj);
2969 --i;
2970 }
2971 }
2972 }
2973 }
2974 }
2975
2976 ShenandoahLoadReferenceBarrierNode::ShenandoahLoadReferenceBarrierNode(Node* ctrl, Node* obj)
2977 : Node(ctrl, obj) {
2978 ShenandoahBarrierSetC2::bsc2()->state()->add_load_reference_barrier(this);
2979 }
2980
2981 const Type* ShenandoahLoadReferenceBarrierNode::bottom_type() const {
2982 if (in(ValueIn) == NULL || in(ValueIn)->is_top()) {
2983 return Type::TOP;
2984 }
2985 const Type* t = in(ValueIn)->bottom_type();
2986 if (t == TypePtr::NULL_PTR) {
2987 return t;
2988 }
2989 return t->is_oopptr();
2990 }
2991
2992 const Type* ShenandoahLoadReferenceBarrierNode::Value(PhaseGVN* phase) const {
2993 // Either input is TOP ==> the result is TOP
2994 const Type *t2 = phase->type(in(ValueIn));
2995 if( t2 == Type::TOP ) return Type::TOP;
2996
2997 if (t2 == TypePtr::NULL_PTR) {
2998 return t2;
2999 }
3000
3001 const Type* type = t2->is_oopptr()/*->cast_to_nonconst()*/;
3002 return type;
3003 }
3004
3005 Node* ShenandoahLoadReferenceBarrierNode::Identity(PhaseGVN* phase) {
3006 Node* value = in(ValueIn);
3007 if (!needs_barrier(phase, value)) {
3008 return value;
3009 }
3010 return this;
3011 }
3012
3013 bool ShenandoahLoadReferenceBarrierNode::needs_barrier(PhaseGVN* phase, Node* n) {
3014 Unique_Node_List visited;
3015 return needs_barrier_impl(phase, n, visited);
3016 }
3017
3018 bool ShenandoahLoadReferenceBarrierNode::needs_barrier_impl(PhaseGVN* phase, Node* n, Unique_Node_List &visited) {
3019 if (n == NULL) return false;
3020 if (visited.member(n)) {
3021 return false; // Been there.
3022 }
3023 visited.push(n);
3024
3025 if (n->is_Allocate()) {
3026 // tty->print_cr("optimize barrier on alloc");
3027 return false;
3028 }
3029 if (n->is_Call()) {
3030 // tty->print_cr("optimize barrier on call");
3031 return false;
3032 }
3033
3034 const Type* type = phase->type(n);
3035 if (type == Type::TOP) {
3036 return false;
3037 }
3038 if (type->make_ptr()->higher_equal(TypePtr::NULL_PTR)) {
3039 // tty->print_cr("optimize barrier on null");
3040 return false;
3041 }
3042 if (type->make_oopptr() && type->make_oopptr()->const_oop() != NULL) {
3043 // tty->print_cr("optimize barrier on constant");
3044 return false;
3045 }
3046
3047 switch (n->Opcode()) {
3048 case Op_AddP:
3049 return true; // TODO: Can refine?
3050 case Op_LoadP:
3051 case Op_ShenandoahCompareAndExchangeN:
3052 case Op_ShenandoahCompareAndExchangeP:
3053 case Op_CompareAndExchangeN:
3054 case Op_CompareAndExchangeP:
3055 case Op_GetAndSetN:
3056 case Op_GetAndSetP:
3057 return true;
3058 case Op_Phi: {
3059 for (uint i = 1; i < n->req(); i++) {
3060 if (needs_barrier_impl(phase, n->in(i), visited)) return true;
3061 }
3062 return false;
3063 }
3064 case Op_CheckCastPP:
3065 case Op_CastPP:
3066 return needs_barrier_impl(phase, n->in(1), visited);
3067 case Op_Proj:
3068 return needs_barrier_impl(phase, n->in(0), visited);
3069 case Op_ShenandoahLoadReferenceBarrier:
3070 // tty->print_cr("optimize barrier on barrier");
3071 return false;
3072 case Op_Parm:
3073 // tty->print_cr("optimize barrier on input arg");
3074 return false;
3075 case Op_DecodeN:
3076 case Op_EncodeP:
3077 return needs_barrier_impl(phase, n->in(1), visited);
3078 case Op_LoadN:
3079 return true;
3080 case Op_CMoveP:
3081 return needs_barrier_impl(phase, n->in(2), visited) ||
3082 needs_barrier_impl(phase, n->in(3), visited);
3083 case Op_ShenandoahEnqueueBarrier:
3084 return needs_barrier_impl(phase, n->in(1), visited);
3085 default:
3086 break;
3087 }
3088 #ifdef ASSERT
3089 tty->print("need barrier on?: ");
3090 tty->print_cr("ins:");
3091 n->dump(2);
3092 tty->print_cr("outs:");
3093 n->dump(-2);
3094 ShouldNotReachHere();
3095 #endif
3096 return true;
3097 }
3098
3099 ShenandoahLoadReferenceBarrierNode::Strength ShenandoahLoadReferenceBarrierNode::get_barrier_strength() {
3100 Unique_Node_List visited;
3101 Node_Stack stack(0);
3102 stack.push(this, 0);
3103 Strength strength = NONE;
3104 while (strength != STRONG && stack.size() > 0) {
3105 Node* n = stack.node();
3106 if (visited.member(n)) {
3107 stack.pop();
3108 continue;
3109 }
3110 visited.push(n);
3111 bool visit_users = false;
3112 switch (n->Opcode()) {
3113 case Op_StoreN:
3114 case Op_StoreP: {
3115 strength = STRONG;
3116 break;
3117 }
3118 case Op_CmpP: {
3119 if (!n->in(1)->bottom_type()->higher_equal(TypePtr::NULL_PTR) &&
3120 !n->in(2)->bottom_type()->higher_equal(TypePtr::NULL_PTR)) {
3121 strength = STRONG;
3122 }
3123 break;
3124 }
3125 case Op_CallStaticJava: {
3126 strength = STRONG;
3127 break;
3128 }
3129 case Op_CallDynamicJava:
3130 case Op_CallLeaf:
3131 case Op_CallLeafNoFP:
3132 case Op_CompareAndSwapL:
3133 case Op_CompareAndSwapI:
3134 case Op_CompareAndSwapB:
3135 case Op_CompareAndSwapS:
3136 case Op_ShenandoahCompareAndSwapN:
3137 case Op_ShenandoahCompareAndSwapP:
3138 case Op_ShenandoahWeakCompareAndSwapN:
3139 case Op_ShenandoahWeakCompareAndSwapP:
3140 case Op_ShenandoahCompareAndExchangeN:
3141 case Op_ShenandoahCompareAndExchangeP:
3142 case Op_CompareAndExchangeL:
3143 case Op_CompareAndExchangeI:
3144 case Op_CompareAndExchangeB:
3145 case Op_CompareAndExchangeS:
3146 case Op_WeakCompareAndSwapL:
3147 case Op_WeakCompareAndSwapI:
3148 case Op_WeakCompareAndSwapB:
3149 case Op_WeakCompareAndSwapS:
3150 case Op_GetAndSetL:
3151 case Op_GetAndSetI:
3152 case Op_GetAndSetB:
3153 case Op_GetAndSetS:
3154 case Op_GetAndSetP:
3155 case Op_GetAndSetN:
3156 case Op_GetAndAddL:
3157 case Op_GetAndAddI:
3158 case Op_GetAndAddB:
3159 case Op_GetAndAddS:
3160 case Op_ShenandoahEnqueueBarrier:
3161 case Op_FastLock:
3162 case Op_FastUnlock:
3163 case Op_Rethrow:
3164 case Op_Return:
3165 case Op_StoreB:
3166 case Op_StoreC:
3167 case Op_StoreD:
3168 case Op_StoreF:
3169 case Op_StoreL:
3170 case Op_StoreLConditional:
3171 case Op_StoreI:
3172 case Op_StoreVector:
3173 case Op_StrInflatedCopy:
3174 case Op_StrCompressedCopy:
3175 case Op_EncodeP:
3176 case Op_CastP2X:
3177 case Op_SafePoint:
3178 case Op_EncodeISOArray:
3179 strength = STRONG;
3180 break;
3181 case Op_LoadB:
3182 case Op_LoadUB:
3183 case Op_LoadUS:
3184 case Op_LoadD:
3185 case Op_LoadF:
3186 case Op_LoadL:
3187 case Op_LoadI:
3188 case Op_LoadS:
3189 case Op_LoadN:
3190 case Op_LoadP:
3191 case Op_LoadVector: {
3192 const TypePtr* adr_type = n->adr_type();
3193 int alias_idx = Compile::current()->get_alias_index(adr_type);
3194 Compile::AliasType* alias_type = Compile::current()->alias_type(alias_idx);
3195 ciField* field = alias_type->field();
3196 bool is_static = field != NULL && field->is_static();
3197 bool is_final = field != NULL && field->is_final();
3198 bool is_stable = field != NULL && field->is_stable();
3199 if (ShenandoahOptimizeStaticFinals && is_static && is_final) {
3200 // Leave strength as is.
3201 } else if (ShenandoahOptimizeInstanceFinals && !is_static && is_final) {
3202 // Leave strength as is.
3203 } else if (ShenandoahOptimizeStableFinals && (is_stable || (adr_type->isa_aryptr() && adr_type->isa_aryptr()->is_stable()))) {
3204 // Leave strength as is.
3205 } else {
3206 strength = WEAK;
3207 }
3208 break;
3209 }
3210 case Op_AryEq: {
3211 Node* n1 = n->in(2);
3212 Node* n2 = n->in(3);
3213 if (!ShenandoahOptimizeStableFinals ||
3214 !n1->bottom_type()->isa_aryptr() || !n1->bottom_type()->isa_aryptr()->is_stable() ||
3215 !n2->bottom_type()->isa_aryptr() || !n2->bottom_type()->isa_aryptr()->is_stable()) {
3216 strength = WEAK;
3217 }
3218 break;
3219 }
3220 case Op_StrEquals:
3221 case Op_StrComp:
3222 case Op_StrIndexOf:
3223 case Op_StrIndexOfChar:
3224 if (!ShenandoahOptimizeStableFinals) {
3225 strength = WEAK;
3226 }
3227 break;
3228 case Op_Conv2B:
3229 case Op_LoadRange:
3230 case Op_LoadKlass:
3231 case Op_LoadNKlass:
3232 // NONE, i.e. leave current strength as is
3233 break;
3234 case Op_AddP:
3235 case Op_CheckCastPP:
3236 case Op_CastPP:
3237 case Op_CMoveP:
3238 case Op_Phi:
3239 case Op_ShenandoahLoadReferenceBarrier:
3240 visit_users = true;
3241 break;
3242 default: {
3243 #ifdef ASSERT
3244 tty->print_cr("Unknown node in get_barrier_strength:");
3245 n->dump(1);
3246 ShouldNotReachHere();
3247 #else
3248 strength = STRONG;
3249 #endif
3250 }
3251 }
3252 #ifdef ASSERT
3253 /*
3254 if (strength == STRONG) {
3255 tty->print("strengthening node: ");
3256 n->dump();
3257 }
3258 */
3259 #endif
3260 stack.pop();
3261 if (visit_users) {
3262 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
3263 Node* user = n->fast_out(i);
3264 if (user != NULL) {
3265 stack.push(user, 0);
3266 }
3267 }
3268 }
3269 }
3270 return strength;
3271 }
3272
3273 CallStaticJavaNode* ShenandoahLoadReferenceBarrierNode::pin_and_expand_null_check(PhaseIterGVN& igvn) {
3274 Node* val = in(ValueIn);
3275
3276 const Type* val_t = igvn.type(val);
3277
3278 if (val_t->meet(TypePtr::NULL_PTR) != val_t &&
3279 val->Opcode() == Op_CastPP &&
3280 val->in(0) != NULL &&
3281 val->in(0)->Opcode() == Op_IfTrue &&
3282 val->in(0)->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none) &&
3283 val->in(0)->in(0)->is_If() &&
3284 val->in(0)->in(0)->in(1)->Opcode() == Op_Bool &&
3285 val->in(0)->in(0)->in(1)->as_Bool()->_test._test == BoolTest::ne &&
3286 val->in(0)->in(0)->in(1)->in(1)->Opcode() == Op_CmpP &&
3287 val->in(0)->in(0)->in(1)->in(1)->in(1) == val->in(1) &&
3288 val->in(0)->in(0)->in(1)->in(1)->in(2)->bottom_type() == TypePtr::NULL_PTR) {
3289 assert(val->in(0)->in(0)->in(1)->in(1)->in(1) == val->in(1), "");
3290 CallStaticJavaNode* unc = val->in(0)->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none);
3291 return unc;
3292 }
3293 return NULL;
3294 }
|