1 /*
2 * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 #include "precompiled.hpp"
25 #include "opto/compile.hpp"
26 #include "opto/castnode.hpp"
27 #include "opto/escape.hpp"
28 #include "opto/graphKit.hpp"
29 #include "opto/idealKit.hpp"
30 #include "opto/loopnode.hpp"
31 #include "opto/macro.hpp"
32 #include "opto/node.hpp"
33 #include "opto/type.hpp"
34 #include "utilities/macros.hpp"
35 #include "gc/z/c2/zBarrierSetC2.hpp"
36 #include "gc/z/zThreadLocalData.hpp"
37 #include "gc/z/zBarrierSetRuntime.hpp"
38
39 ZBarrierSetC2State::ZBarrierSetC2State(Arena* comp_arena)
40 : _load_barrier_nodes(new (comp_arena) GrowableArray<LoadBarrierNode*>(comp_arena, 8, 0, NULL)) {}
41
42 int ZBarrierSetC2State::load_barrier_count() const {
43 return _load_barrier_nodes->length();
44 }
45
46 void ZBarrierSetC2State::add_load_barrier_node(LoadBarrierNode * n) {
47 assert(!_load_barrier_nodes->contains(n), " duplicate entry in expand list");
48 _load_barrier_nodes->append(n);
49 }
50
51 void ZBarrierSetC2State::remove_load_barrier_node(LoadBarrierNode * n) {
52 // this function may be called twice for a node so check
53 // that the node is in the array before attempting to remove it
54 if (_load_barrier_nodes->contains(n)) {
55 _load_barrier_nodes->remove(n);
56 }
57 }
58
59 LoadBarrierNode* ZBarrierSetC2State::load_barrier_node(int idx) const {
60 return _load_barrier_nodes->at(idx);
61 }
62
63 void* ZBarrierSetC2::create_barrier_state(Arena* comp_arena) const {
64 return new(comp_arena) ZBarrierSetC2State(comp_arena);
65 }
66
67 ZBarrierSetC2State* ZBarrierSetC2::state() const {
68 return reinterpret_cast<ZBarrierSetC2State*>(Compile::current()->barrier_set_state());
69 }
70
71 bool ZBarrierSetC2::is_gc_barrier_node(Node* node) const {
72 // 1. This step follows potential oop projections of a load barrier before expansion
73 if (node->is_Proj()) {
74 node = node->in(0);
75 }
76
77 // 2. This step checks for unexpanded load barriers
78 if (node->is_LoadBarrier()) {
79 return true;
80 }
81
82 // 3. This step checks for the phi corresponding to an optimized load barrier expansion
83 if (node->is_Phi()) {
84 PhiNode* phi = node->as_Phi();
85 Node* n = phi->in(1);
86 if (n != NULL && (n->is_LoadBarrierSlowReg() || n->is_LoadBarrierWeakSlowReg())) {
87 return true;
88 }
89 }
90
91 return false;
92 }
93
94 void ZBarrierSetC2::register_potential_barrier_node(Node* node) const {
95 if (node->is_LoadBarrier()) {
96 state()->add_load_barrier_node(node->as_LoadBarrier());
97 }
98 }
99
100 void ZBarrierSetC2::unregister_potential_barrier_node(Node* node) const {
101 if (node->is_LoadBarrier()) {
102 state()->remove_load_barrier_node(node->as_LoadBarrier());
103 }
104 }
105
106 void ZBarrierSetC2::eliminate_useless_gc_barriers(Unique_Node_List &useful, Compile* C) const {
107 // Remove useless LoadBarrier nodes
108 ZBarrierSetC2State* s = state();
109 for (int i = s->load_barrier_count()-1; i >= 0; i--) {
110 LoadBarrierNode* n = s->load_barrier_node(i);
111 if (!useful.member(n)) {
112 unregister_potential_barrier_node(n);
113 }
114 }
115 }
116
117 void ZBarrierSetC2::enqueue_useful_gc_barrier(PhaseIterGVN* igvn, Node* node) const {
118 if (node->is_LoadBarrier() && !node->as_LoadBarrier()->has_true_uses()) {
119 igvn->_worklist.push(node);
120 }
121 }
122
123 void ZBarrierSetC2::find_dominating_barriers(PhaseIterGVN& igvn) {
124 // Look for dominating barriers on the same address only once all
125 // other loop opts are over: loop opts may cause a safepoint to be
126 // inserted between a barrier and its dominating barrier.
127 Compile* C = Compile::current();
128 ZBarrierSetC2* bs = (ZBarrierSetC2*)BarrierSet::barrier_set()->barrier_set_c2();
129 ZBarrierSetC2State* s = bs->state();
130 if (s->load_barrier_count() >= 2) {
131 Compile::TracePhase tp("idealLoop", &C->timers[Phase::_t_idealLoop]);
132 PhaseIdealLoop ideal_loop(igvn, LoopOptsLastRound);
133 if (C->major_progress()) C->print_method(PHASE_PHASEIDEALLOOP_ITERATIONS, 2);
134 }
135 }
136
137 void ZBarrierSetC2::add_users_to_worklist(Unique_Node_List* worklist) const {
138 // Permanent temporary workaround
139 // Loadbarriers may have non-obvious dead uses keeping them alive during parsing. The use is
140 // removed by RemoveUseless (after parsing, before optimize) but the barriers won't be added to
141 // the worklist. Unless we add them explicitly they are not guaranteed to end up there.
142 ZBarrierSetC2State* s = state();
143
144 for (int i = 0; i < s->load_barrier_count(); i++) {
145 LoadBarrierNode* n = s->load_barrier_node(i);
146 worklist->push(n);
147 }
148 }
149
150 const TypeFunc* ZBarrierSetC2::load_barrier_Type() const {
151 const Type** fields;
152
153 // Create input types (domain)
154 fields = TypeTuple::fields(2);
155 fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL;
156 fields[TypeFunc::Parms+1] = TypeOopPtr::BOTTOM;
157 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2, fields);
158
159 // Create result type (range)
160 fields = TypeTuple::fields(1);
161 fields[TypeFunc::Parms+0] = TypeInstPtr::BOTTOM;
162 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+1, fields);
163
164 return TypeFunc::make(domain, range);
165 }
166
167 // == LoadBarrierNode ==
168
169 LoadBarrierNode::LoadBarrierNode(Compile* C,
170 Node* c,
171 Node* mem,
172 Node* val,
173 Node* adr,
174 bool weak,
175 bool writeback,
176 bool oop_reload_allowed) :
177 MultiNode(Number_of_Inputs),
178 _weak(weak),
179 _writeback(writeback),
180 _oop_reload_allowed(oop_reload_allowed) {
181 init_req(Control, c);
182 init_req(Memory, mem);
183 init_req(Oop, val);
184 init_req(Address, adr);
185 init_req(Similar, C->top());
186
187 init_class_id(Class_LoadBarrier);
188 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
189 bs->register_potential_barrier_node(this);
190 }
191
192 const Type *LoadBarrierNode::bottom_type() const {
193 const Type** floadbarrier = (const Type **)(Compile::current()->type_arena()->Amalloc_4((Number_of_Outputs)*sizeof(Type*)));
194 Node* in_oop = in(Oop);
195 floadbarrier[Control] = Type::CONTROL;
196 floadbarrier[Memory] = Type::MEMORY;
197 floadbarrier[Oop] = in_oop == NULL ? Type::TOP : in_oop->bottom_type();
198 return TypeTuple::make(Number_of_Outputs, floadbarrier);
199 }
200
201 const Type *LoadBarrierNode::Value(PhaseGVN *phase) const {
202 const Type** floadbarrier = (const Type **)(phase->C->type_arena()->Amalloc_4((Number_of_Outputs)*sizeof(Type*)));
203 const Type* val_t = phase->type(in(Oop));
204 floadbarrier[Control] = Type::CONTROL;
205 floadbarrier[Memory] = Type::MEMORY;
206 floadbarrier[Oop] = val_t;
207 return TypeTuple::make(Number_of_Outputs, floadbarrier);
208 }
209
210 bool LoadBarrierNode::is_dominator(PhaseIdealLoop* phase, bool linear_only, Node *d, Node *n) {
211 if (phase != NULL) {
212 return phase->is_dominator(d, n);
213 }
214
215 for (int i = 0; i < 10 && n != NULL; i++) {
216 n = IfNode::up_one_dom(n, linear_only);
217 if (n == d) {
218 return true;
219 }
220 }
221
222 return false;
223 }
224
225 LoadBarrierNode* LoadBarrierNode::has_dominating_barrier(PhaseIdealLoop* phase, bool linear_only, bool look_for_similar) {
226 Node* val = in(LoadBarrierNode::Oop);
227 if (in(Similar)->is_Proj() && in(Similar)->in(0)->is_LoadBarrier()) {
228 LoadBarrierNode* lb = in(Similar)->in(0)->as_LoadBarrier();
229 assert(lb->in(Address) == in(Address), "");
230 // Load barrier on Similar edge dominates so if it now has the Oop field it can replace this barrier.
231 if (lb->in(Oop) == in(Oop)) {
232 return lb;
233 }
234 // Follow chain of load barrier through Similar edges
235 while (!lb->in(Similar)->is_top()) {
236 lb = lb->in(Similar)->in(0)->as_LoadBarrier();
237 assert(lb->in(Address) == in(Address), "");
238 }
239 if (lb != in(Similar)->in(0)) {
240 return lb;
241 }
242 }
243 for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) {
244 Node* u = val->fast_out(i);
245 if (u != this && u->is_LoadBarrier() && u->in(Oop) == val && u->as_LoadBarrier()->has_true_uses()) {
246 Node* this_ctrl = in(LoadBarrierNode::Control);
247 Node* other_ctrl = u->in(LoadBarrierNode::Control);
248 if (is_dominator(phase, linear_only, other_ctrl, this_ctrl)) {
249 return u->as_LoadBarrier();
250 }
251 }
252 }
253
254 if (ZVerifyLoadBarriers || can_be_eliminated()) {
255 return NULL;
256 }
257
258 if (!look_for_similar) {
259 return NULL;
260 }
261
262 Node* addr = in(LoadBarrierNode::Address);
263 for (DUIterator_Fast imax, i = addr->fast_outs(imax); i < imax; i++) {
264 Node* u = addr->fast_out(i);
265 if (u != this && u->is_LoadBarrier() && u->as_LoadBarrier()->has_true_uses()) {
266 Node* this_ctrl = in(LoadBarrierNode::Control);
267 Node* other_ctrl = u->in(LoadBarrierNode::Control);
268 if (is_dominator(phase, linear_only, other_ctrl, this_ctrl)) {
269 ResourceMark rm;
270 Unique_Node_List wq;
271 wq.push(in(LoadBarrierNode::Control));
272 bool ok = true;
273 bool dom_found = false;
274 for (uint next = 0; next < wq.size(); ++next) {
275 Node *n = wq.at(next);
276 if (n->is_top()) {
277 return NULL;
278 }
279 assert(n->is_CFG(), "");
280 if (n->is_SafePoint()) {
281 ok = false;
282 break;
283 }
284 if (n == u) {
285 dom_found = true;
286 continue;
287 }
288 if (n->is_Region()) {
289 for (uint i = 1; i < n->req(); i++) {
290 Node* m = n->in(i);
291 if (m != NULL) {
292 wq.push(m);
293 }
294 }
295 } else {
296 Node* m = n->in(0);
297 if (m != NULL) {
298 wq.push(m);
299 }
300 }
301 }
302 if (ok) {
303 assert(dom_found, "");
304 return u->as_LoadBarrier();;
305 }
306 break;
307 }
308 }
309 }
310
311 return NULL;
312 }
313
314 void LoadBarrierNode::push_dominated_barriers(PhaseIterGVN* igvn) const {
315 // Change to that barrier may affect a dominated barrier so re-push those
316 Node* val = in(LoadBarrierNode::Oop);
317
318 for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) {
319 Node* u = val->fast_out(i);
320 if (u != this && u->is_LoadBarrier() && u->in(Oop) == val) {
321 Node* this_ctrl = in(Control);
322 Node* other_ctrl = u->in(Control);
323 if (is_dominator(NULL, false, this_ctrl, other_ctrl)) {
324 igvn->_worklist.push(u);
325 }
326 }
327
328 Node* addr = in(LoadBarrierNode::Address);
329 for (DUIterator_Fast imax, i = addr->fast_outs(imax); i < imax; i++) {
330 Node* u = addr->fast_out(i);
331 if (u != this && u->is_LoadBarrier() && u->in(Similar)->is_top()) {
332 Node* this_ctrl = in(Control);
333 Node* other_ctrl = u->in(Control);
334 if (is_dominator(NULL, false, this_ctrl, other_ctrl)) {
335 igvn->_worklist.push(u);
336 }
337 }
338 }
339 }
340 }
341
342 Node *LoadBarrierNode::Identity(PhaseGVN *phase) {
343 if (!phase->C->directive()->ZOptimizeLoadBarriersOption) {
344 return this;
345 }
346
347 bool redundant_addr = false;
348 LoadBarrierNode* dominating_barrier = has_dominating_barrier(NULL, true, false);
349 if (dominating_barrier != NULL) {
350 assert(dominating_barrier->in(Oop) == in(Oop), "");
351 return dominating_barrier;
352 }
353
354 return this;
355 }
356
357 Node *LoadBarrierNode::Ideal(PhaseGVN *phase, bool can_reshape) {
358 if (remove_dead_region(phase, can_reshape)) {
359 return this;
360 }
361
362 Node* val = in(Oop);
363 Node* mem = in(Memory);
364 Node* ctrl = in(Control);
365 Node* adr = in(Address);
366 assert(val->Opcode() != Op_LoadN, "");
367
368 if (mem->is_MergeMem()) {
369 Node* new_mem = mem->as_MergeMem()->memory_at(Compile::AliasIdxRaw);
370 set_req(Memory, new_mem);
371 if (mem->outcnt() == 0 && can_reshape) {
372 phase->is_IterGVN()->_worklist.push(mem);
373 }
374
375 return this;
376 }
377
378 bool optimizeLoadBarriers = phase->C->directive()->ZOptimizeLoadBarriersOption;
379 LoadBarrierNode* dominating_barrier = optimizeLoadBarriers ? has_dominating_barrier(NULL, !can_reshape, !phase->C->major_progress()) : NULL;
380 if (dominating_barrier != NULL && dominating_barrier->in(Oop) != in(Oop)) {
381 assert(in(Address) == dominating_barrier->in(Address), "");
382 set_req(Similar, dominating_barrier->proj_out(Oop));
383 return this;
384 }
385
386 bool eliminate = (optimizeLoadBarriers && !(val->is_Phi() || val->Opcode() == Op_LoadP || val->Opcode() == Op_GetAndSetP || val->is_DecodeN())) ||
387 (can_reshape && (dominating_barrier != NULL || !has_true_uses()));
388
389 if (eliminate) {
390 if (can_reshape) {
391 PhaseIterGVN* igvn = phase->is_IterGVN();
392 Node* out_ctrl = proj_out_or_null(Control);
393 Node* out_res = proj_out_or_null(Oop);
394
395 if (out_ctrl != NULL) {
396 igvn->replace_node(out_ctrl, ctrl);
397 }
398
399 // That transformation may cause the Similar edge on the load barrier to be invalid
400 fix_similar_in_uses(igvn);
401 if (out_res != NULL) {
402 if (dominating_barrier != NULL) {
403 igvn->replace_node(out_res, dominating_barrier->proj_out(Oop));
404 } else {
405 igvn->replace_node(out_res, val);
406 }
407 }
408 }
409
410 return new ConINode(TypeInt::ZERO);
411 }
412
413 // If the Similar edge is no longer a load barrier, clear it
414 Node* similar = in(Similar);
415 if (!similar->is_top() && !(similar->is_Proj() && similar->in(0)->is_LoadBarrier())) {
416 set_req(Similar, phase->C->top());
417 return this;
418 }
419
420 if (can_reshape) {
421 // If this barrier is linked through the Similar edge by a
422 // dominated barrier and both barriers have the same Oop field,
423 // the dominated barrier can go away, so push it for reprocessing.
424 // We also want to avoid a barrier to depend on another dominating
425 // barrier through its Similar edge that itself depend on another
426 // barrier through its Similar edge and rather have the first
427 // depend on the third.
428 PhaseIterGVN* igvn = phase->is_IterGVN();
429 Node* out_res = proj_out(Oop);
430 for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) {
431 Node* u = out_res->fast_out(i);
432 if (u->is_LoadBarrier() && u->in(Similar) == out_res &&
433 (u->in(Oop) == val || !u->in(Similar)->is_top())) {
434 igvn->_worklist.push(u);
435 }
436 }
437
438 push_dominated_barriers(igvn);
439 }
440
441 return NULL;
442 }
443
444 void LoadBarrierNode::fix_similar_in_uses(PhaseIterGVN* igvn) {
445 Node* out_res = proj_out_or_null(Oop);
446 if (out_res == NULL) {
447 return;
448 }
449
450 for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) {
451 Node* u = out_res->fast_out(i);
452 if (u->is_LoadBarrier() && u->in(Similar) == out_res) {
453 igvn->replace_input_of(u, Similar, igvn->C->top());
454 --i;
455 --imax;
456 }
457 }
458 }
459
460 bool LoadBarrierNode::has_true_uses() const {
461 Node* out_res = proj_out_or_null(Oop);
462 if (out_res == NULL) {
463 return false;
464 }
465
466 for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) {
467 Node* u = out_res->fast_out(i);
468 if (!u->is_LoadBarrier() || u->in(Similar) != out_res) {
469 return true;
470 }
471 }
472
473 return false;
474 }
475
476 // == Accesses ==
477
478 Node* ZBarrierSetC2::make_cas_loadbarrier(C2AtomicParseAccess& access) const {
479 assert(!UseCompressedOops, "Not allowed");
480 CompareAndSwapNode* cas = (CompareAndSwapNode*)access.raw_access();
481 PhaseGVN& gvn = access.gvn();
482 Compile* C = Compile::current();
483 GraphKit* kit = access.kit();
484
485 Node* in_ctrl = cas->in(MemNode::Control);
486 Node* in_mem = cas->in(MemNode::Memory);
487 Node* in_adr = cas->in(MemNode::Address);
488 Node* in_val = cas->in(MemNode::ValueIn);
489 Node* in_expected = cas->in(LoadStoreConditionalNode::ExpectedIn);
490
491 float likely = PROB_LIKELY(0.999);
492
493 const TypePtr *adr_type = gvn.type(in_adr)->isa_ptr();
494 Compile::AliasType* alias_type = C->alias_type(adr_type);
495 int alias_idx = C->get_alias_index(adr_type);
496
497 // Outer check - true: continue, false: load and check
498 Node* region = new RegionNode(3);
499 Node* phi = new PhiNode(region, TypeInt::BOOL);
500 Node* phi_mem = new PhiNode(region, Type::MEMORY, adr_type);
501
502 // Inner check - is the healed ref equal to the expected
503 Node* region2 = new RegionNode(3);
504 Node* phi2 = new PhiNode(region2, TypeInt::BOOL);
505 Node* phi_mem2 = new PhiNode(region2, Type::MEMORY, adr_type);
506
507 // CAS node returns 0 or 1
508 Node* cmp = gvn.transform(new CmpINode(cas, kit->intcon(0)));
509 Node* bol = gvn.transform(new BoolNode(cmp, BoolTest::ne))->as_Bool();
510 IfNode* iff = gvn.transform(new IfNode(in_ctrl, bol, likely, COUNT_UNKNOWN))->as_If();
511 Node* then = gvn.transform(new IfTrueNode(iff));
512 Node* elsen = gvn.transform(new IfFalseNode(iff));
513
514 Node* scmemproj1 = gvn.transform(new SCMemProjNode(cas));
515
516 kit->set_memory(scmemproj1, alias_idx);
517 phi_mem->init_req(1, scmemproj1);
518 phi_mem2->init_req(2, scmemproj1);
519
520 // CAS fail - reload and heal oop
521 Node* reload = kit->make_load(elsen, in_adr, TypeOopPtr::BOTTOM, T_OBJECT, MemNode::unordered);
522 Node* barrier = gvn.transform(new LoadBarrierNode(C, elsen, scmemproj1, reload, in_adr, false, true, false));
523 Node* barrierctrl = gvn.transform(new ProjNode(barrier, LoadBarrierNode::Control));
524 Node* barrierdata = gvn.transform(new ProjNode(barrier, LoadBarrierNode::Oop));
525
526 // Check load
527 Node* tmpX = gvn.transform(new CastP2XNode(NULL, barrierdata));
528 Node* in_expX = gvn.transform(new CastP2XNode(NULL, in_expected));
529 Node* cmp2 = gvn.transform(new CmpXNode(tmpX, in_expX));
530 Node *bol2 = gvn.transform(new BoolNode(cmp2, BoolTest::ne))->as_Bool();
531 IfNode* iff2 = gvn.transform(new IfNode(barrierctrl, bol2, likely, COUNT_UNKNOWN))->as_If();
532 Node* then2 = gvn.transform(new IfTrueNode(iff2));
533 Node* elsen2 = gvn.transform(new IfFalseNode(iff2));
534
535 // redo CAS
536 Node* cas2 = gvn.transform(new CompareAndSwapPNode(elsen2, kit->memory(alias_idx), in_adr, in_val, in_expected, cas->order()));
537 Node* scmemproj2 = gvn.transform(new SCMemProjNode(cas2));
538 kit->set_control(elsen2);
539 kit->set_memory(scmemproj2, alias_idx);
540
541 // Merge inner flow - check if healed oop was equal too expected.
542 region2->set_req(1, kit->control());
543 region2->set_req(2, then2);
544 phi2->set_req(1, cas2);
545 phi2->set_req(2, kit->intcon(0));
546 phi_mem2->init_req(1, scmemproj2);
547 kit->set_memory(phi_mem2, alias_idx);
548
549 // Merge outer flow - then check if first CAS succeeded
550 region->set_req(1, then);
551 region->set_req(2, region2);
552 phi->set_req(1, kit->intcon(1));
553 phi->set_req(2, phi2);
554 phi_mem->init_req(2, phi_mem2);
555 kit->set_memory(phi_mem, alias_idx);
556
557 gvn.transform(region2);
558 gvn.transform(phi2);
559 gvn.transform(phi_mem2);
560 gvn.transform(region);
561 gvn.transform(phi);
562 gvn.transform(phi_mem);
563
564 kit->set_control(region);
565 kit->insert_mem_bar(Op_MemBarCPUOrder);
566
567 return phi;
568 }
569
570 Node* ZBarrierSetC2::make_cmpx_loadbarrier(C2AtomicParseAccess& access) const {
571 CompareAndExchangePNode* cmpx = (CompareAndExchangePNode*)access.raw_access();
572 GraphKit* kit = access.kit();
573 PhaseGVN& gvn = kit->gvn();
574 Compile* C = Compile::current();
575
576 Node* in_ctrl = cmpx->in(MemNode::Control);
577 Node* in_mem = cmpx->in(MemNode::Memory);
578 Node* in_adr = cmpx->in(MemNode::Address);
579 Node* in_val = cmpx->in(MemNode::ValueIn);
580 Node* in_expected = cmpx->in(LoadStoreConditionalNode::ExpectedIn);
581
582 float likely = PROB_LIKELY(0.999);
583
584 const TypePtr *adr_type = cmpx->get_ptr_type();
585 Compile::AliasType* alias_type = C->alias_type(adr_type);
586 int alias_idx = C->get_alias_index(adr_type);
587
588 // Outer check - true: continue, false: load and check
589 Node* region = new RegionNode(3);
590 Node* phi = new PhiNode(region, adr_type);
591
592 // Inner check - is the healed ref equal to the expected
593 Node* region2 = new RegionNode(3);
594 Node* phi2 = new PhiNode(region2, adr_type);
595
596 // Check if cmpx succeeded
597 Node* cmp = gvn.transform(new CmpPNode(cmpx, in_expected));
598 Node* bol = gvn.transform(new BoolNode(cmp, BoolTest::eq))->as_Bool();
599 IfNode* iff = gvn.transform(new IfNode(in_ctrl, bol, likely, COUNT_UNKNOWN))->as_If();
600 Node* then = gvn.transform(new IfTrueNode(iff));
601 Node* elsen = gvn.transform(new IfFalseNode(iff));
602
603 Node* scmemproj1 = gvn.transform(new SCMemProjNode(cmpx));
604 kit->set_memory(scmemproj1, alias_idx);
605
606 // CAS fail - reload and heal oop
607 Node* reload = kit->make_load(elsen, in_adr, TypeOopPtr::BOTTOM, T_OBJECT, MemNode::unordered);
608 Node* barrier = gvn.transform(new LoadBarrierNode(C, elsen, scmemproj1, reload, in_adr, false, true, false));
609 Node* barrierctrl = gvn.transform(new ProjNode(barrier, LoadBarrierNode::Control));
610 Node* barrierdata = gvn.transform(new ProjNode(barrier, LoadBarrierNode::Oop));
611
612 // Check load
613 Node* tmpX = gvn.transform(new CastP2XNode(NULL, barrierdata));
614 Node* in_expX = gvn.transform(new CastP2XNode(NULL, in_expected));
615 Node* cmp2 = gvn.transform(new CmpXNode(tmpX, in_expX));
616 Node *bol2 = gvn.transform(new BoolNode(cmp2, BoolTest::ne))->as_Bool();
617 IfNode* iff2 = gvn.transform(new IfNode(barrierctrl, bol2, likely, COUNT_UNKNOWN))->as_If();
618 Node* then2 = gvn.transform(new IfTrueNode(iff2));
619 Node* elsen2 = gvn.transform(new IfFalseNode(iff2));
620
621 // Redo CAS
622 Node* cmpx2 = gvn.transform(new CompareAndExchangePNode(elsen2, kit->memory(alias_idx), in_adr, in_val, in_expected, adr_type, cmpx->get_ptr_type(), cmpx->order()));
623 Node* scmemproj2 = gvn.transform(new SCMemProjNode(cmpx2));
624 kit->set_control(elsen2);
625 kit->set_memory(scmemproj2, alias_idx);
626
627 // Merge inner flow - check if healed oop was equal too expected.
628 region2->set_req(1, kit->control());
629 region2->set_req(2, then2);
630 phi2->set_req(1, cmpx2);
631 phi2->set_req(2, barrierdata);
632
633 // Merge outer flow - then check if first cas succeeded
634 region->set_req(1, then);
635 region->set_req(2, region2);
636 phi->set_req(1, cmpx);
637 phi->set_req(2, phi2);
638
639 gvn.transform(region2);
640 gvn.transform(phi2);
641 gvn.transform(region);
642 gvn.transform(phi);
643
644 kit->set_control(region);
645 kit->set_memory(in_mem, alias_idx);
646 kit->insert_mem_bar(Op_MemBarCPUOrder);
647
648 return phi;
649 }
650
651 Node* ZBarrierSetC2::load_barrier(GraphKit* kit, Node* val, Node* adr, bool weak, bool writeback, bool oop_reload_allowed) const {
652 PhaseGVN& gvn = kit->gvn();
653 Node* barrier = new LoadBarrierNode(Compile::current(), kit->control(), kit->memory(TypeRawPtr::BOTTOM), val, adr, weak, writeback, oop_reload_allowed);
654 Node* transformed_barrier = gvn.transform(barrier);
655
656 if (transformed_barrier->is_LoadBarrier()) {
657 if (barrier == transformed_barrier) {
658 kit->set_control(gvn.transform(new ProjNode(barrier, LoadBarrierNode::Control)));
659 }
660 Node* result = gvn.transform(new ProjNode(transformed_barrier, LoadBarrierNode::Oop));
661 assert(is_gc_barrier_node(result), "sanity");
662 assert(step_over_gc_barrier(result) == val, "sanity");
663 return result;
664 } else {
665 return val;
666 }
667 }
668
669 static bool barrier_needed(C2Access& access) {
670 return ZBarrierSet::barrier_needed(access.decorators(), access.type());
671 }
672
673 Node* ZBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const {
674 Node* p = BarrierSetC2::load_at_resolved(access, val_type);
675 if (!barrier_needed(access)) {
676 return p;
677 }
678
679 bool weak = (access.decorators() & ON_WEAK_OOP_REF) != 0;
680
681 assert(access.is_parse_access(), "entry not supported at optimization time");
682 C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
683 GraphKit* kit = parse_access.kit();
684 PhaseGVN& gvn = kit->gvn();
685 Node* adr = access.addr().node();
686 Node* heap_base_oop = access.base();
687 bool unsafe = (access.decorators() & C2_UNSAFE_ACCESS) != 0;
688 if (unsafe) {
689 if (!ZVerifyLoadBarriers) {
690 p = load_barrier(kit, p, adr);
691 } else {
692 if (!TypePtr::NULL_PTR->higher_equal(gvn.type(heap_base_oop))) {
693 p = load_barrier(kit, p, adr);
694 } else {
695 IdealKit ideal(kit);
696 IdealVariable res(ideal);
697 #define __ ideal.
698 __ declarations_done();
699 __ set(res, p);
700 __ if_then(heap_base_oop, BoolTest::ne, kit->null(), PROB_UNLIKELY(0.999)); {
701 kit->sync_kit(ideal);
702 p = load_barrier(kit, p, adr);
703 __ set(res, p);
704 __ sync_kit(kit);
705 } __ end_if();
706 kit->final_sync(ideal);
707 p = __ value(res);
708 #undef __
709 }
710 }
711 return p;
712 } else {
713 return load_barrier(parse_access.kit(), p, access.addr().node(), weak, true, true);
714 }
715 }
716
717 Node* ZBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
718 Node* new_val, const Type* val_type) const {
719 Node* result = BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, val_type);
720 if (!barrier_needed(access)) {
721 return result;
722 }
723
724 access.set_needs_pinning(false);
725 return make_cmpx_loadbarrier(access);
726 }
727
728 Node* ZBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
729 Node* new_val, const Type* value_type) const {
730 Node* result = BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type);
731 if (!barrier_needed(access)) {
732 return result;
733 }
734
735 Node* load_store = access.raw_access();
736 bool weak_cas = (access.decorators() & C2_WEAK_CMPXCHG) != 0;
737 bool expected_is_null = (expected_val->get_ptr_type() == TypePtr::NULL_PTR);
738
739 if (!expected_is_null) {
740 if (weak_cas) {
741 access.set_needs_pinning(false);
742 load_store = make_cas_loadbarrier(access);
743 } else {
744 access.set_needs_pinning(false);
745 load_store = make_cas_loadbarrier(access);
746 }
747 }
748
749 return load_store;
750 }
751
752 Node* ZBarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* val_type) const {
753 Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, new_val, val_type);
754 if (!barrier_needed(access)) {
755 return result;
756 }
757
758 Node* load_store = access.raw_access();
759 Node* adr = access.addr().node();
760
761 assert(access.is_parse_access(), "entry not supported at optimization time");
762 C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
763 return load_barrier(parse_access.kit(), load_store, adr, false, false, false);
764 }
765
766 // == Macro Expansion ==
767
768 void ZBarrierSetC2::expand_loadbarrier_node(PhaseMacroExpand* phase, LoadBarrierNode* barrier) const {
769 Node* in_ctrl = barrier->in(LoadBarrierNode::Control);
770 Node* in_mem = barrier->in(LoadBarrierNode::Memory);
771 Node* in_val = barrier->in(LoadBarrierNode::Oop);
772 Node* in_adr = barrier->in(LoadBarrierNode::Address);
773
774 Node* out_ctrl = barrier->proj_out(LoadBarrierNode::Control);
775 Node* out_res = barrier->proj_out(LoadBarrierNode::Oop);
776
777 PhaseIterGVN &igvn = phase->igvn();
778
779 if (ZVerifyLoadBarriers) {
780 igvn.replace_node(out_res, in_val);
781 igvn.replace_node(out_ctrl, in_ctrl);
782 return;
783 }
784
785 if (barrier->can_be_eliminated()) {
786 // Clone and pin the load for this barrier below the dominating
787 // barrier: the load cannot be allowed to float above the
788 // dominating barrier
789 Node* load = in_val;
790
791 if (load->is_Load()) {
792 Node* new_load = load->clone();
793 Node* addp = new_load->in(MemNode::Address);
794 assert(addp->is_AddP() || addp->is_Phi() || addp->is_Load(), "bad address");
795 Node* cast = new CastPPNode(addp, igvn.type(addp), true);
796 Node* ctrl = NULL;
797 Node* similar = barrier->in(LoadBarrierNode::Similar);
798 if (similar->is_Phi()) {
799 // already expanded
800 ctrl = similar->in(0);
801 } else {
802 assert(similar->is_Proj() && similar->in(0)->is_LoadBarrier(), "unexpected graph shape");
803 ctrl = similar->in(0)->as_LoadBarrier()->proj_out(LoadBarrierNode::Control);
804 }
805 assert(ctrl != NULL, "bad control");
806 cast->set_req(0, ctrl);
807 igvn.transform(cast);
808 new_load->set_req(MemNode::Address, cast);
809 igvn.transform(new_load);
810
811 igvn.replace_node(out_res, new_load);
812 igvn.replace_node(out_ctrl, in_ctrl);
813 return;
814 }
815 // cannot eliminate
816 }
817
818 // There are two cases that require the basic loadbarrier
819 // 1) When the writeback of a healed oop must be avoided (swap)
820 // 2) When we must guarantee that no reload of is done (swap, cas, cmpx)
821 if (!barrier->is_writeback()) {
822 assert(!barrier->oop_reload_allowed(), "writeback barriers should be marked as requires oop");
823 }
824
825 if (!barrier->oop_reload_allowed()) {
826 expand_loadbarrier_basic(phase, barrier);
827 } else {
828 expand_loadbarrier_optimized(phase, barrier);
829 }
830 }
831
832 // Basic loadbarrier using conventional argument passing
833 void ZBarrierSetC2::expand_loadbarrier_basic(PhaseMacroExpand* phase, LoadBarrierNode *barrier) const {
834 PhaseIterGVN &igvn = phase->igvn();
835
836 Node* in_ctrl = barrier->in(LoadBarrierNode::Control);
837 Node* in_mem = barrier->in(LoadBarrierNode::Memory);
838 Node* in_val = barrier->in(LoadBarrierNode::Oop);
839 Node* in_adr = barrier->in(LoadBarrierNode::Address);
840
841 Node* out_ctrl = barrier->proj_out(LoadBarrierNode::Control);
842 Node* out_res = barrier->proj_out(LoadBarrierNode::Oop);
843
844 float unlikely = PROB_UNLIKELY(0.999);
845 const Type* in_val_maybe_null_t = igvn.type(in_val);
846
847 Node* jthread = igvn.transform(new ThreadLocalNode());
848 Node* adr = phase->basic_plus_adr(jthread, in_bytes(ZThreadLocalData::address_bad_mask_offset()));
849 Node* bad_mask = igvn.transform(LoadNode::make(igvn, in_ctrl, in_mem, adr, TypeRawPtr::BOTTOM, TypeX_X, TypeX_X->basic_type(), MemNode::unordered));
850 Node* cast = igvn.transform(new CastP2XNode(in_ctrl, in_val));
851 Node* obj_masked = igvn.transform(new AndXNode(cast, bad_mask));
852 Node* cmp = igvn.transform(new CmpXNode(obj_masked, igvn.zerocon(TypeX_X->basic_type())));
853 Node *bol = igvn.transform(new BoolNode(cmp, BoolTest::ne))->as_Bool();
854 IfNode* iff = igvn.transform(new IfNode(in_ctrl, bol, unlikely, COUNT_UNKNOWN))->as_If();
855 Node* then = igvn.transform(new IfTrueNode(iff));
856 Node* elsen = igvn.transform(new IfFalseNode(iff));
857
858 Node* result_region;
859 Node* result_val;
860
861 result_region = new RegionNode(3);
862 result_val = new PhiNode(result_region, TypeInstPtr::BOTTOM);
863
864 result_region->set_req(1, elsen);
865 Node* res = igvn.transform(new CastPPNode(in_val, in_val_maybe_null_t));
866 res->init_req(0, elsen);
867 result_val->set_req(1, res);
868
869 const TypeFunc *tf = load_barrier_Type();
870 Node* call;
871 if (barrier->is_weak()) {
872 call = new CallLeafNode(tf,
873 ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded_addr(),
874 "ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded",
875 TypeRawPtr::BOTTOM);
876 } else {
877 call = new CallLeafNode(tf,
878 ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(),
879 "ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded",
880 TypeRawPtr::BOTTOM);
881 }
882
883 call->init_req(TypeFunc::Control, then);
884 call->init_req(TypeFunc::I_O , phase->top());
885 call->init_req(TypeFunc::Memory , in_mem);
886 call->init_req(TypeFunc::FramePtr, phase->top());
887 call->init_req(TypeFunc::ReturnAdr, phase->top());
888 call->init_req(TypeFunc::Parms+0, in_val);
889 if (barrier->is_writeback()) {
890 call->init_req(TypeFunc::Parms+1, in_adr);
891 } else {
892 // When slow path is called with a null address, the healed oop will not be written back
893 call->init_req(TypeFunc::Parms+1, igvn.zerocon(T_OBJECT));
894 }
895 call = igvn.transform(call);
896
897 Node* ctrl = igvn.transform(new ProjNode(call, TypeFunc::Control));
898 res = igvn.transform(new ProjNode(call, TypeFunc::Parms));
899 res = igvn.transform(new CheckCastPPNode(ctrl, res, in_val_maybe_null_t));
900
901 result_region->set_req(2, ctrl);
902 result_val->set_req(2, res);
903
904 result_region = igvn.transform(result_region);
905 result_val = igvn.transform(result_val);
906
907 if (out_ctrl != NULL) { // Added if cond
908 igvn.replace_node(out_ctrl, result_region);
909 }
910 igvn.replace_node(out_res, result_val);
911 }
912
913 // Optimized, low spill, loadbarrier variant using stub specialized on register used
914 void ZBarrierSetC2::expand_loadbarrier_optimized(PhaseMacroExpand* phase, LoadBarrierNode *barrier) const {
915 PhaseIterGVN &igvn = phase->igvn();
916 #ifdef PRINT_NODE_TRAVERSALS
917 Node* preceding_barrier_node = barrier->in(LoadBarrierNode::Oop);
918 #endif
919
920 Node* in_ctrl = barrier->in(LoadBarrierNode::Control);
921 Node* in_mem = barrier->in(LoadBarrierNode::Memory);
922 Node* in_val = barrier->in(LoadBarrierNode::Oop);
923 Node* in_adr = barrier->in(LoadBarrierNode::Address);
924
925 Node* out_ctrl = barrier->proj_out(LoadBarrierNode::Control);
926 Node* out_res = barrier->proj_out(LoadBarrierNode::Oop);
927
928 assert(barrier->in(LoadBarrierNode::Oop) != NULL, "oop to loadbarrier node cannot be null");
929
930 #ifdef PRINT_NODE_TRAVERSALS
931 tty->print("\n\n\nBefore barrier optimization:\n");
932 traverse(barrier, out_ctrl, out_res, -1);
933
934 tty->print("\nBefore barrier optimization: preceding_barrier_node\n");
935 traverse(preceding_barrier_node, out_ctrl, out_res, -1);
936 #endif
937
938 float unlikely = PROB_UNLIKELY(0.999);
939
940 Node* jthread = igvn.transform(new ThreadLocalNode());
941 Node* adr = phase->basic_plus_adr(jthread, in_bytes(ZThreadLocalData::address_bad_mask_offset()));
942 Node* bad_mask = igvn.transform(LoadNode::make(igvn, in_ctrl, in_mem, adr,
943 TypeRawPtr::BOTTOM, TypeX_X, TypeX_X->basic_type(),
944 MemNode::unordered));
945 Node* cast = igvn.transform(new CastP2XNode(in_ctrl, in_val));
946 Node* obj_masked = igvn.transform(new AndXNode(cast, bad_mask));
947 Node* cmp = igvn.transform(new CmpXNode(obj_masked, igvn.zerocon(TypeX_X->basic_type())));
948 Node *bol = igvn.transform(new BoolNode(cmp, BoolTest::ne))->as_Bool();
949 IfNode* iff = igvn.transform(new IfNode(in_ctrl, bol, unlikely, COUNT_UNKNOWN))->as_If();
950 Node* then = igvn.transform(new IfTrueNode(iff));
951 Node* elsen = igvn.transform(new IfFalseNode(iff));
952
953 Node* slow_path_surrogate;
954 if (!barrier->is_weak()) {
955 slow_path_surrogate = igvn.transform(new LoadBarrierSlowRegNode(then, in_mem, in_adr, in_val->adr_type(),
956 (const TypePtr*) in_val->bottom_type(), MemNode::unordered));
957 } else {
958 slow_path_surrogate = igvn.transform(new LoadBarrierWeakSlowRegNode(then, in_mem, in_adr, in_val->adr_type(),
959 (const TypePtr*) in_val->bottom_type(), MemNode::unordered));
960 }
961
962 Node *new_loadp;
963 new_loadp = slow_path_surrogate;
964 // Create the final region/phi pair to converge cntl/data paths to downstream code
965 Node* result_region = igvn.transform(new RegionNode(3));
966 result_region->set_req(1, then);
967 result_region->set_req(2, elsen);
968
969 Node* result_phi = igvn.transform(new PhiNode(result_region, TypeInstPtr::BOTTOM));
970 result_phi->set_req(1, new_loadp);
971 result_phi->set_req(2, barrier->in(LoadBarrierNode::Oop));
972
973 // Finally, connect the original outputs to the barrier region and phi to complete the expansion/substitution
974 // igvn.replace_node(out_ctrl, result_region);
975 if (out_ctrl != NULL) { // added if cond
976 igvn.replace_node(out_ctrl, result_region);
977 }
978 igvn.replace_node(out_res, result_phi);
979
980 assert(barrier->outcnt() == 0,"LoadBarrier macro node has non-null outputs after expansion!");
981
982 #ifdef PRINT_NODE_TRAVERSALS
983 tty->print("\nAfter barrier optimization: old out_ctrl\n");
984 traverse(out_ctrl, out_ctrl, out_res, -1);
985 tty->print("\nAfter barrier optimization: old out_res\n");
986 traverse(out_res, out_ctrl, out_res, -1);
987 tty->print("\nAfter barrier optimization: old barrier\n");
988 traverse(barrier, out_ctrl, out_res, -1);
989 tty->print("\nAfter barrier optimization: preceding_barrier_node\n");
990 traverse(preceding_barrier_node, result_region, result_phi, -1);
991 #endif
992
993 assert(is_gc_barrier_node(result_phi), "sanity");
994 assert(step_over_gc_barrier(result_phi) == in_val, "sanity");
995
996 return;
997 }
998
999 bool ZBarrierSetC2::expand_macro_nodes(PhaseMacroExpand* macro) const {
1000 Compile* C = Compile::current();
1001 PhaseIterGVN &igvn = macro->igvn();
1002 ZBarrierSetC2State* s = state();
1003 if (s->load_barrier_count() > 0) {
1004 #ifdef ASSERT
1005 verify_gc_barriers(false);
1006 #endif
1007 igvn.set_delay_transform(true);
1008 int skipped = 0;
1009 while (s->load_barrier_count() > skipped) {
1010 int load_barrier_count = s->load_barrier_count();
1011 LoadBarrierNode * n = s->load_barrier_node(load_barrier_count-1-skipped);
1012 if (igvn.type(n) == Type::TOP || (n->in(0) != NULL && n->in(0)->is_top())) {
1013 // Node is unreachable, so don't try to expand it
1014 s->remove_load_barrier_node(n);
1015 continue;
1016 }
1017 if (!n->can_be_eliminated()) {
1018 skipped++;
1019 continue;
1020 }
1021 expand_loadbarrier_node(macro, n);
1022 assert(s->load_barrier_count() < load_barrier_count, "must have deleted a node from load barrier list");
1023 if (C->failing()) return true;
1024 }
1025 while (s->load_barrier_count() > 0) {
1026 int load_barrier_count = s->load_barrier_count();
1027 LoadBarrierNode* n = s->load_barrier_node(load_barrier_count - 1);
1028 assert(!(igvn.type(n) == Type::TOP || (n->in(0) != NULL && n->in(0)->is_top())), "should have been processed already");
1029 assert(!n->can_be_eliminated(), "should have been processed already");
1030 expand_loadbarrier_node(macro, n);
1031 assert(s->load_barrier_count() < load_barrier_count, "must have deleted a node from load barrier list");
1032 if (C->failing()) return true;
1033 }
1034 igvn.set_delay_transform(false);
1035 igvn.optimize();
1036 if (C->failing()) return true;
1037 }
1038 return false;
1039 }
1040
1041 // == Loop optimization ==
1042
1043 static bool replace_with_dominating_barrier(PhaseIdealLoop* phase, LoadBarrierNode* lb, bool last_round) {
1044 PhaseIterGVN &igvn = phase->igvn();
1045 Compile* C = Compile::current();
1046
1047 LoadBarrierNode* lb2 = lb->has_dominating_barrier(phase, false, last_round);
1048 if (lb2 != NULL) {
1049 if (lb->in(LoadBarrierNode::Oop) != lb2->in(LoadBarrierNode::Oop)) {
1050 assert(lb->in(LoadBarrierNode::Address) == lb2->in(LoadBarrierNode::Address), "");
1051 igvn.replace_input_of(lb, LoadBarrierNode::Similar, lb2->proj_out(LoadBarrierNode::Oop));
1052 C->set_major_progress();
1053 } else {
1054 // That transformation may cause the Similar edge on dominated load barriers to be invalid
1055 lb->fix_similar_in_uses(&igvn);
1056
1057 Node* val = lb->proj_out(LoadBarrierNode::Oop);
1058 assert(lb2->has_true_uses(), "");
1059 assert(lb2->in(LoadBarrierNode::Oop) == lb->in(LoadBarrierNode::Oop), "");
1060
1061 phase->lazy_update(lb, lb->in(LoadBarrierNode::Control));
1062 phase->lazy_replace(lb->proj_out(LoadBarrierNode::Control), lb->in(LoadBarrierNode::Control));
1063 igvn.replace_node(val, lb2->proj_out(LoadBarrierNode::Oop));
1064
1065 return true;
1066 }
1067 }
1068 return false;
1069 }
1070
1071 static Node* find_dominating_memory(PhaseIdealLoop* phase, Node* mem, Node* dom, int i) {
1072 assert(dom->is_Region() || i == -1, "");
1073 Node* m = mem;
1074 while(phase->is_dominator(dom, phase->has_ctrl(m) ? phase->get_ctrl(m) : m->in(0))) {
1075 if (m->is_Mem()) {
1076 assert(m->as_Mem()->adr_type() == TypeRawPtr::BOTTOM, "");
1077 m = m->in(MemNode::Memory);
1078 } else if (m->is_MergeMem()) {
1079 m = m->as_MergeMem()->memory_at(Compile::AliasIdxRaw);
1080 } else if (m->is_Phi()) {
1081 if (m->in(0) == dom && i != -1) {
1082 m = m->in(i);
1083 break;
1084 } else {
1085 m = m->in(LoopNode::EntryControl);
1086 }
1087 } else if (m->is_Proj()) {
1088 m = m->in(0);
1089 } else if (m->is_SafePoint() || m->is_MemBar()) {
1090 m = m->in(TypeFunc::Memory);
1091 } else {
1092 #ifdef ASSERT
1093 m->dump();
1094 #endif
1095 ShouldNotReachHere();
1096 }
1097 }
1098 return m;
1099 }
1100
1101 static LoadBarrierNode* clone_load_barrier(PhaseIdealLoop* phase, LoadBarrierNode* lb, Node* ctl, Node* mem, Node* oop_in) {
1102 PhaseIterGVN &igvn = phase->igvn();
1103 Compile* C = Compile::current();
1104 Node* the_clone = lb->clone();
1105 the_clone->set_req(LoadBarrierNode::Control, ctl);
1106 the_clone->set_req(LoadBarrierNode::Memory, mem);
1107 if (oop_in != NULL) {
1108 the_clone->set_req(LoadBarrierNode::Oop, oop_in);
1109 }
1110
1111 LoadBarrierNode* new_lb = the_clone->as_LoadBarrier();
1112 igvn.register_new_node_with_optimizer(new_lb);
1113 IdealLoopTree *loop = phase->get_loop(new_lb->in(0));
1114 phase->set_ctrl(new_lb, new_lb->in(0));
1115 phase->set_loop(new_lb, loop);
1116 phase->set_idom(new_lb, new_lb->in(0), phase->dom_depth(new_lb->in(0))+1);
1117 if (!loop->_child) {
1118 loop->_body.push(new_lb);
1119 }
1120
1121 Node* proj_ctl = new ProjNode(new_lb, LoadBarrierNode::Control);
1122 igvn.register_new_node_with_optimizer(proj_ctl);
1123 phase->set_ctrl(proj_ctl, proj_ctl->in(0));
1124 phase->set_loop(proj_ctl, loop);
1125 phase->set_idom(proj_ctl, new_lb, phase->dom_depth(new_lb)+1);
1126 if (!loop->_child) {
1127 loop->_body.push(proj_ctl);
1128 }
1129
1130 Node* proj_oop = new ProjNode(new_lb, LoadBarrierNode::Oop);
1131 phase->register_new_node(proj_oop, new_lb);
1132
1133 if (!new_lb->in(LoadBarrierNode::Similar)->is_top()) {
1134 LoadBarrierNode* similar = new_lb->in(LoadBarrierNode::Similar)->in(0)->as_LoadBarrier();
1135 if (!phase->is_dominator(similar, ctl)) {
1136 igvn.replace_input_of(new_lb, LoadBarrierNode::Similar, C->top());
1137 }
1138 }
1139
1140 return new_lb;
1141 }
1142
1143 static void replace_barrier(PhaseIdealLoop* phase, LoadBarrierNode* lb, Node* new_val) {
1144 PhaseIterGVN &igvn = phase->igvn();
1145 Node* val = lb->proj_out(LoadBarrierNode::Oop);
1146 igvn.replace_node(val, new_val);
1147 phase->lazy_update(lb, lb->in(LoadBarrierNode::Control));
1148 phase->lazy_replace(lb->proj_out(LoadBarrierNode::Control), lb->in(LoadBarrierNode::Control));
1149 }
1150
1151 static bool split_barrier_thru_phi(PhaseIdealLoop* phase, LoadBarrierNode* lb) {
1152 PhaseIterGVN &igvn = phase->igvn();
1153 Compile* C = Compile::current();
1154
1155 if (lb->in(LoadBarrierNode::Oop)->is_Phi()) {
1156 Node* oop_phi = lb->in(LoadBarrierNode::Oop);
1157
1158 if (oop_phi->in(2) == oop_phi) {
1159 // Ignore phis with only one input
1160 return false;
1161 }
1162
1163 if (phase->is_dominator(phase->get_ctrl(lb->in(LoadBarrierNode::Address)),
1164 oop_phi->in(0)) && phase->get_ctrl(lb->in(LoadBarrierNode::Address)) != oop_phi->in(0)) {
1165 // That transformation may cause the Similar edge on dominated load barriers to be invalid
1166 lb->fix_similar_in_uses(&igvn);
1167
1168 RegionNode* region = oop_phi->in(0)->as_Region();
1169
1170 int backedge = LoopNode::LoopBackControl;
1171 if (region->is_Loop() && region->in(backedge)->is_Proj() && region->in(backedge)->in(0)->is_If()) {
1172 Node* c = region->in(backedge)->in(0)->in(0);
1173 assert(c->unique_ctrl_out() == region->in(backedge)->in(0), "");
1174 Node* oop = lb->in(LoadBarrierNode::Oop)->in(backedge);
1175 Node* oop_c = phase->has_ctrl(oop) ? phase->get_ctrl(oop) : oop;
1176 if (!phase->is_dominator(oop_c, c)) {
1177 return false;
1178 }
1179 }
1180
1181 // If the node on the backedge above the phi is the node itself - we have a self loop.
1182 // Don't clone - this will be folded later.
1183 if (oop_phi->in(LoopNode::LoopBackControl) == lb->proj_out(LoadBarrierNode::Oop)) {
1184 return false;
1185 }
1186
1187 bool is_strip_mined = region->is_CountedLoop() && region->as_CountedLoop()->is_strip_mined();
1188 Node *phi = oop_phi->clone();
1189
1190 for (uint i = 1; i < region->req(); i++) {
1191 Node* ctrl = region->in(i);
1192 if (ctrl != C->top()) {
1193 assert(!phase->is_dominator(ctrl, region) || region->is_Loop(), "");
1194
1195 Node* mem = lb->in(LoadBarrierNode::Memory);
1196 Node* m = find_dominating_memory(phase, mem, region, i);
1197
1198 if (region->is_Loop() && i == LoopNode::LoopBackControl && ctrl->is_Proj() && ctrl->in(0)->is_If()) {
1199 ctrl = ctrl->in(0)->in(0);
1200 } else if (region->is_Loop() && is_strip_mined) {
1201 // If this is a strip mined loop, control must move above OuterStripMinedLoop
1202 assert(i == LoopNode::EntryControl, "check");
1203 assert(ctrl->is_OuterStripMinedLoop(), "sanity");
1204 ctrl = ctrl->as_OuterStripMinedLoop()->in(LoopNode::EntryControl);
1205 }
1206
1207 LoadBarrierNode* new_lb = clone_load_barrier(phase, lb, ctrl, m, lb->in(LoadBarrierNode::Oop)->in(i));
1208 Node* out_ctrl = new_lb->proj_out(LoadBarrierNode::Control);
1209
1210 if (is_strip_mined && (i == LoopNode::EntryControl)) {
1211 assert(region->in(i)->is_OuterStripMinedLoop(), "");
1212 igvn.replace_input_of(region->in(i), i, out_ctrl);
1213 phase->set_idom(region->in(i), out_ctrl, phase->dom_depth(out_ctrl));
1214 } else if (ctrl == region->in(i)) {
1215 igvn.replace_input_of(region, i, out_ctrl);
1216 // Only update the idom if is the loop entry we are updating
1217 // - A loop backedge doesn't change the idom
1218 if (region->is_Loop() && i == LoopNode::EntryControl) {
1219 phase->set_idom(region, out_ctrl, phase->dom_depth(out_ctrl));
1220 }
1221 } else {
1222 Node* iff = region->in(i)->in(0);
1223 igvn.replace_input_of(iff, 0, out_ctrl);
1224 phase->set_idom(iff, out_ctrl, phase->dom_depth(out_ctrl)+1);
1225 }
1226 phi->set_req(i, new_lb->proj_out(LoadBarrierNode::Oop));
1227 }
1228 }
1229 phase->register_new_node(phi, region);
1230 replace_barrier(phase, lb, phi);
1231
1232 if (region->is_Loop()) {
1233 // Load barrier moved to the back edge of the Loop may now
1234 // have a safepoint on the path to the barrier on the Similar
1235 // edge
1236 igvn.replace_input_of(phi->in(LoopNode::LoopBackControl)->in(0), LoadBarrierNode::Similar, C->top());
1237 Node* head = region->in(LoopNode::EntryControl);
1238 phase->set_idom(region, head, phase->dom_depth(head)+1);
1239 phase->recompute_dom_depth();
1240 if (head->is_CountedLoop() && head->as_CountedLoop()->is_main_loop()) {
1241 head->as_CountedLoop()->set_normal_loop();
1242 }
1243 }
1244
1245 return true;
1246 }
1247 }
1248
1249 return false;
1250 }
1251
1252 static bool move_out_of_loop(PhaseIdealLoop* phase, LoadBarrierNode* lb) {
1253 PhaseIterGVN &igvn = phase->igvn();
1254 IdealLoopTree *lb_loop = phase->get_loop(lb->in(0));
1255 if (lb_loop != phase->ltree_root() && !lb_loop->_irreducible) {
1256 Node* oop_ctrl = phase->get_ctrl(lb->in(LoadBarrierNode::Oop));
1257 IdealLoopTree *oop_loop = phase->get_loop(oop_ctrl);
1258 IdealLoopTree* adr_loop = phase->get_loop(phase->get_ctrl(lb->in(LoadBarrierNode::Address)));
1259 if (!lb_loop->is_member(oop_loop) && !lb_loop->is_member(adr_loop)) {
1260 // That transformation may cause the Similar edge on dominated load barriers to be invalid
1261 lb->fix_similar_in_uses(&igvn);
1262
1263 Node* head = lb_loop->_head;
1264 assert(head->is_Loop(), "");
1265
1266 if (phase->is_dominator(head, oop_ctrl)) {
1267 assert(oop_ctrl->Opcode() == Op_CProj && oop_ctrl->in(0)->Opcode() == Op_NeverBranch, "");
1268 assert(lb_loop->is_member(phase->get_loop(oop_ctrl->in(0)->in(0))), "");
1269 return false;
1270 }
1271
1272 if (head->is_CountedLoop()) {
1273 CountedLoopNode* cloop = head->as_CountedLoop();
1274 if (cloop->is_main_loop()) {
1275 cloop->set_normal_loop();
1276 }
1277 // When we are moving barrier out of a counted loop,
1278 // make sure we move it all the way out of the strip mined outer loop.
1279 if (cloop->is_strip_mined()) {
1280 head = cloop->outer_loop();
1281 }
1282 }
1283
1284 Node* mem = lb->in(LoadBarrierNode::Memory);
1285 Node* m = find_dominating_memory(phase, mem, head, -1);
1286
1287 LoadBarrierNode* new_lb = clone_load_barrier(phase, lb, head->in(LoopNode::EntryControl), m, NULL);
1288
1289 assert(phase->idom(head) == head->in(LoopNode::EntryControl), "");
1290 Node* proj_ctl = new_lb->proj_out(LoadBarrierNode::Control);
1291 igvn.replace_input_of(head, LoopNode::EntryControl, proj_ctl);
1292 phase->set_idom(head, proj_ctl, phase->dom_depth(proj_ctl) + 1);
1293
1294 replace_barrier(phase, lb, new_lb->proj_out(LoadBarrierNode::Oop));
1295
1296 phase->recompute_dom_depth();
1297
1298 return true;
1299 }
1300 }
1301
1302 return false;
1303 }
1304
1305 static bool common_barriers(PhaseIdealLoop* phase, LoadBarrierNode* lb) {
1306 PhaseIterGVN &igvn = phase->igvn();
1307 Node* in_val = lb->in(LoadBarrierNode::Oop);
1308 for (DUIterator_Fast imax, i = in_val->fast_outs(imax); i < imax; i++) {
1309 Node* u = in_val->fast_out(i);
1310 if (u != lb && u->is_LoadBarrier() && u->as_LoadBarrier()->has_true_uses()) {
1311 Node* this_ctrl = lb->in(LoadBarrierNode::Control);
1312 Node* other_ctrl = u->in(LoadBarrierNode::Control);
1313
1314 Node* lca = phase->dom_lca(this_ctrl, other_ctrl);
1315 bool ok = true;
1316
1317 Node* proj1 = NULL;
1318 Node* proj2 = NULL;
1319
1320 while (this_ctrl != lca && ok) {
1321 if (this_ctrl->in(0) != NULL &&
1322 this_ctrl->in(0)->is_MultiBranch()) {
1323 if (this_ctrl->in(0)->in(0) == lca) {
1324 assert(proj1 == NULL, "");
1325 assert(this_ctrl->is_Proj(), "");
1326 proj1 = this_ctrl;
1327 } else if (!(this_ctrl->in(0)->is_If() && this_ctrl->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none))) {
1328 ok = false;
1329 }
1330 }
1331 this_ctrl = phase->idom(this_ctrl);
1332 }
1333 while (other_ctrl != lca && ok) {
1334 if (other_ctrl->in(0) != NULL &&
1335 other_ctrl->in(0)->is_MultiBranch()) {
1336 if (other_ctrl->in(0)->in(0) == lca) {
1337 assert(other_ctrl->is_Proj(), "");
1338 assert(proj2 == NULL, "");
1339 proj2 = other_ctrl;
1340 } else if (!(other_ctrl->in(0)->is_If() && other_ctrl->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none))) {
1341 ok = false;
1342 }
1343 }
1344 other_ctrl = phase->idom(other_ctrl);
1345 }
1346 assert(proj1 == NULL || proj2 == NULL || proj1->in(0) == proj2->in(0), "");
1347 if (ok && proj1 && proj2 && proj1 != proj2 && proj1->in(0)->is_If()) {
1348 // That transformation may cause the Similar edge on dominated load barriers to be invalid
1349 lb->fix_similar_in_uses(&igvn);
1350 u->as_LoadBarrier()->fix_similar_in_uses(&igvn);
1351
1352 Node* split = lca->unique_ctrl_out();
1353 assert(split->in(0) == lca, "");
1354
1355 Node* mem = lb->in(LoadBarrierNode::Memory);
1356 Node* m = find_dominating_memory(phase, mem, split, -1);
1357 LoadBarrierNode* new_lb = clone_load_barrier(phase, lb, lca, m, NULL);
1358
1359 Node* proj_ctl = new_lb->proj_out(LoadBarrierNode::Control);
1360 igvn.replace_input_of(split, 0, new_lb->proj_out(LoadBarrierNode::Control));
1361 phase->set_idom(split, proj_ctl, phase->dom_depth(proj_ctl)+1);
1362
1363 Node* proj_oop = new_lb->proj_out(LoadBarrierNode::Oop);
1364 replace_barrier(phase, lb, proj_oop);
1365 replace_barrier(phase, u->as_LoadBarrier(), proj_oop);
1366
1367 phase->recompute_dom_depth();
1368
1369 return true;
1370 }
1371 }
1372 }
1373
1374 return false;
1375 }
1376
1377 static void optimize_load_barrier(PhaseIdealLoop* phase, LoadBarrierNode* lb, bool last_round) {
1378 Compile* C = Compile::current();
1379
1380 if (!C->directive()->ZOptimizeLoadBarriersOption) {
1381 return;
1382 }
1383
1384 if (lb->has_true_uses()) {
1385 if (replace_with_dominating_barrier(phase, lb, last_round)) {
1386 return;
1387 }
1388
1389 if (split_barrier_thru_phi(phase, lb)) {
1390 return;
1391 }
1392
1393 if (move_out_of_loop(phase, lb)) {
1394 return;
1395 }
1396
1397 if (common_barriers(phase, lb)) {
1398 return;
1399 }
1400 }
1401 }
1402
1403 void ZBarrierSetC2::loop_optimize_gc_barrier(PhaseIdealLoop* phase, Node* node, bool last_round) {
1404 if (node->is_LoadBarrier()) {
1405 optimize_load_barrier(phase, node->as_LoadBarrier(), last_round);
1406 }
1407 }
1408
1409 Node* ZBarrierSetC2::step_over_gc_barrier(Node* c) const {
1410 Node* node = c;
1411
1412 // 1. This step follows potential oop projections of a load barrier before expansion
1413 if (node->is_Proj()) {
1414 node = node->in(0);
1415 }
1416
1417 // 2. This step checks for unexpanded load barriers
1418 if (node->is_LoadBarrier()) {
1419 return node->in(LoadBarrierNode::Oop);
1420 }
1421
1422 // 3. This step checks for the phi corresponding to an optimized load barrier expansion
1423 if (node->is_Phi()) {
1424 PhiNode* phi = node->as_Phi();
1425 Node* n = phi->in(1);
1426 if (n != NULL && (n->is_LoadBarrierSlowReg() || n->is_LoadBarrierWeakSlowReg())) {
1427 assert(c == node, "projections from step 1 should only be seen before macro expansion");
1428 return phi->in(2);
1429 }
1430 }
1431
1432 return c;
1433 }
1434
1435 bool ZBarrierSetC2::array_copy_requires_gc_barriers(bool tightly_coupled_alloc, BasicType type, bool is_clone, ArrayCopyPhase phase) const {
1436 return type == T_OBJECT || type == T_ARRAY;
1437 }
1438
1439 bool ZBarrierSetC2::final_graph_reshaping(Compile* compile, Node* n, uint opcode) const {
1440 bool handled;
1441 switch (opcode) {
1442 case Op_LoadBarrierSlowReg:
1443 case Op_LoadBarrierWeakSlowReg:
1444 #ifdef ASSERT
1445 if (VerifyOptoOopOffsets) {
1446 MemNode* mem = n->as_Mem();
1447 // Check to see if address types have grounded out somehow.
1448 const TypeInstPtr* tp = mem->in(MemNode::Address)->bottom_type()->isa_instptr();
1449 ciInstanceKlass* k = tp->klass()->as_instance_klass();
1450 bool oop_offset_is_sane = k->contains_field_offset(tp->offset());
1451 assert(!tp || oop_offset_is_sane, "");
1452 }
1453 #endif
1454 handled = true;
1455 break;
1456 default:
1457 handled = false;
1458 }
1459 return handled;
1460 }
1461
1462 bool ZBarrierSetC2::matcher_find_shared_visit(Matcher* matcher, Matcher::MStack& mstack, Node* n, uint opcode, bool& mem_op, int& mem_addr_idx) const {
1463 if (opcode == Op_CallLeaf &&
1464 (n->as_Call()->entry_point() == ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr() ||
1465 n->as_Call()->entry_point() == ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded_addr())) {
1466 mem_op = true;
1467 mem_addr_idx = TypeFunc::Parms + 1;
1468 return true;
1469 }
1470 return false;
1471 }
1472
1473 // == Verification ==
1474
1475 #ifdef ASSERT
1476
1477 static bool look_for_barrier(Node* n, bool post_parse, VectorSet& visited) {
1478 if (visited.test_set(n->_idx)) {
1479 return true;
1480 }
1481
1482 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
1483 Node* u = n->fast_out(i);
1484 if (u->is_LoadBarrier()) {
1485 } else if ((u->is_Phi() || u->is_CMove()) && !post_parse) {
1486 if (!look_for_barrier(u, post_parse, visited)) {
1487 return false;
1488 }
1489 } else if (u->Opcode() == Op_EncodeP || u->Opcode() == Op_DecodeN) {
1490 if (!look_for_barrier(u, post_parse, visited)) {
1491 return false;
1492 }
1493 } else if (u->Opcode() != Op_SCMemProj) {
1494 tty->print("bad use"); u->dump();
1495 return false;
1496 }
1497 }
1498
1499 return true;
1500 }
1501
1502 void ZBarrierSetC2::verify_gc_barriers(Compile* compile, CompilePhase phase) const {
1503 if (phase == BarrierSetC2::BeforeCodeGen) return;
1504 bool post_parse = phase == BarrierSetC2::BeforeOptimize;
1505 verify_gc_barriers(post_parse);
1506 }
1507
1508 void ZBarrierSetC2::verify_gc_barriers(bool post_parse) const {
1509 ZBarrierSetC2State* s = state();
1510 Compile* C = Compile::current();
1511 ResourceMark rm;
1512 VectorSet visited(Thread::current()->resource_area());
1513 for (int i = 0; i < s->load_barrier_count(); i++) {
1514 LoadBarrierNode* n = s->load_barrier_node(i);
1515
1516 // The dominating barrier on the same address if it exists and
1517 // this barrier must not be applied on the value from the same
1518 // load otherwise the value is not reloaded before it's used the
1519 // second time.
1520 assert(n->in(LoadBarrierNode::Similar)->is_top() ||
1521 (n->in(LoadBarrierNode::Similar)->in(0)->is_LoadBarrier() &&
1522 n->in(LoadBarrierNode::Similar)->in(0)->in(LoadBarrierNode::Address) == n->in(LoadBarrierNode::Address) &&
1523 n->in(LoadBarrierNode::Similar)->in(0)->in(LoadBarrierNode::Oop) != n->in(LoadBarrierNode::Oop)),
1524 "broken similar edge");
1525
1526 assert(post_parse || n->as_LoadBarrier()->has_true_uses(),
1527 "found unneeded load barrier");
1528
1529 // Several load barrier nodes chained through their Similar edge
1530 // break the code that remove the barriers in final graph reshape.
1531 assert(n->in(LoadBarrierNode::Similar)->is_top() ||
1532 (n->in(LoadBarrierNode::Similar)->in(0)->is_LoadBarrier() &&
1533 n->in(LoadBarrierNode::Similar)->in(0)->in(LoadBarrierNode::Similar)->is_top()),
1534 "chain of Similar load barriers");
1535
1536 if (!n->in(LoadBarrierNode::Similar)->is_top()) {
1537 ResourceMark rm;
1538 Unique_Node_List wq;
1539 Node* other = n->in(LoadBarrierNode::Similar)->in(0);
1540 wq.push(n);
1541 bool ok = true;
1542 bool dom_found = false;
1543 for (uint next = 0; next < wq.size(); ++next) {
1544 Node *n = wq.at(next);
1545 assert(n->is_CFG(), "");
1546 assert(!n->is_SafePoint(), "");
1547
1548 if (n == other) {
1549 continue;
1550 }
1551
1552 if (n->is_Region()) {
1553 for (uint i = 1; i < n->req(); i++) {
1554 Node* m = n->in(i);
1555 if (m != NULL) {
1556 wq.push(m);
1557 }
1558 }
1559 } else {
1560 Node* m = n->in(0);
1561 if (m != NULL) {
1562 wq.push(m);
1563 }
1564 }
1565 }
1566 }
1567
1568 if (ZVerifyLoadBarriers) {
1569 if ((n->is_Load() || n->is_LoadStore()) && n->bottom_type()->make_oopptr() != NULL) {
1570 visited.Clear();
1571 bool found = look_for_barrier(n, post_parse, visited);
1572 if (!found) {
1573 n->dump(1);
1574 n->dump(-3);
1575 stringStream ss;
1576 C->method()->print_short_name(&ss);
1577 tty->print_cr("-%s-", ss.as_string());
1578 assert(found, "");
1579 }
1580 }
1581 }
1582 }
1583 }
1584
1585 #endif
1586
1587 bool ZBarrierSetC2::escape_add_to_con_graph(ConnectionGraph* conn_graph, PhaseGVN* gvn, Unique_Node_List* delayed_worklist, Node* n, uint opcode) const {
1588 switch (opcode) {
1589 case Op_LoadBarrierSlowReg:
1590 case Op_LoadBarrierWeakSlowReg:
1591 conn_graph->add_objload_to_connection_graph(n, delayed_worklist);
1592 return true;
1593 case Op_Proj:
1594 if (n->as_Proj()->_con == LoadBarrierNode::Oop && n->in(0)->is_LoadBarrier()) {
1595 conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(0)->in(LoadBarrierNode::Oop),
1596 delayed_worklist);
1597 return true;
1598 }
1599 default:
1600 break;
1601 }
1602 return false;
1603 }
1604
1605 bool ZBarrierSetC2::escape_add_final_edges(ConnectionGraph* conn_graph, PhaseGVN* gvn, Node* n, uint opcode) const {
1606 switch (opcode) {
1607 case Op_LoadBarrierSlowReg:
1608 case Op_LoadBarrierWeakSlowReg: {
1609 const Type *t = gvn->type(n);
1610 if (t->make_ptr() != NULL) {
1611 Node *adr = n->in(MemNode::Address);
1612 conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, adr, NULL);
1613 return true;
1614 }
1615 }
1616 case Op_Proj: {
1617 if (n->as_Proj()->_con == LoadBarrierNode::Oop && n->in(0)->is_LoadBarrier()) {
1618 conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(0)->in(LoadBarrierNode::Oop), NULL);
1619 return true;
1620 }
1621 }
1622 default:
1623 break;
1624 }
1625 return false;
1626 }