1 /*
2 * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 #include "precompiled.hpp"
25 #include "opto/compile.hpp"
26 #include "opto/castnode.hpp"
27 #include "opto/escape.hpp"
28 #include "opto/graphKit.hpp"
29 #include "opto/idealKit.hpp"
30 #include "opto/loopnode.hpp"
31 #include "opto/macro.hpp"
32 #include "opto/node.hpp"
33 #include "opto/type.hpp"
34 #include "utilities/macros.hpp"
35 #include "gc/z/zBarrierSet.hpp"
36 #include "gc/z/c2/zBarrierSetC2.hpp"
37 #include "gc/z/zThreadLocalData.hpp"
38 #include "gc/z/zBarrierSetRuntime.hpp"
39
40 ZBarrierSetC2State::ZBarrierSetC2State(Arena* comp_arena)
41 : _load_barrier_nodes(new (comp_arena) GrowableArray<LoadBarrierNode*>(comp_arena, 8, 0, NULL)) {}
42
43 int ZBarrierSetC2State::load_barrier_count() const {
44 return _load_barrier_nodes->length();
45 }
46
47 void ZBarrierSetC2State::add_load_barrier_node(LoadBarrierNode * n) {
48 assert(!_load_barrier_nodes->contains(n), " duplicate entry in expand list");
49 _load_barrier_nodes->append(n);
50 }
51
52 void ZBarrierSetC2State::remove_load_barrier_node(LoadBarrierNode * n) {
53 // this function may be called twice for a node so check
54 // that the node is in the array before attempting to remove it
55 if (_load_barrier_nodes->contains(n)) {
56 _load_barrier_nodes->remove(n);
57 }
58 }
59
60 LoadBarrierNode* ZBarrierSetC2State::load_barrier_node(int idx) const {
61 return _load_barrier_nodes->at(idx);
62 }
63
64 void* ZBarrierSetC2::create_barrier_state(Arena* comp_arena) const {
65 return new(comp_arena) ZBarrierSetC2State(comp_arena);
66 }
67
68 ZBarrierSetC2State* ZBarrierSetC2::state() const {
69 return reinterpret_cast<ZBarrierSetC2State*>(Compile::current()->barrier_set_state());
70 }
71
72 bool ZBarrierSetC2::is_gc_barrier_node(Node* node) const {
73 // 1. This step follows potential oop projections of a load barrier before expansion
74 if (node->is_Proj()) {
75 node = node->in(0);
76 }
77
78 // 2. This step checks for unexpanded load barriers
79 if (node->is_LoadBarrier()) {
80 return true;
81 }
82
83 // 3. This step checks for the phi corresponding to an optimized load barrier expansion
84 if (node->is_Phi()) {
85 PhiNode* phi = node->as_Phi();
86 Node* n = phi->in(1);
87 if (n != NULL && (n->is_LoadBarrierSlowReg() || n->is_LoadBarrierWeakSlowReg())) {
88 return true;
89 }
90 }
91
92 return false;
93 }
94
95 void ZBarrierSetC2::register_potential_barrier_node(Node* node) const {
96 if (node->is_LoadBarrier()) {
97 state()->add_load_barrier_node(node->as_LoadBarrier());
98 }
99 }
100
101 void ZBarrierSetC2::unregister_potential_barrier_node(Node* node) const {
102 if (node->is_LoadBarrier()) {
103 state()->remove_load_barrier_node(node->as_LoadBarrier());
104 }
105 }
106
107 void ZBarrierSetC2::eliminate_useless_gc_barriers(Unique_Node_List &useful, Compile* C) const {
108 // Remove useless LoadBarrier nodes
109 ZBarrierSetC2State* s = state();
110 for (int i = s->load_barrier_count()-1; i >= 0; i--) {
111 LoadBarrierNode* n = s->load_barrier_node(i);
112 if (!useful.member(n)) {
113 unregister_potential_barrier_node(n);
114 }
115 }
116 }
117
118 void ZBarrierSetC2::enqueue_useful_gc_barrier(PhaseIterGVN* igvn, Node* node) const {
119 if (node->is_LoadBarrier() && !node->as_LoadBarrier()->has_true_uses()) {
120 igvn->_worklist.push(node);
121 }
122 }
123
124 void ZBarrierSetC2::find_dominating_barriers(PhaseIterGVN& igvn) {
125 // Look for dominating barriers on the same address only once all
126 // other loop opts are over: loop opts may cause a safepoint to be
127 // inserted between a barrier and its dominating barrier.
128 Compile* C = Compile::current();
129 ZBarrierSetC2* bs = (ZBarrierSetC2*)BarrierSet::barrier_set()->barrier_set_c2();
130 ZBarrierSetC2State* s = bs->state();
131 if (s->load_barrier_count() >= 2) {
132 Compile::TracePhase tp("idealLoop", &C->timers[Phase::_t_idealLoop]);
133 PhaseIdealLoop ideal_loop(igvn, LoopOptsLastRound);
134 if (C->major_progress()) C->print_method(PHASE_PHASEIDEALLOOP_ITERATIONS, 2);
135 }
136 }
137
138 void ZBarrierSetC2::add_users_to_worklist(Unique_Node_List* worklist) const {
139 // Permanent temporary workaround
140 // Loadbarriers may have non-obvious dead uses keeping them alive during parsing. The use is
141 // removed by RemoveUseless (after parsing, before optimize) but the barriers won't be added to
142 // the worklist. Unless we add them explicitly they are not guaranteed to end up there.
143 ZBarrierSetC2State* s = state();
144
145 for (int i = 0; i < s->load_barrier_count(); i++) {
146 LoadBarrierNode* n = s->load_barrier_node(i);
147 worklist->push(n);
148 }
149 }
150
151 const TypeFunc* ZBarrierSetC2::load_barrier_Type() const {
152 const Type** fields;
153
154 // Create input types (domain)
155 fields = TypeTuple::fields(2);
156 fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL;
157 fields[TypeFunc::Parms+1] = TypeOopPtr::BOTTOM;
158 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2, fields);
159
160 // Create result type (range)
161 fields = TypeTuple::fields(1);
162 fields[TypeFunc::Parms+0] = TypeInstPtr::BOTTOM;
163 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+1, fields);
164
165 return TypeFunc::make(domain, range);
166 }
167
168 // == LoadBarrierNode ==
169
170 LoadBarrierNode::LoadBarrierNode(Compile* C,
171 Node* c,
172 Node* mem,
173 Node* val,
174 Node* adr,
175 bool weak,
176 bool writeback,
177 bool oop_reload_allowed) :
178 MultiNode(Number_of_Inputs),
179 _weak(weak),
180 _writeback(writeback),
181 _oop_reload_allowed(oop_reload_allowed) {
182 init_req(Control, c);
183 init_req(Memory, mem);
184 init_req(Oop, val);
185 init_req(Address, adr);
186 init_req(Similar, C->top());
187
188 init_class_id(Class_LoadBarrier);
189 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
190 bs->register_potential_barrier_node(this);
191 }
192
193 uint LoadBarrierNode::size_of() const {
194 return sizeof(*this);
195 }
196
197 uint LoadBarrierNode::cmp(const Node& n) const {
198 ShouldNotReachHere();
199 return 0;
200 }
201
202 const Type *LoadBarrierNode::bottom_type() const {
203 const Type** floadbarrier = (const Type **)(Compile::current()->type_arena()->Amalloc_4((Number_of_Outputs)*sizeof(Type*)));
204 Node* in_oop = in(Oop);
205 floadbarrier[Control] = Type::CONTROL;
206 floadbarrier[Memory] = Type::MEMORY;
207 floadbarrier[Oop] = in_oop == NULL ? Type::TOP : in_oop->bottom_type();
208 return TypeTuple::make(Number_of_Outputs, floadbarrier);
209 }
210
211 const TypePtr* LoadBarrierNode::adr_type() const {
212 ShouldNotReachHere();
213 return NULL;
214 }
215
216 const Type *LoadBarrierNode::Value(PhaseGVN *phase) const {
217 const Type** floadbarrier = (const Type **)(phase->C->type_arena()->Amalloc_4((Number_of_Outputs)*sizeof(Type*)));
218 const Type* val_t = phase->type(in(Oop));
219 floadbarrier[Control] = Type::CONTROL;
220 floadbarrier[Memory] = Type::MEMORY;
221 floadbarrier[Oop] = val_t;
222 return TypeTuple::make(Number_of_Outputs, floadbarrier);
223 }
224
225 bool LoadBarrierNode::is_dominator(PhaseIdealLoop* phase, bool linear_only, Node *d, Node *n) {
226 if (phase != NULL) {
227 return phase->is_dominator(d, n);
228 }
229
230 for (int i = 0; i < 10 && n != NULL; i++) {
231 n = IfNode::up_one_dom(n, linear_only);
232 if (n == d) {
233 return true;
234 }
235 }
236
237 return false;
238 }
239
240 LoadBarrierNode* LoadBarrierNode::has_dominating_barrier(PhaseIdealLoop* phase, bool linear_only, bool look_for_similar) {
241 Node* val = in(LoadBarrierNode::Oop);
242 if (in(Similar)->is_Proj() && in(Similar)->in(0)->is_LoadBarrier()) {
243 LoadBarrierNode* lb = in(Similar)->in(0)->as_LoadBarrier();
244 assert(lb->in(Address) == in(Address), "");
245 // Load barrier on Similar edge dominates so if it now has the Oop field it can replace this barrier.
246 if (lb->in(Oop) == in(Oop)) {
247 return lb;
248 }
249 // Follow chain of load barrier through Similar edges
250 while (!lb->in(Similar)->is_top()) {
251 lb = lb->in(Similar)->in(0)->as_LoadBarrier();
252 assert(lb->in(Address) == in(Address), "");
253 }
254 if (lb != in(Similar)->in(0)) {
255 return lb;
256 }
257 }
258 for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) {
259 Node* u = val->fast_out(i);
260 if (u != this && u->is_LoadBarrier() && u->in(Oop) == val && u->as_LoadBarrier()->has_true_uses()) {
261 Node* this_ctrl = in(LoadBarrierNode::Control);
262 Node* other_ctrl = u->in(LoadBarrierNode::Control);
263 if (is_dominator(phase, linear_only, other_ctrl, this_ctrl)) {
264 return u->as_LoadBarrier();
265 }
266 }
267 }
268
269 if (ZVerifyLoadBarriers || can_be_eliminated()) {
270 return NULL;
271 }
272
273 if (!look_for_similar) {
274 return NULL;
275 }
276
277 Node* addr = in(LoadBarrierNode::Address);
278 for (DUIterator_Fast imax, i = addr->fast_outs(imax); i < imax; i++) {
279 Node* u = addr->fast_out(i);
280 if (u != this && u->is_LoadBarrier() && u->as_LoadBarrier()->has_true_uses()) {
281 Node* this_ctrl = in(LoadBarrierNode::Control);
282 Node* other_ctrl = u->in(LoadBarrierNode::Control);
283 if (is_dominator(phase, linear_only, other_ctrl, this_ctrl)) {
284 ResourceMark rm;
285 Unique_Node_List wq;
286 wq.push(in(LoadBarrierNode::Control));
287 bool ok = true;
288 bool dom_found = false;
289 for (uint next = 0; next < wq.size(); ++next) {
290 Node *n = wq.at(next);
291 if (n->is_top()) {
292 return NULL;
293 }
294 assert(n->is_CFG(), "");
295 if (n->is_SafePoint()) {
296 ok = false;
297 break;
298 }
299 if (n == u) {
300 dom_found = true;
301 continue;
302 }
303 if (n->is_Region()) {
304 for (uint i = 1; i < n->req(); i++) {
305 Node* m = n->in(i);
306 if (m != NULL) {
307 wq.push(m);
308 }
309 }
310 } else {
311 Node* m = n->in(0);
312 if (m != NULL) {
313 wq.push(m);
314 }
315 }
316 }
317 if (ok) {
318 assert(dom_found, "");
319 return u->as_LoadBarrier();;
320 }
321 break;
322 }
323 }
324 }
325
326 return NULL;
327 }
328
329 void LoadBarrierNode::push_dominated_barriers(PhaseIterGVN* igvn) const {
330 // Change to that barrier may affect a dominated barrier so re-push those
331 Node* val = in(LoadBarrierNode::Oop);
332
333 for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) {
334 Node* u = val->fast_out(i);
335 if (u != this && u->is_LoadBarrier() && u->in(Oop) == val) {
336 Node* this_ctrl = in(Control);
337 Node* other_ctrl = u->in(Control);
338 if (is_dominator(NULL, false, this_ctrl, other_ctrl)) {
339 igvn->_worklist.push(u);
340 }
341 }
342
343 Node* addr = in(LoadBarrierNode::Address);
344 for (DUIterator_Fast imax, i = addr->fast_outs(imax); i < imax; i++) {
345 Node* u = addr->fast_out(i);
346 if (u != this && u->is_LoadBarrier() && u->in(Similar)->is_top()) {
347 Node* this_ctrl = in(Control);
348 Node* other_ctrl = u->in(Control);
349 if (is_dominator(NULL, false, this_ctrl, other_ctrl)) {
350 igvn->_worklist.push(u);
351 }
352 }
353 }
354 }
355 }
356
357 Node *LoadBarrierNode::Identity(PhaseGVN *phase) {
358 if (!phase->C->directive()->ZOptimizeLoadBarriersOption) {
359 return this;
360 }
361
362 bool redundant_addr = false;
363 LoadBarrierNode* dominating_barrier = has_dominating_barrier(NULL, true, false);
364 if (dominating_barrier != NULL) {
365 assert(dominating_barrier->in(Oop) == in(Oop), "");
366 return dominating_barrier;
367 }
368
369 return this;
370 }
371
372 Node *LoadBarrierNode::Ideal(PhaseGVN *phase, bool can_reshape) {
373 if (remove_dead_region(phase, can_reshape)) {
374 return this;
375 }
376
377 Node* val = in(Oop);
378 Node* mem = in(Memory);
379 Node* ctrl = in(Control);
380 Node* adr = in(Address);
381 assert(val->Opcode() != Op_LoadN, "");
382
383 if (mem->is_MergeMem()) {
384 Node* new_mem = mem->as_MergeMem()->memory_at(Compile::AliasIdxRaw);
385 set_req(Memory, new_mem);
386 if (mem->outcnt() == 0 && can_reshape) {
387 phase->is_IterGVN()->_worklist.push(mem);
388 }
389
390 return this;
391 }
392
393 bool optimizeLoadBarriers = phase->C->directive()->ZOptimizeLoadBarriersOption;
394 LoadBarrierNode* dominating_barrier = optimizeLoadBarriers ? has_dominating_barrier(NULL, !can_reshape, !phase->C->major_progress()) : NULL;
395 if (dominating_barrier != NULL && dominating_barrier->in(Oop) != in(Oop)) {
396 assert(in(Address) == dominating_barrier->in(Address), "");
397 set_req(Similar, dominating_barrier->proj_out(Oop));
398 return this;
399 }
400
401 bool eliminate = (optimizeLoadBarriers && !(val->is_Phi() || val->Opcode() == Op_LoadP || val->Opcode() == Op_GetAndSetP || val->is_DecodeN())) ||
402 (can_reshape && (dominating_barrier != NULL || !has_true_uses()));
403
404 if (eliminate) {
405 if (can_reshape) {
406 PhaseIterGVN* igvn = phase->is_IterGVN();
407 Node* out_ctrl = proj_out_or_null(Control);
408 Node* out_res = proj_out_or_null(Oop);
409
410 if (out_ctrl != NULL) {
411 igvn->replace_node(out_ctrl, ctrl);
412 }
413
414 // That transformation may cause the Similar edge on the load barrier to be invalid
415 fix_similar_in_uses(igvn);
416 if (out_res != NULL) {
417 if (dominating_barrier != NULL) {
418 igvn->replace_node(out_res, dominating_barrier->proj_out(Oop));
419 } else {
420 igvn->replace_node(out_res, val);
421 }
422 }
423 }
424
425 return new ConINode(TypeInt::ZERO);
426 }
427
428 // If the Similar edge is no longer a load barrier, clear it
429 Node* similar = in(Similar);
430 if (!similar->is_top() && !(similar->is_Proj() && similar->in(0)->is_LoadBarrier())) {
431 set_req(Similar, phase->C->top());
432 return this;
433 }
434
435 if (can_reshape) {
436 // If this barrier is linked through the Similar edge by a
437 // dominated barrier and both barriers have the same Oop field,
438 // the dominated barrier can go away, so push it for reprocessing.
439 // We also want to avoid a barrier to depend on another dominating
440 // barrier through its Similar edge that itself depend on another
441 // barrier through its Similar edge and rather have the first
442 // depend on the third.
443 PhaseIterGVN* igvn = phase->is_IterGVN();
444 Node* out_res = proj_out(Oop);
445 for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) {
446 Node* u = out_res->fast_out(i);
447 if (u->is_LoadBarrier() && u->in(Similar) == out_res &&
448 (u->in(Oop) == val || !u->in(Similar)->is_top())) {
449 igvn->_worklist.push(u);
450 }
451 }
452
453 push_dominated_barriers(igvn);
454 }
455
456 return NULL;
457 }
458
459 uint LoadBarrierNode::match_edge(uint idx) const {
460 ShouldNotReachHere();
461 return 0;
462 }
463
464 void LoadBarrierNode::fix_similar_in_uses(PhaseIterGVN* igvn) {
465 Node* out_res = proj_out_or_null(Oop);
466 if (out_res == NULL) {
467 return;
468 }
469
470 for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) {
471 Node* u = out_res->fast_out(i);
472 if (u->is_LoadBarrier() && u->in(Similar) == out_res) {
473 igvn->replace_input_of(u, Similar, igvn->C->top());
474 --i;
475 --imax;
476 }
477 }
478 }
479
480 bool LoadBarrierNode::has_true_uses() const {
481 Node* out_res = proj_out_or_null(Oop);
482 if (out_res == NULL) {
483 return false;
484 }
485
486 for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) {
487 Node* u = out_res->fast_out(i);
488 if (!u->is_LoadBarrier() || u->in(Similar) != out_res) {
489 return true;
490 }
491 }
492
493 return false;
494 }
495
496 // == Accesses ==
497
498 Node* ZBarrierSetC2::make_cas_loadbarrier(C2AtomicParseAccess& access) const {
499 assert(!UseCompressedOops, "Not allowed");
500 CompareAndSwapNode* cas = (CompareAndSwapNode*)access.raw_access();
501 PhaseGVN& gvn = access.gvn();
502 Compile* C = Compile::current();
503 GraphKit* kit = access.kit();
504
505 Node* in_ctrl = cas->in(MemNode::Control);
506 Node* in_mem = cas->in(MemNode::Memory);
507 Node* in_adr = cas->in(MemNode::Address);
508 Node* in_val = cas->in(MemNode::ValueIn);
509 Node* in_expected = cas->in(LoadStoreConditionalNode::ExpectedIn);
510
511 float likely = PROB_LIKELY(0.999);
512
513 const TypePtr *adr_type = gvn.type(in_adr)->isa_ptr();
514 Compile::AliasType* alias_type = C->alias_type(adr_type);
515 int alias_idx = C->get_alias_index(adr_type);
516
517 // Outer check - true: continue, false: load and check
518 Node* region = new RegionNode(3);
519 Node* phi = new PhiNode(region, TypeInt::BOOL);
520 Node* phi_mem = new PhiNode(region, Type::MEMORY, adr_type);
521
522 // Inner check - is the healed ref equal to the expected
523 Node* region2 = new RegionNode(3);
524 Node* phi2 = new PhiNode(region2, TypeInt::BOOL);
525 Node* phi_mem2 = new PhiNode(region2, Type::MEMORY, adr_type);
526
527 // CAS node returns 0 or 1
528 Node* cmp = gvn.transform(new CmpINode(cas, kit->intcon(0)));
529 Node* bol = gvn.transform(new BoolNode(cmp, BoolTest::ne))->as_Bool();
530 IfNode* iff = gvn.transform(new IfNode(in_ctrl, bol, likely, COUNT_UNKNOWN))->as_If();
531 Node* then = gvn.transform(new IfTrueNode(iff));
532 Node* elsen = gvn.transform(new IfFalseNode(iff));
533
534 Node* scmemproj1 = gvn.transform(new SCMemProjNode(cas));
535
536 kit->set_memory(scmemproj1, alias_idx);
537 phi_mem->init_req(1, scmemproj1);
538 phi_mem2->init_req(2, scmemproj1);
539
540 // CAS fail - reload and heal oop
541 Node* reload = kit->make_load(elsen, in_adr, TypeOopPtr::BOTTOM, T_OBJECT, MemNode::unordered);
542 Node* barrier = gvn.transform(new LoadBarrierNode(C, elsen, scmemproj1, reload, in_adr, false, true, false));
543 Node* barrierctrl = gvn.transform(new ProjNode(barrier, LoadBarrierNode::Control));
544 Node* barrierdata = gvn.transform(new ProjNode(barrier, LoadBarrierNode::Oop));
545
546 // Check load
547 Node* tmpX = gvn.transform(new CastP2XNode(NULL, barrierdata));
548 Node* in_expX = gvn.transform(new CastP2XNode(NULL, in_expected));
549 Node* cmp2 = gvn.transform(new CmpXNode(tmpX, in_expX));
550 Node *bol2 = gvn.transform(new BoolNode(cmp2, BoolTest::ne))->as_Bool();
551 IfNode* iff2 = gvn.transform(new IfNode(barrierctrl, bol2, likely, COUNT_UNKNOWN))->as_If();
552 Node* then2 = gvn.transform(new IfTrueNode(iff2));
553 Node* elsen2 = gvn.transform(new IfFalseNode(iff2));
554
555 // redo CAS
556 Node* cas2 = gvn.transform(new CompareAndSwapPNode(elsen2, kit->memory(alias_idx), in_adr, in_val, in_expected, cas->order()));
557 Node* scmemproj2 = gvn.transform(new SCMemProjNode(cas2));
558 kit->set_control(elsen2);
559 kit->set_memory(scmemproj2, alias_idx);
560
561 // Merge inner flow - check if healed oop was equal too expected.
562 region2->set_req(1, kit->control());
563 region2->set_req(2, then2);
564 phi2->set_req(1, cas2);
565 phi2->set_req(2, kit->intcon(0));
566 phi_mem2->init_req(1, scmemproj2);
567 kit->set_memory(phi_mem2, alias_idx);
568
569 // Merge outer flow - then check if first CAS succeeded
570 region->set_req(1, then);
571 region->set_req(2, region2);
572 phi->set_req(1, kit->intcon(1));
573 phi->set_req(2, phi2);
574 phi_mem->init_req(2, phi_mem2);
575 kit->set_memory(phi_mem, alias_idx);
576
577 gvn.transform(region2);
578 gvn.transform(phi2);
579 gvn.transform(phi_mem2);
580 gvn.transform(region);
581 gvn.transform(phi);
582 gvn.transform(phi_mem);
583
584 kit->set_control(region);
585 kit->insert_mem_bar(Op_MemBarCPUOrder);
586
587 return phi;
588 }
589
590 Node* ZBarrierSetC2::make_cmpx_loadbarrier(C2AtomicParseAccess& access) const {
591 CompareAndExchangePNode* cmpx = (CompareAndExchangePNode*)access.raw_access();
592 GraphKit* kit = access.kit();
593 PhaseGVN& gvn = kit->gvn();
594 Compile* C = Compile::current();
595
596 Node* in_ctrl = cmpx->in(MemNode::Control);
597 Node* in_mem = cmpx->in(MemNode::Memory);
598 Node* in_adr = cmpx->in(MemNode::Address);
599 Node* in_val = cmpx->in(MemNode::ValueIn);
600 Node* in_expected = cmpx->in(LoadStoreConditionalNode::ExpectedIn);
601
602 float likely = PROB_LIKELY(0.999);
603
604 const TypePtr *adr_type = cmpx->get_ptr_type();
605 Compile::AliasType* alias_type = C->alias_type(adr_type);
606 int alias_idx = C->get_alias_index(adr_type);
607
608 // Outer check - true: continue, false: load and check
609 Node* region = new RegionNode(3);
610 Node* phi = new PhiNode(region, adr_type);
611
612 // Inner check - is the healed ref equal to the expected
613 Node* region2 = new RegionNode(3);
614 Node* phi2 = new PhiNode(region2, adr_type);
615
616 // Check if cmpx succeeded
617 Node* cmp = gvn.transform(new CmpPNode(cmpx, in_expected));
618 Node* bol = gvn.transform(new BoolNode(cmp, BoolTest::eq))->as_Bool();
619 IfNode* iff = gvn.transform(new IfNode(in_ctrl, bol, likely, COUNT_UNKNOWN))->as_If();
620 Node* then = gvn.transform(new IfTrueNode(iff));
621 Node* elsen = gvn.transform(new IfFalseNode(iff));
622
623 Node* scmemproj1 = gvn.transform(new SCMemProjNode(cmpx));
624 kit->set_memory(scmemproj1, alias_idx);
625
626 // CAS fail - reload and heal oop
627 Node* reload = kit->make_load(elsen, in_adr, TypeOopPtr::BOTTOM, T_OBJECT, MemNode::unordered);
628 Node* barrier = gvn.transform(new LoadBarrierNode(C, elsen, scmemproj1, reload, in_adr, false, true, false));
629 Node* barrierctrl = gvn.transform(new ProjNode(barrier, LoadBarrierNode::Control));
630 Node* barrierdata = gvn.transform(new ProjNode(barrier, LoadBarrierNode::Oop));
631
632 // Check load
633 Node* tmpX = gvn.transform(new CastP2XNode(NULL, barrierdata));
634 Node* in_expX = gvn.transform(new CastP2XNode(NULL, in_expected));
635 Node* cmp2 = gvn.transform(new CmpXNode(tmpX, in_expX));
636 Node *bol2 = gvn.transform(new BoolNode(cmp2, BoolTest::ne))->as_Bool();
637 IfNode* iff2 = gvn.transform(new IfNode(barrierctrl, bol2, likely, COUNT_UNKNOWN))->as_If();
638 Node* then2 = gvn.transform(new IfTrueNode(iff2));
639 Node* elsen2 = gvn.transform(new IfFalseNode(iff2));
640
641 // Redo CAS
642 Node* cmpx2 = gvn.transform(new CompareAndExchangePNode(elsen2, kit->memory(alias_idx), in_adr, in_val, in_expected, adr_type, cmpx->get_ptr_type(), cmpx->order()));
643 Node* scmemproj2 = gvn.transform(new SCMemProjNode(cmpx2));
644 kit->set_control(elsen2);
645 kit->set_memory(scmemproj2, alias_idx);
646
647 // Merge inner flow - check if healed oop was equal too expected.
648 region2->set_req(1, kit->control());
649 region2->set_req(2, then2);
650 phi2->set_req(1, cmpx2);
651 phi2->set_req(2, barrierdata);
652
653 // Merge outer flow - then check if first cas succeeded
654 region->set_req(1, then);
655 region->set_req(2, region2);
656 phi->set_req(1, cmpx);
657 phi->set_req(2, phi2);
658
659 gvn.transform(region2);
660 gvn.transform(phi2);
661 gvn.transform(region);
662 gvn.transform(phi);
663
664 kit->set_control(region);
665 kit->set_memory(in_mem, alias_idx);
666 kit->insert_mem_bar(Op_MemBarCPUOrder);
667
668 return phi;
669 }
670
671 Node* ZBarrierSetC2::load_barrier(GraphKit* kit, Node* val, Node* adr, bool weak, bool writeback, bool oop_reload_allowed) const {
672 PhaseGVN& gvn = kit->gvn();
673 Node* barrier = new LoadBarrierNode(Compile::current(), kit->control(), kit->memory(TypeRawPtr::BOTTOM), val, adr, weak, writeback, oop_reload_allowed);
674 Node* transformed_barrier = gvn.transform(barrier);
675
676 if (transformed_barrier->is_LoadBarrier()) {
677 if (barrier == transformed_barrier) {
678 kit->set_control(gvn.transform(new ProjNode(barrier, LoadBarrierNode::Control)));
679 }
680 Node* result = gvn.transform(new ProjNode(transformed_barrier, LoadBarrierNode::Oop));
681 return result;
682 } else {
683 return val;
684 }
685 }
686
687 static bool barrier_needed(C2Access& access) {
688 return ZBarrierSet::barrier_needed(access.decorators(), access.type());
689 }
690
691 Node* ZBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const {
692 Node* p = BarrierSetC2::load_at_resolved(access, val_type);
693 if (!barrier_needed(access)) {
694 return p;
695 }
696
697 bool weak = (access.decorators() & ON_WEAK_OOP_REF) != 0;
698
699 assert(access.is_parse_access(), "entry not supported at optimization time");
700 C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
701 GraphKit* kit = parse_access.kit();
702 PhaseGVN& gvn = kit->gvn();
703 Node* adr = access.addr().node();
704 Node* heap_base_oop = access.base();
705 bool unsafe = (access.decorators() & C2_UNSAFE_ACCESS) != 0;
706 if (unsafe) {
707 if (!ZVerifyLoadBarriers) {
708 p = load_barrier(kit, p, adr);
709 } else {
710 if (!TypePtr::NULL_PTR->higher_equal(gvn.type(heap_base_oop))) {
711 p = load_barrier(kit, p, adr);
712 } else {
713 IdealKit ideal(kit);
714 IdealVariable res(ideal);
715 #define __ ideal.
716 __ declarations_done();
717 __ set(res, p);
718 __ if_then(heap_base_oop, BoolTest::ne, kit->null(), PROB_UNLIKELY(0.999)); {
719 kit->sync_kit(ideal);
720 p = load_barrier(kit, p, adr);
721 __ set(res, p);
722 __ sync_kit(kit);
723 } __ end_if();
724 kit->final_sync(ideal);
725 p = __ value(res);
726 #undef __
727 }
728 }
729 return p;
730 } else {
731 return load_barrier(parse_access.kit(), p, access.addr().node(), weak, true, true);
732 }
733 }
734
735 Node* ZBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
736 Node* new_val, const Type* val_type) const {
737 Node* result = BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, val_type);
738 if (!barrier_needed(access)) {
739 return result;
740 }
741
742 access.set_needs_pinning(false);
743 return make_cmpx_loadbarrier(access);
744 }
745
746 Node* ZBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
747 Node* new_val, const Type* value_type) const {
748 Node* result = BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type);
749 if (!barrier_needed(access)) {
750 return result;
751 }
752
753 Node* load_store = access.raw_access();
754 bool weak_cas = (access.decorators() & C2_WEAK_CMPXCHG) != 0;
755 bool expected_is_null = (expected_val->get_ptr_type() == TypePtr::NULL_PTR);
756
757 if (!expected_is_null) {
758 if (weak_cas) {
759 access.set_needs_pinning(false);
760 load_store = make_cas_loadbarrier(access);
761 } else {
762 access.set_needs_pinning(false);
763 load_store = make_cas_loadbarrier(access);
764 }
765 }
766
767 return load_store;
768 }
769
770 Node* ZBarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* val_type) const {
771 Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, new_val, val_type);
772 if (!barrier_needed(access)) {
773 return result;
774 }
775
776 Node* load_store = access.raw_access();
777 Node* adr = access.addr().node();
778
779 assert(access.is_parse_access(), "entry not supported at optimization time");
780 C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
781 return load_barrier(parse_access.kit(), load_store, adr, false, false, false);
782 }
783
784 // == Macro Expansion ==
785
786 void ZBarrierSetC2::expand_loadbarrier_node(PhaseMacroExpand* phase, LoadBarrierNode* barrier) const {
787 Node* in_ctrl = barrier->in(LoadBarrierNode::Control);
788 Node* in_mem = barrier->in(LoadBarrierNode::Memory);
789 Node* in_val = barrier->in(LoadBarrierNode::Oop);
790 Node* in_adr = barrier->in(LoadBarrierNode::Address);
791
792 Node* out_ctrl = barrier->proj_out(LoadBarrierNode::Control);
793 Node* out_res = barrier->proj_out(LoadBarrierNode::Oop);
794
795 PhaseIterGVN &igvn = phase->igvn();
796
797 if (ZVerifyLoadBarriers) {
798 igvn.replace_node(out_res, in_val);
799 igvn.replace_node(out_ctrl, in_ctrl);
800 return;
801 }
802
803 if (barrier->can_be_eliminated()) {
804 // Clone and pin the load for this barrier below the dominating
805 // barrier: the load cannot be allowed to float above the
806 // dominating barrier
807 Node* load = in_val;
808
809 if (load->is_Load()) {
810 Node* new_load = load->clone();
811 Node* addp = new_load->in(MemNode::Address);
812 assert(addp->is_AddP() || addp->is_Phi() || addp->is_Load(), "bad address");
813 Node* cast = new CastPPNode(addp, igvn.type(addp), true);
814 Node* ctrl = NULL;
815 Node* similar = barrier->in(LoadBarrierNode::Similar);
816 if (similar->is_Phi()) {
817 // already expanded
818 ctrl = similar->in(0);
819 } else {
820 assert(similar->is_Proj() && similar->in(0)->is_LoadBarrier(), "unexpected graph shape");
821 ctrl = similar->in(0)->as_LoadBarrier()->proj_out(LoadBarrierNode::Control);
822 }
823 assert(ctrl != NULL, "bad control");
824 cast->set_req(0, ctrl);
825 igvn.transform(cast);
826 new_load->set_req(MemNode::Address, cast);
827 igvn.transform(new_load);
828
829 igvn.replace_node(out_res, new_load);
830 igvn.replace_node(out_ctrl, in_ctrl);
831 return;
832 }
833 // cannot eliminate
834 }
835
836 // There are two cases that require the basic loadbarrier
837 // 1) When the writeback of a healed oop must be avoided (swap)
838 // 2) When we must guarantee that no reload of is done (swap, cas, cmpx)
839 if (!barrier->is_writeback()) {
840 assert(!barrier->oop_reload_allowed(), "writeback barriers should be marked as requires oop");
841 }
842
843 if (!barrier->oop_reload_allowed()) {
844 expand_loadbarrier_basic(phase, barrier);
845 } else {
846 expand_loadbarrier_optimized(phase, barrier);
847 }
848 }
849
850 // Basic loadbarrier using conventional argument passing
851 void ZBarrierSetC2::expand_loadbarrier_basic(PhaseMacroExpand* phase, LoadBarrierNode *barrier) const {
852 PhaseIterGVN &igvn = phase->igvn();
853
854 Node* in_ctrl = barrier->in(LoadBarrierNode::Control);
855 Node* in_mem = barrier->in(LoadBarrierNode::Memory);
856 Node* in_val = barrier->in(LoadBarrierNode::Oop);
857 Node* in_adr = barrier->in(LoadBarrierNode::Address);
858
859 Node* out_ctrl = barrier->proj_out(LoadBarrierNode::Control);
860 Node* out_res = barrier->proj_out(LoadBarrierNode::Oop);
861
862 float unlikely = PROB_UNLIKELY(0.999);
863 const Type* in_val_maybe_null_t = igvn.type(in_val);
864
865 Node* jthread = igvn.transform(new ThreadLocalNode());
866 Node* adr = phase->basic_plus_adr(jthread, in_bytes(ZThreadLocalData::address_bad_mask_offset()));
867 Node* bad_mask = igvn.transform(LoadNode::make(igvn, in_ctrl, in_mem, adr, TypeRawPtr::BOTTOM, TypeX_X, TypeX_X->basic_type(), MemNode::unordered));
868 Node* cast = igvn.transform(new CastP2XNode(in_ctrl, in_val));
869 Node* obj_masked = igvn.transform(new AndXNode(cast, bad_mask));
870 Node* cmp = igvn.transform(new CmpXNode(obj_masked, igvn.zerocon(TypeX_X->basic_type())));
871 Node *bol = igvn.transform(new BoolNode(cmp, BoolTest::ne))->as_Bool();
872 IfNode* iff = igvn.transform(new IfNode(in_ctrl, bol, unlikely, COUNT_UNKNOWN))->as_If();
873 Node* then = igvn.transform(new IfTrueNode(iff));
874 Node* elsen = igvn.transform(new IfFalseNode(iff));
875
876 Node* result_region;
877 Node* result_val;
878
879 result_region = new RegionNode(3);
880 result_val = new PhiNode(result_region, TypeInstPtr::BOTTOM);
881
882 result_region->set_req(1, elsen);
883 Node* res = igvn.transform(new CastPPNode(in_val, in_val_maybe_null_t));
884 res->init_req(0, elsen);
885 result_val->set_req(1, res);
886
887 const TypeFunc *tf = load_barrier_Type();
888 Node* call;
889 if (barrier->is_weak()) {
890 call = new CallLeafNode(tf,
891 ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded_addr(),
892 "ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded",
893 TypeRawPtr::BOTTOM);
894 } else {
895 call = new CallLeafNode(tf,
896 ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(),
897 "ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded",
898 TypeRawPtr::BOTTOM);
899 }
900
901 call->init_req(TypeFunc::Control, then);
902 call->init_req(TypeFunc::I_O , phase->top());
903 call->init_req(TypeFunc::Memory , in_mem);
904 call->init_req(TypeFunc::FramePtr, phase->top());
905 call->init_req(TypeFunc::ReturnAdr, phase->top());
906 call->init_req(TypeFunc::Parms+0, in_val);
907 if (barrier->is_writeback()) {
908 call->init_req(TypeFunc::Parms+1, in_adr);
909 } else {
910 // When slow path is called with a null address, the healed oop will not be written back
911 call->init_req(TypeFunc::Parms+1, igvn.zerocon(T_OBJECT));
912 }
913 call = igvn.transform(call);
914
915 Node* ctrl = igvn.transform(new ProjNode(call, TypeFunc::Control));
916 res = igvn.transform(new ProjNode(call, TypeFunc::Parms));
917 res = igvn.transform(new CheckCastPPNode(ctrl, res, in_val_maybe_null_t));
918
919 result_region->set_req(2, ctrl);
920 result_val->set_req(2, res);
921
922 result_region = igvn.transform(result_region);
923 result_val = igvn.transform(result_val);
924
925 if (out_ctrl != NULL) { // Added if cond
926 igvn.replace_node(out_ctrl, result_region);
927 }
928 igvn.replace_node(out_res, result_val);
929 }
930
931 // Optimized, low spill, loadbarrier variant using stub specialized on register used
932 void ZBarrierSetC2::expand_loadbarrier_optimized(PhaseMacroExpand* phase, LoadBarrierNode *barrier) const {
933 PhaseIterGVN &igvn = phase->igvn();
934 #ifdef PRINT_NODE_TRAVERSALS
935 Node* preceding_barrier_node = barrier->in(LoadBarrierNode::Oop);
936 #endif
937
938 Node* in_ctrl = barrier->in(LoadBarrierNode::Control);
939 Node* in_mem = barrier->in(LoadBarrierNode::Memory);
940 Node* in_val = barrier->in(LoadBarrierNode::Oop);
941 Node* in_adr = barrier->in(LoadBarrierNode::Address);
942
943 Node* out_ctrl = barrier->proj_out(LoadBarrierNode::Control);
944 Node* out_res = barrier->proj_out(LoadBarrierNode::Oop);
945
946 assert(barrier->in(LoadBarrierNode::Oop) != NULL, "oop to loadbarrier node cannot be null");
947
948 #ifdef PRINT_NODE_TRAVERSALS
949 tty->print("\n\n\nBefore barrier optimization:\n");
950 traverse(barrier, out_ctrl, out_res, -1);
951
952 tty->print("\nBefore barrier optimization: preceding_barrier_node\n");
953 traverse(preceding_barrier_node, out_ctrl, out_res, -1);
954 #endif
955
956 float unlikely = PROB_UNLIKELY(0.999);
957
958 Node* jthread = igvn.transform(new ThreadLocalNode());
959 Node* adr = phase->basic_plus_adr(jthread, in_bytes(ZThreadLocalData::address_bad_mask_offset()));
960 Node* bad_mask = igvn.transform(LoadNode::make(igvn, in_ctrl, in_mem, adr,
961 TypeRawPtr::BOTTOM, TypeX_X, TypeX_X->basic_type(),
962 MemNode::unordered));
963 Node* cast = igvn.transform(new CastP2XNode(in_ctrl, in_val));
964 Node* obj_masked = igvn.transform(new AndXNode(cast, bad_mask));
965 Node* cmp = igvn.transform(new CmpXNode(obj_masked, igvn.zerocon(TypeX_X->basic_type())));
966 Node *bol = igvn.transform(new BoolNode(cmp, BoolTest::ne))->as_Bool();
967 IfNode* iff = igvn.transform(new IfNode(in_ctrl, bol, unlikely, COUNT_UNKNOWN))->as_If();
968 Node* then = igvn.transform(new IfTrueNode(iff));
969 Node* elsen = igvn.transform(new IfFalseNode(iff));
970
971 Node* slow_path_surrogate;
972 if (!barrier->is_weak()) {
973 slow_path_surrogate = igvn.transform(new LoadBarrierSlowRegNode(then, in_mem, in_adr, in_val->adr_type(),
974 (const TypePtr*) in_val->bottom_type(), MemNode::unordered));
975 } else {
976 slow_path_surrogate = igvn.transform(new LoadBarrierWeakSlowRegNode(then, in_mem, in_adr, in_val->adr_type(),
977 (const TypePtr*) in_val->bottom_type(), MemNode::unordered));
978 }
979
980 Node *new_loadp;
981 new_loadp = slow_path_surrogate;
982 // Create the final region/phi pair to converge cntl/data paths to downstream code
983 Node* result_region = igvn.transform(new RegionNode(3));
984 result_region->set_req(1, then);
985 result_region->set_req(2, elsen);
986
987 Node* result_phi = igvn.transform(new PhiNode(result_region, TypeInstPtr::BOTTOM));
988 result_phi->set_req(1, new_loadp);
989 result_phi->set_req(2, barrier->in(LoadBarrierNode::Oop));
990
991 // Finally, connect the original outputs to the barrier region and phi to complete the expansion/substitution
992 // igvn.replace_node(out_ctrl, result_region);
993 if (out_ctrl != NULL) { // added if cond
994 igvn.replace_node(out_ctrl, result_region);
995 }
996 igvn.replace_node(out_res, result_phi);
997
998 assert(barrier->outcnt() == 0,"LoadBarrier macro node has non-null outputs after expansion!");
999
1000 #ifdef PRINT_NODE_TRAVERSALS
1001 tty->print("\nAfter barrier optimization: old out_ctrl\n");
1002 traverse(out_ctrl, out_ctrl, out_res, -1);
1003 tty->print("\nAfter barrier optimization: old out_res\n");
1004 traverse(out_res, out_ctrl, out_res, -1);
1005 tty->print("\nAfter barrier optimization: old barrier\n");
1006 traverse(barrier, out_ctrl, out_res, -1);
1007 tty->print("\nAfter barrier optimization: preceding_barrier_node\n");
1008 traverse(preceding_barrier_node, result_region, result_phi, -1);
1009 #endif
1010
1011 assert(is_gc_barrier_node(result_phi), "sanity");
1012 assert(step_over_gc_barrier(result_phi) == in_val, "sanity");
1013
1014 return;
1015 }
1016
1017 bool ZBarrierSetC2::expand_barriers(Compile* C, PhaseIterGVN& igvn) const {
1018 ZBarrierSetC2State* s = state();
1019 if (s->load_barrier_count() > 0) {
1020 PhaseMacroExpand macro(igvn);
1021 #ifdef ASSERT
1022 verify_gc_barriers(false);
1023 #endif
1024 int skipped = 0;
1025 while (s->load_barrier_count() > skipped) {
1026 int load_barrier_count = s->load_barrier_count();
1027 LoadBarrierNode * n = s->load_barrier_node(load_barrier_count-1-skipped);
1028 if (igvn.type(n) == Type::TOP || (n->in(0) != NULL && n->in(0)->is_top())) {
1029 // Node is unreachable, so don't try to expand it
1030 s->remove_load_barrier_node(n);
1031 continue;
1032 }
1033 if (!n->can_be_eliminated()) {
1034 skipped++;
1035 continue;
1036 }
1037 expand_loadbarrier_node(¯o, n);
1038 assert(s->load_barrier_count() < load_barrier_count, "must have deleted a node from load barrier list");
1039 if (C->failing()) return true;
1040 }
1041 while (s->load_barrier_count() > 0) {
1042 int load_barrier_count = s->load_barrier_count();
1043 LoadBarrierNode* n = s->load_barrier_node(load_barrier_count - 1);
1044 assert(!(igvn.type(n) == Type::TOP || (n->in(0) != NULL && n->in(0)->is_top())), "should have been processed already");
1045 assert(!n->can_be_eliminated(), "should have been processed already");
1046 expand_loadbarrier_node(¯o, n);
1047 assert(s->load_barrier_count() < load_barrier_count, "must have deleted a node from load barrier list");
1048 if (C->failing()) return true;
1049 }
1050 igvn.set_delay_transform(false);
1051 igvn.optimize();
1052 if (C->failing()) return true;
1053 }
1054 return false;
1055 }
1056
1057 // == Loop optimization ==
1058
1059 static bool replace_with_dominating_barrier(PhaseIdealLoop* phase, LoadBarrierNode* lb, bool last_round) {
1060 PhaseIterGVN &igvn = phase->igvn();
1061 Compile* C = Compile::current();
1062
1063 LoadBarrierNode* lb2 = lb->has_dominating_barrier(phase, false, last_round);
1064 if (lb2 != NULL) {
1065 if (lb->in(LoadBarrierNode::Oop) != lb2->in(LoadBarrierNode::Oop)) {
1066 assert(lb->in(LoadBarrierNode::Address) == lb2->in(LoadBarrierNode::Address), "");
1067 igvn.replace_input_of(lb, LoadBarrierNode::Similar, lb2->proj_out(LoadBarrierNode::Oop));
1068 C->set_major_progress();
1069 } else {
1070 // That transformation may cause the Similar edge on dominated load barriers to be invalid
1071 lb->fix_similar_in_uses(&igvn);
1072
1073 Node* val = lb->proj_out(LoadBarrierNode::Oop);
1074 assert(lb2->has_true_uses(), "");
1075 assert(lb2->in(LoadBarrierNode::Oop) == lb->in(LoadBarrierNode::Oop), "");
1076
1077 phase->lazy_update(lb, lb->in(LoadBarrierNode::Control));
1078 phase->lazy_replace(lb->proj_out(LoadBarrierNode::Control), lb->in(LoadBarrierNode::Control));
1079 igvn.replace_node(val, lb2->proj_out(LoadBarrierNode::Oop));
1080
1081 return true;
1082 }
1083 }
1084 return false;
1085 }
1086
1087 static Node* find_dominating_memory(PhaseIdealLoop* phase, Node* mem, Node* dom, int i) {
1088 assert(dom->is_Region() || i == -1, "");
1089 Node* m = mem;
1090 while(phase->is_dominator(dom, phase->has_ctrl(m) ? phase->get_ctrl(m) : m->in(0))) {
1091 if (m->is_Mem()) {
1092 assert(m->as_Mem()->adr_type() == TypeRawPtr::BOTTOM, "");
1093 m = m->in(MemNode::Memory);
1094 } else if (m->is_MergeMem()) {
1095 m = m->as_MergeMem()->memory_at(Compile::AliasIdxRaw);
1096 } else if (m->is_Phi()) {
1097 if (m->in(0) == dom && i != -1) {
1098 m = m->in(i);
1099 break;
1100 } else {
1101 m = m->in(LoopNode::EntryControl);
1102 }
1103 } else if (m->is_Proj()) {
1104 m = m->in(0);
1105 } else if (m->is_SafePoint() || m->is_MemBar()) {
1106 m = m->in(TypeFunc::Memory);
1107 } else {
1108 #ifdef ASSERT
1109 m->dump();
1110 #endif
1111 ShouldNotReachHere();
1112 }
1113 }
1114 return m;
1115 }
1116
1117 static LoadBarrierNode* clone_load_barrier(PhaseIdealLoop* phase, LoadBarrierNode* lb, Node* ctl, Node* mem, Node* oop_in) {
1118 PhaseIterGVN &igvn = phase->igvn();
1119 Compile* C = Compile::current();
1120 Node* the_clone = lb->clone();
1121 the_clone->set_req(LoadBarrierNode::Control, ctl);
1122 the_clone->set_req(LoadBarrierNode::Memory, mem);
1123 if (oop_in != NULL) {
1124 the_clone->set_req(LoadBarrierNode::Oop, oop_in);
1125 }
1126
1127 LoadBarrierNode* new_lb = the_clone->as_LoadBarrier();
1128 igvn.register_new_node_with_optimizer(new_lb);
1129 IdealLoopTree *loop = phase->get_loop(new_lb->in(0));
1130 phase->set_ctrl(new_lb, new_lb->in(0));
1131 phase->set_loop(new_lb, loop);
1132 phase->set_idom(new_lb, new_lb->in(0), phase->dom_depth(new_lb->in(0))+1);
1133 if (!loop->_child) {
1134 loop->_body.push(new_lb);
1135 }
1136
1137 Node* proj_ctl = new ProjNode(new_lb, LoadBarrierNode::Control);
1138 igvn.register_new_node_with_optimizer(proj_ctl);
1139 phase->set_ctrl(proj_ctl, proj_ctl->in(0));
1140 phase->set_loop(proj_ctl, loop);
1141 phase->set_idom(proj_ctl, new_lb, phase->dom_depth(new_lb)+1);
1142 if (!loop->_child) {
1143 loop->_body.push(proj_ctl);
1144 }
1145
1146 Node* proj_oop = new ProjNode(new_lb, LoadBarrierNode::Oop);
1147 phase->register_new_node(proj_oop, new_lb);
1148
1149 if (!new_lb->in(LoadBarrierNode::Similar)->is_top()) {
1150 LoadBarrierNode* similar = new_lb->in(LoadBarrierNode::Similar)->in(0)->as_LoadBarrier();
1151 if (!phase->is_dominator(similar, ctl)) {
1152 igvn.replace_input_of(new_lb, LoadBarrierNode::Similar, C->top());
1153 }
1154 }
1155
1156 return new_lb;
1157 }
1158
1159 static void replace_barrier(PhaseIdealLoop* phase, LoadBarrierNode* lb, Node* new_val) {
1160 PhaseIterGVN &igvn = phase->igvn();
1161 Node* val = lb->proj_out(LoadBarrierNode::Oop);
1162 igvn.replace_node(val, new_val);
1163 phase->lazy_update(lb, lb->in(LoadBarrierNode::Control));
1164 phase->lazy_replace(lb->proj_out(LoadBarrierNode::Control), lb->in(LoadBarrierNode::Control));
1165 }
1166
1167 static bool split_barrier_thru_phi(PhaseIdealLoop* phase, LoadBarrierNode* lb) {
1168 PhaseIterGVN &igvn = phase->igvn();
1169 Compile* C = Compile::current();
1170
1171 if (lb->in(LoadBarrierNode::Oop)->is_Phi()) {
1172 Node* oop_phi = lb->in(LoadBarrierNode::Oop);
1173
1174 if ((oop_phi->req() != 3) || (oop_phi->in(2) == oop_phi)) {
1175 // Ignore phis with only one input
1176 return false;
1177 }
1178
1179 if (phase->is_dominator(phase->get_ctrl(lb->in(LoadBarrierNode::Address)),
1180 oop_phi->in(0)) && phase->get_ctrl(lb->in(LoadBarrierNode::Address)) != oop_phi->in(0)) {
1181 // That transformation may cause the Similar edge on dominated load barriers to be invalid
1182 lb->fix_similar_in_uses(&igvn);
1183
1184 RegionNode* region = oop_phi->in(0)->as_Region();
1185
1186 int backedge = LoopNode::LoopBackControl;
1187 if (region->is_Loop() && region->in(backedge)->is_Proj() && region->in(backedge)->in(0)->is_If()) {
1188 Node* c = region->in(backedge)->in(0)->in(0);
1189 assert(c->unique_ctrl_out() == region->in(backedge)->in(0), "");
1190 Node* oop = lb->in(LoadBarrierNode::Oop)->in(backedge);
1191 Node* oop_c = phase->has_ctrl(oop) ? phase->get_ctrl(oop) : oop;
1192 if (!phase->is_dominator(oop_c, c)) {
1193 return false;
1194 }
1195 }
1196
1197 // If the node on the backedge above the phi is the node itself - we have a self loop.
1198 // Don't clone - this will be folded later.
1199 if (oop_phi->in(LoopNode::LoopBackControl) == lb->proj_out(LoadBarrierNode::Oop)) {
1200 return false;
1201 }
1202
1203 bool is_strip_mined = region->is_CountedLoop() && region->as_CountedLoop()->is_strip_mined();
1204 Node *phi = oop_phi->clone();
1205
1206 for (uint i = 1; i < region->req(); i++) {
1207 Node* ctrl = region->in(i);
1208 if (ctrl != C->top()) {
1209 assert(!phase->is_dominator(ctrl, region) || region->is_Loop(), "");
1210
1211 Node* mem = lb->in(LoadBarrierNode::Memory);
1212 Node* m = find_dominating_memory(phase, mem, region, i);
1213
1214 if (region->is_Loop() && i == LoopNode::LoopBackControl && ctrl->is_Proj() && ctrl->in(0)->is_If()) {
1215 ctrl = ctrl->in(0)->in(0);
1216 } else if (region->is_Loop() && is_strip_mined) {
1217 // If this is a strip mined loop, control must move above OuterStripMinedLoop
1218 assert(i == LoopNode::EntryControl, "check");
1219 assert(ctrl->is_OuterStripMinedLoop(), "sanity");
1220 ctrl = ctrl->as_OuterStripMinedLoop()->in(LoopNode::EntryControl);
1221 }
1222
1223 LoadBarrierNode* new_lb = clone_load_barrier(phase, lb, ctrl, m, lb->in(LoadBarrierNode::Oop)->in(i));
1224 Node* out_ctrl = new_lb->proj_out(LoadBarrierNode::Control);
1225
1226 if (is_strip_mined && (i == LoopNode::EntryControl)) {
1227 assert(region->in(i)->is_OuterStripMinedLoop(), "");
1228 igvn.replace_input_of(region->in(i), i, out_ctrl);
1229 phase->set_idom(region->in(i), out_ctrl, phase->dom_depth(out_ctrl));
1230 } else if (ctrl == region->in(i)) {
1231 igvn.replace_input_of(region, i, out_ctrl);
1232 // Only update the idom if is the loop entry we are updating
1233 // - A loop backedge doesn't change the idom
1234 if (region->is_Loop() && i == LoopNode::EntryControl) {
1235 phase->set_idom(region, out_ctrl, phase->dom_depth(out_ctrl));
1236 }
1237 } else {
1238 Node* iff = region->in(i)->in(0);
1239 igvn.replace_input_of(iff, 0, out_ctrl);
1240 phase->set_idom(iff, out_ctrl, phase->dom_depth(out_ctrl)+1);
1241 }
1242 phi->set_req(i, new_lb->proj_out(LoadBarrierNode::Oop));
1243 }
1244 }
1245 phase->register_new_node(phi, region);
1246 replace_barrier(phase, lb, phi);
1247
1248 if (region->is_Loop()) {
1249 // Load barrier moved to the back edge of the Loop may now
1250 // have a safepoint on the path to the barrier on the Similar
1251 // edge
1252 igvn.replace_input_of(phi->in(LoopNode::LoopBackControl)->in(0), LoadBarrierNode::Similar, C->top());
1253 Node* head = region->in(LoopNode::EntryControl);
1254 phase->set_idom(region, head, phase->dom_depth(head)+1);
1255 phase->recompute_dom_depth();
1256 if (head->is_CountedLoop() && head->as_CountedLoop()->is_main_loop()) {
1257 head->as_CountedLoop()->set_normal_loop();
1258 }
1259 }
1260
1261 return true;
1262 }
1263 }
1264
1265 return false;
1266 }
1267
1268 static bool move_out_of_loop(PhaseIdealLoop* phase, LoadBarrierNode* lb) {
1269 PhaseIterGVN &igvn = phase->igvn();
1270 IdealLoopTree *lb_loop = phase->get_loop(lb->in(0));
1271 if (lb_loop != phase->ltree_root() && !lb_loop->_irreducible) {
1272 Node* oop_ctrl = phase->get_ctrl(lb->in(LoadBarrierNode::Oop));
1273 IdealLoopTree *oop_loop = phase->get_loop(oop_ctrl);
1274 IdealLoopTree* adr_loop = phase->get_loop(phase->get_ctrl(lb->in(LoadBarrierNode::Address)));
1275 if (!lb_loop->is_member(oop_loop) && !lb_loop->is_member(adr_loop)) {
1276 // That transformation may cause the Similar edge on dominated load barriers to be invalid
1277 lb->fix_similar_in_uses(&igvn);
1278
1279 Node* head = lb_loop->_head;
1280 assert(head->is_Loop(), "");
1281
1282 if (phase->is_dominator(head, oop_ctrl)) {
1283 assert(oop_ctrl->Opcode() == Op_CProj && oop_ctrl->in(0)->Opcode() == Op_NeverBranch, "");
1284 assert(lb_loop->is_member(phase->get_loop(oop_ctrl->in(0)->in(0))), "");
1285 return false;
1286 }
1287
1288 if (head->is_CountedLoop()) {
1289 CountedLoopNode* cloop = head->as_CountedLoop();
1290 if (cloop->is_main_loop()) {
1291 cloop->set_normal_loop();
1292 }
1293 // When we are moving barrier out of a counted loop,
1294 // make sure we move it all the way out of the strip mined outer loop.
1295 if (cloop->is_strip_mined()) {
1296 head = cloop->outer_loop();
1297 }
1298 }
1299
1300 Node* mem = lb->in(LoadBarrierNode::Memory);
1301 Node* m = find_dominating_memory(phase, mem, head, -1);
1302
1303 LoadBarrierNode* new_lb = clone_load_barrier(phase, lb, head->in(LoopNode::EntryControl), m, NULL);
1304
1305 assert(phase->idom(head) == head->in(LoopNode::EntryControl), "");
1306 Node* proj_ctl = new_lb->proj_out(LoadBarrierNode::Control);
1307 igvn.replace_input_of(head, LoopNode::EntryControl, proj_ctl);
1308 phase->set_idom(head, proj_ctl, phase->dom_depth(proj_ctl) + 1);
1309
1310 replace_barrier(phase, lb, new_lb->proj_out(LoadBarrierNode::Oop));
1311
1312 phase->recompute_dom_depth();
1313
1314 return true;
1315 }
1316 }
1317
1318 return false;
1319 }
1320
1321 static bool common_barriers(PhaseIdealLoop* phase, LoadBarrierNode* lb) {
1322 PhaseIterGVN &igvn = phase->igvn();
1323 Node* in_val = lb->in(LoadBarrierNode::Oop);
1324 for (DUIterator_Fast imax, i = in_val->fast_outs(imax); i < imax; i++) {
1325 Node* u = in_val->fast_out(i);
1326 if (u != lb && u->is_LoadBarrier() && u->as_LoadBarrier()->has_true_uses()) {
1327 Node* this_ctrl = lb->in(LoadBarrierNode::Control);
1328 Node* other_ctrl = u->in(LoadBarrierNode::Control);
1329
1330 Node* lca = phase->dom_lca(this_ctrl, other_ctrl);
1331 bool ok = true;
1332
1333 Node* proj1 = NULL;
1334 Node* proj2 = NULL;
1335
1336 while (this_ctrl != lca && ok) {
1337 if (this_ctrl->in(0) != NULL &&
1338 this_ctrl->in(0)->is_MultiBranch()) {
1339 if (this_ctrl->in(0)->in(0) == lca) {
1340 assert(proj1 == NULL, "");
1341 assert(this_ctrl->is_Proj(), "");
1342 proj1 = this_ctrl;
1343 } else if (!(this_ctrl->in(0)->is_If() && this_ctrl->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none))) {
1344 ok = false;
1345 }
1346 }
1347 this_ctrl = phase->idom(this_ctrl);
1348 }
1349 while (other_ctrl != lca && ok) {
1350 if (other_ctrl->in(0) != NULL &&
1351 other_ctrl->in(0)->is_MultiBranch()) {
1352 if (other_ctrl->in(0)->in(0) == lca) {
1353 assert(other_ctrl->is_Proj(), "");
1354 assert(proj2 == NULL, "");
1355 proj2 = other_ctrl;
1356 } else if (!(other_ctrl->in(0)->is_If() && other_ctrl->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none))) {
1357 ok = false;
1358 }
1359 }
1360 other_ctrl = phase->idom(other_ctrl);
1361 }
1362 assert(proj1 == NULL || proj2 == NULL || proj1->in(0) == proj2->in(0), "");
1363 if (ok && proj1 && proj2 && proj1 != proj2 && proj1->in(0)->is_If()) {
1364 // That transformation may cause the Similar edge on dominated load barriers to be invalid
1365 lb->fix_similar_in_uses(&igvn);
1366 u->as_LoadBarrier()->fix_similar_in_uses(&igvn);
1367
1368 Node* split = lca->unique_ctrl_out();
1369 assert(split->in(0) == lca, "");
1370
1371 Node* mem = lb->in(LoadBarrierNode::Memory);
1372 Node* m = find_dominating_memory(phase, mem, split, -1);
1373 LoadBarrierNode* new_lb = clone_load_barrier(phase, lb, lca, m, NULL);
1374
1375 Node* proj_ctl = new_lb->proj_out(LoadBarrierNode::Control);
1376 igvn.replace_input_of(split, 0, new_lb->proj_out(LoadBarrierNode::Control));
1377 phase->set_idom(split, proj_ctl, phase->dom_depth(proj_ctl)+1);
1378
1379 Node* proj_oop = new_lb->proj_out(LoadBarrierNode::Oop);
1380 replace_barrier(phase, lb, proj_oop);
1381 replace_barrier(phase, u->as_LoadBarrier(), proj_oop);
1382
1383 phase->recompute_dom_depth();
1384
1385 return true;
1386 }
1387 }
1388 }
1389
1390 return false;
1391 }
1392
1393 static void optimize_load_barrier(PhaseIdealLoop* phase, LoadBarrierNode* lb, bool last_round) {
1394 Compile* C = Compile::current();
1395
1396 if (!C->directive()->ZOptimizeLoadBarriersOption) {
1397 return;
1398 }
1399
1400 if (lb->has_true_uses()) {
1401 if (replace_with_dominating_barrier(phase, lb, last_round)) {
1402 return;
1403 }
1404
1405 if (split_barrier_thru_phi(phase, lb)) {
1406 return;
1407 }
1408
1409 if (move_out_of_loop(phase, lb)) {
1410 return;
1411 }
1412
1413 if (common_barriers(phase, lb)) {
1414 return;
1415 }
1416 }
1417 }
1418
1419 void ZBarrierSetC2::loop_optimize_gc_barrier(PhaseIdealLoop* phase, Node* node, bool last_round) {
1420 if (node->is_LoadBarrier()) {
1421 optimize_load_barrier(phase, node->as_LoadBarrier(), last_round);
1422 }
1423 }
1424
1425 Node* ZBarrierSetC2::step_over_gc_barrier(Node* c) const {
1426 Node* node = c;
1427
1428 // 1. This step follows potential oop projections of a load barrier before expansion
1429 if (node->is_Proj()) {
1430 node = node->in(0);
1431 }
1432
1433 // 2. This step checks for unexpanded load barriers
1434 if (node->is_LoadBarrier()) {
1435 return node->in(LoadBarrierNode::Oop);
1436 }
1437
1438 // 3. This step checks for the phi corresponding to an optimized load barrier expansion
1439 if (node->is_Phi()) {
1440 PhiNode* phi = node->as_Phi();
1441 Node* n = phi->in(1);
1442 if (n != NULL && (n->is_LoadBarrierSlowReg() || n->is_LoadBarrierWeakSlowReg())) {
1443 assert(c == node, "projections from step 1 should only be seen before macro expansion");
1444 return phi->in(2);
1445 }
1446 }
1447
1448 return c;
1449 }
1450
1451 bool ZBarrierSetC2::array_copy_requires_gc_barriers(bool tightly_coupled_alloc, BasicType type, bool is_clone, ArrayCopyPhase phase) const {
1452 return type == T_OBJECT || type == T_ARRAY;
1453 }
1454
1455 bool ZBarrierSetC2::final_graph_reshaping(Compile* compile, Node* n, uint opcode) const {
1456 bool handled;
1457 switch (opcode) {
1458 case Op_LoadBarrierSlowReg:
1459 case Op_LoadBarrierWeakSlowReg:
1460 #ifdef ASSERT
1461 if (VerifyOptoOopOffsets) {
1462 MemNode* mem = n->as_Mem();
1463 // Check to see if address types have grounded out somehow.
1464 const TypeInstPtr* tp = mem->in(MemNode::Address)->bottom_type()->isa_instptr();
1465 ciInstanceKlass* k = tp->klass()->as_instance_klass();
1466 bool oop_offset_is_sane = k->contains_field_offset(tp->offset());
1467 assert(!tp || oop_offset_is_sane, "");
1468 }
1469 #endif
1470 handled = true;
1471 break;
1472 default:
1473 handled = false;
1474 }
1475 return handled;
1476 }
1477
1478 bool ZBarrierSetC2::matcher_find_shared_visit(Matcher* matcher, Matcher::MStack& mstack, Node* n, uint opcode, bool& mem_op, int& mem_addr_idx) const {
1479 if (opcode == Op_CallLeaf &&
1480 (n->as_Call()->entry_point() == ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr() ||
1481 n->as_Call()->entry_point() == ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded_addr())) {
1482 mem_op = true;
1483 mem_addr_idx = TypeFunc::Parms + 1;
1484 return true;
1485 }
1486 return false;
1487 }
1488
1489 // == Verification ==
1490
1491 #ifdef ASSERT
1492
1493 static bool look_for_barrier(Node* n, bool post_parse, VectorSet& visited) {
1494 if (visited.test_set(n->_idx)) {
1495 return true;
1496 }
1497
1498 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
1499 Node* u = n->fast_out(i);
1500 if (u->is_LoadBarrier()) {
1501 } else if ((u->is_Phi() || u->is_CMove()) && !post_parse) {
1502 if (!look_for_barrier(u, post_parse, visited)) {
1503 return false;
1504 }
1505 } else if (u->Opcode() == Op_EncodeP || u->Opcode() == Op_DecodeN) {
1506 if (!look_for_barrier(u, post_parse, visited)) {
1507 return false;
1508 }
1509 } else if (u->Opcode() != Op_SCMemProj) {
1510 tty->print("bad use"); u->dump();
1511 return false;
1512 }
1513 }
1514
1515 return true;
1516 }
1517
1518 void ZBarrierSetC2::verify_gc_barriers(Compile* compile, CompilePhase phase) const {
1519 if (phase == BarrierSetC2::BeforeCodeGen) return;
1520 bool post_parse = phase == BarrierSetC2::BeforeOptimize;
1521 verify_gc_barriers(post_parse);
1522 }
1523
1524 void ZBarrierSetC2::verify_gc_barriers(bool post_parse) const {
1525 ZBarrierSetC2State* s = state();
1526 Compile* C = Compile::current();
1527 ResourceMark rm;
1528 VectorSet visited(Thread::current()->resource_area());
1529 for (int i = 0; i < s->load_barrier_count(); i++) {
1530 LoadBarrierNode* n = s->load_barrier_node(i);
1531
1532 // The dominating barrier on the same address if it exists and
1533 // this barrier must not be applied on the value from the same
1534 // load otherwise the value is not reloaded before it's used the
1535 // second time.
1536 assert(n->in(LoadBarrierNode::Similar)->is_top() ||
1537 (n->in(LoadBarrierNode::Similar)->in(0)->is_LoadBarrier() &&
1538 n->in(LoadBarrierNode::Similar)->in(0)->in(LoadBarrierNode::Address) == n->in(LoadBarrierNode::Address) &&
1539 n->in(LoadBarrierNode::Similar)->in(0)->in(LoadBarrierNode::Oop) != n->in(LoadBarrierNode::Oop)),
1540 "broken similar edge");
1541
1542 assert(post_parse || n->as_LoadBarrier()->has_true_uses(),
1543 "found unneeded load barrier");
1544
1545 // Several load barrier nodes chained through their Similar edge
1546 // break the code that remove the barriers in final graph reshape.
1547 assert(n->in(LoadBarrierNode::Similar)->is_top() ||
1548 (n->in(LoadBarrierNode::Similar)->in(0)->is_LoadBarrier() &&
1549 n->in(LoadBarrierNode::Similar)->in(0)->in(LoadBarrierNode::Similar)->is_top()),
1550 "chain of Similar load barriers");
1551
1552 if (!n->in(LoadBarrierNode::Similar)->is_top()) {
1553 ResourceMark rm;
1554 Unique_Node_List wq;
1555 Node* other = n->in(LoadBarrierNode::Similar)->in(0);
1556 wq.push(n);
1557 bool ok = true;
1558 bool dom_found = false;
1559 for (uint next = 0; next < wq.size(); ++next) {
1560 Node *n = wq.at(next);
1561 assert(n->is_CFG(), "");
1562 assert(!n->is_SafePoint(), "");
1563
1564 if (n == other) {
1565 continue;
1566 }
1567
1568 if (n->is_Region()) {
1569 for (uint i = 1; i < n->req(); i++) {
1570 Node* m = n->in(i);
1571 if (m != NULL) {
1572 wq.push(m);
1573 }
1574 }
1575 } else {
1576 Node* m = n->in(0);
1577 if (m != NULL) {
1578 wq.push(m);
1579 }
1580 }
1581 }
1582 }
1583
1584 if (ZVerifyLoadBarriers) {
1585 if ((n->is_Load() || n->is_LoadStore()) && n->bottom_type()->make_oopptr() != NULL) {
1586 visited.Clear();
1587 bool found = look_for_barrier(n, post_parse, visited);
1588 if (!found) {
1589 n->dump(1);
1590 n->dump(-3);
1591 stringStream ss;
1592 C->method()->print_short_name(&ss);
1593 tty->print_cr("-%s-", ss.as_string());
1594 assert(found, "");
1595 }
1596 }
1597 }
1598 }
1599 }
1600
1601 #endif
1602
1603 bool ZBarrierSetC2::escape_add_to_con_graph(ConnectionGraph* conn_graph, PhaseGVN* gvn, Unique_Node_List* delayed_worklist, Node* n, uint opcode) const {
1604 switch (opcode) {
1605 case Op_LoadBarrierSlowReg:
1606 case Op_LoadBarrierWeakSlowReg:
1607 conn_graph->add_objload_to_connection_graph(n, delayed_worklist);
1608 return true;
1609 case Op_Proj:
1610 if (n->as_Proj()->_con == LoadBarrierNode::Oop && n->in(0)->is_LoadBarrier()) {
1611 conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(0)->in(LoadBarrierNode::Oop),
1612 delayed_worklist);
1613 return true;
1614 }
1615 default:
1616 break;
1617 }
1618 return false;
1619 }
1620
1621 bool ZBarrierSetC2::escape_add_final_edges(ConnectionGraph* conn_graph, PhaseGVN* gvn, Node* n, uint opcode) const {
1622 switch (opcode) {
1623 case Op_LoadBarrierSlowReg:
1624 case Op_LoadBarrierWeakSlowReg: {
1625 const Type *t = gvn->type(n);
1626 if (t->make_ptr() != NULL) {
1627 Node *adr = n->in(MemNode::Address);
1628 conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, adr, NULL);
1629 return true;
1630 }
1631 }
1632 case Op_Proj: {
1633 if (n->as_Proj()->_con == LoadBarrierNode::Oop && n->in(0)->is_LoadBarrier()) {
1634 conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(0)->in(LoadBarrierNode::Oop), NULL);
1635 return true;
1636 }
1637 }
1638 default:
1639 break;
1640 }
1641 return false;
1642 }