1 /*
2 * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 #include "precompiled.hpp"
25 #include "opto/compile.hpp"
26 #include "opto/castnode.hpp"
27 #include "opto/graphKit.hpp"
28 #include "opto/idealKit.hpp"
29 #include "opto/loopnode.hpp"
30 #include "opto/macro.hpp"
31 #include "opto/node.hpp"
32 #include "opto/type.hpp"
33 #include "utilities/macros.hpp"
34 #include "gc/z/c2/zBarrierSetC2.hpp"
35 #include "gc/z/zThreadLocalData.hpp"
36 #include "gc/z/zBarrierSetRuntime.hpp"
37
38 ZBarrierSetC2State::ZBarrierSetC2State(Arena* comp_arena)
39 : _load_barrier_nodes(new (comp_arena) GrowableArray<LoadBarrierNode*>(comp_arena, 8, 0, NULL)) {}
40
41 int ZBarrierSetC2State::load_barrier_count() const {
42 return _load_barrier_nodes->length();
43 }
44
45 void ZBarrierSetC2State::add_load_barrier_node(LoadBarrierNode * n) {
46 assert(!_load_barrier_nodes->contains(n), " duplicate entry in expand list");
47 _load_barrier_nodes->append(n);
48 }
49
50 void ZBarrierSetC2State::remove_load_barrier_node(LoadBarrierNode * n) {
51 // this function may be called twice for a node so check
52 // that the node is in the array before attempting to remove it
53 if (_load_barrier_nodes->contains(n)) {
54 _load_barrier_nodes->remove(n);
55 }
56 }
57
58 LoadBarrierNode* ZBarrierSetC2State::load_barrier_node(int idx) const {
59 return _load_barrier_nodes->at(idx);
60 }
61
62 void* ZBarrierSetC2::create_barrier_state(Arena* comp_arena) const {
63 return new(comp_arena) ZBarrierSetC2State(comp_arena);
64 }
65
66 ZBarrierSetC2State* ZBarrierSetC2::state() const {
67 return reinterpret_cast<ZBarrierSetC2State*>(Compile::current()->barrier_set_state());
68 }
69
70 bool ZBarrierSetC2::is_gc_barrier_node(Node* node) const {
71 return node->is_LoadBarrier();
72 }
73
74 void ZBarrierSetC2::register_potential_barrier_node(Node* node) const {
75 if (node->is_LoadBarrier()) {
76 state()->add_load_barrier_node(node->as_LoadBarrier());
77 }
78 }
79
80 void ZBarrierSetC2::unregister_potential_barrier_node(Node* node) const {
81 if (node->is_LoadBarrier()) {
82 state()->remove_load_barrier_node(node->as_LoadBarrier());
83 }
84 }
85
86 void ZBarrierSetC2::eliminate_useless_gc_barriers(Unique_Node_List &useful) const {
87 // Remove useless LoadBarrier nodes
88 ZBarrierSetC2State* s = state();
89 for (int i = s->load_barrier_count()-1; i >= 0; i--) {
90 LoadBarrierNode* n = s->load_barrier_node(i);
91 if (!useful.member(n)) {
92 unregister_potential_barrier_node(n);
93 }
94 }
95 }
96
97 void ZBarrierSetC2::enqueue_useful_gc_barrier(Unique_Node_List &worklist, Node* node) const {
98 if (node->is_LoadBarrier() && !node->as_LoadBarrier()->has_true_uses()) {
99 worklist.push(node);
100 }
101 }
102
103 void ZBarrierSetC2::find_dominating_barriers(PhaseIterGVN& igvn) {
104 // Look for dominating barriers on the same address only once all
105 // other loop opts are over: loop opts may cause a safepoint to be
106 // inserted between a barrier and its dominating barrier.
107 Compile* C = Compile::current();
108 ZBarrierSetC2* bs = (ZBarrierSetC2*)BarrierSet::barrier_set()->barrier_set_c2();
109 ZBarrierSetC2State* s = bs->state();
110 if (s->load_barrier_count() >= 2) {
111 Compile::TracePhase tp("idealLoop", &C->timers[Phase::_t_idealLoop]);
112 PhaseIdealLoop ideal_loop(igvn, true, false, true);
113 if (C->major_progress()) C->print_method(PHASE_PHASEIDEALLOOP_ITERATIONS, 2);
114 }
115 }
116
117 void ZBarrierSetC2::add_users_to_worklist(Unique_Node_List* worklist) const {
118 // Permanent temporary workaround
119 // Loadbarriers may have non-obvious dead uses keeping them alive during parsing. The use is
120 // removed by RemoveUseless (after parsing, before optimize) but the barriers won't be added to
121 // the worklist. Unless we add them explicitly they are not guaranteed to end up there.
122 ZBarrierSetC2State* s = state();
123
124 for (int i = 0; i < s->load_barrier_count(); i++) {
125 LoadBarrierNode* n = s->load_barrier_node(i);
126 worklist->push(n);
127 }
128 }
129
130 const TypeFunc* ZBarrierSetC2::load_barrier_Type() const {
131 const Type** fields;
132
133 // Create input types (domain)
134 fields = TypeTuple::fields(2);
135 fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL;
136 fields[TypeFunc::Parms+1] = TypeOopPtr::BOTTOM;
137 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2, fields);
138
139 // Create result type (range)
140 fields = TypeTuple::fields(1);
141 fields[TypeFunc::Parms+0] = TypeInstPtr::BOTTOM;
142 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+1, fields);
143
144 return TypeFunc::make(domain, range);
145 }
146
147 // == LoadBarrierNode ==
148
149 LoadBarrierNode::LoadBarrierNode(Compile* C,
150 Node* c,
151 Node* mem,
152 Node* val,
153 Node* adr,
154 bool weak,
155 bool writeback,
156 bool oop_reload_allowed) :
157 MultiNode(Number_of_Inputs),
158 _weak(weak),
159 _writeback(writeback),
160 _oop_reload_allowed(oop_reload_allowed) {
161 init_req(Control, c);
162 init_req(Memory, mem);
163 init_req(Oop, val);
164 init_req(Address, adr);
165 init_req(Similar, C->top());
166
167 init_class_id(Class_LoadBarrier);
168 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
169 bs->register_potential_barrier_node(this);
170 }
171
172 const Type *LoadBarrierNode::bottom_type() const {
173 const Type** floadbarrier = (const Type **)(Compile::current()->type_arena()->Amalloc_4((Number_of_Outputs)*sizeof(Type*)));
174 Node* in_oop = in(Oop);
175 floadbarrier[Control] = Type::CONTROL;
176 floadbarrier[Memory] = Type::MEMORY;
177 floadbarrier[Oop] = in_oop == NULL ? Type::TOP : in_oop->bottom_type();
178 return TypeTuple::make(Number_of_Outputs, floadbarrier);
179 }
180
181 const Type *LoadBarrierNode::Value(PhaseGVN *phase) const {
182 const Type** floadbarrier = (const Type **)(phase->C->type_arena()->Amalloc_4((Number_of_Outputs)*sizeof(Type*)));
183 const Type* val_t = phase->type(in(Oop));
184 floadbarrier[Control] = Type::CONTROL;
185 floadbarrier[Memory] = Type::MEMORY;
186 floadbarrier[Oop] = val_t;
187 return TypeTuple::make(Number_of_Outputs, floadbarrier);
188 }
189
190 bool LoadBarrierNode::is_dominator(PhaseIdealLoop* phase, bool linear_only, Node *d, Node *n) {
191 if (phase != NULL) {
192 return phase->is_dominator(d, n);
193 }
194
195 for (int i = 0; i < 10 && n != NULL; i++) {
196 n = IfNode::up_one_dom(n, linear_only);
197 if (n == d) {
198 return true;
199 }
200 }
201
202 return false;
203 }
204
205 LoadBarrierNode* LoadBarrierNode::has_dominating_barrier(PhaseIdealLoop* phase, bool linear_only, bool look_for_similar) {
206 Node* val = in(LoadBarrierNode::Oop);
207 if (in(Similar)->is_Proj() && in(Similar)->in(0)->is_LoadBarrier()) {
208 LoadBarrierNode* lb = in(Similar)->in(0)->as_LoadBarrier();
209 assert(lb->in(Address) == in(Address), "");
210 // Load barrier on Similar edge dominates so if it now has the Oop field it can replace this barrier.
211 if (lb->in(Oop) == in(Oop)) {
212 return lb;
213 }
214 // Follow chain of load barrier through Similar edges
215 while (!lb->in(Similar)->is_top()) {
216 lb = lb->in(Similar)->in(0)->as_LoadBarrier();
217 assert(lb->in(Address) == in(Address), "");
218 }
219 if (lb != in(Similar)->in(0)) {
220 return lb;
221 }
222 }
223 for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) {
224 Node* u = val->fast_out(i);
225 if (u != this && u->is_LoadBarrier() && u->in(Oop) == val && u->as_LoadBarrier()->has_true_uses()) {
226 Node* this_ctrl = in(LoadBarrierNode::Control);
227 Node* other_ctrl = u->in(LoadBarrierNode::Control);
228 if (is_dominator(phase, linear_only, other_ctrl, this_ctrl)) {
229 return u->as_LoadBarrier();
230 }
231 }
232 }
233
234 if (ZVerifyLoadBarriers || can_be_eliminated()) {
235 return NULL;
236 }
237
238 if (!look_for_similar) {
239 return NULL;
240 }
241
242 Node* addr = in(LoadBarrierNode::Address);
243 for (DUIterator_Fast imax, i = addr->fast_outs(imax); i < imax; i++) {
244 Node* u = addr->fast_out(i);
245 if (u != this && u->is_LoadBarrier() && u->as_LoadBarrier()->has_true_uses()) {
246 Node* this_ctrl = in(LoadBarrierNode::Control);
247 Node* other_ctrl = u->in(LoadBarrierNode::Control);
248 if (is_dominator(phase, linear_only, other_ctrl, this_ctrl)) {
249 ResourceMark rm;
250 Unique_Node_List wq;
251 wq.push(in(LoadBarrierNode::Control));
252 bool ok = true;
253 bool dom_found = false;
254 for (uint next = 0; next < wq.size(); ++next) {
255 Node *n = wq.at(next);
256 if (n->is_top()) {
257 return NULL;
258 }
259 assert(n->is_CFG(), "");
260 if (n->is_SafePoint()) {
261 ok = false;
262 break;
263 }
264 if (n == u) {
265 dom_found = true;
266 continue;
267 }
268 if (n->is_Region()) {
269 for (uint i = 1; i < n->req(); i++) {
270 Node* m = n->in(i);
271 if (m != NULL) {
272 wq.push(m);
273 }
274 }
275 } else {
276 Node* m = n->in(0);
277 if (m != NULL) {
278 wq.push(m);
279 }
280 }
281 }
282 if (ok) {
283 assert(dom_found, "");
284 return u->as_LoadBarrier();;
285 }
286 break;
287 }
288 }
289 }
290
291 return NULL;
292 }
293
294 void LoadBarrierNode::push_dominated_barriers(PhaseIterGVN* igvn) const {
295 // change to that barrier may affect a dominated barrier so re-push those
296 Node* val = in(LoadBarrierNode::Oop);
297
298 for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) {
299 Node* u = val->fast_out(i);
300 if (u != this && u->is_LoadBarrier() && u->in(Oop) == val) {
301 Node* this_ctrl = in(Control);
302 Node* other_ctrl = u->in(Control);
303 if (is_dominator(NULL, false, this_ctrl, other_ctrl)) {
304 igvn->_worklist.push(u);
305 }
306 }
307
308 Node* addr = in(LoadBarrierNode::Address);
309 for (DUIterator_Fast imax, i = addr->fast_outs(imax); i < imax; i++) {
310 Node* u = addr->fast_out(i);
311 if (u != this && u->is_LoadBarrier() && u->in(Similar)->is_top()) {
312 Node* this_ctrl = in(Control);
313 Node* other_ctrl = u->in(Control);
314 if (is_dominator(NULL, false, this_ctrl, other_ctrl)) {
315 igvn->_worklist.push(u);
316 }
317 }
318 }
319 }
320 }
321
322 Node *LoadBarrierNode::Identity(PhaseGVN *phase) {
323 if (!phase->C->directive()->ZOptimizeLoadBarriersOption) {
324 return this;
325 }
326
327 bool redundant_addr = false;
328 LoadBarrierNode* dominating_barrier = has_dominating_barrier(NULL, true, false);
329 if (dominating_barrier != NULL) {
330 assert(dominating_barrier->in(Oop) == in(Oop), "");
331 return dominating_barrier;
332 }
333
334 return this;
335 }
336
337 Node *LoadBarrierNode::Ideal(PhaseGVN *phase, bool can_reshape) {
338 if (remove_dead_region(phase, can_reshape)) {
339 return this;
340 }
341
342 Node* val = in(Oop);
343 Node* mem = in(Memory);
344 Node* ctrl = in(Control);
345 Node* adr = in(Address);
346 assert(val->Opcode() != Op_LoadN, "");
347
348 if (mem->is_MergeMem()) {
349 Node* new_mem = mem->as_MergeMem()->memory_at(Compile::AliasIdxRaw);
350 set_req(Memory, new_mem);
351 if (mem->outcnt() == 0 && can_reshape) {
352 phase->is_IterGVN()->_worklist.push(mem);
353 }
354
355 return this;
356 }
357
358 bool optimizeLoadBarriers = phase->C->directive()->ZOptimizeLoadBarriersOption;
359 LoadBarrierNode* dominating_barrier = optimizeLoadBarriers ? has_dominating_barrier(NULL, !can_reshape, !phase->C->major_progress()) : NULL;
360 if (dominating_barrier != NULL && dominating_barrier->in(Oop) != in(Oop)) {
361 assert(in(Address) == dominating_barrier->in(Address), "");
362 set_req(Similar, dominating_barrier->proj_out(Oop));
363 return this;
364 }
365
366 bool eliminate = (optimizeLoadBarriers && !(val->is_Phi() || val->Opcode() == Op_LoadP || val->Opcode() == Op_GetAndSetP || val->is_DecodeN())) ||
367 (can_reshape && (dominating_barrier != NULL || !has_true_uses()));
368
369 if (eliminate) {
370 if (can_reshape) {
371 PhaseIterGVN* igvn = phase->is_IterGVN();
372 Node* out_ctrl = proj_out_or_null(Control);
373 Node* out_res = proj_out_or_null(Oop);
374
375 if (out_ctrl != NULL) {
376 igvn->replace_node(out_ctrl, ctrl);
377 }
378
379 // That transformation may cause the Similar edge on the load barrier to be invalid
380 fix_similar_in_uses(igvn);
381 if (out_res != NULL) {
382 if (dominating_barrier != NULL) {
383 igvn->replace_node(out_res, dominating_barrier->proj_out(Oop));
384 } else {
385 igvn->replace_node(out_res, val);
386 }
387 }
388 }
389
390 return new ConINode(TypeInt::ZERO);
391 }
392
393 // If the Similar edge is no longer a load barrier, clear it
394 Node* similar = in(Similar);
395 if (!similar->is_top() && !(similar->is_Proj() && similar->in(0)->is_LoadBarrier())) {
396 set_req(Similar, phase->C->top());
397 return this;
398 }
399
400 if (can_reshape) {
401 // If this barrier is linked through the Similar edge by a
402 // dominated barrier and both barriers have the same Oop field,
403 // the dominated barrier can go away, so push it for reprocessing.
404 // We also want to avoid a barrier to depend on another dominating
405 // barrier through its Similar edge that itself depend on another
406 // barrier through its Similar edge and rather have the first
407 // depend on the third.
408 PhaseIterGVN* igvn = phase->is_IterGVN();
409 Node* out_res = proj_out(Oop);
410 for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) {
411 Node* u = out_res->fast_out(i);
412 if (u->is_LoadBarrier() && u->in(Similar) == out_res &&
413 (u->in(Oop) == val || !u->in(Similar)->is_top())) {
414 igvn->_worklist.push(u);
415 }
416 }
417
418 push_dominated_barriers(igvn);
419 }
420
421 return NULL;
422 }
423
424 void LoadBarrierNode::fix_similar_in_uses(PhaseIterGVN* igvn) {
425 Node* out_res = proj_out_or_null(Oop);
426 if (out_res == NULL) {
427 return;
428 }
429
430 for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) {
431 Node* u = out_res->fast_out(i);
432 if (u->is_LoadBarrier() && u->in(Similar) == out_res) {
433 igvn->replace_input_of(u, Similar, igvn->C->top());
434 --i;
435 --imax;
436 }
437 }
438 }
439
440 bool LoadBarrierNode::has_true_uses() const {
441 Node* out_res = proj_out_or_null(Oop);
442 if (out_res == NULL) {
443 return false;
444 }
445
446 for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) {
447 Node* u = out_res->fast_out(i);
448 if (!u->is_LoadBarrier() || u->in(Similar) != out_res) {
449 return true;
450 }
451 }
452
453 return false;
454 }
455
456 // == Accesses ==
457
458 Node* ZBarrierSetC2::make_cas_loadbarrier(C2AtomicAccess& access) const {
459 assert(!UseCompressedOops, "Not allowed");
460 CompareAndSwapNode* cas = (CompareAndSwapNode*)access.raw_access();
461 PhaseGVN& gvn = access.kit()->gvn();
462 Compile* C = Compile::current();
463 GraphKit* kit = access.kit();
464
465 Node* in_ctrl = cas->in(MemNode::Control);
466 Node* in_mem = cas->in(MemNode::Memory);
467 Node* in_adr = cas->in(MemNode::Address);
468 Node* in_val = cas->in(MemNode::ValueIn);
469 Node* in_expected = cas->in(LoadStoreConditionalNode::ExpectedIn);
470
471 float likely = PROB_LIKELY(0.999);
472
473 const TypePtr *adr_type = gvn.type(in_adr)->isa_ptr();
474 Compile::AliasType* alias_type = C->alias_type(adr_type);
475 int alias_idx = C->get_alias_index(adr_type);
476
477 // Outer check - true: continue, false: load and check
478 Node* region = new RegionNode(3);
479 Node* phi = new PhiNode(region, TypeInt::BOOL);
480 Node* phi_mem = new PhiNode(region, Type::MEMORY, adr_type);
481
482 // Inner check - is the healed ref equal to the expected
483 Node* region2 = new RegionNode(3);
484 Node* phi2 = new PhiNode(region2, TypeInt::BOOL);
485 Node* phi_mem2 = new PhiNode(region2, Type::MEMORY, adr_type);
486
487 // CAS node returns 0 or 1
488 Node* cmp = gvn.transform(new CmpINode(cas, kit->intcon(0)));
489 Node* bol = gvn.transform(new BoolNode(cmp, BoolTest::ne))->as_Bool();
490 IfNode* iff = gvn.transform(new IfNode(in_ctrl, bol, likely, COUNT_UNKNOWN))->as_If();
491 Node* then = gvn.transform(new IfTrueNode(iff));
492 Node* elsen = gvn.transform(new IfFalseNode(iff));
493
494 Node* scmemproj1 = gvn.transform(new SCMemProjNode(cas));
495
496 kit->set_memory(scmemproj1, alias_idx);
497 phi_mem->init_req(1, scmemproj1);
498 phi_mem2->init_req(2, scmemproj1);
499
500 // CAS fail - reload and heal oop
501 Node* reload = kit->make_load(elsen, in_adr, TypeOopPtr::BOTTOM, T_OBJECT, MemNode::unordered);
502 Node* barrier = gvn.transform(new LoadBarrierNode(C, elsen, scmemproj1, reload, in_adr, false, true, false));
503 Node* barrierctrl = gvn.transform(new ProjNode(barrier, LoadBarrierNode::Control));
504 Node* barrierdata = gvn.transform(new ProjNode(barrier, LoadBarrierNode::Oop));
505
506 // Check load
507 Node* tmpX = gvn.transform(new CastP2XNode(NULL, barrierdata));
508 Node* in_expX = gvn.transform(new CastP2XNode(NULL, in_expected));
509 Node* cmp2 = gvn.transform(new CmpXNode(tmpX, in_expX));
510 Node *bol2 = gvn.transform(new BoolNode(cmp2, BoolTest::ne))->as_Bool();
511 IfNode* iff2 = gvn.transform(new IfNode(barrierctrl, bol2, likely, COUNT_UNKNOWN))->as_If();
512 Node* then2 = gvn.transform(new IfTrueNode(iff2));
513 Node* elsen2 = gvn.transform(new IfFalseNode(iff2));
514
515 // redo CAS
516 Node* cas2 = gvn.transform(new CompareAndSwapPNode(elsen2, kit->memory(alias_idx), in_adr, in_val, in_expected, cas->order()));
517 Node* scmemproj2 = gvn.transform(new SCMemProjNode(cas2));
518 kit->set_control(elsen2);
519 kit->set_memory(scmemproj2, alias_idx);
520
521 // Merge inner flow - check if healed oop was equal too expected.
522 region2->set_req(1, kit->control());
523 region2->set_req(2, then2);
524 phi2->set_req(1, cas2);
525 phi2->set_req(2, kit->intcon(0));
526 phi_mem2->init_req(1, scmemproj2);
527 kit->set_memory(phi_mem2, alias_idx);
528
529 // Merge outer flow - then check if first cas succeded
530 region->set_req(1, then);
531 region->set_req(2, region2);
532 phi->set_req(1, kit->intcon(1));
533 phi->set_req(2, phi2);
534 phi_mem->init_req(2, phi_mem2);
535 kit->set_memory(phi_mem, alias_idx);
536
537 gvn.transform(region2);
538 gvn.transform(phi2);
539 gvn.transform(phi_mem2);
540 gvn.transform(region);
541 gvn.transform(phi);
542 gvn.transform(phi_mem);
543
544 kit->set_control(region);
545 kit->insert_mem_bar(Op_MemBarCPUOrder);
546
547 return phi;
548 }
549
550 Node* ZBarrierSetC2::make_cmpx_loadbarrier(C2AtomicAccess& access) const {
551 CompareAndExchangePNode* cmpx = (CompareAndExchangePNode*)access.raw_access();
552 GraphKit* kit = access.kit();
553 PhaseGVN& gvn = kit->gvn();
554 Compile* C = Compile::current();
555
556 Node* in_ctrl = cmpx->in(MemNode::Control);
557 Node* in_mem = cmpx->in(MemNode::Memory);
558 Node* in_adr = cmpx->in(MemNode::Address);
559 Node* in_val = cmpx->in(MemNode::ValueIn);
560 Node* in_expected = cmpx->in(LoadStoreConditionalNode::ExpectedIn);
561
562 float likely = PROB_LIKELY(0.999);
563
564 const TypePtr *adr_type = cmpx->get_ptr_type();
565 Compile::AliasType* alias_type = C->alias_type(adr_type);
566 int alias_idx = C->get_alias_index(adr_type);
567
568 // Outer check - true: continue, false: load and check
569 Node* region = new RegionNode(3);
570 Node* phi = new PhiNode(region, adr_type);
571
572 // Inner check - is the healed ref equal to the expected
573 Node* region2 = new RegionNode(3);
574 Node* phi2 = new PhiNode(region2, adr_type);
575
576 // Check if cmpx succeded
577 Node* cmp = gvn.transform(new CmpPNode(cmpx, in_expected));
578 Node* bol = gvn.transform(new BoolNode(cmp, BoolTest::eq))->as_Bool();
579 IfNode* iff = gvn.transform(new IfNode(in_ctrl, bol, likely, COUNT_UNKNOWN))->as_If();
580 Node* then = gvn.transform(new IfTrueNode(iff));
581 Node* elsen = gvn.transform(new IfFalseNode(iff));
582
583 Node* scmemproj1 = gvn.transform(new SCMemProjNode(cmpx));
584 kit->set_memory(scmemproj1, alias_idx);
585
586 // CAS fail - reload and heal oop
587 Node* reload = kit->make_load(elsen, in_adr, TypeOopPtr::BOTTOM, T_OBJECT, MemNode::unordered);
588 Node* barrier = gvn.transform(new LoadBarrierNode(C, elsen, scmemproj1, reload, in_adr, false, true, false));
589 Node* barrierctrl = gvn.transform(new ProjNode(barrier, LoadBarrierNode::Control));
590 Node* barrierdata = gvn.transform(new ProjNode(barrier, LoadBarrierNode::Oop));
591
592 // Check load
593 Node* tmpX = gvn.transform(new CastP2XNode(NULL, barrierdata));
594 Node* in_expX = gvn.transform(new CastP2XNode(NULL, in_expected));
595 Node* cmp2 = gvn.transform(new CmpXNode(tmpX, in_expX));
596 Node *bol2 = gvn.transform(new BoolNode(cmp2, BoolTest::ne))->as_Bool();
597 IfNode* iff2 = gvn.transform(new IfNode(barrierctrl, bol2, likely, COUNT_UNKNOWN))->as_If();
598 Node* then2 = gvn.transform(new IfTrueNode(iff2));
599 Node* elsen2 = gvn.transform(new IfFalseNode(iff2));
600
601 // Redo CAS
602 Node* cmpx2 = gvn.transform(new CompareAndExchangePNode(elsen2, kit->memory(alias_idx), in_adr, in_val, in_expected, adr_type, cmpx->get_ptr_type(), cmpx->order()));
603 Node* scmemproj2 = gvn.transform(new SCMemProjNode(cmpx2));
604 kit->set_control(elsen2);
605 kit->set_memory(scmemproj2, alias_idx);
606
607 // Merge inner flow - check if healed oop was equal too expected.
608 region2->set_req(1, kit->control());
609 region2->set_req(2, then2);
610 phi2->set_req(1, cmpx2);
611 phi2->set_req(2, barrierdata);
612
613 // Merge outer flow - then check if first cas succeded
614 region->set_req(1, then);
615 region->set_req(2, region2);
616 phi->set_req(1, cmpx);
617 phi->set_req(2, phi2);
618
619 gvn.transform(region2);
620 gvn.transform(phi2);
621 gvn.transform(region);
622 gvn.transform(phi);
623
624 kit->set_control(region);
625 kit->set_memory(in_mem, alias_idx);
626 kit->insert_mem_bar(Op_MemBarCPUOrder);
627
628 return phi;
629 }
630
631 Node* ZBarrierSetC2::load_barrier(GraphKit* kit, Node* val, Node* adr, bool weak, bool writeback, bool oop_reload_allowed) const {
632 PhaseGVN& gvn = kit->gvn();
633 Node* barrier = new LoadBarrierNode(Compile::current(), kit->control(), kit->memory(TypeRawPtr::BOTTOM), val, adr, weak, writeback, oop_reload_allowed);
634 Node* transformed_barrier = gvn.transform(barrier);
635
636 if (transformed_barrier->is_LoadBarrier()) {
637 if (barrier == transformed_barrier) {
638 kit->set_control(gvn.transform(new ProjNode(barrier, LoadBarrierNode::Control)));
639 }
640 return gvn.transform(new ProjNode(transformed_barrier, LoadBarrierNode::Oop));
641 } else {
642 return val;
643 }
644 }
645
646 static bool barrier_needed(C2Access access) {
647 return ZBarrierSet::barrier_needed(access.decorators(), access.type());
648 }
649
650 Node* ZBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const {
651 Node* p = BarrierSetC2::load_at_resolved(access, val_type);
652 if (!barrier_needed(access)) {
653 return p;
654 }
655
656 bool conc_root = (access.decorators() & IN_CONCURRENT_ROOT) != 0;
657 bool weak = (access.decorators() & ON_WEAK_OOP_REF) != 0;
658
659 GraphKit* kit = access.kit();
660 PhaseGVN& gvn = kit->gvn();
661 Node* adr = access.addr().node();
662 Node* heap_base_oop = access.base();
663 bool unsafe = (access.decorators() & C2_UNSAFE_ACCESS) != 0;
664 if (unsafe) {
665 if (!ZVerifyLoadBarriers) {
666 p = load_barrier(kit, p, adr);
667 } else {
668 if (!TypePtr::NULL_PTR->higher_equal(gvn.type(heap_base_oop))) {
669 p = load_barrier(kit, p, adr);
670 } else {
671 IdealKit ideal(kit);
672 IdealVariable res(ideal);
673 #define __ ideal.
674 __ declarations_done();
675 __ set(res, p);
676 __ if_then(heap_base_oop, BoolTest::ne, kit->null(), PROB_UNLIKELY(0.999)); {
677 kit->sync_kit(ideal);
678 p = load_barrier(kit, p, adr);
679 __ set(res, p);
680 __ sync_kit(kit);
681 } __ end_if();
682 kit->final_sync(ideal);
683 p = __ value(res);
684 #undef __
685 }
686 }
687 return p;
688 } else {
689 return load_barrier(access.kit(), p, access.addr().node(), weak, true, true);
690 }
691 }
692
693 Node* ZBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicAccess& access, Node* expected_val,
694 Node* new_val, const Type* val_type) const {
695 Node* result = BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, val_type);
696 if (!barrier_needed(access)) {
697 return result;
698 }
699
700 access.set_needs_pinning(false);
701 return make_cmpx_loadbarrier(access);
702 }
703
704 Node* ZBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicAccess& access, Node* expected_val,
705 Node* new_val, const Type* value_type) const {
706 Node* result = BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type);
707 if (!barrier_needed(access)) {
708 return result;
709 }
710
711 Node* load_store = access.raw_access();
712 bool weak_cas = (access.decorators() & C2_WEAK_CMPXCHG) != 0;
713 bool expected_is_null = (expected_val->get_ptr_type() == TypePtr::NULL_PTR);
714
715 if (!expected_is_null) {
716 if (weak_cas) {
717 access.set_needs_pinning(false);
718 load_store = make_cas_loadbarrier(access);
719 } else {
720 access.set_needs_pinning(false);
721 load_store = make_cas_loadbarrier(access);
722 }
723 }
724
725 return load_store;
726 }
727
728 Node* ZBarrierSetC2::atomic_xchg_at_resolved(C2AtomicAccess& access, Node* new_val, const Type* val_type) const {
729 Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, new_val, val_type);
730 if (!barrier_needed(access)) {
731 return result;
732 }
733
734 Node* load_store = access.raw_access();
735 Node* adr = access.addr().node();
736
737 return load_barrier(access.kit(), load_store, adr, false, false, false);
738 }
739
740 // == Macro Expansion ==
741
742 void ZBarrierSetC2::expand_loadbarrier_node(PhaseMacroExpand* phase, LoadBarrierNode* barrier) const {
743 Node* in_ctrl = barrier->in(LoadBarrierNode::Control);
744 Node* in_mem = barrier->in(LoadBarrierNode::Memory);
745 Node* in_val = barrier->in(LoadBarrierNode::Oop);
746 Node* in_adr = barrier->in(LoadBarrierNode::Address);
747
748 Node* out_ctrl = barrier->proj_out(LoadBarrierNode::Control);
749 Node* out_res = barrier->proj_out(LoadBarrierNode::Oop);
750
751 PhaseIterGVN &igvn = phase->igvn();
752
753 if (ZVerifyLoadBarriers) {
754 igvn.replace_node(out_res, in_val);
755 igvn.replace_node(out_ctrl, in_ctrl);
756 return;
757 }
758
759 if (barrier->can_be_eliminated()) {
760 // Clone and pin the load for this barrier below the dominating
761 // barrier: the load cannot be allowed to float above the
762 // dominating barrier
763 Node* load = in_val;
764 Node* decode = NULL;
765 if (load->is_DecodeN()) {
766 decode = load;
767 load = load->in(1);
768 }
769 if (load->is_Load()) {
770 Node* new_load = load->clone();
771 Node* addp = new_load->in(MemNode::Address);
772 assert(addp->is_AddP() || addp->is_Phi(), "bad address");
773 Node* cast = new CastPPNode(addp, igvn.type(addp), true);
774 Node* ctrl = NULL;
775 Node* similar = barrier->in(LoadBarrierNode::Similar);
776 if (similar->is_Phi()) {
777 // already expanded
778 ctrl = similar->in(0);
779 } else {
780 assert(similar->is_Proj() && similar->in(0)->is_LoadBarrier(), "unexpected graph shape");
781 ctrl = similar->in(0)->as_LoadBarrier()->proj_out(LoadBarrierNode::Control);
782 }
783 assert(ctrl != NULL, "bad control");
784 cast->set_req(0, ctrl);
785 igvn.transform(cast);
786 new_load->set_req(MemNode::Address, cast);
787 igvn.transform(new_load);
788
789 Node* new_in_val = new_load;
790 if (decode != NULL) {
791 new_in_val = decode->clone();
792 new_in_val->set_req(1, new_load);
793 igvn.transform(new_in_val);
794 }
795
796 igvn.replace_node(out_res, new_in_val);
797 igvn.replace_node(out_ctrl, in_ctrl);
798 return;
799 }
800 // cannot eliminate
801 }
802
803 // There are two cases that require the basic loadbarrier
804 // 1) When the writeback of a healed oop must be avoided (swap)
805 // 2) When we must guarantee that no reload of is done (swap, cas, cmpx)
806 if (!barrier->is_writeback()) {
807 assert(!barrier->oop_reload_allowed(), "writeback barriers should be marked as requires oop");
808 }
809
810 if (!barrier->oop_reload_allowed()) {
811 expand_loadbarrier_basic(phase, barrier);
812 } else {
813 expand_loadbarrier_optimized(phase, barrier);
814 }
815 }
816
817 // Basic loadbarrier using conventional arg passing
818 void ZBarrierSetC2::expand_loadbarrier_basic(PhaseMacroExpand* phase, LoadBarrierNode *barrier) const {
819 PhaseIterGVN &igvn = phase->igvn();
820
821 Node* in_ctrl = barrier->in(LoadBarrierNode::Control);
822 Node* in_mem = barrier->in(LoadBarrierNode::Memory);
823 Node* in_val = barrier->in(LoadBarrierNode::Oop);
824 Node* in_adr = barrier->in(LoadBarrierNode::Address);
825
826 Node* out_ctrl = barrier->proj_out(LoadBarrierNode::Control);
827 Node* out_res = barrier->proj_out(LoadBarrierNode::Oop);
828
829 float unlikely = PROB_UNLIKELY(0.999);
830 const Type* in_val_maybe_null_t = igvn.type(in_val);
831
832 Node* jthread = igvn.transform(new ThreadLocalNode());
833 Node* adr = phase->basic_plus_adr(jthread, in_bytes(ZThreadLocalData::address_bad_mask_offset()));
834 Node* bad_mask = igvn.transform(LoadNode::make(igvn, in_ctrl, in_mem, adr, TypeRawPtr::BOTTOM, TypeX_X, TypeX_X->basic_type(), MemNode::unordered));
835 Node* cast = igvn.transform(new CastP2XNode(in_ctrl, in_val));
836 Node* obj_masked = igvn.transform(new AndXNode(cast, bad_mask));
837 Node* cmp = igvn.transform(new CmpXNode(obj_masked, igvn.zerocon(TypeX_X->basic_type())));
838 Node *bol = igvn.transform(new BoolNode(cmp, BoolTest::ne))->as_Bool();
839 IfNode* iff = igvn.transform(new IfNode(in_ctrl, bol, unlikely, COUNT_UNKNOWN))->as_If();
840 Node* then = igvn.transform(new IfTrueNode(iff));
841 Node* elsen = igvn.transform(new IfFalseNode(iff));
842
843 Node* result_region;
844 Node* result_val;
845
846 result_region = new RegionNode(3);
847 result_val = new PhiNode(result_region, TypeInstPtr::BOTTOM);
848
849 result_region->set_req(1, elsen);
850 Node* res = igvn.transform(new CastPPNode(in_val, in_val_maybe_null_t));
851 res->init_req(0, elsen);
852 result_val->set_req(1, res);
853
854 const TypeFunc *tf = load_barrier_Type();
855 Node* call;
856 if (barrier->is_weak()) {
857 call = new CallLeafNode(tf,
858 ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded_addr(),
859 "ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded",
860 TypeRawPtr::BOTTOM);
861 } else {
862 call = new CallLeafNode(tf,
863 ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(),
864 "ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded",
865 TypeRawPtr::BOTTOM);
866 }
867
868 call->init_req(TypeFunc::Control, then);
869 call->init_req(TypeFunc::I_O , phase->top());
870 call->init_req(TypeFunc::Memory , in_mem);
871 call->init_req(TypeFunc::FramePtr, phase->top());
872 call->init_req(TypeFunc::ReturnAdr, phase->top());
873 call->init_req(TypeFunc::Parms+0, in_val);
874 if (barrier->is_writeback()) {
875 call->init_req(TypeFunc::Parms+1, in_adr);
876 } else {
877 // when slow path is called with a null adr, the healed oop will not be written back
878 call->init_req(TypeFunc::Parms+1, igvn.zerocon(T_OBJECT));
879 }
880 call = igvn.transform(call);
881
882 Node* ctrl = igvn.transform(new ProjNode(call, TypeFunc::Control));
883 res = igvn.transform(new ProjNode(call, TypeFunc::Parms));
884 res = igvn.transform(new CheckCastPPNode(ctrl, res, in_val_maybe_null_t));
885
886 result_region->set_req(2, ctrl);
887 result_val->set_req(2, res);
888
889 result_region = igvn.transform(result_region);
890 result_val = igvn.transform(result_val);
891
892 if (out_ctrl != NULL) { // added if cond
893 igvn.replace_node(out_ctrl, result_region);
894 }
895 igvn.replace_node(out_res, result_val);
896 }
897
898 // Optimized, low spill, loadbarrier variant using stub specialized on register used
899 void ZBarrierSetC2::expand_loadbarrier_optimized(PhaseMacroExpand* phase, LoadBarrierNode *barrier) const {
900 PhaseIterGVN &igvn = phase->igvn();
901 #ifdef PRINT_NODE_TRAVERSALS
902 Node* preceding_barrier_node = barrier->in(LoadBarrierNode::Oop);
903 #endif
904
905 Node* in_ctrl = barrier->in(LoadBarrierNode::Control);
906 Node* in_mem = barrier->in(LoadBarrierNode::Memory);
907 Node* in_val = barrier->in(LoadBarrierNode::Oop);
908 Node* in_adr = barrier->in(LoadBarrierNode::Address);
909
910 Node* out_ctrl = barrier->proj_out(LoadBarrierNode::Control);
911 Node* out_res = barrier->proj_out(LoadBarrierNode::Oop);
912
913 assert(barrier->in(LoadBarrierNode::Oop) != NULL, "oop to loadbarrier node cannot be null");
914
915 #ifdef PRINT_NODE_TRAVERSALS
916 tty->print("\n\n\nBefore barrier optimization:\n");
917 traverse(barrier, out_ctrl, out_res, -1);
918
919 tty->print("\nBefore barrier optimization: preceding_barrier_node\n");
920 traverse(preceding_barrier_node, out_ctrl, out_res, -1);
921 #endif
922
923 float unlikely = PROB_UNLIKELY(0.999);
924
925 Node* jthread = igvn.transform(new ThreadLocalNode());
926 Node* adr = phase->basic_plus_adr(jthread, in_bytes(ZThreadLocalData::address_bad_mask_offset()));
927 Node* bad_mask = igvn.transform(LoadNode::make(igvn, in_ctrl, in_mem, adr,
928 TypeRawPtr::BOTTOM, TypeX_X, TypeX_X->basic_type(),
929 MemNode::unordered));
930 Node* cast = igvn.transform(new CastP2XNode(in_ctrl, in_val));
931 Node* obj_masked = igvn.transform(new AndXNode(cast, bad_mask));
932 Node* cmp = igvn.transform(new CmpXNode(obj_masked, igvn.zerocon(TypeX_X->basic_type())));
933 Node *bol = igvn.transform(new BoolNode(cmp, BoolTest::ne))->as_Bool();
934 IfNode* iff = igvn.transform(new IfNode(in_ctrl, bol, unlikely, COUNT_UNKNOWN))->as_If();
935 Node* then = igvn.transform(new IfTrueNode(iff));
936 Node* elsen = igvn.transform(new IfFalseNode(iff));
937
938 Node* slow_path_surrogate;
939 if (!barrier->is_weak()) {
940 slow_path_surrogate = igvn.transform(new LoadBarrierSlowRegNode(then, in_mem, in_adr, in_val->adr_type(),
941 (const TypePtr*) in_val->bottom_type(), MemNode::unordered));
942 } else {
943 slow_path_surrogate = igvn.transform(new LoadBarrierWeakSlowRegNode(then, in_mem, in_adr, in_val->adr_type(),
944 (const TypePtr*) in_val->bottom_type(), MemNode::unordered));
945 }
946
947 Node *new_loadp;
948 new_loadp = slow_path_surrogate;
949 // create the final region/phi pair to converge cntl/data paths to downstream code
950 Node* result_region = igvn.transform(new RegionNode(3));
951 result_region->set_req(1, then);
952 result_region->set_req(2, elsen);
953
954 Node* result_phi = igvn.transform(new PhiNode(result_region, TypeInstPtr::BOTTOM));
955 result_phi->set_req(1, new_loadp);
956 result_phi->set_req(2, barrier->in(LoadBarrierNode::Oop));
957
958 // finally, connect the original outputs to the barrier region and phi to complete the expansion/substitution
959 // igvn.replace_node(out_ctrl, result_region);
960 if (out_ctrl != NULL) { // added if cond
961 igvn.replace_node(out_ctrl, result_region);
962 }
963 igvn.replace_node(out_res, result_phi);
964
965 assert(barrier->outcnt() == 0,"LoadBarrier macro node has non-null outputs after expansion!");
966
967 #ifdef PRINT_NODE_TRAVERSALS
968 tty->print("\nAfter barrier optimization: old out_ctrl\n");
969 traverse(out_ctrl, out_ctrl, out_res, -1);
970 tty->print("\nAfter barrier optimization: old out_res\n");
971 traverse(out_res, out_ctrl, out_res, -1);
972 tty->print("\nAfter barrier optimization: old barrier\n");
973 traverse(barrier, out_ctrl, out_res, -1);
974 tty->print("\nAfter barrier optimization: preceding_barrier_node\n");
975 traverse(preceding_barrier_node, result_region, result_phi, -1);
976 #endif
977
978 return;
979 }
980
981 bool ZBarrierSetC2::expand_macro_nodes(PhaseMacroExpand* macro) const {
982 Compile* C = Compile::current();
983 PhaseIterGVN &igvn = macro->igvn();
984 ZBarrierSetC2State* s = state();
985 if (s->load_barrier_count() > 0) {
986 #ifdef ASSERT
987 verify_gc_barriers(false);
988 #endif
989 igvn.set_delay_transform(true);
990 int skipped = 0;
991 while (s->load_barrier_count() > skipped) {
992 int load_barrier_count = s->load_barrier_count();
993 LoadBarrierNode * n = s->load_barrier_node(load_barrier_count-1-skipped);
994 if (igvn.type(n) == Type::TOP || (n->in(0) != NULL && n->in(0)->is_top())) {
995 // node is unreachable, so don't try to expand it
996 s->remove_load_barrier_node(n);
997 continue;
998 }
999 if (!n->can_be_eliminated()) {
1000 skipped++;
1001 continue;
1002 }
1003 expand_loadbarrier_node(macro, n);
1004 assert(s->load_barrier_count() < load_barrier_count, "must have deleted a node from load barrier list");
1005 if (C->failing()) return true;
1006 }
1007 while (s->load_barrier_count() > 0) {
1008 int load_barrier_count = s->load_barrier_count();
1009 LoadBarrierNode* n = s->load_barrier_node(load_barrier_count - 1);
1010 assert(!(igvn.type(n) == Type::TOP || (n->in(0) != NULL && n->in(0)->is_top())), "should have been processed already");
1011 assert(!n->can_be_eliminated(), "should have been processed already");
1012 expand_loadbarrier_node(macro, n);
1013 assert(s->load_barrier_count() < load_barrier_count, "must have deleted a node from load barrier list");
1014 if (C->failing()) return true;
1015 }
1016 igvn.set_delay_transform(false);
1017 igvn.optimize();
1018 if (C->failing()) return true;
1019 }
1020 return false;
1021 }
1022
1023 // == Loop optimization ==
1024
1025 static bool replace_with_dominating_barrier(PhaseIdealLoop* phase, LoadBarrierNode* lb, bool last_round) {
1026 PhaseIterGVN &igvn = phase->igvn();
1027 Compile* C = Compile::current();
1028
1029 LoadBarrierNode* lb2 = lb->has_dominating_barrier(phase, false, last_round);
1030 if (lb2 != NULL) {
1031 if (lb->in(LoadBarrierNode::Oop) != lb2->in(LoadBarrierNode::Oop)) {
1032 assert(lb->in(LoadBarrierNode::Address) == lb2->in(LoadBarrierNode::Address), "");
1033 igvn.replace_input_of(lb, LoadBarrierNode::Similar, lb2->proj_out(LoadBarrierNode::Oop));
1034 C->set_major_progress();
1035 } else {
1036 // That transformation may cause the Similar edge on dominated load barriers to be invalid
1037 lb->fix_similar_in_uses(&igvn);
1038
1039 Node* val = lb->proj_out(LoadBarrierNode::Oop);
1040 assert(lb2->has_true_uses(), "");
1041 assert(lb2->in(LoadBarrierNode::Oop) == lb->in(LoadBarrierNode::Oop), "");
1042
1043 phase->lazy_update(lb, lb->in(LoadBarrierNode::Control));
1044 phase->lazy_replace(lb->proj_out(LoadBarrierNode::Control), lb->in(LoadBarrierNode::Control));
1045 igvn.replace_node(val, lb2->proj_out(LoadBarrierNode::Oop));
1046
1047 return true;
1048 }
1049 }
1050 return false;
1051 }
1052
1053 static Node* find_dominating_memory(PhaseIdealLoop* phase, Node* mem, Node* dom, int i) {
1054 assert(dom->is_Region() || i == -1, "");
1055 Node* m = mem;
1056 while(phase->is_dominator(dom, phase->has_ctrl(m) ? phase->get_ctrl(m) : m->in(0))) {
1057 if (m->is_Mem()) {
1058 assert(m->as_Mem()->adr_type() == TypeRawPtr::BOTTOM, "");
1059 m = m->in(MemNode::Memory);
1060 } else if (m->is_MergeMem()) {
1061 m = m->as_MergeMem()->memory_at(Compile::AliasIdxRaw);
1062 } else if (m->is_Phi()) {
1063 if (m->in(0) == dom && i != -1) {
1064 m = m->in(i);
1065 break;
1066 } else {
1067 m = m->in(LoopNode::EntryControl);
1068 }
1069 } else if (m->is_Proj()) {
1070 m = m->in(0);
1071 } else if (m->is_SafePoint() || m->is_MemBar()) {
1072 m = m->in(TypeFunc::Memory);
1073 } else {
1074 #ifdef ASSERT
1075 m->dump();
1076 #endif
1077 ShouldNotReachHere();
1078 }
1079 }
1080 return m;
1081 }
1082
1083 static LoadBarrierNode* clone_load_barrier(PhaseIdealLoop* phase, LoadBarrierNode* lb, Node* ctl, Node* mem, Node* oop_in) {
1084 PhaseIterGVN &igvn = phase->igvn();
1085 Compile* C = Compile::current();
1086 Node* the_clone = lb->clone();
1087 the_clone->set_req(LoadBarrierNode::Control, ctl);
1088 the_clone->set_req(LoadBarrierNode::Memory, mem);
1089 if (oop_in != NULL) {
1090 the_clone->set_req(LoadBarrierNode::Oop, oop_in);
1091 }
1092
1093 LoadBarrierNode* new_lb = the_clone->as_LoadBarrier();
1094 igvn.register_new_node_with_optimizer(new_lb);
1095 IdealLoopTree *loop = phase->get_loop(new_lb->in(0));
1096 phase->set_ctrl(new_lb, new_lb->in(0));
1097 phase->set_loop(new_lb, loop);
1098 phase->set_idom(new_lb, new_lb->in(0), phase->dom_depth(new_lb->in(0))+1);
1099 if (!loop->_child) {
1100 loop->_body.push(new_lb);
1101 }
1102
1103 Node* proj_ctl = new ProjNode(new_lb, LoadBarrierNode::Control);
1104 igvn.register_new_node_with_optimizer(proj_ctl);
1105 phase->set_ctrl(proj_ctl, proj_ctl->in(0));
1106 phase->set_loop(proj_ctl, loop);
1107 phase->set_idom(proj_ctl, new_lb, phase->dom_depth(new_lb)+1);
1108 if (!loop->_child) {
1109 loop->_body.push(proj_ctl);
1110 }
1111
1112 Node* proj_oop = new ProjNode(new_lb, LoadBarrierNode::Oop);
1113 phase->register_new_node(proj_oop, new_lb);
1114
1115 if (!new_lb->in(LoadBarrierNode::Similar)->is_top()) {
1116 LoadBarrierNode* similar = new_lb->in(LoadBarrierNode::Similar)->in(0)->as_LoadBarrier();
1117 if (!phase->is_dominator(similar, ctl)) {
1118 igvn.replace_input_of(new_lb, LoadBarrierNode::Similar, C->top());
1119 }
1120 }
1121
1122 return new_lb;
1123 }
1124
1125 static void replace_barrier(PhaseIdealLoop* phase, LoadBarrierNode* lb, Node* new_val) {
1126 PhaseIterGVN &igvn = phase->igvn();
1127 Node* val = lb->proj_out(LoadBarrierNode::Oop);
1128 igvn.replace_node(val, new_val);
1129 phase->lazy_update(lb, lb->in(LoadBarrierNode::Control));
1130 phase->lazy_replace(lb->proj_out(LoadBarrierNode::Control), lb->in(LoadBarrierNode::Control));
1131 }
1132
1133 static bool split_barrier_thru_phi(PhaseIdealLoop* phase, LoadBarrierNode* lb) {
1134 PhaseIterGVN &igvn = phase->igvn();
1135 Compile* C = Compile::current();
1136
1137 if (lb->in(LoadBarrierNode::Oop)->is_Phi()) {
1138 Node* oop_phi = lb->in(LoadBarrierNode::Oop);
1139
1140 if (oop_phi->req() == 2) {
1141 // Ignore phis with only one input
1142 return false;
1143 }
1144
1145 if (phase->is_dominator(phase->get_ctrl(lb->in(LoadBarrierNode::Address)),
1146 oop_phi->in(0)) && phase->get_ctrl(lb->in(LoadBarrierNode::Address)) != oop_phi->in(0)) {
1147 // That transformation may cause the Similar edge on dominated load barriers to be invalid
1148 lb->fix_similar_in_uses(&igvn);
1149
1150 RegionNode* region = oop_phi->in(0)->as_Region();
1151
1152 int backedge = LoopNode::LoopBackControl;
1153 if (region->is_Loop() && region->in(backedge)->is_Proj() && region->in(backedge)->in(0)->is_If()) {
1154 Node* c = region->in(backedge)->in(0)->in(0);
1155 assert(c->unique_ctrl_out() == region->in(backedge)->in(0), "");
1156 Node* oop = lb->in(LoadBarrierNode::Oop)->in(backedge);
1157 Node* oop_c = phase->has_ctrl(oop) ? phase->get_ctrl(oop) : oop;
1158 if (!phase->is_dominator(oop_c, c)) {
1159 return false;
1160 }
1161 }
1162
1163 // If the node on the backedge above the phi is the node itself - we have a self loop.
1164 // Don't clone - this will be folded later.
1165 if (oop_phi->in(LoopNode::LoopBackControl) == lb->proj_out(LoadBarrierNode::Oop)) {
1166 return false;
1167 }
1168
1169 bool is_strip_mined = region->is_CountedLoop() && region->as_CountedLoop()->is_strip_mined();
1170 Node *phi = oop_phi->clone();
1171
1172 for (uint i = 1; i < region->req(); i++) {
1173 Node* ctrl = region->in(i);
1174 if (ctrl != C->top()) {
1175 assert(!phase->is_dominator(ctrl, region) || region->is_Loop(), "");
1176
1177 Node* mem = lb->in(LoadBarrierNode::Memory);
1178 Node* m = find_dominating_memory(phase, mem, region, i);
1179
1180 if (region->is_Loop() && i == LoopNode::LoopBackControl && ctrl->is_Proj() && ctrl->in(0)->is_If()) {
1181 ctrl = ctrl->in(0)->in(0);
1182 } else if (region->is_Loop() && is_strip_mined) {
1183 // If this is a strip mined loop, control must move above OuterStripMinedLoop
1184 assert(i == LoopNode::EntryControl, "check");
1185 assert(ctrl->is_OuterStripMinedLoop(), "sanity");
1186 ctrl = ctrl->as_OuterStripMinedLoop()->in(LoopNode::EntryControl);
1187 }
1188
1189 LoadBarrierNode* new_lb = clone_load_barrier(phase, lb, ctrl, m, lb->in(LoadBarrierNode::Oop)->in(i));
1190 Node* out_ctrl = new_lb->proj_out(LoadBarrierNode::Control);
1191
1192 if (is_strip_mined && (i == LoopNode::EntryControl)) {
1193 assert(region->in(i)->is_OuterStripMinedLoop(), "");
1194 igvn.replace_input_of(region->in(i), i, out_ctrl);
1195 } else if (ctrl == region->in(i)) {
1196 igvn.replace_input_of(region, i, out_ctrl);
1197 } else {
1198 Node* iff = region->in(i)->in(0);
1199 igvn.replace_input_of(iff, 0, out_ctrl);
1200 phase->set_idom(iff, out_ctrl, phase->dom_depth(out_ctrl)+1);
1201 }
1202 phi->set_req(i, new_lb->proj_out(LoadBarrierNode::Oop));
1203 }
1204 }
1205 phase->register_new_node(phi, region);
1206 replace_barrier(phase, lb, phi);
1207
1208 if (region->is_Loop()) {
1209 // Load barrier moved to the back edge of the Loop may now
1210 // have a safepoint on the path to the barrier on the Similar
1211 // edge
1212 igvn.replace_input_of(phi->in(LoopNode::LoopBackControl)->in(0), LoadBarrierNode::Similar, C->top());
1213 Node* head = region->in(LoopNode::EntryControl);
1214 phase->set_idom(region, head, phase->dom_depth(head)+1);
1215 phase->recompute_dom_depth();
1216 if (head->is_CountedLoop() && head->as_CountedLoop()->is_main_loop()) {
1217 head->as_CountedLoop()->set_normal_loop();
1218 }
1219 }
1220
1221 return true;
1222 }
1223 }
1224
1225 return false;
1226 }
1227
1228 static bool move_out_of_loop(PhaseIdealLoop* phase, LoadBarrierNode* lb) {
1229 PhaseIterGVN &igvn = phase->igvn();
1230 IdealLoopTree *lb_loop = phase->get_loop(lb->in(0));
1231 if (lb_loop != phase->ltree_root() && !lb_loop->_irreducible) {
1232 Node* oop_ctrl = phase->get_ctrl(lb->in(LoadBarrierNode::Oop));
1233 IdealLoopTree *oop_loop = phase->get_loop(oop_ctrl);
1234 IdealLoopTree* adr_loop = phase->get_loop(phase->get_ctrl(lb->in(LoadBarrierNode::Address)));
1235 if (!lb_loop->is_member(oop_loop) && !lb_loop->is_member(adr_loop)) {
1236 // That transformation may cause the Similar edge on dominated load barriers to be invalid
1237 lb->fix_similar_in_uses(&igvn);
1238
1239 Node* head = lb_loop->_head;
1240 assert(head->is_Loop(), "");
1241
1242 if (phase->is_dominator(head, oop_ctrl)) {
1243 assert(oop_ctrl->Opcode() == Op_CProj && oop_ctrl->in(0)->Opcode() == Op_NeverBranch, "");
1244 assert(lb_loop->is_member(phase->get_loop(oop_ctrl->in(0)->in(0))), "");
1245 return false;
1246 }
1247
1248 if (head->is_CountedLoop()) {
1249 CountedLoopNode* cloop = head->as_CountedLoop();
1250 if (cloop->is_main_loop()) {
1251 cloop->set_normal_loop();
1252 }
1253 // When we are moving barrier out of a counted loop,
1254 // make sure we move it all the way out of the strip mined outer loop.
1255 if (cloop->is_strip_mined()) {
1256 head = cloop->outer_loop();
1257 }
1258 }
1259
1260 Node* mem = lb->in(LoadBarrierNode::Memory);
1261 Node* m = find_dominating_memory(phase, mem, head, -1);
1262
1263 LoadBarrierNode* new_lb = clone_load_barrier(phase, lb, head->in(LoopNode::EntryControl), m, NULL);
1264
1265 assert(phase->idom(head) == head->in(LoopNode::EntryControl), "");
1266 Node* proj_ctl = new_lb->proj_out(LoadBarrierNode::Control);
1267 igvn.replace_input_of(head, LoopNode::EntryControl, proj_ctl);
1268 phase->set_idom(head, proj_ctl, phase->dom_depth(proj_ctl) + 1);
1269
1270 replace_barrier(phase, lb, new_lb->proj_out(LoadBarrierNode::Oop));
1271
1272 phase->recompute_dom_depth();
1273
1274 return true;
1275 }
1276 }
1277
1278 return false;
1279 }
1280
1281 static bool common_barriers(PhaseIdealLoop* phase, LoadBarrierNode* lb) {
1282 PhaseIterGVN &igvn = phase->igvn();
1283 Node* in_val = lb->in(LoadBarrierNode::Oop);
1284 for (DUIterator_Fast imax, i = in_val->fast_outs(imax); i < imax; i++) {
1285 Node* u = in_val->fast_out(i);
1286 if (u != lb && u->is_LoadBarrier() && u->as_LoadBarrier()->has_true_uses()) {
1287 Node* this_ctrl = lb->in(LoadBarrierNode::Control);
1288 Node* other_ctrl = u->in(LoadBarrierNode::Control);
1289
1290 Node* lca = phase->dom_lca(this_ctrl, other_ctrl);
1291 bool ok = true;
1292
1293 Node* proj1 = NULL;
1294 Node* proj2 = NULL;
1295
1296 while (this_ctrl != lca && ok) {
1297 if (this_ctrl->in(0) != NULL &&
1298 this_ctrl->in(0)->is_MultiBranch()) {
1299 if (this_ctrl->in(0)->in(0) == lca) {
1300 assert(proj1 == NULL, "");
1301 assert(this_ctrl->is_Proj(), "");
1302 proj1 = this_ctrl;
1303 } else if (!(this_ctrl->in(0)->is_If() && this_ctrl->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none))) {
1304 ok = false;
1305 }
1306 }
1307 this_ctrl = phase->idom(this_ctrl);
1308 }
1309 while (other_ctrl != lca && ok) {
1310 if (other_ctrl->in(0) != NULL &&
1311 other_ctrl->in(0)->is_MultiBranch()) {
1312 if (other_ctrl->in(0)->in(0) == lca) {
1313 assert(other_ctrl->is_Proj(), "");
1314 assert(proj2 == NULL, "");
1315 proj2 = other_ctrl;
1316 } else if (!(other_ctrl->in(0)->is_If() && other_ctrl->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none))) {
1317 ok = false;
1318 }
1319 }
1320 other_ctrl = phase->idom(other_ctrl);
1321 }
1322 assert(proj1 == NULL || proj2 == NULL || proj1->in(0) == proj2->in(0), "");
1323 if (ok && proj1 && proj2 && proj1 != proj2 && proj1->in(0)->is_If()) {
1324 // That transformation may cause the Similar edge on dominated load barriers to be invalid
1325 lb->fix_similar_in_uses(&igvn);
1326 u->as_LoadBarrier()->fix_similar_in_uses(&igvn);
1327
1328 Node* split = lca->unique_ctrl_out();
1329 assert(split->in(0) == lca, "");
1330
1331 Node* mem = lb->in(LoadBarrierNode::Memory);
1332 Node* m = find_dominating_memory(phase, mem, split, -1);
1333 LoadBarrierNode* new_lb = clone_load_barrier(phase, lb, lca, m, NULL);
1334
1335 Node* proj_ctl = new_lb->proj_out(LoadBarrierNode::Control);
1336 igvn.replace_input_of(split, 0, new_lb->proj_out(LoadBarrierNode::Control));
1337 phase->set_idom(split, proj_ctl, phase->dom_depth(proj_ctl)+1);
1338
1339 Node* proj_oop = new_lb->proj_out(LoadBarrierNode::Oop);
1340 replace_barrier(phase, lb, proj_oop);
1341 replace_barrier(phase, u->as_LoadBarrier(), proj_oop);
1342
1343 phase->recompute_dom_depth();
1344
1345 return true;
1346 }
1347 }
1348 }
1349
1350 return false;
1351 }
1352
1353 static void optimize_load_barrier(PhaseIdealLoop* phase, LoadBarrierNode* lb, bool last_round) {
1354 Compile* C = Compile::current();
1355
1356 if (!C->directive()->ZOptimizeLoadBarriersOption) {
1357 return;
1358 }
1359
1360 if (lb->has_true_uses()) {
1361 if (replace_with_dominating_barrier(phase, lb, last_round)) {
1362 return;
1363 }
1364
1365 if (split_barrier_thru_phi(phase, lb)) {
1366 return;
1367 }
1368
1369 if (move_out_of_loop(phase, lb)) {
1370 return;
1371 }
1372
1373 if (common_barriers(phase, lb)) {
1374 return;
1375 }
1376 }
1377 }
1378
1379 void ZBarrierSetC2::loop_optimize_gc_barrier(PhaseIdealLoop* phase, Node* node, bool last_round) {
1380 if (node->is_LoadBarrier()) {
1381 optimize_load_barrier(phase, node->as_LoadBarrier(), last_round);
1382 }
1383 }
1384
1385 // == Verification ==
1386
1387 #ifdef ASSERT
1388
1389 static bool look_for_barrier(Node* n, bool post_parse, VectorSet& visited) {
1390 if (visited.test_set(n->_idx)) {
1391 return true;
1392 }
1393
1394 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
1395 Node* u = n->fast_out(i);
1396 if (u->is_LoadBarrier()) {
1397 } else if ((u->is_Phi() || u->is_CMove()) && !post_parse) {
1398 if (!look_for_barrier(u, post_parse, visited)) {
1399 return false;
1400 }
1401 } else if (u->Opcode() == Op_EncodeP || u->Opcode() == Op_DecodeN) {
1402 if (!look_for_barrier(u, post_parse, visited)) {
1403 return false;
1404 }
1405 } else if (u->Opcode() != Op_SCMemProj) {
1406 tty->print("bad use"); u->dump();
1407 return false;
1408 }
1409 }
1410
1411 return true;
1412 }
1413
1414 void ZBarrierSetC2::verify_gc_barriers(bool post_parse) const {
1415 ZBarrierSetC2State* s = state();
1416 Compile* C = Compile::current();
1417 ResourceMark rm;
1418 VectorSet visited(Thread::current()->resource_area());
1419 for (int i = 0; i < s->load_barrier_count(); i++) {
1420 LoadBarrierNode* n = s->load_barrier_node(i);
1421
1422 // The dominating barrier on the same address if it exists and
1423 // this barrier must not be applied on the value from the same
1424 // load otherwise the value is not reloaded before it's used the
1425 // second time.
1426 assert(n->in(LoadBarrierNode::Similar)->is_top() ||
1427 (n->in(LoadBarrierNode::Similar)->in(0)->is_LoadBarrier() &&
1428 n->in(LoadBarrierNode::Similar)->in(0)->in(LoadBarrierNode::Address) == n->in(LoadBarrierNode::Address) &&
1429 n->in(LoadBarrierNode::Similar)->in(0)->in(LoadBarrierNode::Oop) != n->in(LoadBarrierNode::Oop)),
1430 "broken similar edge");
1431
1432 assert(post_parse || n->as_LoadBarrier()->has_true_uses(),
1433 "found unneeded load barrier");
1434
1435 // Several load barrier nodes chained through their Similar edge
1436 // break the code that remove the barriers in final graph reshape.
1437 assert(n->in(LoadBarrierNode::Similar)->is_top() ||
1438 (n->in(LoadBarrierNode::Similar)->in(0)->is_LoadBarrier() &&
1439 n->in(LoadBarrierNode::Similar)->in(0)->in(LoadBarrierNode::Similar)->is_top()),
1440 "chain of Similar load barriers");
1441
1442 if (!n->in(LoadBarrierNode::Similar)->is_top()) {
1443 ResourceMark rm;
1444 Unique_Node_List wq;
1445 Node* other = n->in(LoadBarrierNode::Similar)->in(0);
1446 wq.push(n);
1447 bool ok = true;
1448 bool dom_found = false;
1449 for (uint next = 0; next < wq.size(); ++next) {
1450 Node *n = wq.at(next);
1451 assert(n->is_CFG(), "");
1452 assert(!n->is_SafePoint(), "");
1453
1454 if (n == other) {
1455 continue;
1456 }
1457
1458 if (n->is_Region()) {
1459 for (uint i = 1; i < n->req(); i++) {
1460 Node* m = n->in(i);
1461 if (m != NULL) {
1462 wq.push(m);
1463 }
1464 }
1465 } else {
1466 Node* m = n->in(0);
1467 if (m != NULL) {
1468 wq.push(m);
1469 }
1470 }
1471 }
1472 }
1473
1474 if (ZVerifyLoadBarriers) {
1475 if ((n->is_Load() || n->is_LoadStore()) && n->bottom_type()->make_oopptr() != NULL) {
1476 visited.Clear();
1477 bool found = look_for_barrier(n, post_parse, visited);
1478 if (!found) {
1479 n->dump(1);
1480 n->dump(-3);
1481 stringStream ss;
1482 C->method()->print_short_name(&ss);
1483 tty->print_cr("-%s-", ss.as_string());
1484 assert(found, "");
1485 }
1486 }
1487 }
1488 }
1489 }
1490
1491 #endif