1 /*
2 * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 #include "precompiled.hpp"
25 #include "opto/compile.hpp"
26 #include "opto/castnode.hpp"
27 #include "opto/graphKit.hpp"
28 #include "opto/idealKit.hpp"
29 #include "opto/loopnode.hpp"
30 #include "opto/macro.hpp"
31 #include "opto/node.hpp"
32 #include "opto/type.hpp"
33 #include "utilities/macros.hpp"
34 #include "gc/z/c2/zBarrierSetC2.hpp"
35 #include "gc/z/zThreadLocalData.hpp"
36 #include "gc/z/zBarrierSetRuntime.hpp"
37
38 ZBarrierSetC2State::ZBarrierSetC2State(Arena* comp_arena)
39 : _load_barrier_nodes(new (comp_arena) GrowableArray<LoadBarrierNode*>(comp_arena, 8, 0, NULL)) {}
40
41 int ZBarrierSetC2State::load_barrier_count() const {
42 return _load_barrier_nodes->length();
43 }
44
45 void ZBarrierSetC2State::add_load_barrier_node(LoadBarrierNode * n) {
46 assert(!_load_barrier_nodes->contains(n), " duplicate entry in expand list");
47 _load_barrier_nodes->append(n);
48 }
49
50 void ZBarrierSetC2State::remove_load_barrier_node(LoadBarrierNode * n) {
51 // this function may be called twice for a node so check
52 // that the node is in the array before attempting to remove it
53 if (_load_barrier_nodes->contains(n)) {
54 _load_barrier_nodes->remove(n);
55 }
56 }
57
58 LoadBarrierNode* ZBarrierSetC2State::load_barrier_node(int idx) const {
59 return _load_barrier_nodes->at(idx);
60 }
61
62 void* ZBarrierSetC2::create_barrier_state(Arena* comp_arena) const {
63 return new(comp_arena) ZBarrierSetC2State(comp_arena);
64 }
65
66 ZBarrierSetC2State* ZBarrierSetC2::state() const {
67 return reinterpret_cast<ZBarrierSetC2State*>(Compile::current()->barrier_set_state());
68 }
69
70 bool ZBarrierSetC2::is_gc_barrier_node(Node* node) const {
71 // 1. This step follows potential oop projections of a load barrier before expansion
72 if (node->is_Proj()) {
73 node = node->in(0);
74 }
75
76 // 2. This step checks for unexpanded load barriers
77 if (node->is_LoadBarrier()) {
78 return true;
79 }
80
81 // 3. This step checks for the phi corresponding to an optimized load barrier expansion
82 if (node->is_Phi()) {
83 PhiNode* phi = node->as_Phi();
84 Node* n = phi->in(1);
85 if (n != NULL && (n->is_LoadBarrierSlowReg() || n->is_LoadBarrierWeakSlowReg())) {
86 return true;
87 }
88 }
89
90 return false;
91 }
92
93 void ZBarrierSetC2::register_potential_barrier_node(Node* node) const {
94 if (node->is_LoadBarrier()) {
95 state()->add_load_barrier_node(node->as_LoadBarrier());
96 }
97 }
98
99 void ZBarrierSetC2::unregister_potential_barrier_node(Node* node) const {
100 if (node->is_LoadBarrier()) {
101 state()->remove_load_barrier_node(node->as_LoadBarrier());
102 }
103 }
104
105 void ZBarrierSetC2::eliminate_useless_gc_barriers(Unique_Node_List &useful, Compile* C) const {
106 // Remove useless LoadBarrier nodes
107 ZBarrierSetC2State* s = state();
108 for (int i = s->load_barrier_count()-1; i >= 0; i--) {
109 LoadBarrierNode* n = s->load_barrier_node(i);
110 if (!useful.member(n)) {
111 unregister_potential_barrier_node(n);
112 }
113 }
114 }
115
116 void ZBarrierSetC2::enqueue_useful_gc_barrier(PhaseIterGVN* igvn, Node* node) const {
117 if (node->is_LoadBarrier() && !node->as_LoadBarrier()->has_true_uses()) {
118 igvn->_worklist.push(node);
119 }
120 }
121
122 void ZBarrierSetC2::find_dominating_barriers(PhaseIterGVN& igvn) {
123 // Look for dominating barriers on the same address only once all
124 // other loop opts are over: loop opts may cause a safepoint to be
125 // inserted between a barrier and its dominating barrier.
126 Compile* C = Compile::current();
127 ZBarrierSetC2* bs = (ZBarrierSetC2*)BarrierSet::barrier_set()->barrier_set_c2();
128 ZBarrierSetC2State* s = bs->state();
129 if (s->load_barrier_count() >= 2) {
130 Compile::TracePhase tp("idealLoop", &C->timers[Phase::_t_idealLoop]);
131 PhaseIdealLoop ideal_loop(igvn, LoopOptsLastRound);
132 if (C->major_progress()) C->print_method(PHASE_PHASEIDEALLOOP_ITERATIONS, 2);
133 }
134 }
135
136 void ZBarrierSetC2::add_users_to_worklist(Unique_Node_List* worklist) const {
137 // Permanent temporary workaround
138 // Loadbarriers may have non-obvious dead uses keeping them alive during parsing. The use is
139 // removed by RemoveUseless (after parsing, before optimize) but the barriers won't be added to
140 // the worklist. Unless we add them explicitly they are not guaranteed to end up there.
141 ZBarrierSetC2State* s = state();
142
143 for (int i = 0; i < s->load_barrier_count(); i++) {
144 LoadBarrierNode* n = s->load_barrier_node(i);
145 worklist->push(n);
146 }
147 }
148
149 const TypeFunc* ZBarrierSetC2::load_barrier_Type() const {
150 const Type** fields;
151
152 // Create input types (domain)
153 fields = TypeTuple::fields(2);
154 fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL;
155 fields[TypeFunc::Parms+1] = TypeOopPtr::BOTTOM;
156 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2, fields);
157
158 // Create result type (range)
159 fields = TypeTuple::fields(1);
160 fields[TypeFunc::Parms+0] = TypeInstPtr::BOTTOM;
161 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+1, fields);
162
163 return TypeFunc::make(domain, range);
164 }
165
166 // == LoadBarrierNode ==
167
168 LoadBarrierNode::LoadBarrierNode(Compile* C,
169 Node* c,
170 Node* mem,
171 Node* val,
172 Node* adr,
173 bool weak,
174 bool writeback,
175 bool oop_reload_allowed) :
176 MultiNode(Number_of_Inputs),
177 _weak(weak),
178 _writeback(writeback),
179 _oop_reload_allowed(oop_reload_allowed) {
180 init_req(Control, c);
181 init_req(Memory, mem);
182 init_req(Oop, val);
183 init_req(Address, adr);
184 init_req(Similar, C->top());
185
186 init_class_id(Class_LoadBarrier);
187 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
188 bs->register_potential_barrier_node(this);
189 }
190
191 const Type *LoadBarrierNode::bottom_type() const {
192 const Type** floadbarrier = (const Type **)(Compile::current()->type_arena()->Amalloc_4((Number_of_Outputs)*sizeof(Type*)));
193 Node* in_oop = in(Oop);
194 floadbarrier[Control] = Type::CONTROL;
195 floadbarrier[Memory] = Type::MEMORY;
196 floadbarrier[Oop] = in_oop == NULL ? Type::TOP : in_oop->bottom_type();
197 return TypeTuple::make(Number_of_Outputs, floadbarrier);
198 }
199
200 const Type *LoadBarrierNode::Value(PhaseGVN *phase) const {
201 const Type** floadbarrier = (const Type **)(phase->C->type_arena()->Amalloc_4((Number_of_Outputs)*sizeof(Type*)));
202 const Type* val_t = phase->type(in(Oop));
203 floadbarrier[Control] = Type::CONTROL;
204 floadbarrier[Memory] = Type::MEMORY;
205 floadbarrier[Oop] = val_t;
206 return TypeTuple::make(Number_of_Outputs, floadbarrier);
207 }
208
209 bool LoadBarrierNode::is_dominator(PhaseIdealLoop* phase, bool linear_only, Node *d, Node *n) {
210 if (phase != NULL) {
211 return phase->is_dominator(d, n);
212 }
213
214 for (int i = 0; i < 10 && n != NULL; i++) {
215 n = IfNode::up_one_dom(n, linear_only);
216 if (n == d) {
217 return true;
218 }
219 }
220
221 return false;
222 }
223
224 LoadBarrierNode* LoadBarrierNode::has_dominating_barrier(PhaseIdealLoop* phase, bool linear_only, bool look_for_similar) {
225 Node* val = in(LoadBarrierNode::Oop);
226 if (in(Similar)->is_Proj() && in(Similar)->in(0)->is_LoadBarrier()) {
227 LoadBarrierNode* lb = in(Similar)->in(0)->as_LoadBarrier();
228 assert(lb->in(Address) == in(Address), "");
229 // Load barrier on Similar edge dominates so if it now has the Oop field it can replace this barrier.
230 if (lb->in(Oop) == in(Oop)) {
231 return lb;
232 }
233 // Follow chain of load barrier through Similar edges
234 while (!lb->in(Similar)->is_top()) {
235 lb = lb->in(Similar)->in(0)->as_LoadBarrier();
236 assert(lb->in(Address) == in(Address), "");
237 }
238 if (lb != in(Similar)->in(0)) {
239 return lb;
240 }
241 }
242 for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) {
243 Node* u = val->fast_out(i);
244 if (u != this && u->is_LoadBarrier() && u->in(Oop) == val && u->as_LoadBarrier()->has_true_uses()) {
245 Node* this_ctrl = in(LoadBarrierNode::Control);
246 Node* other_ctrl = u->in(LoadBarrierNode::Control);
247 if (is_dominator(phase, linear_only, other_ctrl, this_ctrl)) {
248 return u->as_LoadBarrier();
249 }
250 }
251 }
252
253 if (ZVerifyLoadBarriers || can_be_eliminated()) {
254 return NULL;
255 }
256
257 if (!look_for_similar) {
258 return NULL;
259 }
260
261 Node* addr = in(LoadBarrierNode::Address);
262 for (DUIterator_Fast imax, i = addr->fast_outs(imax); i < imax; i++) {
263 Node* u = addr->fast_out(i);
264 if (u != this && u->is_LoadBarrier() && u->as_LoadBarrier()->has_true_uses()) {
265 Node* this_ctrl = in(LoadBarrierNode::Control);
266 Node* other_ctrl = u->in(LoadBarrierNode::Control);
267 if (is_dominator(phase, linear_only, other_ctrl, this_ctrl)) {
268 ResourceMark rm;
269 Unique_Node_List wq;
270 wq.push(in(LoadBarrierNode::Control));
271 bool ok = true;
272 bool dom_found = false;
273 for (uint next = 0; next < wq.size(); ++next) {
274 Node *n = wq.at(next);
275 if (n->is_top()) {
276 return NULL;
277 }
278 assert(n->is_CFG(), "");
279 if (n->is_SafePoint()) {
280 ok = false;
281 break;
282 }
283 if (n == u) {
284 dom_found = true;
285 continue;
286 }
287 if (n->is_Region()) {
288 for (uint i = 1; i < n->req(); i++) {
289 Node* m = n->in(i);
290 if (m != NULL) {
291 wq.push(m);
292 }
293 }
294 } else {
295 Node* m = n->in(0);
296 if (m != NULL) {
297 wq.push(m);
298 }
299 }
300 }
301 if (ok) {
302 assert(dom_found, "");
303 return u->as_LoadBarrier();;
304 }
305 break;
306 }
307 }
308 }
309
310 return NULL;
311 }
312
313 void LoadBarrierNode::push_dominated_barriers(PhaseIterGVN* igvn) const {
314 // Change to that barrier may affect a dominated barrier so re-push those
315 Node* val = in(LoadBarrierNode::Oop);
316
317 for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) {
318 Node* u = val->fast_out(i);
319 if (u != this && u->is_LoadBarrier() && u->in(Oop) == val) {
320 Node* this_ctrl = in(Control);
321 Node* other_ctrl = u->in(Control);
322 if (is_dominator(NULL, false, this_ctrl, other_ctrl)) {
323 igvn->_worklist.push(u);
324 }
325 }
326
327 Node* addr = in(LoadBarrierNode::Address);
328 for (DUIterator_Fast imax, i = addr->fast_outs(imax); i < imax; i++) {
329 Node* u = addr->fast_out(i);
330 if (u != this && u->is_LoadBarrier() && u->in(Similar)->is_top()) {
331 Node* this_ctrl = in(Control);
332 Node* other_ctrl = u->in(Control);
333 if (is_dominator(NULL, false, this_ctrl, other_ctrl)) {
334 igvn->_worklist.push(u);
335 }
336 }
337 }
338 }
339 }
340
341 Node *LoadBarrierNode::Identity(PhaseGVN *phase) {
342 if (!phase->C->directive()->ZOptimizeLoadBarriersOption) {
343 return this;
344 }
345
346 bool redundant_addr = false;
347 LoadBarrierNode* dominating_barrier = has_dominating_barrier(NULL, true, false);
348 if (dominating_barrier != NULL) {
349 assert(dominating_barrier->in(Oop) == in(Oop), "");
350 return dominating_barrier;
351 }
352
353 return this;
354 }
355
356 Node *LoadBarrierNode::Ideal(PhaseGVN *phase, bool can_reshape) {
357 if (remove_dead_region(phase, can_reshape)) {
358 return this;
359 }
360
361 Node* val = in(Oop);
362 Node* mem = in(Memory);
363 Node* ctrl = in(Control);
364 Node* adr = in(Address);
365 assert(val->Opcode() != Op_LoadN, "");
366
367 if (mem->is_MergeMem()) {
368 Node* new_mem = mem->as_MergeMem()->memory_at(Compile::AliasIdxRaw);
369 set_req(Memory, new_mem);
370 if (mem->outcnt() == 0 && can_reshape) {
371 phase->is_IterGVN()->_worklist.push(mem);
372 }
373
374 return this;
375 }
376
377 bool optimizeLoadBarriers = phase->C->directive()->ZOptimizeLoadBarriersOption;
378 LoadBarrierNode* dominating_barrier = optimizeLoadBarriers ? has_dominating_barrier(NULL, !can_reshape, !phase->C->major_progress()) : NULL;
379 if (dominating_barrier != NULL && dominating_barrier->in(Oop) != in(Oop)) {
380 assert(in(Address) == dominating_barrier->in(Address), "");
381 set_req(Similar, dominating_barrier->proj_out(Oop));
382 return this;
383 }
384
385 bool eliminate = (optimizeLoadBarriers && !(val->is_Phi() || val->Opcode() == Op_LoadP || val->Opcode() == Op_GetAndSetP || val->is_DecodeN())) ||
386 (can_reshape && (dominating_barrier != NULL || !has_true_uses()));
387
388 if (eliminate) {
389 if (can_reshape) {
390 PhaseIterGVN* igvn = phase->is_IterGVN();
391 Node* out_ctrl = proj_out_or_null(Control);
392 Node* out_res = proj_out_or_null(Oop);
393
394 if (out_ctrl != NULL) {
395 igvn->replace_node(out_ctrl, ctrl);
396 }
397
398 // That transformation may cause the Similar edge on the load barrier to be invalid
399 fix_similar_in_uses(igvn);
400 if (out_res != NULL) {
401 if (dominating_barrier != NULL) {
402 igvn->replace_node(out_res, dominating_barrier->proj_out(Oop));
403 } else {
404 igvn->replace_node(out_res, val);
405 }
406 }
407 }
408
409 return new ConINode(TypeInt::ZERO);
410 }
411
412 // If the Similar edge is no longer a load barrier, clear it
413 Node* similar = in(Similar);
414 if (!similar->is_top() && !(similar->is_Proj() && similar->in(0)->is_LoadBarrier())) {
415 set_req(Similar, phase->C->top());
416 return this;
417 }
418
419 if (can_reshape) {
420 // If this barrier is linked through the Similar edge by a
421 // dominated barrier and both barriers have the same Oop field,
422 // the dominated barrier can go away, so push it for reprocessing.
423 // We also want to avoid a barrier to depend on another dominating
424 // barrier through its Similar edge that itself depend on another
425 // barrier through its Similar edge and rather have the first
426 // depend on the third.
427 PhaseIterGVN* igvn = phase->is_IterGVN();
428 Node* out_res = proj_out(Oop);
429 for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) {
430 Node* u = out_res->fast_out(i);
431 if (u->is_LoadBarrier() && u->in(Similar) == out_res &&
432 (u->in(Oop) == val || !u->in(Similar)->is_top())) {
433 igvn->_worklist.push(u);
434 }
435 }
436
437 push_dominated_barriers(igvn);
438 }
439
440 return NULL;
441 }
442
443 void LoadBarrierNode::fix_similar_in_uses(PhaseIterGVN* igvn) {
444 Node* out_res = proj_out_or_null(Oop);
445 if (out_res == NULL) {
446 return;
447 }
448
449 for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) {
450 Node* u = out_res->fast_out(i);
451 if (u->is_LoadBarrier() && u->in(Similar) == out_res) {
452 igvn->replace_input_of(u, Similar, igvn->C->top());
453 --i;
454 --imax;
455 }
456 }
457 }
458
459 bool LoadBarrierNode::has_true_uses() const {
460 Node* out_res = proj_out_or_null(Oop);
461 if (out_res == NULL) {
462 return false;
463 }
464
465 for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) {
466 Node* u = out_res->fast_out(i);
467 if (!u->is_LoadBarrier() || u->in(Similar) != out_res) {
468 return true;
469 }
470 }
471
472 return false;
473 }
474
475 // == Accesses ==
476
477 Node* ZBarrierSetC2::make_cas_loadbarrier(C2AtomicAccess& access) const {
478 assert(!UseCompressedOops, "Not allowed");
479 CompareAndSwapNode* cas = (CompareAndSwapNode*)access.raw_access();
480 PhaseGVN& gvn = access.kit()->gvn();
481 Compile* C = Compile::current();
482 GraphKit* kit = access.kit();
483
484 Node* in_ctrl = cas->in(MemNode::Control);
485 Node* in_mem = cas->in(MemNode::Memory);
486 Node* in_adr = cas->in(MemNode::Address);
487 Node* in_val = cas->in(MemNode::ValueIn);
488 Node* in_expected = cas->in(LoadStoreConditionalNode::ExpectedIn);
489
490 float likely = PROB_LIKELY(0.999);
491
492 const TypePtr *adr_type = gvn.type(in_adr)->isa_ptr();
493 Compile::AliasType* alias_type = C->alias_type(adr_type);
494 int alias_idx = C->get_alias_index(adr_type);
495
496 // Outer check - true: continue, false: load and check
497 Node* region = new RegionNode(3);
498 Node* phi = new PhiNode(region, TypeInt::BOOL);
499 Node* phi_mem = new PhiNode(region, Type::MEMORY, adr_type);
500
501 // Inner check - is the healed ref equal to the expected
502 Node* region2 = new RegionNode(3);
503 Node* phi2 = new PhiNode(region2, TypeInt::BOOL);
504 Node* phi_mem2 = new PhiNode(region2, Type::MEMORY, adr_type);
505
506 // CAS node returns 0 or 1
507 Node* cmp = gvn.transform(new CmpINode(cas, kit->intcon(0)));
508 Node* bol = gvn.transform(new BoolNode(cmp, BoolTest::ne))->as_Bool();
509 IfNode* iff = gvn.transform(new IfNode(in_ctrl, bol, likely, COUNT_UNKNOWN))->as_If();
510 Node* then = gvn.transform(new IfTrueNode(iff));
511 Node* elsen = gvn.transform(new IfFalseNode(iff));
512
513 Node* scmemproj1 = gvn.transform(new SCMemProjNode(cas));
514
515 kit->set_memory(scmemproj1, alias_idx);
516 phi_mem->init_req(1, scmemproj1);
517 phi_mem2->init_req(2, scmemproj1);
518
519 // CAS fail - reload and heal oop
520 Node* reload = kit->make_load(elsen, in_adr, TypeOopPtr::BOTTOM, T_OBJECT, MemNode::unordered);
521 Node* barrier = gvn.transform(new LoadBarrierNode(C, elsen, scmemproj1, reload, in_adr, false, true, false));
522 Node* barrierctrl = gvn.transform(new ProjNode(barrier, LoadBarrierNode::Control));
523 Node* barrierdata = gvn.transform(new ProjNode(barrier, LoadBarrierNode::Oop));
524
525 // Check load
526 Node* tmpX = gvn.transform(new CastP2XNode(NULL, barrierdata));
527 Node* in_expX = gvn.transform(new CastP2XNode(NULL, in_expected));
528 Node* cmp2 = gvn.transform(new CmpXNode(tmpX, in_expX));
529 Node *bol2 = gvn.transform(new BoolNode(cmp2, BoolTest::ne))->as_Bool();
530 IfNode* iff2 = gvn.transform(new IfNode(barrierctrl, bol2, likely, COUNT_UNKNOWN))->as_If();
531 Node* then2 = gvn.transform(new IfTrueNode(iff2));
532 Node* elsen2 = gvn.transform(new IfFalseNode(iff2));
533
534 // redo CAS
535 Node* cas2 = gvn.transform(new CompareAndSwapPNode(elsen2, kit->memory(alias_idx), in_adr, in_val, in_expected, cas->order()));
536 Node* scmemproj2 = gvn.transform(new SCMemProjNode(cas2));
537 kit->set_control(elsen2);
538 kit->set_memory(scmemproj2, alias_idx);
539
540 // Merge inner flow - check if healed oop was equal too expected.
541 region2->set_req(1, kit->control());
542 region2->set_req(2, then2);
543 phi2->set_req(1, cas2);
544 phi2->set_req(2, kit->intcon(0));
545 phi_mem2->init_req(1, scmemproj2);
546 kit->set_memory(phi_mem2, alias_idx);
547
548 // Merge outer flow - then check if first CAS succeeded
549 region->set_req(1, then);
550 region->set_req(2, region2);
551 phi->set_req(1, kit->intcon(1));
552 phi->set_req(2, phi2);
553 phi_mem->init_req(2, phi_mem2);
554 kit->set_memory(phi_mem, alias_idx);
555
556 gvn.transform(region2);
557 gvn.transform(phi2);
558 gvn.transform(phi_mem2);
559 gvn.transform(region);
560 gvn.transform(phi);
561 gvn.transform(phi_mem);
562
563 kit->set_control(region);
564 kit->insert_mem_bar(Op_MemBarCPUOrder);
565
566 return phi;
567 }
568
569 Node* ZBarrierSetC2::make_cmpx_loadbarrier(C2AtomicAccess& access) const {
570 CompareAndExchangePNode* cmpx = (CompareAndExchangePNode*)access.raw_access();
571 GraphKit* kit = access.kit();
572 PhaseGVN& gvn = kit->gvn();
573 Compile* C = Compile::current();
574
575 Node* in_ctrl = cmpx->in(MemNode::Control);
576 Node* in_mem = cmpx->in(MemNode::Memory);
577 Node* in_adr = cmpx->in(MemNode::Address);
578 Node* in_val = cmpx->in(MemNode::ValueIn);
579 Node* in_expected = cmpx->in(LoadStoreConditionalNode::ExpectedIn);
580
581 float likely = PROB_LIKELY(0.999);
582
583 const TypePtr *adr_type = cmpx->get_ptr_type();
584 Compile::AliasType* alias_type = C->alias_type(adr_type);
585 int alias_idx = C->get_alias_index(adr_type);
586
587 // Outer check - true: continue, false: load and check
588 Node* region = new RegionNode(3);
589 Node* phi = new PhiNode(region, adr_type);
590
591 // Inner check - is the healed ref equal to the expected
592 Node* region2 = new RegionNode(3);
593 Node* phi2 = new PhiNode(region2, adr_type);
594
595 // Check if cmpx succeeded
596 Node* cmp = gvn.transform(new CmpPNode(cmpx, in_expected));
597 Node* bol = gvn.transform(new BoolNode(cmp, BoolTest::eq))->as_Bool();
598 IfNode* iff = gvn.transform(new IfNode(in_ctrl, bol, likely, COUNT_UNKNOWN))->as_If();
599 Node* then = gvn.transform(new IfTrueNode(iff));
600 Node* elsen = gvn.transform(new IfFalseNode(iff));
601
602 Node* scmemproj1 = gvn.transform(new SCMemProjNode(cmpx));
603 kit->set_memory(scmemproj1, alias_idx);
604
605 // CAS fail - reload and heal oop
606 Node* reload = kit->make_load(elsen, in_adr, TypeOopPtr::BOTTOM, T_OBJECT, MemNode::unordered);
607 Node* barrier = gvn.transform(new LoadBarrierNode(C, elsen, scmemproj1, reload, in_adr, false, true, false));
608 Node* barrierctrl = gvn.transform(new ProjNode(barrier, LoadBarrierNode::Control));
609 Node* barrierdata = gvn.transform(new ProjNode(barrier, LoadBarrierNode::Oop));
610
611 // Check load
612 Node* tmpX = gvn.transform(new CastP2XNode(NULL, barrierdata));
613 Node* in_expX = gvn.transform(new CastP2XNode(NULL, in_expected));
614 Node* cmp2 = gvn.transform(new CmpXNode(tmpX, in_expX));
615 Node *bol2 = gvn.transform(new BoolNode(cmp2, BoolTest::ne))->as_Bool();
616 IfNode* iff2 = gvn.transform(new IfNode(barrierctrl, bol2, likely, COUNT_UNKNOWN))->as_If();
617 Node* then2 = gvn.transform(new IfTrueNode(iff2));
618 Node* elsen2 = gvn.transform(new IfFalseNode(iff2));
619
620 // Redo CAS
621 Node* cmpx2 = gvn.transform(new CompareAndExchangePNode(elsen2, kit->memory(alias_idx), in_adr, in_val, in_expected, adr_type, cmpx->get_ptr_type(), cmpx->order()));
622 Node* scmemproj2 = gvn.transform(new SCMemProjNode(cmpx2));
623 kit->set_control(elsen2);
624 kit->set_memory(scmemproj2, alias_idx);
625
626 // Merge inner flow - check if healed oop was equal too expected.
627 region2->set_req(1, kit->control());
628 region2->set_req(2, then2);
629 phi2->set_req(1, cmpx2);
630 phi2->set_req(2, barrierdata);
631
632 // Merge outer flow - then check if first cas succeeded
633 region->set_req(1, then);
634 region->set_req(2, region2);
635 phi->set_req(1, cmpx);
636 phi->set_req(2, phi2);
637
638 gvn.transform(region2);
639 gvn.transform(phi2);
640 gvn.transform(region);
641 gvn.transform(phi);
642
643 kit->set_control(region);
644 kit->set_memory(in_mem, alias_idx);
645 kit->insert_mem_bar(Op_MemBarCPUOrder);
646
647 return phi;
648 }
649
650 Node* ZBarrierSetC2::load_barrier(GraphKit* kit, Node* val, Node* adr, bool weak, bool writeback, bool oop_reload_allowed) const {
651 PhaseGVN& gvn = kit->gvn();
652 Node* barrier = new LoadBarrierNode(Compile::current(), kit->control(), kit->memory(TypeRawPtr::BOTTOM), val, adr, weak, writeback, oop_reload_allowed);
653 Node* transformed_barrier = gvn.transform(barrier);
654
655 if (transformed_barrier->is_LoadBarrier()) {
656 if (barrier == transformed_barrier) {
657 kit->set_control(gvn.transform(new ProjNode(barrier, LoadBarrierNode::Control)));
658 }
659 Node* result = gvn.transform(new ProjNode(transformed_barrier, LoadBarrierNode::Oop));
660 assert(is_gc_barrier_node(result), "sanity");
661 assert(step_over_gc_barrier(result) == val, "sanity");
662 return result;
663 } else {
664 return val;
665 }
666 }
667
668 static bool barrier_needed(C2Access access) {
669 return ZBarrierSet::barrier_needed(access.decorators(), access.type());
670 }
671
672 Node* ZBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const {
673 Node* p = BarrierSetC2::load_at_resolved(access, val_type);
674 if (!barrier_needed(access)) {
675 return p;
676 }
677
678 bool weak = (access.decorators() & ON_WEAK_OOP_REF) != 0;
679
680 GraphKit* kit = access.kit();
681 PhaseGVN& gvn = kit->gvn();
682 Node* adr = access.addr().node();
683 Node* heap_base_oop = access.base();
684 bool unsafe = (access.decorators() & C2_UNSAFE_ACCESS) != 0;
685 if (unsafe) {
686 if (!ZVerifyLoadBarriers) {
687 p = load_barrier(kit, p, adr);
688 } else {
689 if (!TypePtr::NULL_PTR->higher_equal(gvn.type(heap_base_oop))) {
690 p = load_barrier(kit, p, adr);
691 } else {
692 IdealKit ideal(kit);
693 IdealVariable res(ideal);
694 #define __ ideal.
695 __ declarations_done();
696 __ set(res, p);
697 __ if_then(heap_base_oop, BoolTest::ne, kit->null(), PROB_UNLIKELY(0.999)); {
698 kit->sync_kit(ideal);
699 p = load_barrier(kit, p, adr);
700 __ set(res, p);
701 __ sync_kit(kit);
702 } __ end_if();
703 kit->final_sync(ideal);
704 p = __ value(res);
705 #undef __
706 }
707 }
708 return p;
709 } else {
710 return load_barrier(access.kit(), p, access.addr().node(), weak, true, true);
711 }
712 }
713
714 Node* ZBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicAccess& access, Node* expected_val,
715 Node* new_val, const Type* val_type) const {
716 Node* result = BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, val_type);
717 if (!barrier_needed(access)) {
718 return result;
719 }
720
721 access.set_needs_pinning(false);
722 return make_cmpx_loadbarrier(access);
723 }
724
725 Node* ZBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicAccess& access, Node* expected_val,
726 Node* new_val, const Type* value_type) const {
727 Node* result = BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type);
728 if (!barrier_needed(access)) {
729 return result;
730 }
731
732 Node* load_store = access.raw_access();
733 bool weak_cas = (access.decorators() & C2_WEAK_CMPXCHG) != 0;
734 bool expected_is_null = (expected_val->get_ptr_type() == TypePtr::NULL_PTR);
735
736 if (!expected_is_null) {
737 if (weak_cas) {
738 access.set_needs_pinning(false);
739 load_store = make_cas_loadbarrier(access);
740 } else {
741 access.set_needs_pinning(false);
742 load_store = make_cas_loadbarrier(access);
743 }
744 }
745
746 return load_store;
747 }
748
749 Node* ZBarrierSetC2::atomic_xchg_at_resolved(C2AtomicAccess& access, Node* new_val, const Type* val_type) const {
750 Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, new_val, val_type);
751 if (!barrier_needed(access)) {
752 return result;
753 }
754
755 Node* load_store = access.raw_access();
756 Node* adr = access.addr().node();
757
758 return load_barrier(access.kit(), load_store, adr, false, false, false);
759 }
760
761 // == Macro Expansion ==
762
763 void ZBarrierSetC2::expand_loadbarrier_node(PhaseMacroExpand* phase, LoadBarrierNode* barrier) const {
764 Node* in_ctrl = barrier->in(LoadBarrierNode::Control);
765 Node* in_mem = barrier->in(LoadBarrierNode::Memory);
766 Node* in_val = barrier->in(LoadBarrierNode::Oop);
767 Node* in_adr = barrier->in(LoadBarrierNode::Address);
768
769 Node* out_ctrl = barrier->proj_out(LoadBarrierNode::Control);
770 Node* out_res = barrier->proj_out(LoadBarrierNode::Oop);
771
772 PhaseIterGVN &igvn = phase->igvn();
773
774 if (ZVerifyLoadBarriers) {
775 igvn.replace_node(out_res, in_val);
776 igvn.replace_node(out_ctrl, in_ctrl);
777 return;
778 }
779
780 if (barrier->can_be_eliminated()) {
781 // Clone and pin the load for this barrier below the dominating
782 // barrier: the load cannot be allowed to float above the
783 // dominating barrier
784 Node* load = in_val;
785
786 if (load->is_Load()) {
787 Node* new_load = load->clone();
788 Node* addp = new_load->in(MemNode::Address);
789 assert(addp->is_AddP() || addp->is_Phi() || addp->is_Load(), "bad address");
790 Node* cast = new CastPPNode(addp, igvn.type(addp), true);
791 Node* ctrl = NULL;
792 Node* similar = barrier->in(LoadBarrierNode::Similar);
793 if (similar->is_Phi()) {
794 // already expanded
795 ctrl = similar->in(0);
796 } else {
797 assert(similar->is_Proj() && similar->in(0)->is_LoadBarrier(), "unexpected graph shape");
798 ctrl = similar->in(0)->as_LoadBarrier()->proj_out(LoadBarrierNode::Control);
799 }
800 assert(ctrl != NULL, "bad control");
801 cast->set_req(0, ctrl);
802 igvn.transform(cast);
803 new_load->set_req(MemNode::Address, cast);
804 igvn.transform(new_load);
805
806 igvn.replace_node(out_res, new_load);
807 igvn.replace_node(out_ctrl, in_ctrl);
808 return;
809 }
810 // cannot eliminate
811 }
812
813 // There are two cases that require the basic loadbarrier
814 // 1) When the writeback of a healed oop must be avoided (swap)
815 // 2) When we must guarantee that no reload of is done (swap, cas, cmpx)
816 if (!barrier->is_writeback()) {
817 assert(!barrier->oop_reload_allowed(), "writeback barriers should be marked as requires oop");
818 }
819
820 if (!barrier->oop_reload_allowed()) {
821 expand_loadbarrier_basic(phase, barrier);
822 } else {
823 expand_loadbarrier_optimized(phase, barrier);
824 }
825 }
826
827 // Basic loadbarrier using conventional argument passing
828 void ZBarrierSetC2::expand_loadbarrier_basic(PhaseMacroExpand* phase, LoadBarrierNode *barrier) const {
829 PhaseIterGVN &igvn = phase->igvn();
830
831 Node* in_ctrl = barrier->in(LoadBarrierNode::Control);
832 Node* in_mem = barrier->in(LoadBarrierNode::Memory);
833 Node* in_val = barrier->in(LoadBarrierNode::Oop);
834 Node* in_adr = barrier->in(LoadBarrierNode::Address);
835
836 Node* out_ctrl = barrier->proj_out(LoadBarrierNode::Control);
837 Node* out_res = barrier->proj_out(LoadBarrierNode::Oop);
838
839 float unlikely = PROB_UNLIKELY(0.999);
840 const Type* in_val_maybe_null_t = igvn.type(in_val);
841
842 Node* jthread = igvn.transform(new ThreadLocalNode());
843 Node* adr = phase->basic_plus_adr(jthread, in_bytes(ZThreadLocalData::address_bad_mask_offset()));
844 Node* bad_mask = igvn.transform(LoadNode::make(igvn, in_ctrl, in_mem, adr, TypeRawPtr::BOTTOM, TypeX_X, TypeX_X->basic_type(), MemNode::unordered));
845 Node* cast = igvn.transform(new CastP2XNode(in_ctrl, in_val));
846 Node* obj_masked = igvn.transform(new AndXNode(cast, bad_mask));
847 Node* cmp = igvn.transform(new CmpXNode(obj_masked, igvn.zerocon(TypeX_X->basic_type())));
848 Node *bol = igvn.transform(new BoolNode(cmp, BoolTest::ne))->as_Bool();
849 IfNode* iff = igvn.transform(new IfNode(in_ctrl, bol, unlikely, COUNT_UNKNOWN))->as_If();
850 Node* then = igvn.transform(new IfTrueNode(iff));
851 Node* elsen = igvn.transform(new IfFalseNode(iff));
852
853 Node* result_region;
854 Node* result_val;
855
856 result_region = new RegionNode(3);
857 result_val = new PhiNode(result_region, TypeInstPtr::BOTTOM);
858
859 result_region->set_req(1, elsen);
860 Node* res = igvn.transform(new CastPPNode(in_val, in_val_maybe_null_t));
861 res->init_req(0, elsen);
862 result_val->set_req(1, res);
863
864 const TypeFunc *tf = load_barrier_Type();
865 Node* call;
866 if (barrier->is_weak()) {
867 call = new CallLeafNode(tf,
868 ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded_addr(),
869 "ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded",
870 TypeRawPtr::BOTTOM);
871 } else {
872 call = new CallLeafNode(tf,
873 ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(),
874 "ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded",
875 TypeRawPtr::BOTTOM);
876 }
877
878 call->init_req(TypeFunc::Control, then);
879 call->init_req(TypeFunc::I_O , phase->top());
880 call->init_req(TypeFunc::Memory , in_mem);
881 call->init_req(TypeFunc::FramePtr, phase->top());
882 call->init_req(TypeFunc::ReturnAdr, phase->top());
883 call->init_req(TypeFunc::Parms+0, in_val);
884 if (barrier->is_writeback()) {
885 call->init_req(TypeFunc::Parms+1, in_adr);
886 } else {
887 // When slow path is called with a null address, the healed oop will not be written back
888 call->init_req(TypeFunc::Parms+1, igvn.zerocon(T_OBJECT));
889 }
890 call = igvn.transform(call);
891
892 Node* ctrl = igvn.transform(new ProjNode(call, TypeFunc::Control));
893 res = igvn.transform(new ProjNode(call, TypeFunc::Parms));
894 res = igvn.transform(new CheckCastPPNode(ctrl, res, in_val_maybe_null_t));
895
896 result_region->set_req(2, ctrl);
897 result_val->set_req(2, res);
898
899 result_region = igvn.transform(result_region);
900 result_val = igvn.transform(result_val);
901
902 if (out_ctrl != NULL) { // Added if cond
903 igvn.replace_node(out_ctrl, result_region);
904 }
905 igvn.replace_node(out_res, result_val);
906 }
907
908 // Optimized, low spill, loadbarrier variant using stub specialized on register used
909 void ZBarrierSetC2::expand_loadbarrier_optimized(PhaseMacroExpand* phase, LoadBarrierNode *barrier) const {
910 PhaseIterGVN &igvn = phase->igvn();
911 #ifdef PRINT_NODE_TRAVERSALS
912 Node* preceding_barrier_node = barrier->in(LoadBarrierNode::Oop);
913 #endif
914
915 Node* in_ctrl = barrier->in(LoadBarrierNode::Control);
916 Node* in_mem = barrier->in(LoadBarrierNode::Memory);
917 Node* in_val = barrier->in(LoadBarrierNode::Oop);
918 Node* in_adr = barrier->in(LoadBarrierNode::Address);
919
920 Node* out_ctrl = barrier->proj_out(LoadBarrierNode::Control);
921 Node* out_res = barrier->proj_out(LoadBarrierNode::Oop);
922
923 assert(barrier->in(LoadBarrierNode::Oop) != NULL, "oop to loadbarrier node cannot be null");
924
925 #ifdef PRINT_NODE_TRAVERSALS
926 tty->print("\n\n\nBefore barrier optimization:\n");
927 traverse(barrier, out_ctrl, out_res, -1);
928
929 tty->print("\nBefore barrier optimization: preceding_barrier_node\n");
930 traverse(preceding_barrier_node, out_ctrl, out_res, -1);
931 #endif
932
933 float unlikely = PROB_UNLIKELY(0.999);
934
935 Node* jthread = igvn.transform(new ThreadLocalNode());
936 Node* adr = phase->basic_plus_adr(jthread, in_bytes(ZThreadLocalData::address_bad_mask_offset()));
937 Node* bad_mask = igvn.transform(LoadNode::make(igvn, in_ctrl, in_mem, adr,
938 TypeRawPtr::BOTTOM, TypeX_X, TypeX_X->basic_type(),
939 MemNode::unordered));
940 Node* cast = igvn.transform(new CastP2XNode(in_ctrl, in_val));
941 Node* obj_masked = igvn.transform(new AndXNode(cast, bad_mask));
942 Node* cmp = igvn.transform(new CmpXNode(obj_masked, igvn.zerocon(TypeX_X->basic_type())));
943 Node *bol = igvn.transform(new BoolNode(cmp, BoolTest::ne))->as_Bool();
944 IfNode* iff = igvn.transform(new IfNode(in_ctrl, bol, unlikely, COUNT_UNKNOWN))->as_If();
945 Node* then = igvn.transform(new IfTrueNode(iff));
946 Node* elsen = igvn.transform(new IfFalseNode(iff));
947
948 Node* slow_path_surrogate;
949 if (!barrier->is_weak()) {
950 slow_path_surrogate = igvn.transform(new LoadBarrierSlowRegNode(then, in_mem, in_adr, in_val->adr_type(),
951 (const TypePtr*) in_val->bottom_type(), MemNode::unordered));
952 } else {
953 slow_path_surrogate = igvn.transform(new LoadBarrierWeakSlowRegNode(then, in_mem, in_adr, in_val->adr_type(),
954 (const TypePtr*) in_val->bottom_type(), MemNode::unordered));
955 }
956
957 Node *new_loadp;
958 new_loadp = slow_path_surrogate;
959 // Create the final region/phi pair to converge cntl/data paths to downstream code
960 Node* result_region = igvn.transform(new RegionNode(3));
961 result_region->set_req(1, then);
962 result_region->set_req(2, elsen);
963
964 Node* result_phi = igvn.transform(new PhiNode(result_region, TypeInstPtr::BOTTOM));
965 result_phi->set_req(1, new_loadp);
966 result_phi->set_req(2, barrier->in(LoadBarrierNode::Oop));
967
968 // Finally, connect the original outputs to the barrier region and phi to complete the expansion/substitution
969 // igvn.replace_node(out_ctrl, result_region);
970 if (out_ctrl != NULL) { // added if cond
971 igvn.replace_node(out_ctrl, result_region);
972 }
973 igvn.replace_node(out_res, result_phi);
974
975 assert(barrier->outcnt() == 0,"LoadBarrier macro node has non-null outputs after expansion!");
976
977 #ifdef PRINT_NODE_TRAVERSALS
978 tty->print("\nAfter barrier optimization: old out_ctrl\n");
979 traverse(out_ctrl, out_ctrl, out_res, -1);
980 tty->print("\nAfter barrier optimization: old out_res\n");
981 traverse(out_res, out_ctrl, out_res, -1);
982 tty->print("\nAfter barrier optimization: old barrier\n");
983 traverse(barrier, out_ctrl, out_res, -1);
984 tty->print("\nAfter barrier optimization: preceding_barrier_node\n");
985 traverse(preceding_barrier_node, result_region, result_phi, -1);
986 #endif
987
988 assert(is_gc_barrier_node(result_phi), "sanity");
989 assert(step_over_gc_barrier(result_phi) == in_val, "sanity");
990
991 return;
992 }
993
994 bool ZBarrierSetC2::expand_macro_nodes(PhaseMacroExpand* macro) const {
995 Compile* C = Compile::current();
996 PhaseIterGVN &igvn = macro->igvn();
997 ZBarrierSetC2State* s = state();
998 if (s->load_barrier_count() > 0) {
999 #ifdef ASSERT
1000 verify_gc_barriers(false);
1001 #endif
1002 igvn.set_delay_transform(true);
1003 int skipped = 0;
1004 while (s->load_barrier_count() > skipped) {
1005 int load_barrier_count = s->load_barrier_count();
1006 LoadBarrierNode * n = s->load_barrier_node(load_barrier_count-1-skipped);
1007 if (igvn.type(n) == Type::TOP || (n->in(0) != NULL && n->in(0)->is_top())) {
1008 // Node is unreachable, so don't try to expand it
1009 s->remove_load_barrier_node(n);
1010 continue;
1011 }
1012 if (!n->can_be_eliminated()) {
1013 skipped++;
1014 continue;
1015 }
1016 expand_loadbarrier_node(macro, n);
1017 assert(s->load_barrier_count() < load_barrier_count, "must have deleted a node from load barrier list");
1018 if (C->failing()) return true;
1019 }
1020 while (s->load_barrier_count() > 0) {
1021 int load_barrier_count = s->load_barrier_count();
1022 LoadBarrierNode* n = s->load_barrier_node(load_barrier_count - 1);
1023 assert(!(igvn.type(n) == Type::TOP || (n->in(0) != NULL && n->in(0)->is_top())), "should have been processed already");
1024 assert(!n->can_be_eliminated(), "should have been processed already");
1025 expand_loadbarrier_node(macro, n);
1026 assert(s->load_barrier_count() < load_barrier_count, "must have deleted a node from load barrier list");
1027 if (C->failing()) return true;
1028 }
1029 igvn.set_delay_transform(false);
1030 igvn.optimize();
1031 if (C->failing()) return true;
1032 }
1033 return false;
1034 }
1035
1036 // == Loop optimization ==
1037
1038 static bool replace_with_dominating_barrier(PhaseIdealLoop* phase, LoadBarrierNode* lb, bool last_round) {
1039 PhaseIterGVN &igvn = phase->igvn();
1040 Compile* C = Compile::current();
1041
1042 LoadBarrierNode* lb2 = lb->has_dominating_barrier(phase, false, last_round);
1043 if (lb2 != NULL) {
1044 if (lb->in(LoadBarrierNode::Oop) != lb2->in(LoadBarrierNode::Oop)) {
1045 assert(lb->in(LoadBarrierNode::Address) == lb2->in(LoadBarrierNode::Address), "");
1046 igvn.replace_input_of(lb, LoadBarrierNode::Similar, lb2->proj_out(LoadBarrierNode::Oop));
1047 C->set_major_progress();
1048 } else {
1049 // That transformation may cause the Similar edge on dominated load barriers to be invalid
1050 lb->fix_similar_in_uses(&igvn);
1051
1052 Node* val = lb->proj_out(LoadBarrierNode::Oop);
1053 assert(lb2->has_true_uses(), "");
1054 assert(lb2->in(LoadBarrierNode::Oop) == lb->in(LoadBarrierNode::Oop), "");
1055
1056 phase->lazy_update(lb, lb->in(LoadBarrierNode::Control));
1057 phase->lazy_replace(lb->proj_out(LoadBarrierNode::Control), lb->in(LoadBarrierNode::Control));
1058 igvn.replace_node(val, lb2->proj_out(LoadBarrierNode::Oop));
1059
1060 return true;
1061 }
1062 }
1063 return false;
1064 }
1065
1066 static Node* find_dominating_memory(PhaseIdealLoop* phase, Node* mem, Node* dom, int i) {
1067 assert(dom->is_Region() || i == -1, "");
1068 Node* m = mem;
1069 while(phase->is_dominator(dom, phase->has_ctrl(m) ? phase->get_ctrl(m) : m->in(0))) {
1070 if (m->is_Mem()) {
1071 assert(m->as_Mem()->adr_type() == TypeRawPtr::BOTTOM, "");
1072 m = m->in(MemNode::Memory);
1073 } else if (m->is_MergeMem()) {
1074 m = m->as_MergeMem()->memory_at(Compile::AliasIdxRaw);
1075 } else if (m->is_Phi()) {
1076 if (m->in(0) == dom && i != -1) {
1077 m = m->in(i);
1078 break;
1079 } else {
1080 m = m->in(LoopNode::EntryControl);
1081 }
1082 } else if (m->is_Proj()) {
1083 m = m->in(0);
1084 } else if (m->is_SafePoint() || m->is_MemBar()) {
1085 m = m->in(TypeFunc::Memory);
1086 } else {
1087 #ifdef ASSERT
1088 m->dump();
1089 #endif
1090 ShouldNotReachHere();
1091 }
1092 }
1093 return m;
1094 }
1095
1096 static LoadBarrierNode* clone_load_barrier(PhaseIdealLoop* phase, LoadBarrierNode* lb, Node* ctl, Node* mem, Node* oop_in) {
1097 PhaseIterGVN &igvn = phase->igvn();
1098 Compile* C = Compile::current();
1099 Node* the_clone = lb->clone();
1100 the_clone->set_req(LoadBarrierNode::Control, ctl);
1101 the_clone->set_req(LoadBarrierNode::Memory, mem);
1102 if (oop_in != NULL) {
1103 the_clone->set_req(LoadBarrierNode::Oop, oop_in);
1104 }
1105
1106 LoadBarrierNode* new_lb = the_clone->as_LoadBarrier();
1107 igvn.register_new_node_with_optimizer(new_lb);
1108 IdealLoopTree *loop = phase->get_loop(new_lb->in(0));
1109 phase->set_ctrl(new_lb, new_lb->in(0));
1110 phase->set_loop(new_lb, loop);
1111 phase->set_idom(new_lb, new_lb->in(0), phase->dom_depth(new_lb->in(0))+1);
1112 if (!loop->_child) {
1113 loop->_body.push(new_lb);
1114 }
1115
1116 Node* proj_ctl = new ProjNode(new_lb, LoadBarrierNode::Control);
1117 igvn.register_new_node_with_optimizer(proj_ctl);
1118 phase->set_ctrl(proj_ctl, proj_ctl->in(0));
1119 phase->set_loop(proj_ctl, loop);
1120 phase->set_idom(proj_ctl, new_lb, phase->dom_depth(new_lb)+1);
1121 if (!loop->_child) {
1122 loop->_body.push(proj_ctl);
1123 }
1124
1125 Node* proj_oop = new ProjNode(new_lb, LoadBarrierNode::Oop);
1126 phase->register_new_node(proj_oop, new_lb);
1127
1128 if (!new_lb->in(LoadBarrierNode::Similar)->is_top()) {
1129 LoadBarrierNode* similar = new_lb->in(LoadBarrierNode::Similar)->in(0)->as_LoadBarrier();
1130 if (!phase->is_dominator(similar, ctl)) {
1131 igvn.replace_input_of(new_lb, LoadBarrierNode::Similar, C->top());
1132 }
1133 }
1134
1135 return new_lb;
1136 }
1137
1138 static void replace_barrier(PhaseIdealLoop* phase, LoadBarrierNode* lb, Node* new_val) {
1139 PhaseIterGVN &igvn = phase->igvn();
1140 Node* val = lb->proj_out(LoadBarrierNode::Oop);
1141 igvn.replace_node(val, new_val);
1142 phase->lazy_update(lb, lb->in(LoadBarrierNode::Control));
1143 phase->lazy_replace(lb->proj_out(LoadBarrierNode::Control), lb->in(LoadBarrierNode::Control));
1144 }
1145
1146 static bool split_barrier_thru_phi(PhaseIdealLoop* phase, LoadBarrierNode* lb) {
1147 PhaseIterGVN &igvn = phase->igvn();
1148 Compile* C = Compile::current();
1149
1150 if (lb->in(LoadBarrierNode::Oop)->is_Phi()) {
1151 Node* oop_phi = lb->in(LoadBarrierNode::Oop);
1152
1153 if (oop_phi->in(2) == oop_phi) {
1154 // Ignore phis with only one input
1155 return false;
1156 }
1157
1158 if (phase->is_dominator(phase->get_ctrl(lb->in(LoadBarrierNode::Address)),
1159 oop_phi->in(0)) && phase->get_ctrl(lb->in(LoadBarrierNode::Address)) != oop_phi->in(0)) {
1160 // That transformation may cause the Similar edge on dominated load barriers to be invalid
1161 lb->fix_similar_in_uses(&igvn);
1162
1163 RegionNode* region = oop_phi->in(0)->as_Region();
1164
1165 int backedge = LoopNode::LoopBackControl;
1166 if (region->is_Loop() && region->in(backedge)->is_Proj() && region->in(backedge)->in(0)->is_If()) {
1167 Node* c = region->in(backedge)->in(0)->in(0);
1168 assert(c->unique_ctrl_out() == region->in(backedge)->in(0), "");
1169 Node* oop = lb->in(LoadBarrierNode::Oop)->in(backedge);
1170 Node* oop_c = phase->has_ctrl(oop) ? phase->get_ctrl(oop) : oop;
1171 if (!phase->is_dominator(oop_c, c)) {
1172 return false;
1173 }
1174 }
1175
1176 // If the node on the backedge above the phi is the node itself - we have a self loop.
1177 // Don't clone - this will be folded later.
1178 if (oop_phi->in(LoopNode::LoopBackControl) == lb->proj_out(LoadBarrierNode::Oop)) {
1179 return false;
1180 }
1181
1182 bool is_strip_mined = region->is_CountedLoop() && region->as_CountedLoop()->is_strip_mined();
1183 Node *phi = oop_phi->clone();
1184
1185 for (uint i = 1; i < region->req(); i++) {
1186 Node* ctrl = region->in(i);
1187 if (ctrl != C->top()) {
1188 assert(!phase->is_dominator(ctrl, region) || region->is_Loop(), "");
1189
1190 Node* mem = lb->in(LoadBarrierNode::Memory);
1191 Node* m = find_dominating_memory(phase, mem, region, i);
1192
1193 if (region->is_Loop() && i == LoopNode::LoopBackControl && ctrl->is_Proj() && ctrl->in(0)->is_If()) {
1194 ctrl = ctrl->in(0)->in(0);
1195 } else if (region->is_Loop() && is_strip_mined) {
1196 // If this is a strip mined loop, control must move above OuterStripMinedLoop
1197 assert(i == LoopNode::EntryControl, "check");
1198 assert(ctrl->is_OuterStripMinedLoop(), "sanity");
1199 ctrl = ctrl->as_OuterStripMinedLoop()->in(LoopNode::EntryControl);
1200 }
1201
1202 LoadBarrierNode* new_lb = clone_load_barrier(phase, lb, ctrl, m, lb->in(LoadBarrierNode::Oop)->in(i));
1203 Node* out_ctrl = new_lb->proj_out(LoadBarrierNode::Control);
1204
1205 if (is_strip_mined && (i == LoopNode::EntryControl)) {
1206 assert(region->in(i)->is_OuterStripMinedLoop(), "");
1207 igvn.replace_input_of(region->in(i), i, out_ctrl);
1208 phase->set_idom(region->in(i), out_ctrl, phase->dom_depth(out_ctrl));
1209 } else if (ctrl == region->in(i)) {
1210 igvn.replace_input_of(region, i, out_ctrl);
1211 // Only update the idom if is the loop entry we are updating
1212 // - A loop backedge doesn't change the idom
1213 if (region->is_Loop() && i == LoopNode::EntryControl) {
1214 phase->set_idom(region, out_ctrl, phase->dom_depth(out_ctrl));
1215 }
1216 } else {
1217 Node* iff = region->in(i)->in(0);
1218 igvn.replace_input_of(iff, 0, out_ctrl);
1219 phase->set_idom(iff, out_ctrl, phase->dom_depth(out_ctrl)+1);
1220 }
1221 phi->set_req(i, new_lb->proj_out(LoadBarrierNode::Oop));
1222 }
1223 }
1224 phase->register_new_node(phi, region);
1225 replace_barrier(phase, lb, phi);
1226
1227 if (region->is_Loop()) {
1228 // Load barrier moved to the back edge of the Loop may now
1229 // have a safepoint on the path to the barrier on the Similar
1230 // edge
1231 igvn.replace_input_of(phi->in(LoopNode::LoopBackControl)->in(0), LoadBarrierNode::Similar, C->top());
1232 Node* head = region->in(LoopNode::EntryControl);
1233 phase->set_idom(region, head, phase->dom_depth(head)+1);
1234 phase->recompute_dom_depth();
1235 if (head->is_CountedLoop() && head->as_CountedLoop()->is_main_loop()) {
1236 head->as_CountedLoop()->set_normal_loop();
1237 }
1238 }
1239
1240 return true;
1241 }
1242 }
1243
1244 return false;
1245 }
1246
1247 static bool move_out_of_loop(PhaseIdealLoop* phase, LoadBarrierNode* lb) {
1248 PhaseIterGVN &igvn = phase->igvn();
1249 IdealLoopTree *lb_loop = phase->get_loop(lb->in(0));
1250 if (lb_loop != phase->ltree_root() && !lb_loop->_irreducible) {
1251 Node* oop_ctrl = phase->get_ctrl(lb->in(LoadBarrierNode::Oop));
1252 IdealLoopTree *oop_loop = phase->get_loop(oop_ctrl);
1253 IdealLoopTree* adr_loop = phase->get_loop(phase->get_ctrl(lb->in(LoadBarrierNode::Address)));
1254 if (!lb_loop->is_member(oop_loop) && !lb_loop->is_member(adr_loop)) {
1255 // That transformation may cause the Similar edge on dominated load barriers to be invalid
1256 lb->fix_similar_in_uses(&igvn);
1257
1258 Node* head = lb_loop->_head;
1259 assert(head->is_Loop(), "");
1260
1261 if (phase->is_dominator(head, oop_ctrl)) {
1262 assert(oop_ctrl->Opcode() == Op_CProj && oop_ctrl->in(0)->Opcode() == Op_NeverBranch, "");
1263 assert(lb_loop->is_member(phase->get_loop(oop_ctrl->in(0)->in(0))), "");
1264 return false;
1265 }
1266
1267 if (head->is_CountedLoop()) {
1268 CountedLoopNode* cloop = head->as_CountedLoop();
1269 if (cloop->is_main_loop()) {
1270 cloop->set_normal_loop();
1271 }
1272 // When we are moving barrier out of a counted loop,
1273 // make sure we move it all the way out of the strip mined outer loop.
1274 if (cloop->is_strip_mined()) {
1275 head = cloop->outer_loop();
1276 }
1277 }
1278
1279 Node* mem = lb->in(LoadBarrierNode::Memory);
1280 Node* m = find_dominating_memory(phase, mem, head, -1);
1281
1282 LoadBarrierNode* new_lb = clone_load_barrier(phase, lb, head->in(LoopNode::EntryControl), m, NULL);
1283
1284 assert(phase->idom(head) == head->in(LoopNode::EntryControl), "");
1285 Node* proj_ctl = new_lb->proj_out(LoadBarrierNode::Control);
1286 igvn.replace_input_of(head, LoopNode::EntryControl, proj_ctl);
1287 phase->set_idom(head, proj_ctl, phase->dom_depth(proj_ctl) + 1);
1288
1289 replace_barrier(phase, lb, new_lb->proj_out(LoadBarrierNode::Oop));
1290
1291 phase->recompute_dom_depth();
1292
1293 return true;
1294 }
1295 }
1296
1297 return false;
1298 }
1299
1300 static bool common_barriers(PhaseIdealLoop* phase, LoadBarrierNode* lb) {
1301 PhaseIterGVN &igvn = phase->igvn();
1302 Node* in_val = lb->in(LoadBarrierNode::Oop);
1303 for (DUIterator_Fast imax, i = in_val->fast_outs(imax); i < imax; i++) {
1304 Node* u = in_val->fast_out(i);
1305 if (u != lb && u->is_LoadBarrier() && u->as_LoadBarrier()->has_true_uses()) {
1306 Node* this_ctrl = lb->in(LoadBarrierNode::Control);
1307 Node* other_ctrl = u->in(LoadBarrierNode::Control);
1308
1309 Node* lca = phase->dom_lca(this_ctrl, other_ctrl);
1310 bool ok = true;
1311
1312 Node* proj1 = NULL;
1313 Node* proj2 = NULL;
1314
1315 while (this_ctrl != lca && ok) {
1316 if (this_ctrl->in(0) != NULL &&
1317 this_ctrl->in(0)->is_MultiBranch()) {
1318 if (this_ctrl->in(0)->in(0) == lca) {
1319 assert(proj1 == NULL, "");
1320 assert(this_ctrl->is_Proj(), "");
1321 proj1 = this_ctrl;
1322 } else if (!(this_ctrl->in(0)->is_If() && this_ctrl->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none))) {
1323 ok = false;
1324 }
1325 }
1326 this_ctrl = phase->idom(this_ctrl);
1327 }
1328 while (other_ctrl != lca && ok) {
1329 if (other_ctrl->in(0) != NULL &&
1330 other_ctrl->in(0)->is_MultiBranch()) {
1331 if (other_ctrl->in(0)->in(0) == lca) {
1332 assert(other_ctrl->is_Proj(), "");
1333 assert(proj2 == NULL, "");
1334 proj2 = other_ctrl;
1335 } else if (!(other_ctrl->in(0)->is_If() && other_ctrl->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none))) {
1336 ok = false;
1337 }
1338 }
1339 other_ctrl = phase->idom(other_ctrl);
1340 }
1341 assert(proj1 == NULL || proj2 == NULL || proj1->in(0) == proj2->in(0), "");
1342 if (ok && proj1 && proj2 && proj1 != proj2 && proj1->in(0)->is_If()) {
1343 // That transformation may cause the Similar edge on dominated load barriers to be invalid
1344 lb->fix_similar_in_uses(&igvn);
1345 u->as_LoadBarrier()->fix_similar_in_uses(&igvn);
1346
1347 Node* split = lca->unique_ctrl_out();
1348 assert(split->in(0) == lca, "");
1349
1350 Node* mem = lb->in(LoadBarrierNode::Memory);
1351 Node* m = find_dominating_memory(phase, mem, split, -1);
1352 LoadBarrierNode* new_lb = clone_load_barrier(phase, lb, lca, m, NULL);
1353
1354 Node* proj_ctl = new_lb->proj_out(LoadBarrierNode::Control);
1355 igvn.replace_input_of(split, 0, new_lb->proj_out(LoadBarrierNode::Control));
1356 phase->set_idom(split, proj_ctl, phase->dom_depth(proj_ctl)+1);
1357
1358 Node* proj_oop = new_lb->proj_out(LoadBarrierNode::Oop);
1359 replace_barrier(phase, lb, proj_oop);
1360 replace_barrier(phase, u->as_LoadBarrier(), proj_oop);
1361
1362 phase->recompute_dom_depth();
1363
1364 return true;
1365 }
1366 }
1367 }
1368
1369 return false;
1370 }
1371
1372 static void optimize_load_barrier(PhaseIdealLoop* phase, LoadBarrierNode* lb, bool last_round) {
1373 Compile* C = Compile::current();
1374
1375 if (!C->directive()->ZOptimizeLoadBarriersOption) {
1376 return;
1377 }
1378
1379 if (lb->has_true_uses()) {
1380 if (replace_with_dominating_barrier(phase, lb, last_round)) {
1381 return;
1382 }
1383
1384 if (split_barrier_thru_phi(phase, lb)) {
1385 return;
1386 }
1387
1388 if (move_out_of_loop(phase, lb)) {
1389 return;
1390 }
1391
1392 if (common_barriers(phase, lb)) {
1393 return;
1394 }
1395 }
1396 }
1397
1398 void ZBarrierSetC2::loop_optimize_gc_barrier(PhaseIdealLoop* phase, Node* node, bool last_round) {
1399 if (node->is_LoadBarrier()) {
1400 optimize_load_barrier(phase, node->as_LoadBarrier(), last_round);
1401 }
1402 }
1403
1404 Node* ZBarrierSetC2::step_over_gc_barrier(Node* c) const {
1405 Node* node = c;
1406
1407 // 1. This step follows potential oop projections of a load barrier before expansion
1408 if (node->is_Proj()) {
1409 node = node->in(0);
1410 }
1411
1412 // 2. This step checks for unexpanded load barriers
1413 if (node->is_LoadBarrier()) {
1414 return node->in(LoadBarrierNode::Oop);
1415 }
1416
1417 // 3. This step checks for the phi corresponding to an optimized load barrier expansion
1418 if (node->is_Phi()) {
1419 PhiNode* phi = node->as_Phi();
1420 Node* n = phi->in(1);
1421 if (n != NULL && (n->is_LoadBarrierSlowReg() || n->is_LoadBarrierWeakSlowReg())) {
1422 assert(c == node, "projections from step 1 should only be seen before macro expansion");
1423 return phi->in(2);
1424 }
1425 }
1426
1427 return c;
1428 }
1429
1430 bool ZBarrierSetC2::array_copy_requires_gc_barriers(bool tightly_coupled_alloc, BasicType type, bool is_clone, ArrayCopyPhase phase) const {
1431 return type == T_OBJECT || type == T_ARRAY;
1432 }
1433
1434 // == Verification ==
1435
1436 #ifdef ASSERT
1437
1438 static bool look_for_barrier(Node* n, bool post_parse, VectorSet& visited) {
1439 if (visited.test_set(n->_idx)) {
1440 return true;
1441 }
1442
1443 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
1444 Node* u = n->fast_out(i);
1445 if (u->is_LoadBarrier()) {
1446 } else if ((u->is_Phi() || u->is_CMove()) && !post_parse) {
1447 if (!look_for_barrier(u, post_parse, visited)) {
1448 return false;
1449 }
1450 } else if (u->Opcode() == Op_EncodeP || u->Opcode() == Op_DecodeN) {
1451 if (!look_for_barrier(u, post_parse, visited)) {
1452 return false;
1453 }
1454 } else if (u->Opcode() != Op_SCMemProj) {
1455 tty->print("bad use"); u->dump();
1456 return false;
1457 }
1458 }
1459
1460 return true;
1461 }
1462
1463 void ZBarrierSetC2::verify_gc_barriers(bool post_parse) const {
1464 ZBarrierSetC2State* s = state();
1465 Compile* C = Compile::current();
1466 ResourceMark rm;
1467 VectorSet visited(Thread::current()->resource_area());
1468 for (int i = 0; i < s->load_barrier_count(); i++) {
1469 LoadBarrierNode* n = s->load_barrier_node(i);
1470
1471 // The dominating barrier on the same address if it exists and
1472 // this barrier must not be applied on the value from the same
1473 // load otherwise the value is not reloaded before it's used the
1474 // second time.
1475 assert(n->in(LoadBarrierNode::Similar)->is_top() ||
1476 (n->in(LoadBarrierNode::Similar)->in(0)->is_LoadBarrier() &&
1477 n->in(LoadBarrierNode::Similar)->in(0)->in(LoadBarrierNode::Address) == n->in(LoadBarrierNode::Address) &&
1478 n->in(LoadBarrierNode::Similar)->in(0)->in(LoadBarrierNode::Oop) != n->in(LoadBarrierNode::Oop)),
1479 "broken similar edge");
1480
1481 assert(post_parse || n->as_LoadBarrier()->has_true_uses(),
1482 "found unneeded load barrier");
1483
1484 // Several load barrier nodes chained through their Similar edge
1485 // break the code that remove the barriers in final graph reshape.
1486 assert(n->in(LoadBarrierNode::Similar)->is_top() ||
1487 (n->in(LoadBarrierNode::Similar)->in(0)->is_LoadBarrier() &&
1488 n->in(LoadBarrierNode::Similar)->in(0)->in(LoadBarrierNode::Similar)->is_top()),
1489 "chain of Similar load barriers");
1490
1491 if (!n->in(LoadBarrierNode::Similar)->is_top()) {
1492 ResourceMark rm;
1493 Unique_Node_List wq;
1494 Node* other = n->in(LoadBarrierNode::Similar)->in(0);
1495 wq.push(n);
1496 bool ok = true;
1497 bool dom_found = false;
1498 for (uint next = 0; next < wq.size(); ++next) {
1499 Node *n = wq.at(next);
1500 assert(n->is_CFG(), "");
1501 assert(!n->is_SafePoint(), "");
1502
1503 if (n == other) {
1504 continue;
1505 }
1506
1507 if (n->is_Region()) {
1508 for (uint i = 1; i < n->req(); i++) {
1509 Node* m = n->in(i);
1510 if (m != NULL) {
1511 wq.push(m);
1512 }
1513 }
1514 } else {
1515 Node* m = n->in(0);
1516 if (m != NULL) {
1517 wq.push(m);
1518 }
1519 }
1520 }
1521 }
1522
1523 if (ZVerifyLoadBarriers) {
1524 if ((n->is_Load() || n->is_LoadStore()) && n->bottom_type()->make_oopptr() != NULL) {
1525 visited.Clear();
1526 bool found = look_for_barrier(n, post_parse, visited);
1527 if (!found) {
1528 n->dump(1);
1529 n->dump(-3);
1530 stringStream ss;
1531 C->method()->print_short_name(&ss);
1532 tty->print_cr("-%s-", ss.as_string());
1533 assert(found, "");
1534 }
1535 }
1536 }
1537 }
1538 }
1539
1540 #endif