rev 50711 : ZGC: C2: Remove dead code

   1 /*
   2  * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  */
  23 
  24 #include "precompiled.hpp"
  25 #include "opto/compile.hpp"
  26 #include "opto/castnode.hpp"
  27 #include "opto/graphKit.hpp"
  28 #include "opto/idealKit.hpp"
  29 #include "opto/loopnode.hpp"
  30 #include "opto/macro.hpp"
  31 #include "opto/node.hpp"
  32 #include "opto/type.hpp"
  33 #include "utilities/macros.hpp"
  34 #include "gc/z/c2/zBarrierSetC2.hpp"
  35 #include "gc/z/zThreadLocalData.hpp"
  36 #include "gc/z/zBarrierSetRuntime.hpp"
  37 
  38 ZBarrierSetC2State::ZBarrierSetC2State(Arena* comp_arena)
  39   : _load_barrier_nodes(new (comp_arena) GrowableArray<LoadBarrierNode*>(comp_arena, 8,  0, NULL)) {}
  40 
  41 int ZBarrierSetC2State::load_barrier_count() const {
  42   return _load_barrier_nodes->length();
  43 }
  44 
  45 void ZBarrierSetC2State::add_load_barrier_node(LoadBarrierNode * n) {
  46   assert(!_load_barrier_nodes->contains(n), " duplicate entry in expand list");
  47   _load_barrier_nodes->append(n);
  48 }
  49 
  50 void ZBarrierSetC2State::remove_load_barrier_node(LoadBarrierNode * n) {
  51   // this function may be called twice for a node so check
  52   // that the node is in the array before attempting to remove it
  53   if (_load_barrier_nodes->contains(n)) {
  54     _load_barrier_nodes->remove(n);
  55   }
  56 }
  57 
  58 LoadBarrierNode* ZBarrierSetC2State::load_barrier_node(int idx) const {
  59   return _load_barrier_nodes->at(idx);
  60 }
  61 
  62 void* ZBarrierSetC2::create_barrier_state(Arena* comp_arena) const {
  63   return new(comp_arena) ZBarrierSetC2State(comp_arena);
  64 }
  65 
  66 ZBarrierSetC2State* ZBarrierSetC2::state() const {
  67   return reinterpret_cast<ZBarrierSetC2State*>(Compile::current()->barrier_set_state());
  68 }
  69 
  70 bool ZBarrierSetC2::is_gc_barrier_node(Node* node) const {
  71   return node->is_LoadBarrier();
  72 }
  73 
  74 void ZBarrierSetC2::register_potential_barrier_node(Node* node) const {
  75   if (node->is_LoadBarrier()) {
  76     state()->add_load_barrier_node(node->as_LoadBarrier());
  77   }
  78 }
  79 
  80 void ZBarrierSetC2::unregister_potential_barrier_node(Node* node) const {
  81   if (node->is_LoadBarrier()) {
  82     state()->remove_load_barrier_node(node->as_LoadBarrier());
  83   }
  84 }
  85 
  86 void ZBarrierSetC2::eliminate_useless_gc_barriers(Unique_Node_List &useful) const {
  87   // Remove useless LoadBarrier nodes
  88   ZBarrierSetC2State* s = state();
  89   for (int i = s->load_barrier_count()-1; i >= 0; i--) {
  90     LoadBarrierNode* n = s->load_barrier_node(i);
  91     if (!useful.member(n)) {
  92       unregister_potential_barrier_node(n);
  93     }
  94   }
  95 }
  96 
  97 void ZBarrierSetC2::enqueue_useful_gc_barrier(Unique_Node_List &worklist, Node* node) const {
  98   if (node->is_LoadBarrier() && !node->as_LoadBarrier()->has_true_uses()) {
  99     worklist.push(node);
 100   }
 101 }
 102 
 103 void ZBarrierSetC2::find_dominating_barriers(PhaseIterGVN& igvn) {
 104   // Look for dominating barriers on the same address only once all
 105   // other loop opts are over: loop opts may cause a safepoint to be
 106   // inserted between a barrier and its dominating barrier.
 107   Compile* C = Compile::current();
 108   ZBarrierSetC2* bs = (ZBarrierSetC2*)BarrierSet::barrier_set()->barrier_set_c2();
 109   ZBarrierSetC2State* s = bs->state();
 110   if (s->load_barrier_count() >= 2) {
 111     Compile::TracePhase tp("idealLoop", &C->timers[Phase::_t_idealLoop]);
 112     PhaseIdealLoop ideal_loop(igvn, true, false, true);
 113     if (C->major_progress()) C->print_method(PHASE_PHASEIDEALLOOP_ITERATIONS, 2);
 114   }
 115 }
 116 
 117 void ZBarrierSetC2::add_users_to_worklist(Unique_Node_List* worklist) const {
 118   // Permanent temporary workaround
 119   // Loadbarriers may have non-obvious dead uses keeping them alive during parsing. The use is
 120   // removed by RemoveUseless (after parsing, before optimize) but the barriers won't be added to
 121   // the worklist. Unless we add them explicitly they are not guaranteed to end up there.
 122   ZBarrierSetC2State* s = state();
 123 
 124   for (int i = 0; i < s->load_barrier_count(); i++) {
 125     LoadBarrierNode* n = s->load_barrier_node(i);
 126     worklist->push(n);
 127   }
 128 }
 129 
 130 const TypeFunc* ZBarrierSetC2::load_barrier_Type() const {
 131   const Type** fields;
 132 
 133   // Create input types (domain)
 134   fields = TypeTuple::fields(2);
 135   fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL;
 136   fields[TypeFunc::Parms+1] = TypeOopPtr::BOTTOM;
 137   const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2, fields);
 138 
 139   // Create result type (range)
 140   fields = TypeTuple::fields(1);
 141   fields[TypeFunc::Parms+0] = TypeInstPtr::BOTTOM;
 142   const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+1, fields);
 143 
 144   return TypeFunc::make(domain, range);
 145 }
 146 
 147 // == LoadBarrierNode ==
 148 
 149 LoadBarrierNode::LoadBarrierNode(Compile* C,
 150                                  Node* c,
 151                                  Node* mem,
 152                                  Node* val,
 153                                  Node* adr,
 154                                  bool weak,
 155                                  bool writeback,
 156                                  bool oop_reload_allowed) :
 157     MultiNode(Number_of_Inputs),
 158     _weak(weak),
 159     _writeback(writeback),
 160     _oop_reload_allowed(oop_reload_allowed) {
 161   init_req(Control, c);
 162   init_req(Memory, mem);
 163   init_req(Oop, val);
 164   init_req(Address, adr);
 165   init_req(Similar, C->top());
 166 
 167   init_class_id(Class_LoadBarrier);
 168   BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
 169   bs->register_potential_barrier_node(this);
 170 }
 171 
 172 const Type *LoadBarrierNode::bottom_type() const {
 173   const Type** floadbarrier = (const Type **)(Compile::current()->type_arena()->Amalloc_4((Number_of_Outputs)*sizeof(Type*)));
 174   Node* in_oop = in(Oop);
 175   floadbarrier[Control] = Type::CONTROL;
 176   floadbarrier[Memory] = Type::MEMORY;
 177   floadbarrier[Oop] = in_oop == NULL ? Type::TOP : in_oop->bottom_type();
 178   return TypeTuple::make(Number_of_Outputs, floadbarrier);
 179 }
 180 
 181 const Type *LoadBarrierNode::Value(PhaseGVN *phase) const {
 182   const Type** floadbarrier = (const Type **)(phase->C->type_arena()->Amalloc_4((Number_of_Outputs)*sizeof(Type*)));
 183   const Type* val_t = phase->type(in(Oop));
 184   floadbarrier[Control] = Type::CONTROL;
 185   floadbarrier[Memory] = Type::MEMORY;
 186   floadbarrier[Oop] = val_t;
 187   return TypeTuple::make(Number_of_Outputs, floadbarrier);
 188 }
 189 
 190 bool LoadBarrierNode::is_dominator(PhaseIdealLoop* phase, bool linear_only, Node *d, Node *n) {
 191   if (phase != NULL) {
 192     return phase->is_dominator(d, n);
 193   }
 194 
 195   for (int i = 0; i < 10 && n != NULL; i++) {
 196     n = IfNode::up_one_dom(n, linear_only);
 197     if (n == d) {
 198       return true;
 199     }
 200   }
 201 
 202   return false;
 203 }
 204 
 205 LoadBarrierNode* LoadBarrierNode::has_dominating_barrier(PhaseIdealLoop* phase, bool linear_only, bool look_for_similar) {
 206   Node* val = in(LoadBarrierNode::Oop);
 207   if (in(Similar)->is_Proj() && in(Similar)->in(0)->is_LoadBarrier()) {
 208     LoadBarrierNode* lb = in(Similar)->in(0)->as_LoadBarrier();
 209     assert(lb->in(Address) == in(Address), "");
 210     // Load barrier on Similar edge dominates so if it now has the Oop field it can replace this barrier.
 211     if (lb->in(Oop) == in(Oop)) {
 212       return lb;
 213     }
 214     // Follow chain of load barrier through Similar edges
 215     while (!lb->in(Similar)->is_top()) {
 216       lb = lb->in(Similar)->in(0)->as_LoadBarrier();
 217       assert(lb->in(Address) == in(Address), "");
 218     }
 219     if (lb != in(Similar)->in(0)) {
 220       return lb;
 221     }
 222   }
 223   for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) {
 224     Node* u = val->fast_out(i);
 225     if (u != this && u->is_LoadBarrier() && u->in(Oop) == val && u->as_LoadBarrier()->has_true_uses()) {
 226       Node* this_ctrl = in(LoadBarrierNode::Control);
 227       Node* other_ctrl = u->in(LoadBarrierNode::Control);
 228       if (is_dominator(phase, linear_only, other_ctrl, this_ctrl)) {
 229         return u->as_LoadBarrier();
 230       }
 231     }
 232   }
 233 
 234   if (ZVerifyLoadBarriers || can_be_eliminated()) {
 235     return NULL;
 236   }
 237 
 238   if (!look_for_similar) {
 239     return NULL;
 240   }
 241 
 242   Node* addr = in(LoadBarrierNode::Address);
 243   for (DUIterator_Fast imax, i = addr->fast_outs(imax); i < imax; i++) {
 244     Node* u = addr->fast_out(i);
 245     if (u != this && u->is_LoadBarrier() && u->as_LoadBarrier()->has_true_uses()) {
 246       Node* this_ctrl = in(LoadBarrierNode::Control);
 247       Node* other_ctrl = u->in(LoadBarrierNode::Control);
 248       if (is_dominator(phase, linear_only, other_ctrl, this_ctrl)) {
 249         ResourceMark rm;
 250         Unique_Node_List wq;
 251         wq.push(in(LoadBarrierNode::Control));
 252         bool ok = true;
 253         bool dom_found = false;
 254         for (uint next = 0; next < wq.size(); ++next) {
 255           Node *n = wq.at(next);
 256           if (n->is_top()) {
 257             return NULL;
 258           }
 259           assert(n->is_CFG(), "");
 260           if (n->is_SafePoint()) {
 261             ok = false;
 262             break;
 263           }
 264           if (n == u) {
 265             dom_found = true;
 266             continue;
 267           }
 268           if (n->is_Region()) {
 269             for (uint i = 1; i < n->req(); i++) {
 270               Node* m = n->in(i);
 271               if (m != NULL) {
 272                 wq.push(m);
 273               }
 274             }
 275           } else {
 276             Node* m = n->in(0);
 277             if (m != NULL) {
 278               wq.push(m);
 279             }
 280           }
 281         }
 282         if (ok) {
 283           assert(dom_found, "");
 284           return u->as_LoadBarrier();;
 285         }
 286         break;
 287       }
 288     }
 289   }
 290 
 291   return NULL;
 292 }
 293 
 294 void LoadBarrierNode::push_dominated_barriers(PhaseIterGVN* igvn) const {
 295   // change to that barrier may affect a dominated barrier so re-push those
 296   Node* val = in(LoadBarrierNode::Oop);
 297 
 298   for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) {
 299     Node* u = val->fast_out(i);
 300     if (u != this && u->is_LoadBarrier() && u->in(Oop) == val) {
 301       Node* this_ctrl = in(Control);
 302       Node* other_ctrl = u->in(Control);
 303       if (is_dominator(NULL, false, this_ctrl, other_ctrl)) {
 304         igvn->_worklist.push(u);
 305       }
 306     }
 307 
 308     Node* addr = in(LoadBarrierNode::Address);
 309     for (DUIterator_Fast imax, i = addr->fast_outs(imax); i < imax; i++) {
 310       Node* u = addr->fast_out(i);
 311       if (u != this && u->is_LoadBarrier() && u->in(Similar)->is_top()) {
 312         Node* this_ctrl = in(Control);
 313         Node* other_ctrl = u->in(Control);
 314         if (is_dominator(NULL, false, this_ctrl, other_ctrl)) {
 315           igvn->_worklist.push(u);
 316         }
 317       }
 318     }
 319   }
 320 }
 321 
 322 Node *LoadBarrierNode::Identity(PhaseGVN *phase) {
 323   if (!phase->C->directive()->ZOptimizeLoadBarriersOption) {
 324     return this;
 325   }
 326 
 327   bool redundant_addr = false;
 328   LoadBarrierNode* dominating_barrier = has_dominating_barrier(NULL, true, false);
 329   if (dominating_barrier != NULL) {
 330     assert(dominating_barrier->in(Oop) == in(Oop), "");
 331     return dominating_barrier;
 332   }
 333 
 334   return this;
 335 }
 336 
 337 Node *LoadBarrierNode::Ideal(PhaseGVN *phase, bool can_reshape) {
 338   if (remove_dead_region(phase, can_reshape)) {
 339     return this;
 340   }
 341 
 342   Node* val = in(Oop);
 343   Node* mem = in(Memory);
 344   Node* ctrl = in(Control);
 345   Node* adr = in(Address);
 346   assert(val->Opcode() != Op_LoadN, "");
 347 
 348   if (mem->is_MergeMem()) {
 349     Node* new_mem = mem->as_MergeMem()->memory_at(Compile::AliasIdxRaw);
 350     set_req(Memory, new_mem);
 351     if (mem->outcnt() == 0 && can_reshape) {
 352       phase->is_IterGVN()->_worklist.push(mem);
 353     }
 354 
 355     return this;
 356   }
 357 
 358   bool optimizeLoadBarriers = phase->C->directive()->ZOptimizeLoadBarriersOption;
 359   LoadBarrierNode* dominating_barrier = optimizeLoadBarriers ? has_dominating_barrier(NULL, !can_reshape, !phase->C->major_progress()) : NULL;
 360   if (dominating_barrier != NULL && dominating_barrier->in(Oop) != in(Oop)) {
 361     assert(in(Address) == dominating_barrier->in(Address), "");
 362     set_req(Similar, dominating_barrier->proj_out(Oop));
 363     return this;
 364   }
 365 
 366   bool eliminate = (optimizeLoadBarriers && !(val->is_Phi() || val->Opcode() == Op_LoadP || val->Opcode() == Op_GetAndSetP || val->is_DecodeN())) ||
 367                    (can_reshape && (dominating_barrier != NULL || !has_true_uses()));
 368 
 369   if (eliminate) {
 370     if (can_reshape) {
 371       PhaseIterGVN* igvn = phase->is_IterGVN();
 372       Node* out_ctrl = proj_out_or_null(Control);
 373       Node* out_res = proj_out_or_null(Oop);
 374 
 375       if (out_ctrl != NULL) {
 376         igvn->replace_node(out_ctrl, ctrl);
 377       }
 378 
 379       // That transformation may cause the Similar edge on the load barrier to be invalid
 380       fix_similar_in_uses(igvn);
 381       if (out_res != NULL) {
 382         if (dominating_barrier != NULL) {
 383           igvn->replace_node(out_res, dominating_barrier->proj_out(Oop));
 384         } else {
 385           igvn->replace_node(out_res, val);
 386         }
 387       }
 388     }
 389 
 390     return new ConINode(TypeInt::ZERO);
 391   }
 392 
 393   // If the Similar edge is no longer a load barrier, clear it
 394   Node* similar = in(Similar);
 395   if (!similar->is_top() && !(similar->is_Proj() && similar->in(0)->is_LoadBarrier())) {
 396     set_req(Similar, phase->C->top());
 397     return this;
 398   }
 399 
 400   if (can_reshape) {
 401     // If this barrier is linked through the Similar edge by a
 402     // dominated barrier and both barriers have the same Oop field,
 403     // the dominated barrier can go away, so push it for reprocessing.
 404     // We also want to avoid a barrier to depend on another dominating
 405     // barrier through its Similar edge that itself depend on another
 406     // barrier through its Similar edge and rather have the first
 407     // depend on the third.
 408     PhaseIterGVN* igvn = phase->is_IterGVN();
 409     Node* out_res = proj_out(Oop);
 410     for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) {
 411       Node* u = out_res->fast_out(i);
 412       if (u->is_LoadBarrier() && u->in(Similar) == out_res &&
 413           (u->in(Oop) == val || !u->in(Similar)->is_top())) {
 414         igvn->_worklist.push(u);
 415       }
 416     }
 417 
 418     push_dominated_barriers(igvn);
 419   }
 420 
 421   return NULL;
 422 }
 423 
 424 void LoadBarrierNode::fix_similar_in_uses(PhaseIterGVN* igvn) {
 425   Node* out_res = proj_out_or_null(Oop);
 426   if (out_res == NULL) {
 427     return;
 428   }
 429 
 430   for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) {
 431     Node* u = out_res->fast_out(i);
 432     if (u->is_LoadBarrier() && u->in(Similar) == out_res) {
 433       igvn->replace_input_of(u, Similar, igvn->C->top());
 434       --i;
 435       --imax;
 436     }
 437   }
 438 }
 439 
 440 bool LoadBarrierNode::has_true_uses() const {
 441   Node* out_res = proj_out_or_null(Oop);
 442   if (out_res == NULL) {
 443     return false;
 444   }
 445 
 446   for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) {
 447     Node* u = out_res->fast_out(i);
 448     if (!u->is_LoadBarrier() || u->in(Similar) != out_res) {
 449       return true;
 450     }
 451   }
 452 
 453   return false;
 454 }
 455 
 456 // == Accesses ==
 457 
 458 Node* ZBarrierSetC2::make_cas_loadbarrier(C2AtomicAccess& access) const {
 459   assert(!UseCompressedOops, "Not allowed");
 460   CompareAndSwapNode* cas = (CompareAndSwapNode*)access.raw_access();
 461   PhaseGVN& gvn = access.kit()->gvn();
 462   Compile* C = Compile::current();
 463   GraphKit* kit = access.kit();
 464 
 465   Node* in_ctrl     = cas->in(MemNode::Control);
 466   Node* in_mem      = cas->in(MemNode::Memory);
 467   Node* in_adr      = cas->in(MemNode::Address);
 468   Node* in_val      = cas->in(MemNode::ValueIn);
 469   Node* in_expected = cas->in(LoadStoreConditionalNode::ExpectedIn);
 470 
 471   float likely                   = PROB_LIKELY(0.999);
 472 
 473   const TypePtr *adr_type        = gvn.type(in_adr)->isa_ptr();
 474   Compile::AliasType* alias_type = C->alias_type(adr_type);
 475   int alias_idx                  = C->get_alias_index(adr_type);
 476 
 477   // Outer check - true: continue, false: load and check
 478   Node* region   = new RegionNode(3);
 479   Node* phi      = new PhiNode(region, TypeInt::BOOL);
 480   Node* phi_mem  = new PhiNode(region, Type::MEMORY, adr_type);
 481 
 482   // Inner check - is the healed ref equal to the expected
 483   Node* region2  = new RegionNode(3);
 484   Node* phi2     = new PhiNode(region2, TypeInt::BOOL);
 485   Node* phi_mem2 = new PhiNode(region2, Type::MEMORY, adr_type);
 486 
 487   // CAS node returns 0 or 1
 488   Node* cmp     = gvn.transform(new CmpINode(cas, kit->intcon(0)));
 489   Node* bol     = gvn.transform(new BoolNode(cmp, BoolTest::ne))->as_Bool();
 490   IfNode* iff   = gvn.transform(new IfNode(in_ctrl, bol, likely, COUNT_UNKNOWN))->as_If();
 491   Node* then    = gvn.transform(new IfTrueNode(iff));
 492   Node* elsen   = gvn.transform(new IfFalseNode(iff));
 493 
 494   Node* scmemproj1   = gvn.transform(new SCMemProjNode(cas));
 495 
 496   kit->set_memory(scmemproj1, alias_idx);
 497   phi_mem->init_req(1, scmemproj1);
 498   phi_mem2->init_req(2, scmemproj1);
 499 
 500   // CAS fail - reload and heal oop
 501   Node* reload      = kit->make_load(elsen, in_adr, TypeOopPtr::BOTTOM, T_OBJECT, MemNode::unordered);
 502   Node* barrier     = gvn.transform(new LoadBarrierNode(C, elsen, scmemproj1, reload, in_adr, false, true, false));
 503   Node* barrierctrl = gvn.transform(new ProjNode(barrier, LoadBarrierNode::Control));
 504   Node* barrierdata = gvn.transform(new ProjNode(barrier, LoadBarrierNode::Oop));
 505 
 506   // Check load
 507   Node* tmpX    = gvn.transform(new CastP2XNode(NULL, barrierdata));
 508   Node* in_expX = gvn.transform(new CastP2XNode(NULL, in_expected));
 509   Node* cmp2    = gvn.transform(new CmpXNode(tmpX, in_expX));
 510   Node *bol2    = gvn.transform(new BoolNode(cmp2, BoolTest::ne))->as_Bool();
 511   IfNode* iff2  = gvn.transform(new IfNode(barrierctrl, bol2, likely, COUNT_UNKNOWN))->as_If();
 512   Node* then2   = gvn.transform(new IfTrueNode(iff2));
 513   Node* elsen2  = gvn.transform(new IfFalseNode(iff2));
 514 
 515   // redo CAS
 516   Node* cas2       = gvn.transform(new CompareAndSwapPNode(elsen2, kit->memory(alias_idx), in_adr, in_val, in_expected, cas->order()));
 517   Node* scmemproj2 = gvn.transform(new SCMemProjNode(cas2));
 518   kit->set_control(elsen2);
 519   kit->set_memory(scmemproj2, alias_idx);
 520 
 521   // Merge inner flow - check if healed oop was equal too expected.
 522   region2->set_req(1, kit->control());
 523   region2->set_req(2, then2);
 524   phi2->set_req(1, cas2);
 525   phi2->set_req(2, kit->intcon(0));
 526   phi_mem2->init_req(1, scmemproj2);
 527   kit->set_memory(phi_mem2, alias_idx);
 528 
 529   // Merge outer flow - then check if first cas succeded
 530   region->set_req(1, then);
 531   region->set_req(2, region2);
 532   phi->set_req(1, kit->intcon(1));
 533   phi->set_req(2, phi2);
 534   phi_mem->init_req(2, phi_mem2);
 535   kit->set_memory(phi_mem, alias_idx);
 536 
 537   gvn.transform(region2);
 538   gvn.transform(phi2);
 539   gvn.transform(phi_mem2);
 540   gvn.transform(region);
 541   gvn.transform(phi);
 542   gvn.transform(phi_mem);
 543 
 544   kit->set_control(region);
 545   kit->insert_mem_bar(Op_MemBarCPUOrder);
 546 
 547   return phi;
 548 }
 549 
 550 Node* ZBarrierSetC2::make_cmpx_loadbarrier(C2AtomicAccess& access) const {
 551   CompareAndExchangePNode* cmpx = (CompareAndExchangePNode*)access.raw_access();
 552   GraphKit* kit = access.kit();
 553   PhaseGVN& gvn = kit->gvn();
 554   Compile* C = Compile::current();
 555 
 556   Node* in_ctrl     = cmpx->in(MemNode::Control);
 557   Node* in_mem      = cmpx->in(MemNode::Memory);
 558   Node* in_adr      = cmpx->in(MemNode::Address);
 559   Node* in_val      = cmpx->in(MemNode::ValueIn);
 560   Node* in_expected = cmpx->in(LoadStoreConditionalNode::ExpectedIn);
 561 
 562   float likely                   = PROB_LIKELY(0.999);
 563 
 564   const TypePtr *adr_type        = cmpx->get_ptr_type();
 565   Compile::AliasType* alias_type = C->alias_type(adr_type);
 566   int alias_idx                  = C->get_alias_index(adr_type);
 567 
 568   // Outer check - true: continue, false: load and check
 569   Node* region  = new RegionNode(3);
 570   Node* phi     = new PhiNode(region, adr_type);
 571 
 572   // Inner check - is the healed ref equal to the expected
 573   Node* region2 = new RegionNode(3);
 574   Node* phi2    = new PhiNode(region2, adr_type);
 575 
 576   // Check if cmpx succeded
 577   Node* cmp     = gvn.transform(new CmpPNode(cmpx, in_expected));
 578   Node* bol     = gvn.transform(new BoolNode(cmp, BoolTest::eq))->as_Bool();
 579   IfNode* iff   = gvn.transform(new IfNode(in_ctrl, bol, likely, COUNT_UNKNOWN))->as_If();
 580   Node* then    = gvn.transform(new IfTrueNode(iff));
 581   Node* elsen   = gvn.transform(new IfFalseNode(iff));
 582 
 583   Node* scmemproj1  = gvn.transform(new SCMemProjNode(cmpx));
 584   kit->set_memory(scmemproj1, alias_idx);
 585 
 586   // CAS fail - reload and heal oop
 587   Node* reload      = kit->make_load(elsen, in_adr, TypeOopPtr::BOTTOM, T_OBJECT, MemNode::unordered);
 588   Node* barrier     = gvn.transform(new LoadBarrierNode(C, elsen, scmemproj1, reload, in_adr, false, true, false));
 589   Node* barrierctrl = gvn.transform(new ProjNode(barrier, LoadBarrierNode::Control));
 590   Node* barrierdata = gvn.transform(new ProjNode(barrier, LoadBarrierNode::Oop));
 591 
 592   // Check load
 593   Node* tmpX    = gvn.transform(new CastP2XNode(NULL, barrierdata));
 594   Node* in_expX = gvn.transform(new CastP2XNode(NULL, in_expected));
 595   Node* cmp2    = gvn.transform(new CmpXNode(tmpX, in_expX));
 596   Node *bol2    = gvn.transform(new BoolNode(cmp2, BoolTest::ne))->as_Bool();
 597   IfNode* iff2  = gvn.transform(new IfNode(barrierctrl, bol2, likely, COUNT_UNKNOWN))->as_If();
 598   Node* then2   = gvn.transform(new IfTrueNode(iff2));
 599   Node* elsen2  = gvn.transform(new IfFalseNode(iff2));
 600 
 601   // Redo CAS
 602   Node* cmpx2      = gvn.transform(new CompareAndExchangePNode(elsen2, kit->memory(alias_idx), in_adr, in_val, in_expected, adr_type, cmpx->get_ptr_type(), cmpx->order()));
 603   Node* scmemproj2 = gvn.transform(new SCMemProjNode(cmpx2));
 604   kit->set_control(elsen2);
 605   kit->set_memory(scmemproj2, alias_idx);
 606 
 607   // Merge inner flow - check if healed oop was equal too expected.
 608   region2->set_req(1, kit->control());
 609   region2->set_req(2, then2);
 610   phi2->set_req(1, cmpx2);
 611   phi2->set_req(2, barrierdata);
 612 
 613   // Merge outer flow - then check if first cas succeded
 614   region->set_req(1, then);
 615   region->set_req(2, region2);
 616   phi->set_req(1, cmpx);
 617   phi->set_req(2, phi2);
 618 
 619   gvn.transform(region2);
 620   gvn.transform(phi2);
 621   gvn.transform(region);
 622   gvn.transform(phi);
 623 
 624   kit->set_control(region);
 625   kit->set_memory(in_mem, alias_idx);
 626   kit->insert_mem_bar(Op_MemBarCPUOrder);
 627 
 628   return phi;
 629 }
 630 
 631 Node* ZBarrierSetC2::load_barrier(GraphKit* kit, Node* val, Node* adr, bool weak, bool writeback, bool oop_reload_allowed) const {
 632   PhaseGVN& gvn = kit->gvn();
 633   Node* barrier = new LoadBarrierNode(Compile::current(), kit->control(), kit->memory(TypeRawPtr::BOTTOM), val, adr, weak, writeback, oop_reload_allowed);
 634   Node* transformed_barrier = gvn.transform(barrier);
 635 
 636   if (transformed_barrier->is_LoadBarrier()) {
 637     if (barrier == transformed_barrier) {
 638       kit->set_control(gvn.transform(new ProjNode(barrier, LoadBarrierNode::Control)));
 639     }
 640     return gvn.transform(new ProjNode(transformed_barrier, LoadBarrierNode::Oop));
 641   } else {
 642     return val;
 643   }
 644 }
 645 
 646 static bool barrier_needed(C2Access access) {
 647   return ZBarrierSet::barrier_needed(access.decorators(), access.type());
 648 }
 649 
 650 Node* ZBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const {
 651   Node* p = BarrierSetC2::load_at_resolved(access, val_type);
 652   if (!barrier_needed(access)) {
 653     return p;
 654   }
 655 
 656   bool conc_root = (access.decorators() & IN_CONCURRENT_ROOT) != 0;
 657   bool weak = (access.decorators() & ON_WEAK_OOP_REF) != 0;
 658 
 659   GraphKit* kit = access.kit();
 660   PhaseGVN& gvn = kit->gvn();
 661   Node* adr = access.addr().node();
 662   Node* heap_base_oop = access.base();
 663   bool unsafe = (access.decorators() & C2_UNSAFE_ACCESS) != 0;
 664   if (unsafe) {
 665     if (!ZVerifyLoadBarriers) {
 666       p = load_barrier(kit, p, adr);
 667     } else {
 668       if (!TypePtr::NULL_PTR->higher_equal(gvn.type(heap_base_oop))) {
 669         p = load_barrier(kit, p, adr);
 670       } else {
 671         IdealKit ideal(kit);
 672         IdealVariable res(ideal);
 673 #define __ ideal.
 674         __ declarations_done();
 675         __ set(res, p);
 676         __ if_then(heap_base_oop, BoolTest::ne, kit->null(), PROB_UNLIKELY(0.999)); {
 677           kit->sync_kit(ideal);
 678           p = load_barrier(kit, p, adr);
 679           __ set(res, p);
 680           __ sync_kit(kit);
 681         } __ end_if();
 682         kit->final_sync(ideal);
 683         p = __ value(res);
 684 #undef __
 685       }
 686     }
 687     return p;
 688   } else {
 689     return load_barrier(access.kit(), p, access.addr().node(), weak, true, true);
 690   }
 691 }
 692 
 693 Node* ZBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicAccess& access, Node* expected_val,
 694                                                     Node* new_val, const Type* val_type) const {
 695   Node* result = BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, val_type);
 696   if (!barrier_needed(access)) {
 697     return result;
 698   }
 699 
 700   access.set_needs_pinning(false);
 701   return make_cmpx_loadbarrier(access);
 702 }
 703 
 704 Node* ZBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicAccess& access, Node* expected_val,
 705                                                      Node* new_val, const Type* value_type) const {
 706   Node* result = BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type);
 707   if (!barrier_needed(access)) {
 708     return result;
 709   }
 710 
 711   Node* load_store = access.raw_access();
 712   bool weak_cas = (access.decorators() & C2_WEAK_CMPXCHG) != 0;
 713   bool expected_is_null = (expected_val->get_ptr_type() == TypePtr::NULL_PTR);
 714 
 715   if (!expected_is_null) {
 716     if (weak_cas) {
 717       access.set_needs_pinning(false);
 718       load_store = make_cas_loadbarrier(access);
 719     } else {
 720       access.set_needs_pinning(false);
 721       load_store = make_cas_loadbarrier(access);
 722     }
 723   }
 724 
 725   return load_store;
 726 }
 727 
 728 Node* ZBarrierSetC2::atomic_xchg_at_resolved(C2AtomicAccess& access, Node* new_val, const Type* val_type) const {
 729   Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, new_val, val_type);
 730   if (!barrier_needed(access)) {
 731     return result;
 732   }
 733 
 734   Node* load_store = access.raw_access();
 735   Node* adr = access.addr().node();
 736 
 737   return load_barrier(access.kit(), load_store, adr, false, false, false);
 738 }
 739 
 740 // == Macro Expansion ==
 741 
 742 void ZBarrierSetC2::expand_loadbarrier_node(PhaseMacroExpand* phase, LoadBarrierNode* barrier) const {
 743   Node* in_ctrl = barrier->in(LoadBarrierNode::Control);
 744   Node* in_mem  = barrier->in(LoadBarrierNode::Memory);
 745   Node* in_val  = barrier->in(LoadBarrierNode::Oop);
 746   Node* in_adr  = barrier->in(LoadBarrierNode::Address);
 747 
 748   Node* out_ctrl = barrier->proj_out(LoadBarrierNode::Control);
 749   Node* out_res  = barrier->proj_out(LoadBarrierNode::Oop);
 750 
 751   PhaseIterGVN &igvn = phase->igvn();
 752 
 753   if (ZVerifyLoadBarriers) {
 754     igvn.replace_node(out_res, in_val);
 755     igvn.replace_node(out_ctrl, in_ctrl);
 756     return;
 757   }
 758 
 759   if (barrier->can_be_eliminated()) {
 760     // Clone and pin the load for this barrier below the dominating
 761     // barrier: the load cannot be allowed to float above the
 762     // dominating barrier
 763     Node* load = in_val;
 764     Node* decode = NULL;
 765     if (load->is_DecodeN()) {
 766       decode = load;
 767       load = load->in(1);
 768     }
 769     if (load->is_Load()) {
 770       Node* new_load = load->clone();
 771       Node* addp = new_load->in(MemNode::Address);
 772       assert(addp->is_AddP() || addp->is_Phi(), "bad address");
 773       Node* cast = new CastPPNode(addp, igvn.type(addp), true);
 774       Node* ctrl = NULL;
 775       Node* similar = barrier->in(LoadBarrierNode::Similar);
 776       if (similar->is_Phi()) {
 777         // already expanded
 778         ctrl = similar->in(0);
 779       } else {
 780         assert(similar->is_Proj() && similar->in(0)->is_LoadBarrier(), "unexpected graph shape");
 781         ctrl = similar->in(0)->as_LoadBarrier()->proj_out(LoadBarrierNode::Control);
 782       }
 783       assert(ctrl != NULL, "bad control");
 784       cast->set_req(0, ctrl);
 785       igvn.transform(cast);
 786       new_load->set_req(MemNode::Address, cast);
 787       igvn.transform(new_load);
 788 
 789       Node* new_in_val = new_load;
 790       if (decode != NULL) {
 791         new_in_val = decode->clone();
 792         new_in_val->set_req(1, new_load);
 793         igvn.transform(new_in_val);
 794       }
 795 
 796       igvn.replace_node(out_res, new_in_val);
 797       igvn.replace_node(out_ctrl, in_ctrl);
 798       return;
 799     }
 800     // cannot eliminate
 801   }
 802 
 803   // There are two cases that require the basic loadbarrier
 804   // 1) When the writeback of a healed oop must be avoided (swap)
 805   // 2) When we must guarantee that no reload of is done (swap, cas, cmpx)
 806   if (!barrier->is_writeback()) {
 807     assert(!barrier->oop_reload_allowed(), "writeback barriers should be marked as requires oop");
 808   }
 809 
 810 #ifdef SPARC
 811   bool basic_load_barrier = true;
 812 #else
 813   bool basic_load_barrier = !barrier->oop_reload_allowed();
 814 #endif
 815 
 816   if (basic_load_barrier) {
 817     expand_loadbarrier_basic(phase, barrier);
 818   } else {
 819     expand_loadbarrier_optimized(phase, barrier);
 820   }
 821 }
 822 
 823 // Basic loadbarrier using conventional arg passing
 824 void ZBarrierSetC2::expand_loadbarrier_basic(PhaseMacroExpand* phase, LoadBarrierNode *barrier) const {
 825   PhaseIterGVN &igvn = phase->igvn();
 826 
 827   Node* in_ctrl = barrier->in(LoadBarrierNode::Control);
 828   Node* in_mem  = barrier->in(LoadBarrierNode::Memory);
 829   Node* in_val  = barrier->in(LoadBarrierNode::Oop);
 830   Node* in_adr  = barrier->in(LoadBarrierNode::Address);
 831 
 832   Node* out_ctrl = barrier->proj_out(LoadBarrierNode::Control);
 833   Node* out_res  = barrier->proj_out(LoadBarrierNode::Oop);
 834 
 835   float unlikely  = PROB_UNLIKELY(0.999);
 836   const Type* in_val_maybe_null_t = igvn.type(in_val);
 837 
 838 #ifdef SPARC
 839   Node* bad_mask = igvn.transform(new AddrBadBitNode());
 840 #else
 841   Node* jthread = igvn.transform(new ThreadLocalNode());
 842   Node* adr = phase->basic_plus_adr(jthread, in_bytes(ZThreadLocalData::address_bad_mask_offset()));
 843   Node* bad_mask = igvn.transform(LoadNode::make(igvn, in_ctrl, in_mem, adr, TypeRawPtr::BOTTOM, TypeX_X, TypeX_X->basic_type(), MemNode::unordered));
 844 #endif
 845   Node* cast = igvn.transform(new CastP2XNode(in_ctrl, in_val));
 846   Node* obj_masked = igvn.transform(new AndXNode(cast, bad_mask));
 847   Node* cmp = igvn.transform(new CmpXNode(obj_masked, igvn.zerocon(TypeX_X->basic_type())));
 848   Node *bol = igvn.transform(new BoolNode(cmp, BoolTest::ne))->as_Bool();
 849   IfNode* iff = igvn.transform(new IfNode(in_ctrl, bol, unlikely, COUNT_UNKNOWN))->as_If();
 850   Node* then = igvn.transform(new IfTrueNode(iff));
 851   Node* elsen = igvn.transform(new IfFalseNode(iff));
 852 
 853   Node* result_region;
 854   Node* result_val;
 855 
 856   result_region = new RegionNode(3);
 857   result_val = new PhiNode(result_region, TypeInstPtr::BOTTOM);
 858 
 859   result_region->set_req(1, elsen);
 860   Node* res = igvn.transform(new CastPPNode(in_val, in_val_maybe_null_t));
 861   res->init_req(0, elsen);
 862   result_val->set_req(1, res);
 863 
 864   const TypeFunc *tf = load_barrier_Type();
 865   Node* call;
 866   if (barrier->is_weak()) {
 867     call = new CallLeafNode(tf,
 868                             ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded_addr(),
 869                             "ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded",
 870                             TypeRawPtr::BOTTOM);
 871   } else {
 872     call = new CallLeafNode(tf,
 873                             ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(),
 874                             "ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded",
 875                             TypeRawPtr::BOTTOM);
 876   }
 877 
 878   call->init_req(TypeFunc::Control, then);
 879   call->init_req(TypeFunc::I_O    , phase->top());
 880   call->init_req(TypeFunc::Memory , in_mem);
 881   call->init_req(TypeFunc::FramePtr, phase->top());
 882   call->init_req(TypeFunc::ReturnAdr, phase->top());
 883   call->init_req(TypeFunc::Parms+0, in_val);
 884   if (barrier->is_writeback()) {
 885     call->init_req(TypeFunc::Parms+1, in_adr);
 886   } else {
 887     // when slow path is called with a null adr, the healed oop will not be written back
 888     call->init_req(TypeFunc::Parms+1, igvn.zerocon(T_OBJECT));
 889   }
 890   call = igvn.transform(call);
 891 
 892   Node* ctrl = igvn.transform(new ProjNode(call, TypeFunc::Control));
 893   res = igvn.transform(new ProjNode(call, TypeFunc::Parms));
 894   res = igvn.transform(new CheckCastPPNode(ctrl, res, in_val_maybe_null_t));
 895 
 896   result_region->set_req(2, ctrl);
 897   result_val->set_req(2, res);
 898 
 899   result_region = igvn.transform(result_region);
 900   result_val = igvn.transform(result_val);
 901 
 902   if (out_ctrl != NULL) { // added if cond
 903     igvn.replace_node(out_ctrl, result_region);
 904   }
 905   igvn.replace_node(out_res, result_val);
 906 }
 907 
 908 // Optimized, low spill, loadbarrier variant using stub specialized on register used
 909 void ZBarrierSetC2::expand_loadbarrier_optimized(PhaseMacroExpand* phase, LoadBarrierNode *barrier) const {
 910   PhaseIterGVN &igvn = phase->igvn();
 911 #ifdef PRINT_NODE_TRAVERSALS
 912   Node* preceding_barrier_node = barrier->in(LoadBarrierNode::Oop);
 913 #endif
 914 
 915   Node* in_ctrl = barrier->in(LoadBarrierNode::Control);
 916   Node* in_mem = barrier->in(LoadBarrierNode::Memory);
 917   Node* in_val = barrier->in(LoadBarrierNode::Oop);
 918   Node* in_adr = barrier->in(LoadBarrierNode::Address);
 919 
 920   Node* out_ctrl = barrier->proj_out(LoadBarrierNode::Control);
 921   Node* out_res = barrier->proj_out(LoadBarrierNode::Oop);
 922 
 923   assert(barrier->in(LoadBarrierNode::Oop) != NULL, "oop to loadbarrier node cannot be null");
 924 
 925 #ifdef PRINT_NODE_TRAVERSALS
 926   tty->print("\n\n\nBefore barrier optimization:\n");
 927   traverse(barrier, out_ctrl, out_res, -1);
 928 
 929   tty->print("\nBefore barrier optimization:  preceding_barrier_node\n");
 930   traverse(preceding_barrier_node, out_ctrl, out_res, -1);
 931 #endif
 932 
 933   float unlikely  = PROB_UNLIKELY(0.999);
 934 
 935   Node* jthread = igvn.transform(new ThreadLocalNode());
 936   Node* adr = phase->basic_plus_adr(jthread, in_bytes(ZThreadLocalData::address_bad_mask_offset()));
 937   Node* bad_mask = igvn.transform(LoadNode::make(igvn, in_ctrl, in_mem, adr,
 938                                                  TypeRawPtr::BOTTOM, TypeX_X, TypeX_X->basic_type(),
 939                                                  MemNode::unordered));
 940   Node* cast = igvn.transform(new CastP2XNode(in_ctrl, in_val));
 941   Node* obj_masked = igvn.transform(new AndXNode(cast, bad_mask));
 942   Node* cmp = igvn.transform(new CmpXNode(obj_masked, igvn.zerocon(TypeX_X->basic_type())));
 943   Node *bol = igvn.transform(new BoolNode(cmp, BoolTest::ne))->as_Bool();
 944   IfNode* iff = igvn.transform(new IfNode(in_ctrl, bol, unlikely, COUNT_UNKNOWN))->as_If();
 945   Node* then = igvn.transform(new IfTrueNode(iff));
 946   Node* elsen = igvn.transform(new IfFalseNode(iff));
 947 
 948   Node* slow_path_surrogate;
 949   if (!barrier->is_weak()) {
 950     slow_path_surrogate = igvn.transform(new LoadBarrierSlowRegNode(then, in_mem, in_adr, in_val->adr_type(),
 951                                                                     (const TypePtr*) in_val->bottom_type(), MemNode::unordered));
 952   } else {
 953     slow_path_surrogate = igvn.transform(new LoadBarrierWeakSlowRegNode(then, in_mem, in_adr, in_val->adr_type(),
 954                                                                         (const TypePtr*) in_val->bottom_type(), MemNode::unordered));
 955   }
 956 
 957   Node *new_loadp;
 958   new_loadp = slow_path_surrogate;
 959   // create the final region/phi pair to converge cntl/data paths to downstream code
 960   Node* result_region = igvn.transform(new RegionNode(3));
 961   result_region->set_req(1, then);
 962   result_region->set_req(2, elsen);
 963 
 964   Node* result_phi = igvn.transform(new PhiNode(result_region, TypeInstPtr::BOTTOM));
 965   result_phi->set_req(1, new_loadp);
 966   result_phi->set_req(2, barrier->in(LoadBarrierNode::Oop));
 967 
 968   // finally, connect the original outputs to the barrier region and phi to complete the expansion/substitution
 969   // igvn.replace_node(out_ctrl, result_region);
 970   if (out_ctrl != NULL) { // added if cond
 971     igvn.replace_node(out_ctrl, result_region);
 972   }
 973   igvn.replace_node(out_res, result_phi);
 974 
 975   assert(barrier->outcnt() == 0,"LoadBarrier macro node has non-null outputs after expansion!");
 976 
 977 #ifdef PRINT_NODE_TRAVERSALS
 978   tty->print("\nAfter barrier optimization:  old out_ctrl\n");
 979   traverse(out_ctrl, out_ctrl, out_res, -1);
 980   tty->print("\nAfter barrier optimization:  old out_res\n");
 981   traverse(out_res, out_ctrl, out_res, -1);
 982   tty->print("\nAfter barrier optimization:  old barrier\n");
 983   traverse(barrier, out_ctrl, out_res, -1);
 984   tty->print("\nAfter barrier optimization:  preceding_barrier_node\n");
 985   traverse(preceding_barrier_node, result_region, result_phi, -1);
 986 #endif
 987 
 988   return;
 989 }
 990 
 991 bool ZBarrierSetC2::expand_macro_nodes(PhaseMacroExpand* macro) const {
 992   Compile* C = Compile::current();
 993   PhaseIterGVN &igvn = macro->igvn();
 994   ZBarrierSetC2State* s = state();
 995   if (s->load_barrier_count() > 0) {
 996 #ifdef ASSERT
 997     verify_gc_barriers(false);
 998 #endif
 999     igvn.set_delay_transform(true);
1000     int skipped = 0;
1001     while (s->load_barrier_count() > skipped) {
1002       int load_barrier_count = s->load_barrier_count();
1003       LoadBarrierNode * n = s->load_barrier_node(load_barrier_count-1-skipped);
1004       if (igvn.type(n) == Type::TOP || (n->in(0) != NULL && n->in(0)->is_top())) {
1005         // node is unreachable, so don't try to expand it
1006         s->remove_load_barrier_node(n);
1007         continue;
1008       }
1009       if (!n->can_be_eliminated()) {
1010         skipped++;
1011         continue;
1012       }
1013       expand_loadbarrier_node(macro, n);
1014       assert(s->load_barrier_count() < load_barrier_count, "must have deleted a node from load barrier list");
1015       if (C->failing())  return true;
1016     }
1017     while (s->load_barrier_count() > 0) {
1018       int load_barrier_count = s->load_barrier_count();
1019       LoadBarrierNode* n = s->load_barrier_node(load_barrier_count - 1);
1020       assert(!(igvn.type(n) == Type::TOP || (n->in(0) != NULL && n->in(0)->is_top())), "should have been processed already");
1021       assert(!n->can_be_eliminated(), "should have been processed already");
1022       expand_loadbarrier_node(macro, n);
1023       assert(s->load_barrier_count() < load_barrier_count, "must have deleted a node from load barrier list");
1024       if (C->failing())  return true;
1025     }
1026     igvn.set_delay_transform(false);
1027     igvn.optimize();
1028     if (C->failing())  return true;
1029   }
1030   return false;
1031 }
1032 
1033 // == Loop optimization ==
1034 
1035 static bool replace_with_dominating_barrier(PhaseIdealLoop* phase, LoadBarrierNode* lb, bool last_round) {
1036   PhaseIterGVN &igvn = phase->igvn();
1037   Compile* C = Compile::current();
1038 
1039   LoadBarrierNode* lb2 = lb->has_dominating_barrier(phase, false, last_round);
1040   if (lb2 != NULL) {
1041     if (lb->in(LoadBarrierNode::Oop) != lb2->in(LoadBarrierNode::Oop)) {
1042       assert(lb->in(LoadBarrierNode::Address) == lb2->in(LoadBarrierNode::Address), "");
1043       igvn.replace_input_of(lb, LoadBarrierNode::Similar, lb2->proj_out(LoadBarrierNode::Oop));
1044       C->set_major_progress();
1045     } else  {
1046       // That transformation may cause the Similar edge on dominated load barriers to be invalid
1047       lb->fix_similar_in_uses(&igvn);
1048 
1049       Node* val = lb->proj_out(LoadBarrierNode::Oop);
1050       assert(lb2->has_true_uses(), "");
1051       assert(lb2->in(LoadBarrierNode::Oop) == lb->in(LoadBarrierNode::Oop), "");
1052 
1053       phase->lazy_update(lb, lb->in(LoadBarrierNode::Control));
1054       phase->lazy_replace(lb->proj_out(LoadBarrierNode::Control), lb->in(LoadBarrierNode::Control));
1055       igvn.replace_node(val, lb2->proj_out(LoadBarrierNode::Oop));
1056 
1057       return true;
1058     }
1059   }
1060   return false;
1061 }
1062 
1063 static Node* find_dominating_memory(PhaseIdealLoop* phase, Node* mem, Node* dom, int i) {
1064   assert(dom->is_Region() || i == -1, "");
1065   Node* m = mem;
1066   while(phase->is_dominator(dom, phase->has_ctrl(m) ? phase->get_ctrl(m) : m->in(0))) {
1067     if (m->is_Mem()) {
1068       assert(m->as_Mem()->adr_type() == TypeRawPtr::BOTTOM, "");
1069       m = m->in(MemNode::Memory);
1070     } else if (m->is_MergeMem()) {
1071       m = m->as_MergeMem()->memory_at(Compile::AliasIdxRaw);
1072     } else if (m->is_Phi()) {
1073       if (m->in(0) == dom && i != -1) {
1074         m = m->in(i);
1075         break;
1076       } else {
1077         m = m->in(LoopNode::EntryControl);
1078       }
1079     } else if (m->is_Proj()) {
1080       m = m->in(0);
1081     } else if (m->is_SafePoint() || m->is_MemBar()) {
1082       m = m->in(TypeFunc::Memory);
1083     } else {
1084 #ifdef ASSERT
1085       m->dump();
1086 #endif
1087       ShouldNotReachHere();
1088     }
1089   }
1090   return m;
1091 }
1092 
1093 static LoadBarrierNode* clone_load_barrier(PhaseIdealLoop* phase, LoadBarrierNode* lb, Node* ctl, Node* mem, Node* oop_in) {
1094   PhaseIterGVN &igvn = phase->igvn();
1095   Compile* C = Compile::current();
1096   Node* the_clone = lb->clone();
1097   the_clone->set_req(LoadBarrierNode::Control, ctl);
1098   the_clone->set_req(LoadBarrierNode::Memory, mem);
1099   if (oop_in != NULL) {
1100     the_clone->set_req(LoadBarrierNode::Oop, oop_in);
1101   }
1102 
1103   LoadBarrierNode* new_lb = the_clone->as_LoadBarrier();
1104   igvn.register_new_node_with_optimizer(new_lb);
1105   IdealLoopTree *loop = phase->get_loop(new_lb->in(0));
1106   phase->set_ctrl(new_lb, new_lb->in(0));
1107   phase->set_loop(new_lb, loop);
1108   phase->set_idom(new_lb, new_lb->in(0), phase->dom_depth(new_lb->in(0))+1);
1109   if (!loop->_child) {
1110     loop->_body.push(new_lb);
1111   }
1112 
1113   Node* proj_ctl = new ProjNode(new_lb, LoadBarrierNode::Control);
1114   igvn.register_new_node_with_optimizer(proj_ctl);
1115   phase->set_ctrl(proj_ctl, proj_ctl->in(0));
1116   phase->set_loop(proj_ctl, loop);
1117   phase->set_idom(proj_ctl, new_lb, phase->dom_depth(new_lb)+1);
1118   if (!loop->_child) {
1119     loop->_body.push(proj_ctl);
1120   }
1121 
1122   Node* proj_oop = new ProjNode(new_lb, LoadBarrierNode::Oop);
1123   phase->register_new_node(proj_oop, new_lb);
1124 
1125   if (!new_lb->in(LoadBarrierNode::Similar)->is_top()) {
1126     LoadBarrierNode* similar = new_lb->in(LoadBarrierNode::Similar)->in(0)->as_LoadBarrier();
1127     if (!phase->is_dominator(similar, ctl)) {
1128       igvn.replace_input_of(new_lb, LoadBarrierNode::Similar, C->top());
1129     }
1130   }
1131 
1132   return new_lb;
1133 }
1134 
1135 static void replace_barrier(PhaseIdealLoop* phase, LoadBarrierNode* lb, Node* new_val) {
1136   PhaseIterGVN &igvn = phase->igvn();
1137   Node* val = lb->proj_out(LoadBarrierNode::Oop);
1138   igvn.replace_node(val, new_val);
1139   phase->lazy_update(lb, lb->in(LoadBarrierNode::Control));
1140   phase->lazy_replace(lb->proj_out(LoadBarrierNode::Control), lb->in(LoadBarrierNode::Control));
1141 }
1142 
1143 static bool split_barrier_thru_phi(PhaseIdealLoop* phase, LoadBarrierNode* lb) {
1144   PhaseIterGVN &igvn = phase->igvn();
1145   Compile* C = Compile::current();
1146 
1147   if (lb->in(LoadBarrierNode::Oop)->is_Phi()) {
1148     Node* oop_phi = lb->in(LoadBarrierNode::Oop);
1149 
1150     if (oop_phi->req() == 2) {
1151       // Ignore phis with only one input
1152       return false;
1153     }
1154 
1155     if (phase->is_dominator(phase->get_ctrl(lb->in(LoadBarrierNode::Address)),
1156                             oop_phi->in(0)) && phase->get_ctrl(lb->in(LoadBarrierNode::Address)) != oop_phi->in(0)) {
1157       // That transformation may cause the Similar edge on dominated load barriers to be invalid
1158       lb->fix_similar_in_uses(&igvn);
1159 
1160       RegionNode* region = oop_phi->in(0)->as_Region();
1161 
1162       int backedge = LoopNode::LoopBackControl;
1163       if (region->is_Loop() && region->in(backedge)->is_Proj() && region->in(backedge)->in(0)->is_If()) {
1164         Node* c = region->in(backedge)->in(0)->in(0);
1165         assert(c->unique_ctrl_out() == region->in(backedge)->in(0), "");
1166         Node* oop = lb->in(LoadBarrierNode::Oop)->in(backedge);
1167         Node* oop_c = phase->has_ctrl(oop) ? phase->get_ctrl(oop) : oop;
1168         if (!phase->is_dominator(oop_c, c)) {
1169           return false;
1170         }
1171       }
1172 
1173       // If the node on the backedge above the phi is the node itself - we have a self loop.
1174       // Don't clone - this will be folded later.
1175       if (oop_phi->in(LoopNode::LoopBackControl) == lb->proj_out(LoadBarrierNode::Oop)) {
1176         return false;
1177       }
1178 
1179       bool is_strip_mined = region->is_CountedLoop() && region->as_CountedLoop()->is_strip_mined();
1180       Node *phi = oop_phi->clone();
1181 
1182       for (uint i = 1; i < region->req(); i++) {
1183         Node* ctrl = region->in(i);
1184         if (ctrl != C->top()) {
1185           assert(!phase->is_dominator(ctrl, region) || region->is_Loop(), "");
1186 
1187           Node* mem = lb->in(LoadBarrierNode::Memory);
1188           Node* m = find_dominating_memory(phase, mem, region, i);
1189 
1190           if (region->is_Loop() && i == LoopNode::LoopBackControl && ctrl->is_Proj() && ctrl->in(0)->is_If()) {
1191             ctrl = ctrl->in(0)->in(0);
1192           } else if (region->is_Loop() && is_strip_mined) {
1193             // If this is a strip mined loop, control must move above OuterStripMinedLoop
1194             assert(i == LoopNode::EntryControl, "check");
1195             assert(ctrl->is_OuterStripMinedLoop(), "sanity");
1196             ctrl = ctrl->as_OuterStripMinedLoop()->in(LoopNode::EntryControl);
1197           }
1198 
1199           LoadBarrierNode* new_lb = clone_load_barrier(phase, lb, ctrl, m, lb->in(LoadBarrierNode::Oop)->in(i));
1200           Node* out_ctrl = new_lb->proj_out(LoadBarrierNode::Control);
1201 
1202           if (is_strip_mined && (i == LoopNode::EntryControl)) {
1203             assert(region->in(i)->is_OuterStripMinedLoop(), "");
1204             igvn.replace_input_of(region->in(i), i, out_ctrl);
1205           } else if (ctrl == region->in(i)) {
1206             igvn.replace_input_of(region, i, out_ctrl);
1207           } else {
1208             Node* iff = region->in(i)->in(0);
1209             igvn.replace_input_of(iff, 0, out_ctrl);
1210             phase->set_idom(iff, out_ctrl, phase->dom_depth(out_ctrl)+1);
1211           }
1212           phi->set_req(i, new_lb->proj_out(LoadBarrierNode::Oop));
1213         }
1214       }
1215       phase->register_new_node(phi, region);
1216       replace_barrier(phase, lb, phi);
1217 
1218       if (region->is_Loop()) {
1219         // Load barrier moved to the back edge of the Loop may now
1220         // have a safepoint on the path to the barrier on the Similar
1221         // edge
1222         igvn.replace_input_of(phi->in(LoopNode::LoopBackControl)->in(0), LoadBarrierNode::Similar, C->top());
1223         Node* head = region->in(LoopNode::EntryControl);
1224         phase->set_idom(region, head, phase->dom_depth(head)+1);
1225         phase->recompute_dom_depth();
1226         if (head->is_CountedLoop() && head->as_CountedLoop()->is_main_loop()) {
1227           head->as_CountedLoop()->set_normal_loop();
1228         }
1229       }
1230 
1231       return true;
1232     }
1233   }
1234 
1235   return false;
1236 }
1237 
1238 static bool move_out_of_loop(PhaseIdealLoop* phase, LoadBarrierNode* lb) {
1239   PhaseIterGVN &igvn = phase->igvn();
1240   IdealLoopTree *lb_loop = phase->get_loop(lb->in(0));
1241   if (lb_loop != phase->ltree_root() && !lb_loop->_irreducible) {
1242     Node* oop_ctrl = phase->get_ctrl(lb->in(LoadBarrierNode::Oop));
1243     IdealLoopTree *oop_loop = phase->get_loop(oop_ctrl);
1244     IdealLoopTree* adr_loop = phase->get_loop(phase->get_ctrl(lb->in(LoadBarrierNode::Address)));
1245     if (!lb_loop->is_member(oop_loop) && !lb_loop->is_member(adr_loop)) {
1246       // That transformation may cause the Similar edge on dominated load barriers to be invalid
1247       lb->fix_similar_in_uses(&igvn);
1248 
1249       Node* head = lb_loop->_head;
1250       assert(head->is_Loop(), "");
1251 
1252       if (phase->is_dominator(head, oop_ctrl)) {
1253         assert(oop_ctrl->Opcode() == Op_CProj && oop_ctrl->in(0)->Opcode() == Op_NeverBranch, "");
1254         assert(lb_loop->is_member(phase->get_loop(oop_ctrl->in(0)->in(0))), "");
1255         return false;
1256       }
1257 
1258       if (head->is_CountedLoop()) {
1259         CountedLoopNode* cloop = head->as_CountedLoop();
1260         if (cloop->is_main_loop()) {
1261           cloop->set_normal_loop();
1262         }
1263         // When we are moving barrier out of a counted loop,
1264         // make sure we move it all the way out of the strip mined outer loop.
1265         if (cloop->is_strip_mined()) {
1266           head = cloop->outer_loop();
1267         }
1268       }
1269 
1270       Node* mem = lb->in(LoadBarrierNode::Memory);
1271       Node* m = find_dominating_memory(phase, mem, head, -1);
1272 
1273       LoadBarrierNode* new_lb = clone_load_barrier(phase, lb, head->in(LoopNode::EntryControl), m, NULL);
1274 
1275       assert(phase->idom(head) == head->in(LoopNode::EntryControl), "");
1276       Node* proj_ctl = new_lb->proj_out(LoadBarrierNode::Control);
1277       igvn.replace_input_of(head, LoopNode::EntryControl, proj_ctl);
1278       phase->set_idom(head, proj_ctl, phase->dom_depth(proj_ctl) + 1);
1279 
1280       replace_barrier(phase, lb, new_lb->proj_out(LoadBarrierNode::Oop));
1281 
1282       phase->recompute_dom_depth();
1283 
1284       return true;
1285     }
1286   }
1287 
1288   return false;
1289 }
1290 
1291 static bool common_barriers(PhaseIdealLoop* phase, LoadBarrierNode* lb) {
1292   PhaseIterGVN &igvn = phase->igvn();
1293   Node* in_val = lb->in(LoadBarrierNode::Oop);
1294   for (DUIterator_Fast imax, i = in_val->fast_outs(imax); i < imax; i++) {
1295     Node* u = in_val->fast_out(i);
1296     if (u != lb && u->is_LoadBarrier() && u->as_LoadBarrier()->has_true_uses()) {
1297       Node* this_ctrl = lb->in(LoadBarrierNode::Control);
1298       Node* other_ctrl = u->in(LoadBarrierNode::Control);
1299 
1300       Node* lca = phase->dom_lca(this_ctrl, other_ctrl);
1301       bool ok = true;
1302 
1303       Node* proj1 = NULL;
1304       Node* proj2 = NULL;
1305 
1306       while (this_ctrl != lca && ok) {
1307         if (this_ctrl->in(0) != NULL &&
1308             this_ctrl->in(0)->is_MultiBranch()) {
1309           if (this_ctrl->in(0)->in(0) == lca) {
1310             assert(proj1 == NULL, "");
1311             assert(this_ctrl->is_Proj(), "");
1312             proj1 = this_ctrl;
1313           } else if (!(this_ctrl->in(0)->is_If() && this_ctrl->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none))) {
1314             ok = false;
1315           }
1316         }
1317         this_ctrl = phase->idom(this_ctrl);
1318       }
1319       while (other_ctrl != lca && ok) {
1320         if (other_ctrl->in(0) != NULL &&
1321             other_ctrl->in(0)->is_MultiBranch()) {
1322           if (other_ctrl->in(0)->in(0) == lca) {
1323             assert(other_ctrl->is_Proj(), "");
1324             assert(proj2 == NULL, "");
1325             proj2 = other_ctrl;
1326           } else if (!(other_ctrl->in(0)->is_If() && other_ctrl->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none))) {
1327             ok = false;
1328           }
1329         }
1330         other_ctrl = phase->idom(other_ctrl);
1331       }
1332       assert(proj1 == NULL || proj2 == NULL || proj1->in(0) == proj2->in(0), "");
1333       if (ok && proj1 && proj2 && proj1 != proj2 && proj1->in(0)->is_If()) {
1334         // That transformation may cause the Similar edge on dominated load barriers to be invalid
1335         lb->fix_similar_in_uses(&igvn);
1336         u->as_LoadBarrier()->fix_similar_in_uses(&igvn);
1337 
1338         Node* split = lca->unique_ctrl_out();
1339         assert(split->in(0) == lca, "");
1340 
1341         Node* mem = lb->in(LoadBarrierNode::Memory);
1342         Node* m = find_dominating_memory(phase, mem, split, -1);
1343         LoadBarrierNode* new_lb = clone_load_barrier(phase, lb, lca, m, NULL);
1344 
1345         Node* proj_ctl = new_lb->proj_out(LoadBarrierNode::Control);
1346         igvn.replace_input_of(split, 0, new_lb->proj_out(LoadBarrierNode::Control));
1347         phase->set_idom(split, proj_ctl, phase->dom_depth(proj_ctl)+1);
1348 
1349         Node* proj_oop = new_lb->proj_out(LoadBarrierNode::Oop);
1350         replace_barrier(phase, lb, proj_oop);
1351         replace_barrier(phase, u->as_LoadBarrier(), proj_oop);
1352 
1353         phase->recompute_dom_depth();
1354 
1355         return true;
1356       }
1357     }
1358   }
1359 
1360   return false;
1361 }
1362 
1363 static void optimize_load_barrier(PhaseIdealLoop* phase, LoadBarrierNode* lb, bool last_round) {
1364   Compile* C = Compile::current();
1365 
1366   if (!C->directive()->ZOptimizeLoadBarriersOption) {
1367     return;
1368   }
1369 
1370   if (lb->has_true_uses()) {
1371     if (replace_with_dominating_barrier(phase, lb, last_round)) {
1372       return;
1373     }
1374 
1375     if (split_barrier_thru_phi(phase, lb)) {
1376       return;
1377     }
1378 
1379     if (move_out_of_loop(phase, lb)) {
1380       return;
1381     }
1382 
1383     if (common_barriers(phase, lb)) {
1384       return;
1385     }
1386   }
1387 }
1388 
1389 void ZBarrierSetC2::loop_optimize_gc_barrier(PhaseIdealLoop* phase, Node* node, bool last_round) {
1390   if (node->is_LoadBarrier()) {
1391     optimize_load_barrier(phase, node->as_LoadBarrier(), last_round);
1392   }
1393 }
1394 
1395 // == Verification ==
1396 
1397 #ifdef ASSERT
1398 
1399 static bool look_for_barrier(Node* n, bool post_parse, VectorSet& visited) {
1400   if (visited.test_set(n->_idx)) {
1401     return true;
1402   }
1403 
1404   for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
1405     Node* u = n->fast_out(i);
1406     if (u->is_LoadBarrier()) {
1407     } else if ((u->is_Phi() || u->is_CMove()) && !post_parse) {
1408       if (!look_for_barrier(u, post_parse, visited)) {
1409         return false;
1410       }
1411     } else if (u->Opcode() == Op_EncodeP || u->Opcode() == Op_DecodeN) {
1412       if (!look_for_barrier(u, post_parse, visited)) {
1413         return false;
1414       }
1415     } else if (u->Opcode() != Op_SCMemProj) {
1416       tty->print("bad use"); u->dump();
1417       return false;
1418     }
1419   }
1420 
1421   return true;
1422 }
1423 
1424 void ZBarrierSetC2::verify_gc_barriers(bool post_parse) const {
1425   ZBarrierSetC2State* s = state();
1426   Compile* C = Compile::current();
1427   ResourceMark rm;
1428   VectorSet visited(Thread::current()->resource_area());
1429   for (int i = 0; i < s->load_barrier_count(); i++) {
1430     LoadBarrierNode* n = s->load_barrier_node(i);
1431 
1432     // The dominating barrier on the same address if it exists and
1433     // this barrier must not be applied on the value from the same
1434     // load otherwise the value is not reloaded before it's used the
1435     // second time.
1436     assert(n->in(LoadBarrierNode::Similar)->is_top() ||
1437            (n->in(LoadBarrierNode::Similar)->in(0)->is_LoadBarrier() &&
1438             n->in(LoadBarrierNode::Similar)->in(0)->in(LoadBarrierNode::Address) == n->in(LoadBarrierNode::Address) &&
1439             n->in(LoadBarrierNode::Similar)->in(0)->in(LoadBarrierNode::Oop) != n->in(LoadBarrierNode::Oop)),
1440            "broken similar edge");
1441 
1442     assert(post_parse || n->as_LoadBarrier()->has_true_uses(),
1443            "found unneeded load barrier");
1444 
1445     // Several load barrier nodes chained through their Similar edge
1446     // break the code that remove the barriers in final graph reshape.
1447     assert(n->in(LoadBarrierNode::Similar)->is_top() ||
1448            (n->in(LoadBarrierNode::Similar)->in(0)->is_LoadBarrier() &&
1449             n->in(LoadBarrierNode::Similar)->in(0)->in(LoadBarrierNode::Similar)->is_top()),
1450            "chain of Similar load barriers");
1451 
1452     if (!n->in(LoadBarrierNode::Similar)->is_top()) {
1453       ResourceMark rm;
1454       Unique_Node_List wq;
1455       Node* other = n->in(LoadBarrierNode::Similar)->in(0);
1456       wq.push(n);
1457       bool ok = true;
1458       bool dom_found = false;
1459       for (uint next = 0; next < wq.size(); ++next) {
1460         Node *n = wq.at(next);
1461         assert(n->is_CFG(), "");
1462         assert(!n->is_SafePoint(), "");
1463 
1464         if (n == other) {
1465           continue;
1466         }
1467 
1468         if (n->is_Region()) {
1469           for (uint i = 1; i < n->req(); i++) {
1470             Node* m = n->in(i);
1471             if (m != NULL) {
1472               wq.push(m);
1473             }
1474           }
1475         } else {
1476           Node* m = n->in(0);
1477           if (m != NULL) {
1478             wq.push(m);
1479           }
1480         }
1481       }
1482     }
1483 
1484     if (ZVerifyLoadBarriers) {
1485       if ((n->is_Load() || n->is_LoadStore()) && n->bottom_type()->make_oopptr() != NULL) {
1486         visited.Clear();
1487         bool found = look_for_barrier(n, post_parse, visited);
1488         if (!found) {
1489           n->dump(1);
1490           n->dump(-3);
1491           stringStream ss;
1492           C->method()->print_short_name(&ss);
1493           tty->print_cr("-%s-", ss.as_string());
1494           assert(found, "");
1495         }
1496       }
1497     }
1498   }
1499 }
1500 
1501 #endif
--- EOF ---