1 /*
2 * Copyright (c) 2012, 2018, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24
25 package org.graalvm.compiler.loop;
26
27 import java.util.ArrayList;
28 import java.util.LinkedList;
29 import java.util.List;
30
31 import jdk.internal.vm.compiler.collections.EconomicMap;
32 import jdk.internal.vm.compiler.collections.Equivalence;
33 import org.graalvm.compiler.core.common.type.IntegerStamp;
34 import org.graalvm.compiler.debug.DebugCloseable;
35 import org.graalvm.compiler.debug.DebugContext;
36 import org.graalvm.compiler.debug.GraalError;
37 import org.graalvm.compiler.graph.Graph.DuplicationReplacement;
38 import org.graalvm.compiler.graph.Node;
39 import org.graalvm.compiler.graph.NodeBitMap;
40 import org.graalvm.compiler.graph.iterators.NodeIterable;
41 import org.graalvm.compiler.nodes.AbstractBeginNode;
42 import org.graalvm.compiler.nodes.AbstractEndNode;
43 import org.graalvm.compiler.nodes.AbstractMergeNode;
44 import org.graalvm.compiler.nodes.BeginNode;
45 import org.graalvm.compiler.nodes.ConstantNode;
46 import org.graalvm.compiler.nodes.EndNode;
47 import org.graalvm.compiler.nodes.FixedNode;
48 import org.graalvm.compiler.nodes.FixedWithNextNode;
49 import org.graalvm.compiler.nodes.FrameState;
50 import org.graalvm.compiler.nodes.GuardPhiNode;
51 import org.graalvm.compiler.nodes.IfNode;
52 import org.graalvm.compiler.nodes.LogicNode;
53 import org.graalvm.compiler.nodes.LoopBeginNode;
54 import org.graalvm.compiler.nodes.LoopEndNode;
55 import org.graalvm.compiler.nodes.LoopExitNode;
56 import org.graalvm.compiler.nodes.MergeNode;
57 import org.graalvm.compiler.nodes.NodeView;
58 import org.graalvm.compiler.nodes.PhiNode;
59 import org.graalvm.compiler.nodes.ProxyNode;
60 import org.graalvm.compiler.nodes.SafepointNode;
61 import org.graalvm.compiler.nodes.StateSplit;
62 import org.graalvm.compiler.nodes.StructuredGraph;
63 import org.graalvm.compiler.nodes.ValueNode;
64 import org.graalvm.compiler.nodes.ValuePhiNode;
65 import org.graalvm.compiler.nodes.VirtualState.NodeClosure;
66 import org.graalvm.compiler.nodes.calc.AddNode;
67 import org.graalvm.compiler.nodes.calc.CompareNode;
68 import org.graalvm.compiler.nodes.calc.ConditionalNode;
69 import org.graalvm.compiler.nodes.calc.IntegerBelowNode;
70 import org.graalvm.compiler.nodes.calc.SubNode;
71 import org.graalvm.compiler.nodes.extended.OpaqueNode;
72 import org.graalvm.compiler.nodes.memory.MemoryPhiNode;
73 import org.graalvm.compiler.nodes.util.GraphUtil;
74 import org.graalvm.compiler.nodes.util.IntegerHelper;
75
76 public class LoopFragmentInside extends LoopFragment {
77
78 /**
79 * mergedInitializers. When an inside fragment's (loop)ends are merged to create a unique exit
80 * point, some phis must be created : they phis together all the back-values of the loop-phis
81 * These can then be used to update the loop-phis' forward edge value ('initializer') in the
82 * peeling case. In the unrolling case they will be used as the value that replace the loop-phis
83 * of the duplicated inside fragment
84 */
85 private EconomicMap<PhiNode, ValueNode> mergedInitializers;
86 private final DuplicationReplacement dataFixBefore = new DuplicationReplacement() {
87
88 @Override
89 public Node replacement(Node oriInput) {
90 if (!(oriInput instanceof ValueNode)) {
91 return oriInput;
92 }
93 return prim((ValueNode) oriInput);
94 }
95 };
96
97 private final DuplicationReplacement dataFixWithinAfter = new DuplicationReplacement() {
98
99 @Override
100 public Node replacement(Node oriInput) {
101 if (!(oriInput instanceof ValueNode)) {
102 return oriInput;
103 }
104 return primAfter((ValueNode) oriInput);
105 }
106 };
107
108 public LoopFragmentInside(LoopEx loop) {
109 super(loop);
110 }
111
112 public LoopFragmentInside(LoopFragmentInside original) {
113 super(null, original);
114 }
115
116 @Override
117 public LoopFragmentInside duplicate() {
118 assert !isDuplicate();
119 return new LoopFragmentInside(this);
120 }
121
122 @Override
123 public LoopFragmentInside original() {
124 return (LoopFragmentInside) super.original();
125 }
126
127 @SuppressWarnings("unused")
128 public void appendInside(LoopEx loop) {
129 // TODO (gd)
130 }
131
132 @Override
133 public LoopEx loop() {
134 assert !this.isDuplicate();
135 return super.loop();
136 }
137
138 @Override
139 public void insertBefore(LoopEx loop) {
140 assert this.isDuplicate() && this.original().loop() == loop;
141
142 patchNodes(dataFixBefore);
143
144 AbstractBeginNode end = mergeEnds();
145
146 mergeEarlyExits();
147
148 original().patchPeeling(this);
149
150 AbstractBeginNode entry = getDuplicatedNode(loop.loopBegin());
151 loop.entryPoint().replaceAtPredecessor(entry);
152 end.setNext(loop.entryPoint());
153 }
154
155 /**
156 * Duplicate the body within the loop after the current copy copy of the body, updating the
157 * iteration limit to account for the duplication.
158 */
159 public void insertWithinAfter(LoopEx loop, EconomicMap<LoopBeginNode, OpaqueNode> opaqueUnrolledStrides) {
160 assert isDuplicate() && original().loop() == loop;
161
162 patchNodes(dataFixWithinAfter);
163
164 /*
165 * Collect any new back edges values before updating them since they might reference each
166 * other.
167 */
168 LoopBeginNode mainLoopBegin = loop.loopBegin();
169 ArrayList<ValueNode> backedgeValues = new ArrayList<>();
170 for (PhiNode mainPhiNode : mainLoopBegin.phis()) {
171 ValueNode originalNode = mainPhiNode.valueAt(1);
172 ValueNode duplicatedNode = getDuplicatedNode(originalNode);
173 if (duplicatedNode == null) {
174 if (mainLoopBegin.isPhiAtMerge(originalNode)) {
175 duplicatedNode = ((PhiNode) (originalNode)).valueAt(1);
176 } else {
177 assert originalNode.isConstant() || loop.isOutsideLoop(originalNode) : "Not duplicated node " + originalNode;
178 }
179 }
180 backedgeValues.add(duplicatedNode);
181 }
182 int index = 0;
183 for (PhiNode mainPhiNode : mainLoopBegin.phis()) {
184 ValueNode duplicatedNode = backedgeValues.get(index++);
185 if (duplicatedNode != null) {
186 mainPhiNode.setValueAt(1, duplicatedNode);
187 }
188 }
189
190 placeNewSegmentAndCleanup(loop);
191
192 // Remove any safepoints from the original copy leaving only the duplicated one
193 assert loop.whole().nodes().filter(SafepointNode.class).count() == nodes().filter(SafepointNode.class).count();
194 for (SafepointNode safepoint : loop.whole().nodes().filter(SafepointNode.class)) {
195 graph().removeFixed(safepoint);
196 }
197
198 StructuredGraph graph = mainLoopBegin.graph();
199 if (opaqueUnrolledStrides != null) {
200 OpaqueNode opaque = opaqueUnrolledStrides.get(loop.loopBegin());
201 CountedLoopInfo counted = loop.counted();
202 ValueNode counterStride = counted.getCounter().strideNode();
203 if (opaque == null) {
204 opaque = new OpaqueNode(AddNode.add(counterStride, counterStride, NodeView.DEFAULT));
205 ValueNode limit = counted.getLimit();
206 int bits = ((IntegerStamp) limit.stamp(NodeView.DEFAULT)).getBits();
207 ValueNode newLimit = SubNode.create(limit, opaque, NodeView.DEFAULT);
208 IntegerHelper helper = counted.getCounterIntegerHelper();
209 LogicNode overflowCheck;
210 ConstantNode extremum;
211 if (counted.getDirection() == InductionVariable.Direction.Up) {
212 // limit - counterStride could overflow negatively if limit - min <
213 // counterStride
214 extremum = ConstantNode.forIntegerBits(bits, helper.minValue());
215 overflowCheck = IntegerBelowNode.create(SubNode.create(limit, extremum, NodeView.DEFAULT), opaque, NodeView.DEFAULT);
216 } else {
217 assert counted.getDirection() == InductionVariable.Direction.Down;
218 // limit - counterStride could overflow if max - limit < -counterStride
219 // i.e., counterStride < limit - max
220 extremum = ConstantNode.forIntegerBits(bits, helper.maxValue());
221 overflowCheck = IntegerBelowNode.create(opaque, SubNode.create(limit, extremum, NodeView.DEFAULT), NodeView.DEFAULT);
222 }
223 newLimit = ConditionalNode.create(overflowCheck, extremum, newLimit, NodeView.DEFAULT);
224 CompareNode compareNode = (CompareNode) counted.getLimitTest().condition();
225 compareNode.replaceFirstInput(limit, graph.addOrUniqueWithInputs(newLimit));
226 opaqueUnrolledStrides.put(loop.loopBegin(), opaque);
227 } else {
228 assert counted.getCounter().isConstantStride();
229 assert Math.addExact(counted.getCounter().constantStride(), counted.getCounter().constantStride()) == counted.getCounter().constantStride() * 2;
230 ValueNode previousValue = opaque.getValue();
231 opaque.setValue(graph.addOrUniqueWithInputs(AddNode.add(counterStride, previousValue, NodeView.DEFAULT)));
232 GraphUtil.tryKillUnused(previousValue);
233 }
234 }
235 mainLoopBegin.setUnrollFactor(mainLoopBegin.getUnrollFactor() * 2);
236 mainLoopBegin.setLoopFrequency(mainLoopBegin.loopFrequency() / 2);
237 graph.getDebug().dump(DebugContext.DETAILED_LEVEL, graph, "LoopPartialUnroll %s", loop);
238
239 mainLoopBegin.getDebug().dump(DebugContext.VERBOSE_LEVEL, mainLoopBegin.graph(), "After insertWithinAfter %s", mainLoopBegin);
240 }
241
242 private void placeNewSegmentAndCleanup(LoopEx loop) {
243 CountedLoopInfo mainCounted = loop.counted();
244 LoopBeginNode mainLoopBegin = loop.loopBegin();
245 // Discard the segment entry and its flow, after if merging it into the loop
246 StructuredGraph graph = mainLoopBegin.graph();
247 IfNode loopTest = mainCounted.getLimitTest();
248 IfNode newSegmentTest = getDuplicatedNode(loopTest);
249 AbstractBeginNode trueSuccessor = loopTest.trueSuccessor();
250 AbstractBeginNode falseSuccessor = loopTest.falseSuccessor();
251 FixedNode firstNode;
252 boolean codeInTrueSide = false;
253 if (trueSuccessor == mainCounted.getBody()) {
254 firstNode = trueSuccessor.next();
255 codeInTrueSide = true;
256 } else {
257 assert (falseSuccessor == mainCounted.getBody());
258 firstNode = falseSuccessor.next();
259 }
260 trueSuccessor = newSegmentTest.trueSuccessor();
261 falseSuccessor = newSegmentTest.falseSuccessor();
262 for (Node usage : falseSuccessor.anchored().snapshot()) {
263 usage.replaceFirstInput(falseSuccessor, loopTest.falseSuccessor());
264 }
265 for (Node usage : trueSuccessor.anchored().snapshot()) {
266 usage.replaceFirstInput(trueSuccessor, loopTest.trueSuccessor());
267 }
268 AbstractBeginNode startBlockNode;
269 if (codeInTrueSide) {
270 startBlockNode = trueSuccessor;
271 } else {
272 graph.getDebug().dump(DebugContext.VERBOSE_LEVEL, mainLoopBegin.graph(), "before");
273 startBlockNode = falseSuccessor;
274 }
275 FixedNode lastNode = getBlockEnd(startBlockNode);
276 LoopEndNode loopEndNode = mainLoopBegin.getSingleLoopEnd();
277 FixedWithNextNode lastCodeNode = (FixedWithNextNode) loopEndNode.predecessor();
278 FixedNode newSegmentFirstNode = getDuplicatedNode(firstNode);
279 FixedWithNextNode newSegmentLastNode = getDuplicatedNode(lastCodeNode);
280 graph.getDebug().dump(DebugContext.DETAILED_LEVEL, loopEndNode.graph(), "Before placing segment");
281 if (firstNode instanceof LoopEndNode) {
282 GraphUtil.killCFG(getDuplicatedNode(mainLoopBegin));
283 } else {
284 newSegmentLastNode.clearSuccessors();
285 startBlockNode.setNext(lastNode);
286 lastCodeNode.replaceFirstSuccessor(loopEndNode, newSegmentFirstNode);
287 newSegmentLastNode.replaceFirstSuccessor(lastNode, loopEndNode);
288 lastCodeNode.setNext(newSegmentFirstNode);
289 newSegmentLastNode.setNext(loopEndNode);
290 startBlockNode.clearSuccessors();
291 lastNode.safeDelete();
292 Node newSegmentTestStart = newSegmentTest.predecessor();
293 LogicNode newSegmentIfTest = newSegmentTest.condition();
294 newSegmentTestStart.clearSuccessors();
295 newSegmentTest.safeDelete();
296 newSegmentIfTest.safeDelete();
297 trueSuccessor.safeDelete();
298 falseSuccessor.safeDelete();
299 newSegmentTestStart.safeDelete();
300 }
301 graph.getDebug().dump(DebugContext.DETAILED_LEVEL, loopEndNode.graph(), "After placing segment");
302 }
303
304 private static EndNode getBlockEnd(FixedNode node) {
305 FixedNode curNode = node;
306 while (curNode instanceof FixedWithNextNode) {
307 curNode = ((FixedWithNextNode) curNode).next();
308 }
309 return (EndNode) curNode;
310 }
311
312 @Override
313 public NodeBitMap nodes() {
314 if (nodes == null) {
315 LoopFragmentWhole whole = loop().whole();
316 whole.nodes(); // init nodes bitmap in whole
317 nodes = whole.nodes.copy();
318 // remove the phis
319 LoopBeginNode loopBegin = loop().loopBegin();
320 for (PhiNode phi : loopBegin.phis()) {
321 nodes.clear(phi);
322 }
323 clearStateNodes(loopBegin);
324 for (LoopExitNode exit : exits()) {
325 clearStateNodes(exit);
326 for (ProxyNode proxy : exit.proxies()) {
327 nodes.clear(proxy);
328 }
329 }
330 }
331 return nodes;
332 }
333
334 private void clearStateNodes(StateSplit stateSplit) {
335 FrameState loopState = stateSplit.stateAfter();
336 if (loopState != null) {
337 loopState.applyToVirtual(v -> {
338 if (v.usages().filter(n -> nodes.isMarked(n) && n != stateSplit).isEmpty()) {
339 nodes.clear(v);
340 }
341 });
342 }
343 }
344
345 public NodeIterable<LoopExitNode> exits() {
346 return loop().loopBegin().loopExits();
347 }
348
349 @Override
350 @SuppressWarnings("try")
351 protected DuplicationReplacement getDuplicationReplacement() {
352 final LoopBeginNode loopBegin = loop().loopBegin();
353 final StructuredGraph graph = graph();
354 return new DuplicationReplacement() {
355
356 private EconomicMap<Node, Node> seenNode = EconomicMap.create(Equivalence.IDENTITY);
357
358 @Override
359 public Node replacement(Node original) {
360 try (DebugCloseable position = original.withNodeSourcePosition()) {
361 if (original == loopBegin) {
362 Node value = seenNode.get(original);
363 if (value != null) {
364 return value;
365 }
366 AbstractBeginNode newValue = graph.add(new BeginNode());
367 seenNode.put(original, newValue);
368 return newValue;
369 }
370 if (original instanceof LoopExitNode && ((LoopExitNode) original).loopBegin() == loopBegin) {
371 Node value = seenNode.get(original);
372 if (value != null) {
373 return value;
374 }
375 AbstractBeginNode newValue = graph.add(new BeginNode());
376 seenNode.put(original, newValue);
377 return newValue;
378 }
379 if (original instanceof LoopEndNode && ((LoopEndNode) original).loopBegin() == loopBegin) {
380 Node value = seenNode.get(original);
381 if (value != null) {
382 return value;
383 }
384 EndNode newValue = graph.add(new EndNode());
385 seenNode.put(original, newValue);
386 return newValue;
387 }
388 return original;
389 }
390 }
391 };
392 }
393
394 @Override
395 protected void beforeDuplication() {
396 // Nothing to do
397 }
398
399 private static PhiNode patchPhi(StructuredGraph graph, PhiNode phi, AbstractMergeNode merge) {
400 PhiNode ret;
401 if (phi instanceof ValuePhiNode) {
402 ret = new ValuePhiNode(phi.stamp(NodeView.DEFAULT), merge);
403 } else if (phi instanceof GuardPhiNode) {
404 ret = new GuardPhiNode(merge);
405 } else if (phi instanceof MemoryPhiNode) {
406 ret = new MemoryPhiNode(merge, ((MemoryPhiNode) phi).getLocationIdentity());
407 } else {
408 throw GraalError.shouldNotReachHere();
409 }
410 return graph.addWithoutUnique(ret);
411 }
412
413 private void patchPeeling(LoopFragmentInside peel) {
414 LoopBeginNode loopBegin = loop().loopBegin();
415 StructuredGraph graph = loopBegin.graph();
416 List<PhiNode> newPhis = new LinkedList<>();
417
418 NodeBitMap usagesToPatch = nodes.copy();
419 for (LoopExitNode exit : exits()) {
420 markStateNodes(exit, usagesToPatch);
421 for (ProxyNode proxy : exit.proxies()) {
422 usagesToPatch.markAndGrow(proxy);
423 }
424 }
425 markStateNodes(loopBegin, usagesToPatch);
426
427 List<PhiNode> oldPhis = loopBegin.phis().snapshot();
428 for (PhiNode phi : oldPhis) {
429 if (phi.hasNoUsages()) {
430 continue;
431 }
432 ValueNode first;
433 if (loopBegin.loopEnds().count() == 1) {
434 ValueNode b = phi.valueAt(loopBegin.loopEnds().first()); // back edge value
435 first = peel.prim(b); // corresponding value in the peel
436 } else {
437 first = peel.mergedInitializers.get(phi);
438 }
439 // create a new phi (we don't patch the old one since some usages of the old one may
440 // still be valid)
441 PhiNode newPhi = patchPhi(graph, phi, loopBegin);
442 newPhi.addInput(first);
443 for (LoopEndNode end : loopBegin.orderedLoopEnds()) {
444 newPhi.addInput(phi.valueAt(end));
445 }
446 peel.putDuplicatedNode(phi, newPhi);
447 newPhis.add(newPhi);
448 for (Node usage : phi.usages().snapshot()) {
449 // patch only usages that should use the new phi ie usages that were peeled
450 if (usagesToPatch.isMarkedAndGrow(usage)) {
451 usage.replaceFirstInput(phi, newPhi);
452 }
453 }
454 }
455 // check new phis to see if they have as input some old phis, replace those inputs with the
456 // new corresponding phis
457 for (PhiNode phi : newPhis) {
458 for (int i = 0; i < phi.valueCount(); i++) {
459 ValueNode v = phi.valueAt(i);
460 if (loopBegin.isPhiAtMerge(v)) {
461 PhiNode newV = peel.getDuplicatedNode((ValuePhiNode) v);
462 if (newV != null) {
463 phi.setValueAt(i, newV);
464 }
465 }
466 }
467 }
468
469 boolean progress = true;
470 while (progress) {
471 progress = false;
472 int i = 0;
473 outer: while (i < oldPhis.size()) {
474 PhiNode oldPhi = oldPhis.get(i);
475 for (Node usage : oldPhi.usages()) {
476 if (usage instanceof PhiNode && oldPhis.contains(usage)) {
477 // Do not mark.
478 } else {
479 // Mark alive by removing from delete set.
480 oldPhis.remove(i);
481 progress = true;
482 continue outer;
483 }
484 }
485 i++;
486 }
487 }
488
489 for (PhiNode deadPhi : oldPhis) {
490 deadPhi.clearInputs();
491 }
492
493 for (PhiNode deadPhi : oldPhis) {
494 if (deadPhi.isAlive()) {
495 GraphUtil.killWithUnusedFloatingInputs(deadPhi);
496 }
497 }
498 }
499
500 private static void markStateNodes(StateSplit stateSplit, NodeBitMap marks) {
501 FrameState exitState = stateSplit.stateAfter();
502 if (exitState != null) {
503 exitState.applyToVirtual(v -> marks.markAndGrow(v));
504 }
505 }
506
507 /**
508 * Gets the corresponding value in this fragment.
509 *
510 * @param b original value
511 * @return corresponding value in the peel
512 */
513 @Override
514 protected ValueNode prim(ValueNode b) {
515 assert isDuplicate();
516 LoopBeginNode loopBegin = original().loop().loopBegin();
517 if (loopBegin.isPhiAtMerge(b)) {
518 PhiNode phi = (PhiNode) b;
519 return phi.valueAt(loopBegin.forwardEnd());
520 } else if (nodesReady) {
521 ValueNode v = getDuplicatedNode(b);
522 if (v == null) {
523 return b;
524 }
525 return v;
526 } else {
527 return b;
528 }
529 }
530
531 protected ValueNode primAfter(ValueNode b) {
532 assert isDuplicate();
533 LoopBeginNode loopBegin = original().loop().loopBegin();
534 if (loopBegin.isPhiAtMerge(b)) {
535 PhiNode phi = (PhiNode) b;
536 assert phi.valueCount() == 2;
537 return phi.valueAt(1);
538 } else if (nodesReady) {
539 ValueNode v = getDuplicatedNode(b);
540 if (v == null) {
541 return b;
542 }
543 return v;
544 } else {
545 return b;
546 }
547 }
548
549 @SuppressWarnings("try")
550 private AbstractBeginNode mergeEnds() {
551 assert isDuplicate();
552 List<EndNode> endsToMerge = new LinkedList<>();
553 // map peel exits to the corresponding loop exits
554 EconomicMap<AbstractEndNode, LoopEndNode> reverseEnds = EconomicMap.create(Equivalence.IDENTITY);
555 LoopBeginNode loopBegin = original().loop().loopBegin();
556 for (LoopEndNode le : loopBegin.loopEnds()) {
557 AbstractEndNode duplicate = getDuplicatedNode(le);
558 if (duplicate != null) {
559 endsToMerge.add((EndNode) duplicate);
560 reverseEnds.put(duplicate, le);
561 }
562 }
563 mergedInitializers = EconomicMap.create(Equivalence.IDENTITY);
564 AbstractBeginNode newExit;
565 StructuredGraph graph = graph();
566 if (endsToMerge.size() == 1) {
567 AbstractEndNode end = endsToMerge.get(0);
568 assert end.hasNoUsages();
569 try (DebugCloseable position = end.withNodeSourcePosition()) {
570 newExit = graph.add(new BeginNode());
571 end.replaceAtPredecessor(newExit);
572 end.safeDelete();
573 }
574 } else {
575 assert endsToMerge.size() > 1;
576 AbstractMergeNode newExitMerge = graph.add(new MergeNode());
577 newExit = newExitMerge;
578 FrameState state = loopBegin.stateAfter();
579 FrameState duplicateState = null;
580 if (state != null) {
581 duplicateState = state.duplicateWithVirtualState();
582 newExitMerge.setStateAfter(duplicateState);
583 }
584 for (EndNode end : endsToMerge) {
585 newExitMerge.addForwardEnd(end);
586 }
587
588 for (final PhiNode phi : loopBegin.phis().snapshot()) {
589 if (phi.hasNoUsages()) {
590 continue;
591 }
592 final PhiNode firstPhi = patchPhi(graph, phi, newExitMerge);
593 for (AbstractEndNode end : newExitMerge.forwardEnds()) {
594 LoopEndNode loopEnd = reverseEnds.get(end);
595 ValueNode prim = prim(phi.valueAt(loopEnd));
596 assert prim != null;
597 firstPhi.addInput(prim);
598 }
599 ValueNode initializer = firstPhi;
600 if (duplicateState != null) {
601 // fix the merge's state after
602 duplicateState.applyToNonVirtual(new NodeClosure<ValueNode>() {
603
604 @Override
605 public void apply(Node from, ValueNode node) {
606 if (node == phi) {
607 from.replaceFirstInput(phi, firstPhi);
608 }
609 }
610 });
611 }
612 mergedInitializers.put(phi, initializer);
613 }
614 }
615 return newExit;
616 }
617 }