54 import org.graalvm.compiler.nodes.LoopEndNode;
55 import org.graalvm.compiler.nodes.LoopExitNode;
56 import org.graalvm.compiler.nodes.MergeNode;
57 import org.graalvm.compiler.nodes.NodeView;
58 import org.graalvm.compiler.nodes.PhiNode;
59 import org.graalvm.compiler.nodes.ProxyNode;
60 import org.graalvm.compiler.nodes.SafepointNode;
61 import org.graalvm.compiler.nodes.StateSplit;
62 import org.graalvm.compiler.nodes.StructuredGraph;
63 import org.graalvm.compiler.nodes.ValueNode;
64 import org.graalvm.compiler.nodes.ValuePhiNode;
65 import org.graalvm.compiler.nodes.VirtualState.NodeClosure;
66 import org.graalvm.compiler.nodes.calc.AddNode;
67 import org.graalvm.compiler.nodes.calc.CompareNode;
68 import org.graalvm.compiler.nodes.calc.ConditionalNode;
69 import org.graalvm.compiler.nodes.calc.IntegerBelowNode;
70 import org.graalvm.compiler.nodes.calc.SubNode;
71 import org.graalvm.compiler.nodes.extended.OpaqueNode;
72 import org.graalvm.compiler.nodes.memory.MemoryPhiNode;
73 import org.graalvm.compiler.nodes.util.GraphUtil;
74
75 import jdk.vm.ci.code.CodeUtil;
76
77 public class LoopFragmentInside extends LoopFragment {
78
79 /**
80 * mergedInitializers. When an inside fragment's (loop)ends are merged to create a unique exit
81 * point, some phis must be created : they phis together all the back-values of the loop-phis
82 * These can then be used to update the loop-phis' forward edge value ('initializer') in the
83 * peeling case. In the unrolling case they will be used as the value that replace the loop-phis
84 * of the duplicated inside fragment
85 */
86 private EconomicMap<PhiNode, ValueNode> mergedInitializers;
87 private final DuplicationReplacement dataFixBefore = new DuplicationReplacement() {
88
89 @Override
90 public Node replacement(Node oriInput) {
91 if (!(oriInput instanceof ValueNode)) {
92 return oriInput;
93 }
94 return prim((ValueNode) oriInput);
95 }
189 }
190
191 placeNewSegmentAndCleanup(loop);
192
193 // Remove any safepoints from the original copy leaving only the duplicated one
194 assert loop.whole().nodes().filter(SafepointNode.class).count() == nodes().filter(SafepointNode.class).count();
195 for (SafepointNode safepoint : loop.whole().nodes().filter(SafepointNode.class)) {
196 graph().removeFixed(safepoint);
197 }
198
199 StructuredGraph graph = mainLoopBegin.graph();
200 if (opaqueUnrolledStrides != null) {
201 OpaqueNode opaque = opaqueUnrolledStrides.get(loop.loopBegin());
202 CountedLoopInfo counted = loop.counted();
203 ValueNode counterStride = counted.getCounter().strideNode();
204 if (opaque == null) {
205 opaque = new OpaqueNode(AddNode.add(counterStride, counterStride, NodeView.DEFAULT));
206 ValueNode limit = counted.getLimit();
207 int bits = ((IntegerStamp) limit.stamp(NodeView.DEFAULT)).getBits();
208 ValueNode newLimit = SubNode.create(limit, opaque, NodeView.DEFAULT);
209 LogicNode overflowCheck;
210 ConstantNode extremum;
211 if (counted.getDirection() == InductionVariable.Direction.Up) {
212 // limit - counterStride could overflow negatively if limit - min <
213 // counterStride
214 extremum = ConstantNode.forIntegerBits(bits, CodeUtil.minValue(bits));
215 overflowCheck = IntegerBelowNode.create(SubNode.create(limit, extremum, NodeView.DEFAULT), opaque, NodeView.DEFAULT);
216 } else {
217 assert counted.getDirection() == InductionVariable.Direction.Down;
218 // limit - counterStride could overflow if max - limit < -counterStride
219 // i.e., counterStride < limit - max
220 extremum = ConstantNode.forIntegerBits(bits, CodeUtil.maxValue(bits));
221 overflowCheck = IntegerBelowNode.create(opaque, SubNode.create(limit, extremum, NodeView.DEFAULT), NodeView.DEFAULT);
222 }
223 newLimit = ConditionalNode.create(overflowCheck, extremum, newLimit, NodeView.DEFAULT);
224 CompareNode compareNode = (CompareNode) counted.getLimitTest().condition();
225 compareNode.replaceFirstInput(limit, graph.addOrUniqueWithInputs(newLimit));
226 opaqueUnrolledStrides.put(loop.loopBegin(), opaque);
227 } else {
228 assert counted.getCounter().isConstantStride();
229 assert Math.addExact(counted.getCounter().constantStride(), counted.getCounter().constantStride()) == counted.getCounter().constantStride() * 2;
230 ValueNode previousValue = opaque.getValue();
231 opaque.setValue(graph.addOrUniqueWithInputs(AddNode.add(counterStride, previousValue, NodeView.DEFAULT)));
232 GraphUtil.tryKillUnused(previousValue);
233 }
234 }
235 mainLoopBegin.setUnrollFactor(mainLoopBegin.getUnrollFactor() * 2);
236 mainLoopBegin.setLoopFrequency(mainLoopBegin.loopFrequency() / 2);
237 graph.getDebug().dump(DebugContext.DETAILED_LEVEL, graph, "LoopPartialUnroll %s", loop);
238
239 mainLoopBegin.getDebug().dump(DebugContext.VERBOSE_LEVEL, mainLoopBegin.graph(), "After insertWithinAfter %s", mainLoopBegin);
240 }
|
54 import org.graalvm.compiler.nodes.LoopEndNode;
55 import org.graalvm.compiler.nodes.LoopExitNode;
56 import org.graalvm.compiler.nodes.MergeNode;
57 import org.graalvm.compiler.nodes.NodeView;
58 import org.graalvm.compiler.nodes.PhiNode;
59 import org.graalvm.compiler.nodes.ProxyNode;
60 import org.graalvm.compiler.nodes.SafepointNode;
61 import org.graalvm.compiler.nodes.StateSplit;
62 import org.graalvm.compiler.nodes.StructuredGraph;
63 import org.graalvm.compiler.nodes.ValueNode;
64 import org.graalvm.compiler.nodes.ValuePhiNode;
65 import org.graalvm.compiler.nodes.VirtualState.NodeClosure;
66 import org.graalvm.compiler.nodes.calc.AddNode;
67 import org.graalvm.compiler.nodes.calc.CompareNode;
68 import org.graalvm.compiler.nodes.calc.ConditionalNode;
69 import org.graalvm.compiler.nodes.calc.IntegerBelowNode;
70 import org.graalvm.compiler.nodes.calc.SubNode;
71 import org.graalvm.compiler.nodes.extended.OpaqueNode;
72 import org.graalvm.compiler.nodes.memory.MemoryPhiNode;
73 import org.graalvm.compiler.nodes.util.GraphUtil;
74 import org.graalvm.compiler.nodes.util.IntegerHelper;
75
76 public class LoopFragmentInside extends LoopFragment {
77
78 /**
79 * mergedInitializers. When an inside fragment's (loop)ends are merged to create a unique exit
80 * point, some phis must be created : they phis together all the back-values of the loop-phis
81 * These can then be used to update the loop-phis' forward edge value ('initializer') in the
82 * peeling case. In the unrolling case they will be used as the value that replace the loop-phis
83 * of the duplicated inside fragment
84 */
85 private EconomicMap<PhiNode, ValueNode> mergedInitializers;
86 private final DuplicationReplacement dataFixBefore = new DuplicationReplacement() {
87
88 @Override
89 public Node replacement(Node oriInput) {
90 if (!(oriInput instanceof ValueNode)) {
91 return oriInput;
92 }
93 return prim((ValueNode) oriInput);
94 }
188 }
189
190 placeNewSegmentAndCleanup(loop);
191
192 // Remove any safepoints from the original copy leaving only the duplicated one
193 assert loop.whole().nodes().filter(SafepointNode.class).count() == nodes().filter(SafepointNode.class).count();
194 for (SafepointNode safepoint : loop.whole().nodes().filter(SafepointNode.class)) {
195 graph().removeFixed(safepoint);
196 }
197
198 StructuredGraph graph = mainLoopBegin.graph();
199 if (opaqueUnrolledStrides != null) {
200 OpaqueNode opaque = opaqueUnrolledStrides.get(loop.loopBegin());
201 CountedLoopInfo counted = loop.counted();
202 ValueNode counterStride = counted.getCounter().strideNode();
203 if (opaque == null) {
204 opaque = new OpaqueNode(AddNode.add(counterStride, counterStride, NodeView.DEFAULT));
205 ValueNode limit = counted.getLimit();
206 int bits = ((IntegerStamp) limit.stamp(NodeView.DEFAULT)).getBits();
207 ValueNode newLimit = SubNode.create(limit, opaque, NodeView.DEFAULT);
208 IntegerHelper helper = counted.getCounterIntegerHelper();
209 LogicNode overflowCheck;
210 ConstantNode extremum;
211 if (counted.getDirection() == InductionVariable.Direction.Up) {
212 // limit - counterStride could overflow negatively if limit - min <
213 // counterStride
214 extremum = ConstantNode.forIntegerBits(bits, helper.minValue());
215 overflowCheck = IntegerBelowNode.create(SubNode.create(limit, extremum, NodeView.DEFAULT), opaque, NodeView.DEFAULT);
216 } else {
217 assert counted.getDirection() == InductionVariable.Direction.Down;
218 // limit - counterStride could overflow if max - limit < -counterStride
219 // i.e., counterStride < limit - max
220 extremum = ConstantNode.forIntegerBits(bits, helper.maxValue());
221 overflowCheck = IntegerBelowNode.create(opaque, SubNode.create(limit, extremum, NodeView.DEFAULT), NodeView.DEFAULT);
222 }
223 newLimit = ConditionalNode.create(overflowCheck, extremum, newLimit, NodeView.DEFAULT);
224 CompareNode compareNode = (CompareNode) counted.getLimitTest().condition();
225 compareNode.replaceFirstInput(limit, graph.addOrUniqueWithInputs(newLimit));
226 opaqueUnrolledStrides.put(loop.loopBegin(), opaque);
227 } else {
228 assert counted.getCounter().isConstantStride();
229 assert Math.addExact(counted.getCounter().constantStride(), counted.getCounter().constantStride()) == counted.getCounter().constantStride() * 2;
230 ValueNode previousValue = opaque.getValue();
231 opaque.setValue(graph.addOrUniqueWithInputs(AddNode.add(counterStride, previousValue, NodeView.DEFAULT)));
232 GraphUtil.tryKillUnused(previousValue);
233 }
234 }
235 mainLoopBegin.setUnrollFactor(mainLoopBegin.getUnrollFactor() * 2);
236 mainLoopBegin.setLoopFrequency(mainLoopBegin.loopFrequency() / 2);
237 graph.getDebug().dump(DebugContext.DETAILED_LEVEL, graph, "LoopPartialUnroll %s", loop);
238
239 mainLoopBegin.getDebug().dump(DebugContext.VERBOSE_LEVEL, mainLoopBegin.graph(), "After insertWithinAfter %s", mainLoopBegin);
240 }
|