--- /dev/null 2016-05-31 09:42:47.975716356 -0700 +++ new/src/jdk.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/FloatingReadPhase.java 2016-12-09 00:56:13.646135863 -0800 @@ -0,0 +1,445 @@ +/* + * Copyright (c) 2011, 2016, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.graalvm.compiler.phases.common; + +import static org.graalvm.compiler.core.common.LocationIdentity.any; +import static org.graalvm.compiler.graph.Graph.NodeEvent.NODE_ADDED; +import static org.graalvm.compiler.graph.Graph.NodeEvent.ZERO_USAGES; + +import java.util.Collection; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.graalvm.compiler.core.common.CollectionsFactory; +import org.graalvm.compiler.core.common.LocationIdentity; +import org.graalvm.compiler.core.common.cfg.Loop; +import org.graalvm.compiler.debug.DebugCloseable; +import org.graalvm.compiler.graph.Graph.NodeEventScope; +import org.graalvm.compiler.graph.Node; +import org.graalvm.compiler.nodes.AbstractBeginNode; +import org.graalvm.compiler.nodes.AbstractMergeNode; +import org.graalvm.compiler.nodes.FixedNode; +import org.graalvm.compiler.nodes.InvokeWithExceptionNode; +import org.graalvm.compiler.nodes.LoopBeginNode; +import org.graalvm.compiler.nodes.LoopEndNode; +import org.graalvm.compiler.nodes.LoopExitNode; +import org.graalvm.compiler.nodes.PhiNode; +import org.graalvm.compiler.nodes.ReturnNode; +import org.graalvm.compiler.nodes.StartNode; +import org.graalvm.compiler.nodes.StructuredGraph; +import org.graalvm.compiler.nodes.ValueNodeUtil; +import org.graalvm.compiler.nodes.calc.FloatingNode; +import org.graalvm.compiler.nodes.cfg.Block; +import org.graalvm.compiler.nodes.cfg.ControlFlowGraph; +import org.graalvm.compiler.nodes.cfg.HIRLoop; +import org.graalvm.compiler.nodes.extended.GuardingNode; +import org.graalvm.compiler.nodes.extended.ValueAnchorNode; +import org.graalvm.compiler.nodes.memory.FloatableAccessNode; +import org.graalvm.compiler.nodes.memory.FloatingAccessNode; +import org.graalvm.compiler.nodes.memory.FloatingReadNode; +import org.graalvm.compiler.nodes.memory.MemoryAccess; +import org.graalvm.compiler.nodes.memory.MemoryAnchorNode; +import org.graalvm.compiler.nodes.memory.MemoryCheckpoint; +import org.graalvm.compiler.nodes.memory.MemoryMap; +import org.graalvm.compiler.nodes.memory.MemoryMapNode; +import org.graalvm.compiler.nodes.memory.MemoryNode; +import org.graalvm.compiler.nodes.memory.MemoryPhiNode; +import org.graalvm.compiler.nodes.memory.ReadNode; +import org.graalvm.compiler.nodes.util.GraphUtil; +import org.graalvm.compiler.phases.Phase; +import org.graalvm.compiler.phases.common.util.HashSetNodeEventListener; +import org.graalvm.compiler.phases.graph.ReentrantNodeIterator; +import org.graalvm.compiler.phases.graph.ReentrantNodeIterator.LoopInfo; +import org.graalvm.compiler.phases.graph.ReentrantNodeIterator.NodeIteratorClosure; + +public class FloatingReadPhase extends Phase { + + private boolean createFloatingReads; + private boolean createMemoryMapNodes; + + public static class MemoryMapImpl implements MemoryMap { + + private final Map lastMemorySnapshot; + + public MemoryMapImpl(MemoryMapImpl memoryMap) { + lastMemorySnapshot = CollectionsFactory.newMap(memoryMap.lastMemorySnapshot); + } + + public MemoryMapImpl(StartNode start) { + lastMemorySnapshot = CollectionsFactory.newMap(); + lastMemorySnapshot.put(any(), start); + } + + public MemoryMapImpl() { + lastMemorySnapshot = CollectionsFactory.newMap(); + } + + @Override + public MemoryNode getLastLocationAccess(LocationIdentity locationIdentity) { + MemoryNode lastLocationAccess; + if (locationIdentity.isImmutable()) { + return null; + } else { + lastLocationAccess = lastMemorySnapshot.get(locationIdentity); + if (lastLocationAccess == null) { + lastLocationAccess = lastMemorySnapshot.get(any()); + assert lastLocationAccess != null; + } + return lastLocationAccess; + } + } + + @Override + public Collection getLocations() { + return lastMemorySnapshot.keySet(); + } + + public Map getMap() { + return lastMemorySnapshot; + } + } + + public FloatingReadPhase() { + this(true, false); + } + + /** + * @param createFloatingReads specifies whether {@link FloatableAccessNode}s like + * {@link ReadNode} should be converted into floating nodes (e.g., + * {@link FloatingReadNode}s) where possible + * @param createMemoryMapNodes a {@link MemoryMapNode} will be created for each return if this + * is true + */ + public FloatingReadPhase(boolean createFloatingReads, boolean createMemoryMapNodes) { + this.createFloatingReads = createFloatingReads; + this.createMemoryMapNodes = createMemoryMapNodes; + } + + @Override + public float codeSizeIncrease() { + return 1.25f; + } + + /** + * Removes nodes from a given set that (transitively) have a usage outside the set. + */ + private static Set removeExternallyUsedNodes(Set set) { + boolean change; + do { + change = false; + for (Iterator iter = set.iterator(); iter.hasNext();) { + Node node = iter.next(); + for (Node usage : node.usages()) { + if (!set.contains(usage)) { + change = true; + iter.remove(); + break; + } + } + } + } while (change); + return set; + } + + protected void processNode(FixedNode node, Set currentState) { + if (node instanceof MemoryCheckpoint.Single) { + currentState.add(((MemoryCheckpoint.Single) node).getLocationIdentity()); + } else if (node instanceof MemoryCheckpoint.Multi) { + for (LocationIdentity identity : ((MemoryCheckpoint.Multi) node).getLocationIdentities()) { + currentState.add(identity); + } + } + } + + protected void processBlock(Block b, Set currentState) { + for (FixedNode n : b.getNodes()) { + processNode(n, currentState); + } + } + + private Set processLoop(HIRLoop loop, Map> modifiedInLoops) { + LoopBeginNode loopBegin = (LoopBeginNode) loop.getHeader().getBeginNode(); + Set result = modifiedInLoops.get(loopBegin); + if (result != null) { + return result; + } + + result = CollectionsFactory.newSet(); + for (Loop inner : loop.getChildren()) { + result.addAll(processLoop((HIRLoop) inner, modifiedInLoops)); + } + + for (Block b : loop.getBlocks()) { + if (b.getLoop() == loop) { + processBlock(b, result); + } + } + + modifiedInLoops.put(loopBegin, result); + return result; + } + + @Override + @SuppressWarnings("try") + protected void run(StructuredGraph graph) { + Map> modifiedInLoops = null; + if (graph.hasLoops()) { + modifiedInLoops = new HashMap<>(); + ControlFlowGraph cfg = ControlFlowGraph.compute(graph, true, true, false, false); + for (Loop l : cfg.getLoops()) { + HIRLoop loop = (HIRLoop) l; + processLoop(loop, modifiedInLoops); + } + } + + HashSetNodeEventListener listener = new HashSetNodeEventListener(EnumSet.of(NODE_ADDED, ZERO_USAGES)); + try (NodeEventScope nes = graph.trackNodeEvents(listener)) { + ReentrantNodeIterator.apply(new FloatingReadClosure(modifiedInLoops, createFloatingReads, createMemoryMapNodes), graph.start(), new MemoryMapImpl(graph.start())); + } + + for (Node n : removeExternallyUsedNodes(listener.getNodes())) { + if (n.isAlive() && n instanceof FloatingNode) { + n.replaceAtUsages(null); + GraphUtil.killWithUnusedFloatingInputs(n); + } + } + if (createFloatingReads) { + assert !graph.isAfterFloatingReadPhase(); + graph.setAfterFloatingReadPhase(true); + } + } + + public static MemoryMapImpl mergeMemoryMaps(AbstractMergeNode merge, List states) { + MemoryMapImpl newState = new MemoryMapImpl(); + + Set keys = CollectionsFactory.newSet(); + for (MemoryMap other : states) { + keys.addAll(other.getLocations()); + } + assert checkNoImmutableLocations(keys); + + for (LocationIdentity key : keys) { + int mergedStatesCount = 0; + boolean isPhi = false; + MemoryNode merged = null; + for (MemoryMap state : states) { + MemoryNode last = state.getLastLocationAccess(key); + if (isPhi) { + ((MemoryPhiNode) merged).addInput(ValueNodeUtil.asNode(last)); + } else { + if (merged == last) { + // nothing to do + } else if (merged == null) { + merged = last; + } else { + MemoryPhiNode phi = merge.graph().addWithoutUnique(new MemoryPhiNode(merge, key)); + for (int j = 0; j < mergedStatesCount; j++) { + phi.addInput(ValueNodeUtil.asNode(merged)); + } + phi.addInput(ValueNodeUtil.asNode(last)); + merged = phi; + isPhi = true; + } + } + mergedStatesCount++; + } + newState.lastMemorySnapshot.put(key, merged); + } + return newState; + + } + + private static boolean checkNoImmutableLocations(Set keys) { + keys.forEach(t -> { + assert !t.isImmutable(); + }); + return true; + } + + public static class FloatingReadClosure extends NodeIteratorClosure { + + private final Map> modifiedInLoops; + private boolean createFloatingReads; + private boolean createMemoryMapNodes; + + public FloatingReadClosure(Map> modifiedInLoops, boolean createFloatingReads, boolean createMemoryMapNodes) { + this.modifiedInLoops = modifiedInLoops; + this.createFloatingReads = createFloatingReads; + this.createMemoryMapNodes = createMemoryMapNodes; + } + + @Override + protected MemoryMapImpl processNode(FixedNode node, MemoryMapImpl state) { + if (node instanceof MemoryAnchorNode) { + processAnchor((MemoryAnchorNode) node, state); + return state; + } + + if (node instanceof MemoryAccess) { + processAccess((MemoryAccess) node, state); + } + + if (createFloatingReads & node instanceof FloatableAccessNode) { + processFloatable((FloatableAccessNode) node, state); + } else if (node instanceof MemoryCheckpoint.Single) { + processCheckpoint((MemoryCheckpoint.Single) node, state); + } else if (node instanceof MemoryCheckpoint.Multi) { + processCheckpoint((MemoryCheckpoint.Multi) node, state); + } + assert MemoryCheckpoint.TypeAssertion.correctType(node) : node; + + if (createMemoryMapNodes && node instanceof ReturnNode) { + ((ReturnNode) node).setMemoryMap(node.graph().unique(new MemoryMapNode(state.lastMemorySnapshot))); + } + return state; + } + + /** + * Improve the memory graph by re-wiring all usages of a {@link MemoryAnchorNode} to the + * real last access location. + */ + private static void processAnchor(MemoryAnchorNode anchor, MemoryMapImpl state) { + for (Node node : anchor.usages().snapshot()) { + if (node instanceof MemoryAccess) { + MemoryAccess access = (MemoryAccess) node; + if (access.getLastLocationAccess() == anchor) { + MemoryNode lastLocationAccess = state.getLastLocationAccess(access.getLocationIdentity()); + assert lastLocationAccess != null; + access.setLastLocationAccess(lastLocationAccess); + } + } + } + + if (anchor.hasNoUsages()) { + anchor.graph().removeFixed(anchor); + } + } + + private static void processAccess(MemoryAccess access, MemoryMapImpl state) { + LocationIdentity locationIdentity = access.getLocationIdentity(); + if (!locationIdentity.equals(LocationIdentity.any())) { + MemoryNode lastLocationAccess = state.getLastLocationAccess(locationIdentity); + access.setLastLocationAccess(lastLocationAccess); + } + } + + private static void processCheckpoint(MemoryCheckpoint.Single checkpoint, MemoryMapImpl state) { + processIdentity(checkpoint.getLocationIdentity(), checkpoint, state); + } + + private static void processCheckpoint(MemoryCheckpoint.Multi checkpoint, MemoryMapImpl state) { + for (LocationIdentity identity : checkpoint.getLocationIdentities()) { + processIdentity(identity, checkpoint, state); + } + } + + private static void processIdentity(LocationIdentity identity, MemoryCheckpoint checkpoint, MemoryMapImpl state) { + if (identity.isAny()) { + state.lastMemorySnapshot.clear(); + } + state.lastMemorySnapshot.put(identity, checkpoint); + } + + @SuppressWarnings("try") + private static void processFloatable(FloatableAccessNode accessNode, MemoryMapImpl state) { + StructuredGraph graph = accessNode.graph(); + LocationIdentity locationIdentity = accessNode.getLocationIdentity(); + if (accessNode.canFloat()) { + assert accessNode.getNullCheck() == false; + MemoryNode lastLocationAccess = state.getLastLocationAccess(locationIdentity); + try (DebugCloseable position = accessNode.withNodeSourcePosition()) { + FloatingAccessNode floatingNode = accessNode.asFloatingNode(lastLocationAccess); + ValueAnchorNode anchor = null; + GuardingNode guard = accessNode.getGuard(); + if (guard != null) { + anchor = graph.add(new ValueAnchorNode(guard.asNode())); + graph.addAfterFixed(accessNode, anchor); + } + graph.replaceFixedWithFloating(accessNode, floatingNode); + } + } + } + + @Override + protected MemoryMapImpl merge(AbstractMergeNode merge, List states) { + return mergeMemoryMaps(merge, states); + } + + @Override + protected MemoryMapImpl afterSplit(AbstractBeginNode node, MemoryMapImpl oldState) { + MemoryMapImpl result = new MemoryMapImpl(oldState); + if (node.predecessor() instanceof InvokeWithExceptionNode) { + /* + * InvokeWithException cannot be the lastLocationAccess for a FloatingReadNode. + * Since it is both the invoke and a control flow split, the scheduler cannot + * schedule anything immediately after the invoke. It can only schedule in the + * normal or exceptional successor - and we have to tell the scheduler here which + * side it needs to choose by putting in the location identity on both successors. + */ + InvokeWithExceptionNode invoke = (InvokeWithExceptionNode) node.predecessor(); + result.lastMemorySnapshot.put(invoke.getLocationIdentity(), (MemoryCheckpoint) node); + } + return result; + } + + @Override + protected Map processLoop(LoopBeginNode loop, MemoryMapImpl initialState) { + Set modifiedLocations = modifiedInLoops.get(loop); + Map phis = CollectionsFactory.newMap(); + if (modifiedLocations.contains(LocationIdentity.any())) { + // create phis for all locations if ANY is modified in the loop + modifiedLocations = CollectionsFactory.newSet(modifiedLocations); + modifiedLocations.addAll(initialState.lastMemorySnapshot.keySet()); + } + + for (LocationIdentity location : modifiedLocations) { + createMemoryPhi(loop, initialState, phis, location); + } + for (Map.Entry entry : phis.entrySet()) { + initialState.lastMemorySnapshot.put(entry.getKey(), entry.getValue()); + } + + LoopInfo loopInfo = ReentrantNodeIterator.processLoop(this, loop, initialState); + + for (Map.Entry entry : loopInfo.endStates.entrySet()) { + int endIndex = loop.phiPredecessorIndex(entry.getKey()); + for (Map.Entry phiEntry : phis.entrySet()) { + LocationIdentity key = phiEntry.getKey(); + PhiNode phi = phiEntry.getValue(); + phi.initializeValueAt(endIndex, ValueNodeUtil.asNode(entry.getValue().getLastLocationAccess(key))); + } + } + return loopInfo.exitStates; + } + + private static void createMemoryPhi(LoopBeginNode loop, MemoryMapImpl initialState, Map phis, LocationIdentity location) { + MemoryPhiNode phi = loop.graph().addWithoutUnique(new MemoryPhiNode(loop, location)); + phi.addInput(ValueNodeUtil.asNode(initialState.getLastLocationAccess(location))); + phis.put(location, phi); + } + } +}