1 /*
2 * Copyright (c) 2011, 2016, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24
25 package org.graalvm.compiler.virtual.phases.ea;
26
27 import static org.graalvm.compiler.core.common.GraalOptions.ReadEliminationMaxLoopVisits;
28 import static org.graalvm.compiler.nodes.NamedLocationIdentity.ARRAY_LENGTH_LOCATION;
29
30 import java.util.EnumMap;
31 import java.util.Iterator;
32 import java.util.List;
33
34 import jdk.internal.vm.compiler.collections.EconomicMap;
35 import jdk.internal.vm.compiler.collections.EconomicSet;
36 import jdk.internal.vm.compiler.collections.Equivalence;
37 import jdk.internal.vm.compiler.collections.MapCursor;
38 import jdk.internal.vm.compiler.collections.Pair;
39 import org.graalvm.compiler.core.common.cfg.Loop;
40 import org.graalvm.compiler.core.common.spi.ConstantFieldProvider;
41 import org.graalvm.compiler.graph.Node;
42 import org.graalvm.compiler.nodes.AbstractBeginNode;
43 import org.graalvm.compiler.nodes.FieldLocationIdentity;
44 import org.graalvm.compiler.nodes.FixedNode;
45 import org.graalvm.compiler.nodes.FixedWithNextNode;
46 import org.graalvm.compiler.nodes.LoopBeginNode;
47 import org.graalvm.compiler.nodes.LoopExitNode;
48 import org.graalvm.compiler.nodes.NamedLocationIdentity;
49 import org.graalvm.compiler.nodes.NodeView;
50 import org.graalvm.compiler.nodes.PhiNode;
51 import org.graalvm.compiler.nodes.ProxyNode;
52 import org.graalvm.compiler.nodes.StructuredGraph.ScheduleResult;
53 import org.graalvm.compiler.nodes.ValueNode;
54 import org.graalvm.compiler.nodes.ValueProxyNode;
55 import org.graalvm.compiler.nodes.cfg.Block;
56 import org.graalvm.compiler.nodes.extended.RawLoadNode;
57 import org.graalvm.compiler.nodes.extended.RawStoreNode;
58 import org.graalvm.compiler.nodes.extended.UnboxNode;
59 import org.graalvm.compiler.nodes.java.ArrayLengthNode;
60 import org.graalvm.compiler.nodes.java.LoadFieldNode;
61 import org.graalvm.compiler.nodes.java.LoadIndexedNode;
62 import org.graalvm.compiler.nodes.java.StoreFieldNode;
63 import org.graalvm.compiler.nodes.java.StoreIndexedNode;
64 import org.graalvm.compiler.nodes.memory.MemoryCheckpoint;
65 import org.graalvm.compiler.nodes.spi.LoweringProvider;
66 import org.graalvm.compiler.nodes.type.StampTool;
67 import org.graalvm.compiler.nodes.util.GraphUtil;
68 import org.graalvm.compiler.nodes.virtual.VirtualArrayNode;
69 import org.graalvm.compiler.options.OptionValues;
70 import org.graalvm.compiler.virtual.phases.ea.PEReadEliminationBlockState.ReadCacheEntry;
71 import jdk.internal.vm.compiler.word.LocationIdentity;
72
73 import jdk.vm.ci.meta.ConstantReflectionProvider;
74 import jdk.vm.ci.meta.JavaConstant;
75 import jdk.vm.ci.meta.JavaKind;
76 import jdk.vm.ci.meta.MetaAccessProvider;
77 import jdk.vm.ci.meta.ResolvedJavaType;
78
79 public final class PEReadEliminationClosure extends PartialEscapeClosure<PEReadEliminationBlockState> {
80
81 private static final EnumMap<JavaKind, LocationIdentity> UNBOX_LOCATIONS;
82
83 static {
84 UNBOX_LOCATIONS = new EnumMap<>(JavaKind.class);
85 for (JavaKind kind : JavaKind.values()) {
86 UNBOX_LOCATIONS.put(kind, NamedLocationIdentity.immutable("PEA unbox " + kind.getJavaName()));
87 }
88 }
89
90 public PEReadEliminationClosure(ScheduleResult schedule, MetaAccessProvider metaAccess, ConstantReflectionProvider constantReflection, ConstantFieldProvider constantFieldProvider,
91 LoweringProvider loweringProvider) {
92 super(schedule, metaAccess, constantReflection, constantFieldProvider, loweringProvider);
93 }
94
95 @Override
96 protected PEReadEliminationBlockState getInitialState() {
97 return new PEReadEliminationBlockState(tool.getOptions(), tool.getDebug());
98 }
99
100 @Override
101 protected boolean processNode(Node node, PEReadEliminationBlockState state, GraphEffectList effects, FixedWithNextNode lastFixedNode) {
102 if (super.processNode(node, state, effects, lastFixedNode)) {
103 return true;
104 }
105
106 if (node instanceof LoadFieldNode) {
107 return processLoadField((LoadFieldNode) node, state, effects);
108 } else if (node instanceof StoreFieldNode) {
109 return processStoreField((StoreFieldNode) node, state, effects);
110 } else if (node instanceof LoadIndexedNode) {
111 return processLoadIndexed((LoadIndexedNode) node, state, effects);
112 } else if (node instanceof StoreIndexedNode) {
113 return processStoreIndexed((StoreIndexedNode) node, state, effects);
114 } else if (node instanceof ArrayLengthNode) {
115 return processArrayLength((ArrayLengthNode) node, state, effects);
116 } else if (node instanceof UnboxNode) {
117 return processUnbox((UnboxNode) node, state, effects);
118 } else if (node instanceof RawLoadNode) {
119 return processUnsafeLoad((RawLoadNode) node, state, effects);
120 } else if (node instanceof RawStoreNode) {
121 return processUnsafeStore((RawStoreNode) node, state, effects);
122 } else if (node instanceof MemoryCheckpoint.Single) {
123 COUNTER_MEMORYCHECKPOINT.increment(node.getDebug());
124 LocationIdentity identity = ((MemoryCheckpoint.Single) node).getLocationIdentity();
125 processIdentity(state, identity);
126 } else if (node instanceof MemoryCheckpoint.Multi) {
127 COUNTER_MEMORYCHECKPOINT.increment(node.getDebug());
128 for (LocationIdentity identity : ((MemoryCheckpoint.Multi) node).getLocationIdentities()) {
129 processIdentity(state, identity);
130 }
131 }
132
133 return false;
134 }
135
136 private boolean processStore(FixedNode store, ValueNode object, LocationIdentity identity, int index, JavaKind accessKind, boolean overflowAccess, ValueNode value,
137 PEReadEliminationBlockState state, GraphEffectList effects) {
138 ValueNode unproxiedObject = GraphUtil.unproxify(object);
139 ValueNode cachedValue = state.getReadCache(object, identity, index, accessKind, this);
140
141 ValueNode finalValue = getScalarAlias(value);
142 boolean result = false;
143 if (GraphUtil.unproxify(finalValue) == GraphUtil.unproxify(cachedValue)) {
144 effects.deleteNode(store);
145 result = true;
146 }
147 state.killReadCache(identity, index);
148 state.addReadCache(unproxiedObject, identity, index, accessKind, overflowAccess, finalValue, this);
149 return result;
150 }
151
152 private boolean processLoad(FixedNode load, ValueNode object, LocationIdentity identity, int index, JavaKind kind, PEReadEliminationBlockState state, GraphEffectList effects) {
153 ValueNode unproxiedObject = GraphUtil.unproxify(object);
154 ValueNode cachedValue = state.getReadCache(unproxiedObject, identity, index, kind, this);
155 if (cachedValue != null) {
156 // perform the read elimination
157 effects.replaceAtUsages(load, cachedValue, load);
158 addScalarAlias(load, cachedValue);
159 return true;
160 } else {
161 state.addReadCache(unproxiedObject, identity, index, kind, false, load, this);
162 return false;
163 }
164 }
165
166 private static boolean isOverflowAccess(JavaKind accessKind, JavaKind declaredKind) {
167 if (accessKind == declaredKind) {
168 return false;
169 }
170 if (accessKind == JavaKind.Object) {
171 switch (declaredKind) {
172 case Object:
173 case Double:
174 case Long:
175 return false;
176 default:
177 return true;
178 }
179 }
180 assert accessKind.isPrimitive() : "Illegal access kind";
181 return declaredKind.isPrimitive() ? accessKind.getBitCount() > declaredKind.getBitCount() : true;
182 }
183
184 private boolean processUnsafeLoad(RawLoadNode load, PEReadEliminationBlockState state, GraphEffectList effects) {
185 if (load.offset().isConstant()) {
186 ResolvedJavaType type = StampTool.typeOrNull(load.object());
187 if (type != null && type.isArray()) {
188 JavaKind accessKind = load.accessKind();
189 JavaKind componentKind = type.getComponentType().getJavaKind();
190 long offset = load.offset().asJavaConstant().asLong();
191 int index = VirtualArrayNode.entryIndexForOffset(tool.getArrayOffsetProvider(), offset, accessKind, type.getComponentType(), Integer.MAX_VALUE);
192 ValueNode object = GraphUtil.unproxify(load.object());
193 LocationIdentity location = NamedLocationIdentity.getArrayLocation(componentKind);
194 ValueNode cachedValue = state.getReadCache(object, location, index, accessKind, this);
195 assert cachedValue == null || load.stamp(NodeView.DEFAULT).isCompatible(cachedValue.stamp(NodeView.DEFAULT)) : "The RawLoadNode's stamp is not compatible with the cached value.";
196 if (cachedValue != null) {
197 effects.replaceAtUsages(load, cachedValue, load);
198 addScalarAlias(load, cachedValue);
199 return true;
200 } else {
201 state.addReadCache(object, location, index, accessKind, isOverflowAccess(accessKind, componentKind), load, this);
202 }
203 }
204 }
205 return false;
206 }
207
208 private boolean processUnsafeStore(RawStoreNode store, PEReadEliminationBlockState state, GraphEffectList effects) {
209 ResolvedJavaType type = StampTool.typeOrNull(store.object());
210 if (type != null && type.isArray()) {
211 JavaKind accessKind = store.accessKind();
212 JavaKind componentKind = type.getComponentType().getJavaKind();
213 LocationIdentity location = NamedLocationIdentity.getArrayLocation(componentKind);
214 if (store.offset().isConstant()) {
215 long offset = store.offset().asJavaConstant().asLong();
216 boolean overflowAccess = isOverflowAccess(accessKind, componentKind);
217 int index = overflowAccess ? -1 : VirtualArrayNode.entryIndexForOffset(tool.getArrayOffsetProvider(), offset, accessKind, type.getComponentType(), Integer.MAX_VALUE);
218 return processStore(store, store.object(), location, index, accessKind, overflowAccess, store.value(), state, effects);
219 } else {
220 processIdentity(state, location);
221 }
222 } else {
223 state.killReadCache();
224 }
225 return false;
226 }
227
228 private boolean processArrayLength(ArrayLengthNode length, PEReadEliminationBlockState state, GraphEffectList effects) {
229 return processLoad(length, length.array(), ARRAY_LENGTH_LOCATION, -1, JavaKind.Int, state, effects);
230 }
231
232 private boolean processStoreField(StoreFieldNode store, PEReadEliminationBlockState state, GraphEffectList effects) {
233 if (store.isVolatile()) {
234 state.killReadCache();
235 return false;
236 }
237 JavaKind kind = store.field().getJavaKind();
238 return processStore(store, store.object(), new FieldLocationIdentity(store.field()), -1, kind, false, store.value(), state, effects);
239 }
240
241 private boolean processLoadField(LoadFieldNode load, PEReadEliminationBlockState state, GraphEffectList effects) {
242 if (load.isVolatile()) {
243 state.killReadCache();
244 return false;
245 }
246 return processLoad(load, load.object(), new FieldLocationIdentity(load.field()), -1, load.field().getJavaKind(), state, effects);
247 }
248
249 private boolean processStoreIndexed(StoreIndexedNode store, PEReadEliminationBlockState state, GraphEffectList effects) {
250 int index = store.index().isConstant() ? ((JavaConstant) store.index().asConstant()).asInt() : -1;
251 JavaKind elementKind = store.elementKind();
252 LocationIdentity arrayLocation = NamedLocationIdentity.getArrayLocation(elementKind);
253 if (index != -1) {
254 return processStore(store, store.array(), arrayLocation, index, elementKind, false, store.value(), state, effects);
255 } else {
256 state.killReadCache(arrayLocation, -1);
257 }
258 return false;
259 }
260
261 private boolean processLoadIndexed(LoadIndexedNode load, PEReadEliminationBlockState state, GraphEffectList effects) {
262 if (load.index().isConstant()) {
263 int index = ((JavaConstant) load.index().asConstant()).asInt();
264 JavaKind elementKind = load.elementKind();
265 LocationIdentity arrayLocation = NamedLocationIdentity.getArrayLocation(elementKind);
266 return processLoad(load, load.array(), arrayLocation, index, elementKind, state, effects);
267 }
268 return false;
269 }
270
271 private boolean processUnbox(UnboxNode unbox, PEReadEliminationBlockState state, GraphEffectList effects) {
272 return processLoad(unbox, unbox.getValue(), UNBOX_LOCATIONS.get(unbox.getBoxingKind()), -1, unbox.getBoxingKind(), state, effects);
273 }
274
275 private static void processIdentity(PEReadEliminationBlockState state, LocationIdentity identity) {
276 if (identity.isAny()) {
277 state.killReadCache();
278 } else {
279 state.killReadCache(identity, -1);
280 }
281 }
282
283 @SuppressWarnings("unchecked")
284 @Override
285 protected void processInitialLoopState(Loop<Block> loop, PEReadEliminationBlockState initialState) {
286 super.processInitialLoopState(loop, initialState);
287
288 if (!initialState.getReadCache().isEmpty()) {
289 EconomicMap<ValueNode, Pair<ValueNode, Object>> firstValueSet = null;
290 for (PhiNode phi : ((LoopBeginNode) loop.getHeader().getBeginNode()).phis()) {
291 ValueNode firstValue = phi.valueAt(0);
292 if (firstValue != null && phi.getStackKind().isObject()) {
293 ValueNode unproxified = GraphUtil.unproxify(firstValue);
294 if (firstValueSet == null) {
295 firstValueSet = EconomicMap.create(Equivalence.IDENTITY_WITH_SYSTEM_HASHCODE);
296 }
297 Pair<ValueNode, Object> pair = Pair.create(unproxified, firstValueSet.get(unproxified));
298 firstValueSet.put(unproxified, pair);
299 }
300 }
301
302 if (firstValueSet != null) {
303 ReadCacheEntry[] entries = new ReadCacheEntry[initialState.getReadCache().size()];
304 int z = 0;
305 for (ReadCacheEntry entry : initialState.getReadCache().getKeys()) {
306 entries[z++] = entry;
307 }
308
309 for (ReadCacheEntry entry : entries) {
310 ValueNode object = entry.object;
311 if (object != null) {
312 Pair<ValueNode, Object> pair = firstValueSet.get(object);
313 while (pair != null) {
314 initialState.addReadCache(pair.getLeft(), entry.identity, entry.index, entry.kind, entry.overflowAccess, initialState.getReadCache().get(entry), this);
315 pair = (Pair<ValueNode, Object>) pair.getRight();
316 }
317 }
318 }
319 }
320 }
321 }
322
323 @Override
324 protected void processLoopExit(LoopExitNode exitNode, PEReadEliminationBlockState initialState, PEReadEliminationBlockState exitState, GraphEffectList effects) {
325 super.processLoopExit(exitNode, initialState, exitState, effects);
326
327 if (exitNode.graph().hasValueProxies()) {
328 MapCursor<ReadCacheEntry, ValueNode> entry = exitState.getReadCache().getEntries();
329 while (entry.advance()) {
330 if (initialState.getReadCache().get(entry.getKey()) != entry.getValue()) {
331 ValueNode value = exitState.getReadCache(entry.getKey().object, entry.getKey().identity, entry.getKey().index, entry.getKey().kind, this);
332 assert value != null : "Got null from read cache, entry's value:" + entry.getValue();
333 if (!(value instanceof ProxyNode) || ((ProxyNode) value).proxyPoint() != exitNode) {
334 ProxyNode proxy = new ValueProxyNode(value, exitNode);
335 effects.addFloatingNode(proxy, "readCacheProxy");
336 exitState.getReadCache().put(entry.getKey(), proxy);
337 }
338 }
339 }
340 }
341 }
342
343 @Override
344 protected PEReadEliminationBlockState cloneState(PEReadEliminationBlockState other) {
345 return new PEReadEliminationBlockState(other);
346 }
347
348 @Override
349 protected MergeProcessor createMergeProcessor(Block merge) {
350 return new ReadEliminationMergeProcessor(merge);
351 }
352
353 private class ReadEliminationMergeProcessor extends MergeProcessor {
354
355 ReadEliminationMergeProcessor(Block mergeBlock) {
356 super(mergeBlock);
357 }
358
359 @Override
360 protected void merge(List<PEReadEliminationBlockState> states) {
361 super.merge(states);
362
363 mergeReadCache(states);
364 }
365
366 private void mergeReadCache(List<PEReadEliminationBlockState> states) {
367 MapCursor<ReadCacheEntry, ValueNode> cursor = states.get(0).readCache.getEntries();
368 while (cursor.advance()) {
369 ReadCacheEntry key = cursor.getKey();
370 ValueNode value = cursor.getValue();
371 boolean phi = false;
372 for (int i = 1; i < states.size(); i++) {
373 ValueNode otherValue = states.get(i).readCache.get(key);
374 // e.g. unsafe loads / stores with different access kinds have different stamps
375 // although location, object and offset are the same, in this case we cannot
376 // create a phi nor can we set a common value
377 if (otherValue == null || !value.stamp(NodeView.DEFAULT).isCompatible(otherValue.stamp(NodeView.DEFAULT))) {
378 value = null;
379 phi = false;
380 break;
381 }
382 if (!phi && otherValue != value) {
383 phi = true;
384 }
385 }
386 if (phi) {
387 PhiNode phiNode = getPhi(key, value.stamp(NodeView.DEFAULT).unrestricted());
388 mergeEffects.addFloatingNode(phiNode, "mergeReadCache");
389 for (int i = 0; i < states.size(); i++) {
390 ValueNode v = states.get(i).getReadCache(key.object, key.identity, key.index, key.kind, PEReadEliminationClosure.this);
391 assert phiNode.stamp(NodeView.DEFAULT).isCompatible(v.stamp(NodeView.DEFAULT)) : "Cannot create read elimination phi for inputs with incompatible stamps.";
392 setPhiInput(phiNode, i, v);
393 }
394 newState.readCache.put(key, phiNode);
395 } else if (value != null) {
396 newState.readCache.put(key, value);
397 }
398 }
399 /*
400 * For object phis, see if there are known reads on all predecessors, for which we could
401 * create new phis.
402 */
403 for (PhiNode phi : getPhis()) {
404 if (phi.getStackKind() == JavaKind.Object) {
405 for (ReadCacheEntry entry : states.get(0).readCache.getKeys()) {
406 if (entry.object == getPhiValueAt(phi, 0)) {
407 mergeReadCachePhi(phi, entry.identity, entry.index, entry.kind, entry.overflowAccess, states);
408 }
409 }
410 }
411 }
412 }
413
414 private void mergeReadCachePhi(PhiNode phi, LocationIdentity identity, int index, JavaKind kind, boolean overflowAccess, List<PEReadEliminationBlockState> states) {
415 ValueNode[] values = new ValueNode[states.size()];
416 values[0] = states.get(0).getReadCache(getPhiValueAt(phi, 0), identity, index, kind, PEReadEliminationClosure.this);
417 if (values[0] != null) {
418 for (int i = 1; i < states.size(); i++) {
419 ValueNode value = states.get(i).getReadCache(getPhiValueAt(phi, i), identity, index, kind, PEReadEliminationClosure.this);
420 // e.g. unsafe loads / stores with same identity and different access kinds see
421 // mergeReadCache(states)
422 if (value == null || !values[i - 1].stamp(NodeView.DEFAULT).isCompatible(value.stamp(NodeView.DEFAULT))) {
423 return;
424 }
425 values[i] = value;
426 }
427
428 PhiNode phiNode = getPhi(new ReadCacheEntry(identity, phi, index, kind, overflowAccess), values[0].stamp(NodeView.DEFAULT).unrestricted());
429 mergeEffects.addFloatingNode(phiNode, "mergeReadCachePhi");
430 for (int i = 0; i < values.length; i++) {
431 setPhiInput(phiNode, i, values[i]);
432 }
433 newState.readCache.put(new ReadCacheEntry(identity, phi, index, kind, overflowAccess), phiNode);
434 }
435 }
436 }
437
438 @Override
439 protected void processKilledLoopLocations(Loop<Block> loop, PEReadEliminationBlockState initialState, PEReadEliminationBlockState mergedStates) {
440 assert initialState != null;
441 assert mergedStates != null;
442 if (initialState.readCache.size() > 0) {
443 LoopKillCache loopKilledLocations = loopLocationKillCache.get(loop);
444 // we have fully processed this loop the first time, remember to cache it the next time
445 // it is visited
446 if (loopKilledLocations == null) {
447 loopKilledLocations = new LoopKillCache(1/* 1.visit */);
448 loopLocationKillCache.put(loop, loopKilledLocations);
449 } else {
450 AbstractBeginNode beginNode = loop.getHeader().getBeginNode();
451 OptionValues options = beginNode.getOptions();
452 if (loopKilledLocations.visits() > ReadEliminationMaxLoopVisits.getValue(options)) {
453 // we have processed the loop too many times, kill all locations so the inner
454 // loop will never be processed more than once again on visit
455 loopKilledLocations.setKillsAll();
456 } else {
457 // we have fully processed this loop >1 times, update the killed locations
458 EconomicSet<LocationIdentity> forwardEndLiveLocations = EconomicSet.create(Equivalence.DEFAULT);
459 for (ReadCacheEntry entry : initialState.readCache.getKeys()) {
460 forwardEndLiveLocations.add(entry.identity);
461 }
462 for (ReadCacheEntry entry : mergedStates.readCache.getKeys()) {
463 forwardEndLiveLocations.remove(entry.identity);
464 }
465 // every location that is alive before the loop but not after is killed by the
466 // loop
467 for (LocationIdentity location : forwardEndLiveLocations) {
468 loopKilledLocations.rememberLoopKilledLocation(location);
469 }
470 if (debug.isLogEnabled() && loopKilledLocations != null) {
471 debug.log("[Early Read Elimination] Setting loop killed locations of loop at node %s with %s",
472 beginNode, forwardEndLiveLocations);
473 }
474 }
475 // remember the loop visit
476 loopKilledLocations.visited();
477 }
478 }
479 }
480
481 @Override
482 protected PEReadEliminationBlockState stripKilledLoopLocations(Loop<Block> loop, PEReadEliminationBlockState originalInitialState) {
483 PEReadEliminationBlockState initialState = super.stripKilledLoopLocations(loop, originalInitialState);
484 LoopKillCache loopKilledLocations = loopLocationKillCache.get(loop);
485 if (loopKilledLocations != null && loopKilledLocations.loopKillsLocations()) {
486 Iterator<ReadCacheEntry> it = initialState.readCache.getKeys().iterator();
487 while (it.hasNext()) {
488 ReadCacheEntry entry = it.next();
489 if (loopKilledLocations.containsLocation(entry.identity)) {
490 it.remove();
491 }
492 }
493 }
494 return initialState;
495 }
496
497 }