--- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/arraycopy/ArrayCopySnippets.java 2017-03-20 17:39:03.000000000 -0700 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/arraycopy/ArrayCopySnippets.java 2017-03-20 17:39:03.000000000 -0700 @@ -22,7 +22,6 @@ */ package org.graalvm.compiler.hotspot.replacements.arraycopy; -import static org.graalvm.compiler.core.common.GraalOptions.SnippetCounters; import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFIG; import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.KLASS_SUPER_CHECK_OFFSET_LOCATION; import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.OBJ_ARRAY_KLASS_ELEMENT_KLASS_LOCATION; @@ -39,7 +38,6 @@ import java.lang.reflect.Method; import java.util.EnumMap; -import java.util.Map; import org.graalvm.compiler.api.directives.GraalDirectives; import org.graalvm.compiler.api.replacements.Fold; @@ -60,19 +58,22 @@ import org.graalvm.compiler.nodes.PiNode; import org.graalvm.compiler.nodes.StructuredGraph; import org.graalvm.compiler.nodes.ValueNode; -import org.graalvm.compiler.nodes.extended.UnsafeLoadNode; +import org.graalvm.compiler.nodes.extended.RawLoadNode; +import org.graalvm.compiler.nodes.extended.RawStoreNode; import org.graalvm.compiler.nodes.java.ArrayLengthNode; import org.graalvm.compiler.nodes.spi.LoweringTool; import org.graalvm.compiler.nodes.type.StampTool; +import org.graalvm.compiler.options.OptionValues; import org.graalvm.compiler.replacements.SnippetCounter; +import org.graalvm.compiler.replacements.SnippetCounter.Group; import org.graalvm.compiler.replacements.SnippetTemplate; import org.graalvm.compiler.replacements.SnippetTemplate.Arguments; import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo; import org.graalvm.compiler.replacements.Snippets; import org.graalvm.compiler.replacements.nodes.BasicArrayCopyNode; -import org.graalvm.compiler.replacements.nodes.DirectObjectStoreNode; import org.graalvm.compiler.replacements.nodes.ExplodeLoopNode; import org.graalvm.compiler.word.Word; +import org.graalvm.util.UnmodifiableEconomicMap; import jdk.vm.ci.code.TargetDescription; import jdk.vm.ci.hotspot.HotSpotResolvedObjectType; @@ -85,64 +86,64 @@ public class ArrayCopySnippets implements Snippets { - private static int checkArrayType(KlassPointer hub) { - int layoutHelper = readLayoutHelper(hub); + private static int checkArrayType(KlassPointer nonNullHub) { + int layoutHelper = readLayoutHelper(nonNullHub); if (probability(SLOW_PATH_PROBABILITY, layoutHelper >= 0)) { DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); } return layoutHelper; } - private static void checkLimits(Object src, int srcPos, Object dest, int destPos, int length) { + private static void checkLimits(Object src, int srcPos, Object dest, int destPos, int length, Counters counters) { if (probability(SLOW_PATH_PROBABILITY, srcPos < 0)) { - checkAIOOBECounter.inc(); + counters.checkAIOOBECounter.inc(); DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); } if (probability(SLOW_PATH_PROBABILITY, destPos < 0)) { - checkAIOOBECounter.inc(); + counters.checkAIOOBECounter.inc(); DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); } if (probability(SLOW_PATH_PROBABILITY, length < 0)) { - checkAIOOBECounter.inc(); + counters.checkAIOOBECounter.inc(); DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); } if (probability(SLOW_PATH_PROBABILITY, srcPos > ArrayLengthNode.arrayLength(src) - length)) { - checkAIOOBECounter.inc(); + counters.checkAIOOBECounter.inc(); DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); } if (probability(SLOW_PATH_PROBABILITY, destPos > ArrayLengthNode.arrayLength(dest) - length)) { - checkAIOOBECounter.inc(); + counters.checkAIOOBECounter.inc(); DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); } - checkSuccessCounter.inc(); + counters.checkSuccessCounter.inc(); } @Snippet - public static void arraycopyZeroLengthIntrinsic(Object src, int srcPos, Object dest, int destPos, int length) { + public static void arraycopyZeroLengthIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter Counters counters) { Object nonNullSrc = GraalDirectives.guardingNonNull(src); Object nonNullDest = GraalDirectives.guardingNonNull(dest); KlassPointer srcHub = loadHub(nonNullSrc); KlassPointer destHub = loadHub(nonNullDest); checkArrayType(srcHub); checkArrayType(destHub); - checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length); - zeroLengthStaticCounter.inc(); + checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters); + counters.zeroLengthStaticCounter.inc(); } @Snippet public static void arraycopyExactIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter JavaKind elementKind, @ConstantParameter SnippetCounter counter, - @ConstantParameter SnippetCounter copiedCounter) { + @ConstantParameter SnippetCounter copiedCounter, @ConstantParameter Counters counters) { Object nonNullSrc = GraalDirectives.guardingNonNull(src); Object nonNullDest = GraalDirectives.guardingNonNull(dest); - checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length); + checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters); counter.inc(); copiedCounter.add(length); ArrayCopyCallNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, elementKind); if (length == 0) { - zeroLengthDynamicCounter.inc(); + counters.zeroLengthDynamicCounter.inc(); } else { - nonZeroLengthDynamicCounter.inc(); - nonZeroLengthDynamicCopiedCounter.add(length); + counters.nonZeroLengthDynamicCounter.inc(); + counters.nonZeroLengthDynamicCopiedCounter.add(length); } } @@ -152,7 +153,7 @@ */ @Snippet public static void arraycopyPredictedExactIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter JavaKind elementKind, - @ConstantParameter SnippetCounter counter, @ConstantParameter SnippetCounter copiedCounter) { + @ConstantParameter SnippetCounter counter, @ConstantParameter SnippetCounter copiedCounter, @ConstantParameter Counters counters) { Object nonNullSrc = GraalDirectives.guardingNonNull(src); Object nonNullDest = GraalDirectives.guardingNonNull(dest); KlassPointer srcHub = loadHub(nonNullSrc); @@ -160,33 +161,33 @@ if (probability(SLOW_PATH_PROBABILITY, srcHub != destHub)) { DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); } - checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length); + checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters); counter.inc(); copiedCounter.add(length); ArrayCopyCallNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, elementKind); if (length == 0) { - zeroLengthDynamicCounter.inc(); + counters.zeroLengthDynamicCounter.inc(); } else { - nonZeroLengthDynamicCounter.inc(); - nonZeroLengthDynamicCopiedCounter.add(length); + counters.nonZeroLengthDynamicCounter.inc(); + counters.nonZeroLengthDynamicCopiedCounter.add(length); } } @Snippet public static void arraycopyPredictedObjectWork(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, int length, KlassPointer objectArrayKlass, - @ConstantParameter SnippetCounter counter, @ConstantParameter SnippetCounter copiedCounter) { + @ConstantParameter SnippetCounter counter, @ConstantParameter SnippetCounter copiedCounter, @ConstantParameter Counters counters) { if (length > 0) { KlassPointer srcHub = loadHub(PiNode.asNonNullObject(nonNullSrc)); KlassPointer destHub = loadHub(PiNode.asNonNullObject(nonNullDest)); if (probability(FAST_PATH_PROBABILITY, srcHub == destHub || destHub == objectArrayKlass)) { counter.inc(); copiedCounter.add(length); - predictedObjectArrayCopyFastPathCounter.inc(); - predictedObjectArrayCopyFastPathCopiedCounter.add(length); + counters.predictedObjectArrayCopyFastPathCounter.inc(); + counters.predictedObjectArrayCopyFastPathCopiedCounter.add(length); ArrayCopyCallNode.arraycopyObjectKillsAny(nonNullSrc, srcPos, nonNullDest, destPos, length); } else { - predictedObjectArrayCopySlowPathCounter.inc(); - predictedObjectArrayCopySlowPathCopiedCounter.add(length); + counters.predictedObjectArrayCopySlowPathCounter.inc(); + counters.predictedObjectArrayCopySlowPathCopiedCounter.add(length); System.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length); } } @@ -198,19 +199,19 @@ */ @Snippet public static void arraycopySlowPathIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, KlassPointer predictedKlass, @ConstantParameter JavaKind elementKind, - @ConstantParameter SnippetInfo slowPath, @ConstantParameter Object slowPathArgument) { + @ConstantParameter SnippetInfo slowPath, @ConstantParameter Object slowPathArgument, @ConstantParameter Counters counters) { Object nonNullSrc = GraalDirectives.guardingNonNull(src); Object nonNullDest = GraalDirectives.guardingNonNull(dest); KlassPointer srcHub = loadHub(nonNullSrc); KlassPointer destHub = loadHub(nonNullDest); checkArrayType(srcHub); checkArrayType(destHub); - checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length); + checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters); if (length == 0) { - zeroLengthDynamicCounter.inc(); + counters.zeroLengthDynamicCounter.inc(); } else { - nonZeroLengthDynamicCounter.inc(); - nonZeroLengthDynamicCopiedCounter.add(length); + counters.nonZeroLengthDynamicCounter.inc(); + counters.nonZeroLengthDynamicCopiedCounter.add(length); } ArrayCopySlowPathNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, predictedKlass, elementKind, slowPath, slowPathArgument); } @@ -219,34 +220,35 @@ * Snippet for unrolled arraycopy. */ @Snippet - public static void arraycopyUnrolledIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter int unrolledLength, @ConstantParameter JavaKind elementKind) { + public static void arraycopyUnrolledIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter int unrolledLength, @ConstantParameter JavaKind elementKind, + @ConstantParameter Counters counters) { Object nonNullSrc = GraalDirectives.guardingNonNull(src); Object nonNullDest = GraalDirectives.guardingNonNull(dest); - checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length); + checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters); if (length == 0) { - zeroLengthDynamicCounter.inc(); + counters.zeroLengthDynamicCounter.inc(); } else { - nonZeroLengthDynamicCounter.inc(); - nonZeroLengthDynamicCopiedCounter.add(length); + counters.nonZeroLengthDynamicCounter.inc(); + counters.nonZeroLengthDynamicCopiedCounter.add(length); } ArrayCopyUnrollNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, unrolledLength, elementKind); } @Snippet - public static void checkcastArraycopyWork(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, int length) { + public static void checkcastArraycopyWork(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, int length, @ConstantParameter Counters counters) { if (length > 0) { KlassPointer destKlass = loadHub(nonNullDest); KlassPointer srcKlass = loadHub(nonNullSrc); if (probability(SLOW_PATH_PROBABILITY, srcKlass == destKlass)) { // no storecheck required. - objectCheckcastSameTypeCounter.inc(); - objectCheckcastSameTypeCopiedCounter.add(length); + counters.objectCheckcastSameTypeCounter.inc(); + counters.objectCheckcastSameTypeCopiedCounter.add(length); ArrayCopyCallNode.arraycopyObjectKillsAny(nonNullSrc, srcPos, nonNullDest, destPos, length); } else { KlassPointer destElemKlass = destKlass.readKlassPointer(arrayClassElementOffset(INJECTED_VMCONFIG), OBJ_ARRAY_KLASS_ELEMENT_KLASS_LOCATION); Word superCheckOffset = Word.signed(destElemKlass.readInt(superCheckOffsetOffset(INJECTED_VMCONFIG), KLASS_SUPER_CHECK_OFFSET_LOCATION)); - objectCheckcastCounter.inc(); - objectCheckcastCopiedCounter.add(length); + counters.objectCheckcastCounter.inc(); + counters.objectCheckcastCopiedCounter.add(length); int copiedElements = CheckcastArrayCopyCallNode.checkcastArraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, superCheckOffset, destElemKlass, false); if (copiedElements != 0) { /* @@ -261,7 +263,7 @@ } @Snippet - public static void arraycopyGeneric(Object src, int srcPos, Object dest, int destPos, int length) { + public static void arraycopyGeneric(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter Counters counters) { Object nonNullSrc = GraalDirectives.guardingNonNull(src); Object nonNullDest = GraalDirectives.guardingNonNull(dest); KlassPointer srcHub = loadHub(nonNullSrc); @@ -269,19 +271,19 @@ if (probability(FAST_PATH_PROBABILITY, srcHub.equal(destHub)) && probability(FAST_PATH_PROBABILITY, nonNullSrc != nonNullDest)) { int layoutHelper = checkArrayType(srcHub); final boolean isObjectArray = ((layoutHelper & layoutHelperElementTypePrimitiveInPlace(INJECTED_VMCONFIG)) == 0); - checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length); + checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters); if (probability(FAST_PATH_PROBABILITY, isObjectArray)) { - genericObjectExactCallCounter.inc(); - genericObjectExactCallCopiedCounter.add(length); + counters.genericObjectExactCallCounter.inc(); + counters.genericObjectExactCallCopiedCounter.add(length); ArrayCopyCallNode.disjointArraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, JavaKind.Object); } else { - genericPrimitiveCallCounter.inc(); - genericPrimitiveCallCopiedCounter.add(length); + counters.genericPrimitiveCallCounter.inc(); + counters.genericPrimitiveCallCopiedCounter.add(length); UnsafeArrayCopyNode.arraycopyPrimitive(nonNullSrc, srcPos, nonNullDest, destPos, length, layoutHelper); } } else { - SystemArraycopyCounter.inc(); - SystemArraycopyCopiedCounter.add(length); + counters.systemArraycopyCounter.inc(); + counters.systemArraycopyCopiedCounter.add(length); System.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length); } } @@ -302,8 +304,8 @@ ExplodeLoopNode.explodeLoop(); for (int iteration = 0; iteration < length; iteration++) { if (i >= 0) { - Object a = UnsafeLoadNode.load(nonNullSrc, arrayBaseOffset + i + (long) srcPos * scale, elementKind, arrayLocation); - DirectObjectStoreNode.storeObject(nonNullDest, arrayBaseOffset, i + (long) destPos * scale, a, arrayLocation, elementKind); + Object a = RawLoadNode.load(nonNullSrc, arrayBaseOffset + i + (long) srcPos * scale, elementKind, arrayLocation); + RawStoreNode.storeObject(nonNullDest, arrayBaseOffset + i + (long) destPos * scale, a, elementKind, arrayLocation, false); i -= scale; } } @@ -313,78 +315,103 @@ ExplodeLoopNode.explodeLoop(); for (int iteration = 0; iteration < length; iteration++) { if (i < end) { - Object a = UnsafeLoadNode.load(nonNullSrc, arrayBaseOffset + i + (long) srcPos * scale, elementKind, arrayLocation); - DirectObjectStoreNode.storeObject(nonNullDest, arrayBaseOffset, i + (long) destPos * scale, a, arrayLocation, elementKind); + Object a = RawLoadNode.load(nonNullSrc, arrayBaseOffset + i + (long) srcPos * scale, elementKind, arrayLocation); + RawStoreNode.storeObject(nonNullDest, arrayBaseOffset + i + (long) destPos * scale, a, elementKind, arrayLocation, false); i += scale; } } } } - private static final SnippetCounter.Group checkCounters = SnippetCounters.getValue() ? new SnippetCounter.Group("System.arraycopy checkInputs") : null; - private static final SnippetCounter checkSuccessCounter = new SnippetCounter(checkCounters, "checkSuccess", "checkSuccess"); - private static final SnippetCounter checkAIOOBECounter = new SnippetCounter(checkCounters, "checkAIOOBE", "checkAIOOBE"); + static class Counters { + final SnippetCounter checkSuccessCounter; + final SnippetCounter checkAIOOBECounter; + + final SnippetCounter objectCheckcastCounter; + final SnippetCounter objectCheckcastSameTypeCounter; + final SnippetCounter predictedObjectArrayCopySlowPathCounter; + final SnippetCounter predictedObjectArrayCopyFastPathCounter; + + final SnippetCounter genericPrimitiveCallCounter; + final SnippetCounter genericObjectExactCallCounter; + final SnippetCounter systemArraycopyCounter; + + final SnippetCounter zeroLengthStaticCounter; + final SnippetCounter zeroLengthDynamicCounter; + final SnippetCounter nonZeroLengthDynamicCounter; + + final SnippetCounter nonZeroLengthDynamicCopiedCounter; + final SnippetCounter genericPrimitiveCallCopiedCounter; + final SnippetCounter genericObjectExactCallCopiedCounter; + final SnippetCounter systemArraycopyCopiedCounter; + + final SnippetCounter objectCheckcastCopiedCounter; + final SnippetCounter objectCheckcastSameTypeCopiedCounter; + final SnippetCounter predictedObjectArrayCopySlowPathCopiedCounter; + final SnippetCounter predictedObjectArrayCopyFastPathCopiedCounter; + + final EnumMap arraycopyCallCounters = new EnumMap<>(JavaKind.class); + final EnumMap arraycopyCounters = new EnumMap<>(JavaKind.class); + + final EnumMap arraycopyCallCopiedCounters = new EnumMap<>(JavaKind.class); + final EnumMap arraycopyCopiedCounters = new EnumMap<>(JavaKind.class); + + Counters(SnippetCounter.Group.Factory factory) { + final Group checkCounters = factory.createSnippetCounterGroup("System.arraycopy checkInputs"); + final Group counters = factory.createSnippetCounterGroup("System.arraycopy"); + final Group copiedCounters = factory.createSnippetCounterGroup("System.arraycopy copied elements"); + final Group lengthCounters = factory.createSnippetCounterGroup("System.arraycopy 0-length checks"); + + checkSuccessCounter = new SnippetCounter(checkCounters, "checkSuccess", "checkSuccess"); + checkAIOOBECounter = new SnippetCounter(checkCounters, "checkAIOOBE", "checkAIOOBE"); + + objectCheckcastCounter = new SnippetCounter(counters, "Object[]{non-exact}", "arraycopy for non-exact Object[] arrays"); + objectCheckcastSameTypeCounter = new SnippetCounter(counters, "Object[]{same-type}", "arraycopy call for src.klass == dest.klass Object[] arrays"); + predictedObjectArrayCopySlowPathCounter = new SnippetCounter(counters, "Object[]{slow-path}", "used System.arraycopy slow path for predicted Object[] arrays"); + predictedObjectArrayCopyFastPathCounter = new SnippetCounter(counters, "Object[]{fast-path}", "used oop_arraycopy for predicted Object[] arrays"); + genericPrimitiveCallCounter = new SnippetCounter(counters, "genericPrimitive", "generic arraycopy snippet for primitive arrays"); + genericObjectExactCallCounter = new SnippetCounter(counters, "genericObjectExact", "generic arraycopy snippet for special object arrays"); + systemArraycopyCounter = new SnippetCounter(counters, "genericObject", "call to System.arraycopy"); + + zeroLengthStaticCounter = new SnippetCounter(lengthCounters, "0-lengthcopy static", "arraycopy where the length is statically 0"); + zeroLengthDynamicCounter = new SnippetCounter(lengthCounters, "0-lengthcopy dynamically", "arraycopy where the length is dynamically 0"); + nonZeroLengthDynamicCounter = new SnippetCounter(lengthCounters, "non-0-lengthcopy dynamically", "arraycopy where the length is dynamically not zero"); + + nonZeroLengthDynamicCopiedCounter = new SnippetCounter(copiedCounters, "non-0-lengthcopy dynamically", "arraycopy where the length is dynamically not zero"); + genericPrimitiveCallCopiedCounter = new SnippetCounter(copiedCounters, "genericPrimitive", "generic arraycopy snippet for primitive arrays"); + genericObjectExactCallCopiedCounter = new SnippetCounter(copiedCounters, "genericObjectExact", "generic arraycopy snippet for special object arrays"); + systemArraycopyCopiedCounter = new SnippetCounter(copiedCounters, "genericObject", "call to System.arraycopy"); + + objectCheckcastCopiedCounter = new SnippetCounter(copiedCounters, "Object[]{non-exact}", "arraycopy for non-exact Object[] arrays"); + objectCheckcastSameTypeCopiedCounter = new SnippetCounter(copiedCounters, "Object[]{same-type}", "arraycopy call for src.klass == dest.klass Object[] arrays"); + predictedObjectArrayCopySlowPathCopiedCounter = new SnippetCounter(copiedCounters, "Object[]{slow-path}", + "used System.arraycopy slow path for predicted Object[] arrays"); + predictedObjectArrayCopyFastPathCopiedCounter = new SnippetCounter(copiedCounters, "Object[]{fast-path}", "used oop_arraycopy for predicted Object[] arrays"); + createArraycopyCounter(JavaKind.Byte, counters, copiedCounters); + createArraycopyCounter(JavaKind.Boolean, counters, copiedCounters); + createArraycopyCounter(JavaKind.Char, counters, copiedCounters); + createArraycopyCounter(JavaKind.Short, counters, copiedCounters); + createArraycopyCounter(JavaKind.Int, counters, copiedCounters); + createArraycopyCounter(JavaKind.Long, counters, copiedCounters); + createArraycopyCounter(JavaKind.Float, counters, copiedCounters); + createArraycopyCounter(JavaKind.Double, counters, copiedCounters); + createArraycopyCounter(JavaKind.Object, counters, copiedCounters); + } + + void createArraycopyCounter(JavaKind kind, Group counters, Group copiedCounters) { + arraycopyCallCounters.put(kind, new SnippetCounter(counters, kind + "[]{stub}", "arraycopy call for " + kind + "[] arrays")); + arraycopyCounters.put(kind, new SnippetCounter(counters, kind + "[]{inline}", "inline arraycopy for " + kind + "[] arrays")); - private static final SnippetCounter.Group counters = SnippetCounters.getValue() ? new SnippetCounter.Group("System.arraycopy") : null; - - private static final SnippetCounter objectCheckcastCounter = new SnippetCounter(counters, "Object[]{non-exact}", "arraycopy for non-exact Object[] arrays"); - private static final SnippetCounter objectCheckcastSameTypeCounter = new SnippetCounter(counters, "Object[]{same-type}", "arraycopy call for src.klass == dest.klass Object[] arrays"); - private static final SnippetCounter predictedObjectArrayCopySlowPathCounter = new SnippetCounter(counters, "Object[]{slow-path}", "used System.arraycopy slow path for predicted Object[] arrays"); - private static final SnippetCounter predictedObjectArrayCopyFastPathCounter = new SnippetCounter(counters, "Object[]{fast-path}", "used oop_arraycopy for predicted Object[] arrays"); - - private static final EnumMap arraycopyCallCounters = new EnumMap<>(JavaKind.class); - private static final EnumMap arraycopyCounters = new EnumMap<>(JavaKind.class); - - private static final EnumMap arraycopyCallCopiedCounters = new EnumMap<>(JavaKind.class); - private static final EnumMap arraycopyCopiedCounters = new EnumMap<>(JavaKind.class); - - static void createArraycopyCounter(JavaKind kind) { - arraycopyCallCounters.put(kind, new SnippetCounter(counters, kind + "[]{stub}", "arraycopy call for " + kind + "[] arrays")); - arraycopyCounters.put(kind, new SnippetCounter(counters, kind + "[]{inline}", "inline arraycopy for " + kind + "[] arrays")); - - arraycopyCallCopiedCounters.put(kind, new SnippetCounter(copiedCounters, kind + "[]{stub}", "arraycopy call for " + kind + "[] arrays")); - arraycopyCopiedCounters.put(kind, new SnippetCounter(copiedCounters, kind + "[]{inline}", "inline arraycopy for " + kind + "[] arrays")); - } - - static { - createArraycopyCounter(JavaKind.Byte); - createArraycopyCounter(JavaKind.Boolean); - createArraycopyCounter(JavaKind.Char); - createArraycopyCounter(JavaKind.Short); - createArraycopyCounter(JavaKind.Int); - createArraycopyCounter(JavaKind.Long); - createArraycopyCounter(JavaKind.Float); - createArraycopyCounter(JavaKind.Double); - createArraycopyCounter(JavaKind.Object); + arraycopyCallCopiedCounters.put(kind, new SnippetCounter(copiedCounters, kind + "[]{stub}", "arraycopy call for " + kind + "[] arrays")); + arraycopyCopiedCounters.put(kind, new SnippetCounter(copiedCounters, kind + "[]{inline}", "inline arraycopy for " + kind + "[] arrays")); + } } - private static final SnippetCounter genericPrimitiveCallCounter = new SnippetCounter(counters, "genericPrimitive", "generic arraycopy snippet for primitive arrays"); - private static final SnippetCounter genericObjectExactCallCounter = new SnippetCounter(counters, "genericObjectExact", "generic arraycopy snippet for special object arrays"); - private static final SnippetCounter SystemArraycopyCounter = new SnippetCounter(counters, "genericObject", "call to System.arraycopy"); - - private static final SnippetCounter.Group lengthCounters = SnippetCounters.getValue() ? new SnippetCounter.Group("System.arraycopy 0-length checks") : null; - - private static final SnippetCounter zeroLengthStaticCounter = new SnippetCounter(lengthCounters, "0-lengthcopy static", "arraycopy where the length is statically 0"); - private static final SnippetCounter zeroLengthDynamicCounter = new SnippetCounter(lengthCounters, "0-lengthcopy dynamically", "arraycopy where the length is dynamically 0"); - private static final SnippetCounter nonZeroLengthDynamicCounter = new SnippetCounter(lengthCounters, "non-0-lengthcopy dynamically", "arraycopy where the length is dynamically not zero"); - - private static final SnippetCounter.Group copiedCounters = SnippetCounters.getValue() ? new SnippetCounter.Group("System.arraycopy copied elements") : null; - - private static final SnippetCounter nonZeroLengthDynamicCopiedCounter = new SnippetCounter(copiedCounters, "non-0-lengthcopy dynamically", "arraycopy where the length is dynamically not zero"); - private static final SnippetCounter genericPrimitiveCallCopiedCounter = new SnippetCounter(copiedCounters, "genericPrimitive", "generic arraycopy snippet for primitive arrays"); - private static final SnippetCounter genericObjectExactCallCopiedCounter = new SnippetCounter(copiedCounters, "genericObjectExact", "generic arraycopy snippet for special object arrays"); - private static final SnippetCounter SystemArraycopyCopiedCounter = new SnippetCounter(copiedCounters, "genericObject", "call to System.arraycopy"); - - private static final SnippetCounter objectCheckcastCopiedCounter = new SnippetCounter(copiedCounters, "Object[]{non-exact}", "arraycopy for non-exact Object[] arrays"); - private static final SnippetCounter objectCheckcastSameTypeCopiedCounter = new SnippetCounter(copiedCounters, "Object[]{same-type}", "arraycopy call for src.klass == dest.klass Object[] arrays"); - private static final SnippetCounter predictedObjectArrayCopySlowPathCopiedCounter = new SnippetCounter(copiedCounters, "Object[]{slow-path}", - "used System.arraycopy slow path for predicted Object[] arrays"); - private static final SnippetCounter predictedObjectArrayCopyFastPathCopiedCounter = new SnippetCounter(copiedCounters, "Object[]{fast-path}", "used oop_arraycopy for predicted Object[] arrays"); - public static class Templates extends SnippetTemplate.AbstractTemplates { - public Templates(HotSpotProviders providers, TargetDescription target) { - super(providers, providers.getSnippetReflection(), target); + public Templates(OptionValues options, SnippetCounter.Group.Factory factory, HotSpotProviders providers, TargetDescription target) { + super(options, providers, providers.getSnippetReflection(), target); + this.counters = new Counters(factory); } private ResolvedJavaMethod originalArraycopy() throws GraalError { @@ -414,6 +441,8 @@ private final SnippetInfo arraycopyUnrolledWorkSnippet = snippet("arraycopyUnrolledWork"); + private final Counters counters; + protected SnippetInfo snippet(String methodName) { SnippetInfo info = snippet(ArrayCopySnippets.class, methodName, LocationIdentity.any()); info.setOriginalMethod(originalArraycopy()); @@ -538,9 +567,10 @@ } else if (snippetInfo == arraycopyExactIntrinsicSnippet || snippetInfo == arraycopyPredictedExactIntrinsicSnippet) { assert componentKind != null; args.addConst("elementKind", componentKind); - args.addConst("counter", arraycopyCallCounters.get(componentKind)); - args.addConst("copiedCounter", arraycopyCallCopiedCounters.get(componentKind)); + args.addConst("counter", counters.arraycopyCallCounters.get(componentKind)); + args.addConst("copiedCounter", counters.arraycopyCallCopiedCounters.get(componentKind)); } + args.addConst("counters", counters); instantiate(args, arraycopy); } @@ -564,8 +594,9 @@ } if (snippetInfo == arraycopyPredictedObjectWorkSnippet) { args.add("objectArrayKlass", arraycopy.getPredictedKlass()); - args.addConst("counter", arraycopyCallCounters.get(JavaKind.Object)); - args.addConst("copiedCounter", arraycopyCallCopiedCounters.get(JavaKind.Object)); + args.addConst("counter", counters.arraycopyCallCounters.get(JavaKind.Object)); + args.addConst("copiedCounter", counters.arraycopyCallCopiedCounters.get(JavaKind.Object)); + args.addConst("counters", counters); } instantiate(args, arraycopy); } @@ -597,8 +628,8 @@ private void instantiate(Arguments args, BasicArrayCopyNode arraycopy) { StructuredGraph graph = arraycopy.graph(); SnippetTemplate template = template(args); - Map replacements = template.instantiate(providers.getMetaAccess(), arraycopy, SnippetTemplate.DEFAULT_REPLACER, args); - for (Node originalNode : replacements.keySet()) { + UnmodifiableEconomicMap replacements = template.instantiate(providers.getMetaAccess(), arraycopy, SnippetTemplate.DEFAULT_REPLACER, args); + for (Node originalNode : replacements.getKeys()) { if (originalNode instanceof Invoke) { Invoke invoke = (Invoke) replacements.get(originalNode); assert invoke.asNode().graph() == graph;