1 /* 2 * Copyright (c) 2011, 2016, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 */ 23 package org.graalvm.compiler.hotspot.replacements.arraycopy; 24 25 import static org.graalvm.compiler.core.common.GraalOptions.SnippetCounters; 26 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFIG; 27 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.KLASS_SUPER_CHECK_OFFSET_LOCATION; 28 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.OBJ_ARRAY_KLASS_ELEMENT_KLASS_LOCATION; 29 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayBaseOffset; 30 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayClassElementOffset; 31 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayIndexScale; 32 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperElementTypePrimitiveInPlace; 33 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.loadHub; 34 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readLayoutHelper; 35 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.superCheckOffsetOffset; 36 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FAST_PATH_PROBABILITY; 37 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY; 38 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability; 39 40 import java.lang.reflect.Method; 41 import java.util.EnumMap; 42 import java.util.Map; 43 44 import org.graalvm.compiler.api.directives.GraalDirectives; 45 import org.graalvm.compiler.api.replacements.Fold; 46 import org.graalvm.compiler.api.replacements.Snippet; 47 import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter; 48 import org.graalvm.compiler.core.common.LocationIdentity; 49 import org.graalvm.compiler.debug.GraalError; 50 import org.graalvm.compiler.graph.Node; 51 import org.graalvm.compiler.hotspot.meta.HotSpotProviders; 52 import org.graalvm.compiler.hotspot.nodes.type.KlassPointerStamp; 53 import org.graalvm.compiler.hotspot.word.KlassPointer; 54 import org.graalvm.compiler.nodes.CallTargetNode; 55 import org.graalvm.compiler.nodes.ConstantNode; 56 import org.graalvm.compiler.nodes.DeoptimizeNode; 57 import org.graalvm.compiler.nodes.Invoke; 58 import org.graalvm.compiler.nodes.InvokeNode; 59 import org.graalvm.compiler.nodes.NamedLocationIdentity; 60 import org.graalvm.compiler.nodes.PiNode; 61 import org.graalvm.compiler.nodes.StructuredGraph; 62 import org.graalvm.compiler.nodes.ValueNode; 63 import org.graalvm.compiler.nodes.extended.UnsafeLoadNode; 64 import org.graalvm.compiler.nodes.java.ArrayLengthNode; 65 import org.graalvm.compiler.nodes.spi.LoweringTool; 66 import org.graalvm.compiler.nodes.type.StampTool; 67 import org.graalvm.compiler.replacements.SnippetCounter; 68 import org.graalvm.compiler.replacements.SnippetTemplate; 69 import org.graalvm.compiler.replacements.SnippetTemplate.Arguments; 70 import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo; 71 import org.graalvm.compiler.replacements.Snippets; 72 import org.graalvm.compiler.replacements.nodes.BasicArrayCopyNode; 73 import org.graalvm.compiler.replacements.nodes.DirectObjectStoreNode; 74 import org.graalvm.compiler.replacements.nodes.ExplodeLoopNode; 75 import org.graalvm.compiler.word.Word; 76 77 import jdk.vm.ci.code.TargetDescription; 78 import jdk.vm.ci.hotspot.HotSpotResolvedObjectType; 79 import jdk.vm.ci.meta.DeoptimizationAction; 80 import jdk.vm.ci.meta.DeoptimizationReason; 81 import jdk.vm.ci.meta.JavaConstant; 82 import jdk.vm.ci.meta.JavaKind; 83 import jdk.vm.ci.meta.ResolvedJavaMethod; 84 import jdk.vm.ci.meta.ResolvedJavaType; 85 86 public class ArrayCopySnippets implements Snippets { 87 88 private static int checkArrayType(KlassPointer hub) { 89 int layoutHelper = readLayoutHelper(hub); 90 if (probability(SLOW_PATH_PROBABILITY, layoutHelper >= 0)) { 91 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); 92 } 93 return layoutHelper; 94 } 95 96 private static void checkLimits(Object src, int srcPos, Object dest, int destPos, int length) { 97 if (probability(SLOW_PATH_PROBABILITY, srcPos < 0)) { 98 checkAIOOBECounter.inc(); 99 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); 100 } 101 if (probability(SLOW_PATH_PROBABILITY, destPos < 0)) { 102 checkAIOOBECounter.inc(); 103 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); 104 } 105 if (probability(SLOW_PATH_PROBABILITY, length < 0)) { 106 checkAIOOBECounter.inc(); 107 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); 108 } 109 if (probability(SLOW_PATH_PROBABILITY, srcPos > ArrayLengthNode.arrayLength(src) - length)) { 110 checkAIOOBECounter.inc(); 111 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); 112 } 113 if (probability(SLOW_PATH_PROBABILITY, destPos > ArrayLengthNode.arrayLength(dest) - length)) { 114 checkAIOOBECounter.inc(); 115 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); 116 } 117 checkSuccessCounter.inc(); 118 } 119 120 @Snippet 121 public static void arraycopyZeroLengthIntrinsic(Object src, int srcPos, Object dest, int destPos, int length) { 122 Object nonNullSrc = GraalDirectives.guardingNonNull(src); 123 Object nonNullDest = GraalDirectives.guardingNonNull(dest); 124 KlassPointer srcHub = loadHub(nonNullSrc); 125 KlassPointer destHub = loadHub(nonNullDest); 126 checkArrayType(srcHub); 127 checkArrayType(destHub); 128 checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length); 129 zeroLengthStaticCounter.inc(); 130 } 131 132 @Snippet 133 public static void arraycopyExactIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter JavaKind elementKind, @ConstantParameter SnippetCounter counter, 134 @ConstantParameter SnippetCounter copiedCounter) { 135 Object nonNullSrc = GraalDirectives.guardingNonNull(src); 136 Object nonNullDest = GraalDirectives.guardingNonNull(dest); 137 checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length); 138 counter.inc(); 139 copiedCounter.add(length); 140 ArrayCopyCallNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, elementKind); 141 if (length == 0) { 142 zeroLengthDynamicCounter.inc(); 143 } else { 144 nonZeroLengthDynamicCounter.inc(); 145 nonZeroLengthDynamicCopiedCounter.add(length); 146 } 147 } 148 149 /** 150 * This intrinsic is useful for the case where we know something statically about one of the 151 * inputs but not the other. 152 */ 153 @Snippet 154 public static void arraycopyPredictedExactIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter JavaKind elementKind, 155 @ConstantParameter SnippetCounter counter, @ConstantParameter SnippetCounter copiedCounter) { 156 Object nonNullSrc = GraalDirectives.guardingNonNull(src); 157 Object nonNullDest = GraalDirectives.guardingNonNull(dest); 158 KlassPointer srcHub = loadHub(nonNullSrc); 159 KlassPointer destHub = loadHub(nonNullDest); 160 if (probability(SLOW_PATH_PROBABILITY, srcHub != destHub)) { 161 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); 162 } 163 checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length); 164 counter.inc(); 165 copiedCounter.add(length); 166 ArrayCopyCallNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, elementKind); 167 if (length == 0) { 168 zeroLengthDynamicCounter.inc(); 169 } else { 170 nonZeroLengthDynamicCounter.inc(); 171 nonZeroLengthDynamicCopiedCounter.add(length); 172 } 173 } 174 175 @Snippet 176 public static void arraycopyPredictedObjectWork(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, int length, KlassPointer objectArrayKlass, 177 @ConstantParameter SnippetCounter counter, @ConstantParameter SnippetCounter copiedCounter) { 178 if (length > 0) { 179 KlassPointer srcHub = loadHub(PiNode.asNonNullObject(nonNullSrc)); 180 KlassPointer destHub = loadHub(PiNode.asNonNullObject(nonNullDest)); 181 if (probability(FAST_PATH_PROBABILITY, srcHub == destHub || destHub == objectArrayKlass)) { 182 counter.inc(); 183 copiedCounter.add(length); 184 predictedObjectArrayCopyFastPathCounter.inc(); 185 predictedObjectArrayCopyFastPathCopiedCounter.add(length); 186 ArrayCopyCallNode.arraycopyObjectKillsAny(nonNullSrc, srcPos, nonNullDest, destPos, length); 187 } else { 188 predictedObjectArrayCopySlowPathCounter.inc(); 189 predictedObjectArrayCopySlowPathCopiedCounter.add(length); 190 System.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length); 191 } 192 } 193 } 194 195 /** 196 * This is the basic template for the full arraycopy checks, including a check that the 197 * underlying type is really an array type. 198 */ 199 @Snippet 200 public static void arraycopySlowPathIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, KlassPointer predictedKlass, @ConstantParameter JavaKind elementKind, 201 @ConstantParameter SnippetInfo slowPath, @ConstantParameter Object slowPathArgument) { 202 Object nonNullSrc = GraalDirectives.guardingNonNull(src); 203 Object nonNullDest = GraalDirectives.guardingNonNull(dest); 204 KlassPointer srcHub = loadHub(nonNullSrc); 205 KlassPointer destHub = loadHub(nonNullDest); 206 checkArrayType(srcHub); 207 checkArrayType(destHub); 208 checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length); 209 if (length == 0) { 210 zeroLengthDynamicCounter.inc(); 211 } else { 212 nonZeroLengthDynamicCounter.inc(); 213 nonZeroLengthDynamicCopiedCounter.add(length); 214 } 215 ArrayCopySlowPathNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, predictedKlass, elementKind, slowPath, slowPathArgument); 216 } 217 218 /** 219 * Snippet for unrolled arraycopy. 220 */ 221 @Snippet 222 public static void arraycopyUnrolledIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter int unrolledLength, @ConstantParameter JavaKind elementKind) { 223 Object nonNullSrc = GraalDirectives.guardingNonNull(src); 224 Object nonNullDest = GraalDirectives.guardingNonNull(dest); 225 checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length); 226 if (length == 0) { 227 zeroLengthDynamicCounter.inc(); 228 } else { 229 nonZeroLengthDynamicCounter.inc(); 230 nonZeroLengthDynamicCopiedCounter.add(length); 231 } 232 ArrayCopyUnrollNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, unrolledLength, elementKind); 233 } 234 235 @Snippet 236 public static void checkcastArraycopyWork(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, int length) { 237 if (length > 0) { 238 KlassPointer destKlass = loadHub(nonNullDest); 239 KlassPointer srcKlass = loadHub(nonNullSrc); 240 if (probability(SLOW_PATH_PROBABILITY, srcKlass == destKlass)) { 241 // no storecheck required. 242 objectCheckcastSameTypeCounter.inc(); 243 objectCheckcastSameTypeCopiedCounter.add(length); 244 ArrayCopyCallNode.arraycopyObjectKillsAny(nonNullSrc, srcPos, nonNullDest, destPos, length); 245 } else { 246 KlassPointer destElemKlass = destKlass.readKlassPointer(arrayClassElementOffset(INJECTED_VMCONFIG), OBJ_ARRAY_KLASS_ELEMENT_KLASS_LOCATION); 247 Word superCheckOffset = Word.signed(destElemKlass.readInt(superCheckOffsetOffset(INJECTED_VMCONFIG), KLASS_SUPER_CHECK_OFFSET_LOCATION)); 248 objectCheckcastCounter.inc(); 249 objectCheckcastCopiedCounter.add(length); 250 int copiedElements = CheckcastArrayCopyCallNode.checkcastArraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, superCheckOffset, destElemKlass, false); 251 if (copiedElements != 0) { 252 /* 253 * the checkcast stub doesn't throw the ArrayStoreException, but returns the 254 * number of copied elements (xor'd with -1). 255 */ 256 copiedElements ^= -1; 257 System.arraycopy(nonNullSrc, srcPos + copiedElements, nonNullDest, destPos + copiedElements, length - copiedElements); 258 } 259 } 260 } 261 } 262 263 @Snippet 264 public static void arraycopyGeneric(Object src, int srcPos, Object dest, int destPos, int length) { 265 Object nonNullSrc = GraalDirectives.guardingNonNull(src); 266 Object nonNullDest = GraalDirectives.guardingNonNull(dest); 267 KlassPointer srcHub = loadHub(nonNullSrc); 268 KlassPointer destHub = loadHub(nonNullDest); 269 if (probability(FAST_PATH_PROBABILITY, srcHub.equal(destHub)) && probability(FAST_PATH_PROBABILITY, nonNullSrc != nonNullDest)) { 270 int layoutHelper = checkArrayType(srcHub); 271 final boolean isObjectArray = ((layoutHelper & layoutHelperElementTypePrimitiveInPlace(INJECTED_VMCONFIG)) == 0); 272 checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length); 273 if (probability(FAST_PATH_PROBABILITY, isObjectArray)) { 274 genericObjectExactCallCounter.inc(); 275 genericObjectExactCallCopiedCounter.add(length); 276 ArrayCopyCallNode.disjointArraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, JavaKind.Object); 277 } else { 278 genericPrimitiveCallCounter.inc(); 279 genericPrimitiveCallCopiedCounter.add(length); 280 UnsafeArrayCopyNode.arraycopyPrimitive(nonNullSrc, srcPos, nonNullDest, destPos, length, layoutHelper); 281 } 282 } else { 283 SystemArraycopyCounter.inc(); 284 SystemArraycopyCopiedCounter.add(length); 285 System.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length); 286 } 287 } 288 289 @Fold 290 static LocationIdentity getArrayLocation(JavaKind kind) { 291 return NamedLocationIdentity.getArrayLocation(kind); 292 } 293 294 @Snippet 295 public static void arraycopyUnrolledWork(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, @ConstantParameter int length, @ConstantParameter JavaKind elementKind) { 296 final int scale = arrayIndexScale(elementKind); 297 int arrayBaseOffset = arrayBaseOffset(elementKind); 298 LocationIdentity arrayLocation = getArrayLocation(elementKind); 299 if (nonNullSrc == nonNullDest && srcPos < destPos) { // bad aliased case 300 long start = (long) (length - 1) * scale; 301 long i = start; 302 ExplodeLoopNode.explodeLoop(); 303 for (int iteration = 0; iteration < length; iteration++) { 304 if (i >= 0) { 305 Object a = UnsafeLoadNode.load(nonNullSrc, arrayBaseOffset + i + (long) srcPos * scale, elementKind, arrayLocation); 306 DirectObjectStoreNode.storeObject(nonNullDest, arrayBaseOffset, i + (long) destPos * scale, a, arrayLocation, elementKind); 307 i -= scale; 308 } 309 } 310 } else { 311 long end = (long) length * scale; 312 long i = 0; 313 ExplodeLoopNode.explodeLoop(); 314 for (int iteration = 0; iteration < length; iteration++) { 315 if (i < end) { 316 Object a = UnsafeLoadNode.load(nonNullSrc, arrayBaseOffset + i + (long) srcPos * scale, elementKind, arrayLocation); 317 DirectObjectStoreNode.storeObject(nonNullDest, arrayBaseOffset, i + (long) destPos * scale, a, arrayLocation, elementKind); 318 i += scale; 319 } 320 } 321 } 322 } 323 324 private static final SnippetCounter.Group checkCounters = SnippetCounters.getValue() ? new SnippetCounter.Group("System.arraycopy checkInputs") : null; 325 private static final SnippetCounter checkSuccessCounter = new SnippetCounter(checkCounters, "checkSuccess", "checkSuccess"); 326 private static final SnippetCounter checkAIOOBECounter = new SnippetCounter(checkCounters, "checkAIOOBE", "checkAIOOBE"); 327 328 private static final SnippetCounter.Group counters = SnippetCounters.getValue() ? new SnippetCounter.Group("System.arraycopy") : null; 329 330 private static final SnippetCounter objectCheckcastCounter = new SnippetCounter(counters, "Object[]{non-exact}", "arraycopy for non-exact Object[] arrays"); 331 private static final SnippetCounter objectCheckcastSameTypeCounter = new SnippetCounter(counters, "Object[]{same-type}", "arraycopy call for src.klass == dest.klass Object[] arrays"); 332 private static final SnippetCounter predictedObjectArrayCopySlowPathCounter = new SnippetCounter(counters, "Object[]{slow-path}", "used System.arraycopy slow path for predicted Object[] arrays"); 333 private static final SnippetCounter predictedObjectArrayCopyFastPathCounter = new SnippetCounter(counters, "Object[]{fast-path}", "used oop_arraycopy for predicted Object[] arrays"); 334 335 private static final EnumMap<JavaKind, SnippetCounter> arraycopyCallCounters = new EnumMap<>(JavaKind.class); 336 private static final EnumMap<JavaKind, SnippetCounter> arraycopyCounters = new EnumMap<>(JavaKind.class); 337 338 private static final EnumMap<JavaKind, SnippetCounter> arraycopyCallCopiedCounters = new EnumMap<>(JavaKind.class); 339 private static final EnumMap<JavaKind, SnippetCounter> arraycopyCopiedCounters = new EnumMap<>(JavaKind.class); 340 341 static void createArraycopyCounter(JavaKind kind) { 342 arraycopyCallCounters.put(kind, new SnippetCounter(counters, kind + "[]{stub}", "arraycopy call for " + kind + "[] arrays")); 343 arraycopyCounters.put(kind, new SnippetCounter(counters, kind + "[]{inline}", "inline arraycopy for " + kind + "[] arrays")); 344 345 arraycopyCallCopiedCounters.put(kind, new SnippetCounter(copiedCounters, kind + "[]{stub}", "arraycopy call for " + kind + "[] arrays")); 346 arraycopyCopiedCounters.put(kind, new SnippetCounter(copiedCounters, kind + "[]{inline}", "inline arraycopy for " + kind + "[] arrays")); 347 } 348 349 static { 350 createArraycopyCounter(JavaKind.Byte); 351 createArraycopyCounter(JavaKind.Boolean); 352 createArraycopyCounter(JavaKind.Char); 353 createArraycopyCounter(JavaKind.Short); 354 createArraycopyCounter(JavaKind.Int); 355 createArraycopyCounter(JavaKind.Long); 356 createArraycopyCounter(JavaKind.Float); 357 createArraycopyCounter(JavaKind.Double); 358 createArraycopyCounter(JavaKind.Object); 359 } 360 361 private static final SnippetCounter genericPrimitiveCallCounter = new SnippetCounter(counters, "genericPrimitive", "generic arraycopy snippet for primitive arrays"); 362 private static final SnippetCounter genericObjectExactCallCounter = new SnippetCounter(counters, "genericObjectExact", "generic arraycopy snippet for special object arrays"); 363 private static final SnippetCounter SystemArraycopyCounter = new SnippetCounter(counters, "genericObject", "call to System.arraycopy"); 364 365 private static final SnippetCounter.Group lengthCounters = SnippetCounters.getValue() ? new SnippetCounter.Group("System.arraycopy 0-length checks") : null; 366 367 private static final SnippetCounter zeroLengthStaticCounter = new SnippetCounter(lengthCounters, "0-lengthcopy static", "arraycopy where the length is statically 0"); 368 private static final SnippetCounter zeroLengthDynamicCounter = new SnippetCounter(lengthCounters, "0-lengthcopy dynamically", "arraycopy where the length is dynamically 0"); 369 private static final SnippetCounter nonZeroLengthDynamicCounter = new SnippetCounter(lengthCounters, "non-0-lengthcopy dynamically", "arraycopy where the length is dynamically not zero"); 370 371 private static final SnippetCounter.Group copiedCounters = SnippetCounters.getValue() ? new SnippetCounter.Group("System.arraycopy copied elements") : null; 372 373 private static final SnippetCounter nonZeroLengthDynamicCopiedCounter = new SnippetCounter(copiedCounters, "non-0-lengthcopy dynamically", "arraycopy where the length is dynamically not zero"); 374 private static final SnippetCounter genericPrimitiveCallCopiedCounter = new SnippetCounter(copiedCounters, "genericPrimitive", "generic arraycopy snippet for primitive arrays"); 375 private static final SnippetCounter genericObjectExactCallCopiedCounter = new SnippetCounter(copiedCounters, "genericObjectExact", "generic arraycopy snippet for special object arrays"); 376 private static final SnippetCounter SystemArraycopyCopiedCounter = new SnippetCounter(copiedCounters, "genericObject", "call to System.arraycopy"); 377 378 private static final SnippetCounter objectCheckcastCopiedCounter = new SnippetCounter(copiedCounters, "Object[]{non-exact}", "arraycopy for non-exact Object[] arrays"); 379 private static final SnippetCounter objectCheckcastSameTypeCopiedCounter = new SnippetCounter(copiedCounters, "Object[]{same-type}", "arraycopy call for src.klass == dest.klass Object[] arrays"); 380 private static final SnippetCounter predictedObjectArrayCopySlowPathCopiedCounter = new SnippetCounter(copiedCounters, "Object[]{slow-path}", 381 "used System.arraycopy slow path for predicted Object[] arrays"); 382 private static final SnippetCounter predictedObjectArrayCopyFastPathCopiedCounter = new SnippetCounter(copiedCounters, "Object[]{fast-path}", "used oop_arraycopy for predicted Object[] arrays"); 383 384 public static class Templates extends SnippetTemplate.AbstractTemplates { 385 386 public Templates(HotSpotProviders providers, TargetDescription target) { 387 super(providers, providers.getSnippetReflection(), target); 388 } 389 390 private ResolvedJavaMethod originalArraycopy() throws GraalError { 391 if (originalArraycopy == null) { 392 Method method; 393 try { 394 method = System.class.getDeclaredMethod("arraycopy", Object.class, int.class, Object.class, int.class, int.class); 395 } catch (NoSuchMethodException | SecurityException e) { 396 throw new GraalError(e); 397 } 398 originalArraycopy = providers.getMetaAccess().lookupJavaMethod(method); 399 } 400 return originalArraycopy; 401 } 402 403 private ResolvedJavaMethod originalArraycopy; 404 405 private final SnippetInfo checkcastArraycopyWorkSnippet = snippet("checkcastArraycopyWork"); 406 private final SnippetInfo arraycopyGenericSnippet = snippet("arraycopyGeneric"); 407 408 private final SnippetInfo arraycopySlowPathIntrinsicSnippet = snippet("arraycopySlowPathIntrinsic"); 409 private final SnippetInfo arraycopyUnrolledIntrinsicSnippet = snippet("arraycopyUnrolledIntrinsic"); 410 private final SnippetInfo arraycopyExactIntrinsicSnippet = snippet("arraycopyExactIntrinsic"); 411 private final SnippetInfo arraycopyZeroLengthIntrinsicSnippet = snippet("arraycopyZeroLengthIntrinsic"); 412 private final SnippetInfo arraycopyPredictedExactIntrinsicSnippet = snippet("arraycopyPredictedExactIntrinsic"); 413 private final SnippetInfo arraycopyPredictedObjectWorkSnippet = snippet("arraycopyPredictedObjectWork"); 414 415 private final SnippetInfo arraycopyUnrolledWorkSnippet = snippet("arraycopyUnrolledWork"); 416 417 protected SnippetInfo snippet(String methodName) { 418 SnippetInfo info = snippet(ArrayCopySnippets.class, methodName, LocationIdentity.any()); 419 info.setOriginalMethod(originalArraycopy()); 420 return info; 421 } 422 423 public static JavaKind selectComponentKind(BasicArrayCopyNode arraycopy) { 424 return selectComponentKind(arraycopy, true); 425 } 426 427 public static JavaKind selectComponentKind(BasicArrayCopyNode arraycopy, boolean exact) { 428 ResolvedJavaType srcType = StampTool.typeOrNull(arraycopy.getSource().stamp()); 429 ResolvedJavaType destType = StampTool.typeOrNull(arraycopy.getDestination().stamp()); 430 431 if (srcType == null || !srcType.isArray() || destType == null || !destType.isArray()) { 432 if (!exact) { 433 JavaKind component = getComponentKind(srcType); 434 if (component != null) { 435 return component; 436 } 437 return getComponentKind(destType); 438 } 439 return null; 440 } 441 if (exact) { 442 if (!destType.getComponentType().isAssignableFrom(srcType.getComponentType())) { 443 return null; 444 } 445 if (!arraycopy.isExact()) { 446 return null; 447 } 448 } 449 return srcType.getComponentType().getJavaKind(); 450 } 451 452 private static JavaKind getComponentKind(ResolvedJavaType type) { 453 if (type != null && type.isArray()) { 454 return type.getComponentType().getJavaKind(); 455 } 456 return null; 457 } 458 459 private static boolean shouldUnroll(ValueNode length) { 460 return length.isConstant() && length.asJavaConstant().asInt() <= 8 && length.asJavaConstant().asInt() != 0; 461 } 462 463 public void lower(ArrayCopyNode arraycopy, LoweringTool tool) { 464 JavaKind componentKind = selectComponentKind(arraycopy); 465 SnippetInfo snippetInfo = null; 466 SnippetInfo slowPathSnippetInfo = null; 467 Object slowPathArgument = null; 468 469 if (arraycopy.getLength().isConstant() && arraycopy.getLength().asJavaConstant().asLong() == 0) { 470 snippetInfo = arraycopyZeroLengthIntrinsicSnippet; 471 } else if (arraycopy.isExact()) { 472 snippetInfo = arraycopyExactIntrinsicSnippet; 473 if (shouldUnroll(arraycopy.getLength())) { 474 snippetInfo = arraycopyUnrolledIntrinsicSnippet; 475 } 476 } else { 477 if (componentKind == JavaKind.Object) { 478 ResolvedJavaType srcType = StampTool.typeOrNull(arraycopy.getSource().stamp()); 479 ResolvedJavaType destType = StampTool.typeOrNull(arraycopy.getDestination().stamp()); 480 ResolvedJavaType srcComponentType = srcType == null ? null : srcType.getComponentType(); 481 ResolvedJavaType destComponentType = destType == null ? null : destType.getComponentType(); 482 if (srcComponentType != null && destComponentType != null && !srcComponentType.isPrimitive() && !destComponentType.isPrimitive()) { 483 snippetInfo = arraycopySlowPathIntrinsicSnippet; 484 slowPathSnippetInfo = checkcastArraycopyWorkSnippet; 485 slowPathArgument = LocationIdentity.any(); 486 /* 487 * Because this snippet has to use Sysytem.arraycopy as a slow path, we must 488 * pretend to kill any() so clear the componentKind. 489 */ 490 componentKind = null; 491 } 492 } 493 if (componentKind == null && snippetInfo == null) { 494 JavaKind predictedKind = selectComponentKind(arraycopy, false); 495 if (predictedKind != null) { 496 /* 497 * At least one array is of a known type requiring no store checks, so 498 * assume the other is of the same type. Generally this is working around 499 * deficiencies in our propagation of type information. 500 */ 501 componentKind = predictedKind; 502 if (predictedKind == JavaKind.Object) { 503 snippetInfo = arraycopySlowPathIntrinsicSnippet; 504 slowPathSnippetInfo = arraycopyPredictedObjectWorkSnippet; 505 slowPathArgument = predictedKind; 506 componentKind = null; 507 } else { 508 snippetInfo = arraycopyPredictedExactIntrinsicSnippet; 509 } 510 } 511 } 512 if (snippetInfo == null) { 513 snippetInfo = arraycopyGenericSnippet; 514 } 515 } 516 Arguments args = new Arguments(snippetInfo, arraycopy.graph().getGuardsStage(), tool.getLoweringStage()); 517 args.add("src", arraycopy.getSource()); 518 args.add("srcPos", arraycopy.getSourcePosition()); 519 args.add("dest", arraycopy.getDestination()); 520 args.add("destPos", arraycopy.getDestinationPosition()); 521 args.add("length", arraycopy.getLength()); 522 if (snippetInfo == arraycopyUnrolledIntrinsicSnippet) { 523 args.addConst("unrolledLength", arraycopy.getLength().asJavaConstant().asInt()); 524 args.addConst("elementKind", componentKind != null ? componentKind : JavaKind.Illegal); 525 } else if (snippetInfo == arraycopySlowPathIntrinsicSnippet) { 526 ValueNode predictedKlass = null; 527 if (slowPathArgument == arraycopyPredictedObjectWorkSnippet) { 528 HotSpotResolvedObjectType arrayClass = (HotSpotResolvedObjectType) tool.getMetaAccess().lookupJavaType(Object[].class); 529 predictedKlass = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), arrayClass.klass(), tool.getMetaAccess(), arraycopy.graph()); 530 } else { 531 predictedKlass = ConstantNode.forConstant(KlassPointerStamp.klassAlwaysNull(), JavaConstant.NULL_POINTER, tool.getMetaAccess(), arraycopy.graph()); 532 } 533 args.add("predictedKlass", predictedKlass); 534 args.addConst("elementKind", componentKind != null ? componentKind : JavaKind.Illegal); 535 args.addConst("slowPath", slowPathSnippetInfo); 536 assert slowPathArgument != null; 537 args.addConst("slowPathArgument", slowPathArgument); 538 } else if (snippetInfo == arraycopyExactIntrinsicSnippet || snippetInfo == arraycopyPredictedExactIntrinsicSnippet) { 539 assert componentKind != null; 540 args.addConst("elementKind", componentKind); 541 args.addConst("counter", arraycopyCallCounters.get(componentKind)); 542 args.addConst("copiedCounter", arraycopyCallCopiedCounters.get(componentKind)); 543 } 544 instantiate(args, arraycopy); 545 } 546 547 public void lower(ArrayCopySlowPathNode arraycopy, LoweringTool tool) { 548 StructuredGraph graph = arraycopy.graph(); 549 if (!graph.getGuardsStage().areFrameStatesAtDeopts()) { 550 // Can't be lowered yet 551 return; 552 } 553 SnippetInfo snippetInfo = arraycopy.getSnippet(); 554 Arguments args = new Arguments(snippetInfo, graph.getGuardsStage(), tool.getLoweringStage()); 555 args.add("nonNullSrc", arraycopy.getSource()); 556 args.add("srcPos", arraycopy.getSourcePosition()); 557 args.add("nonNullDest", arraycopy.getDestination()); 558 args.add("destPos", arraycopy.getDestinationPosition()); 559 if (snippetInfo == arraycopyUnrolledWorkSnippet) { 560 args.addConst("length", ((Integer) arraycopy.getArgument()).intValue()); 561 args.addConst("elementKind", arraycopy.getElementKind()); 562 } else { 563 args.add("length", arraycopy.getLength()); 564 } 565 if (snippetInfo == arraycopyPredictedObjectWorkSnippet) { 566 args.add("objectArrayKlass", arraycopy.getPredictedKlass()); 567 args.addConst("counter", arraycopyCallCounters.get(JavaKind.Object)); 568 args.addConst("copiedCounter", arraycopyCallCopiedCounters.get(JavaKind.Object)); 569 } 570 instantiate(args, arraycopy); 571 } 572 573 public void lower(ArrayCopyUnrollNode arraycopy, LoweringTool tool) { 574 StructuredGraph graph = arraycopy.graph(); 575 if (!graph.getGuardsStage().areFrameStatesAtDeopts()) { 576 // Can't be lowered yet 577 return; 578 } 579 SnippetInfo snippetInfo = arraycopyUnrolledWorkSnippet; 580 Arguments args = new Arguments(snippetInfo, graph.getGuardsStage(), tool.getLoweringStage()); 581 args.add("nonNullSrc", arraycopy.getSource()); 582 args.add("srcPos", arraycopy.getSourcePosition()); 583 args.add("nonNullDest", arraycopy.getDestination()); 584 args.add("destPos", arraycopy.getDestinationPosition()); 585 args.addConst("length", arraycopy.getUnrollLength()); 586 args.addConst("elementKind", arraycopy.getElementKind()); 587 template(args).instantiate(providers.getMetaAccess(), arraycopy, SnippetTemplate.DEFAULT_REPLACER, args); 588 } 589 590 /** 591 * Instantiate the snippet template and fix up the FrameState of any Invokes of 592 * System.arraycopy and propagate the captured bci in the ArrayCopySlowPathNode. 593 * 594 * @param args 595 * @param arraycopy 596 */ 597 private void instantiate(Arguments args, BasicArrayCopyNode arraycopy) { 598 StructuredGraph graph = arraycopy.graph(); 599 SnippetTemplate template = template(args); 600 Map<Node, Node> replacements = template.instantiate(providers.getMetaAccess(), arraycopy, SnippetTemplate.DEFAULT_REPLACER, args); 601 for (Node originalNode : replacements.keySet()) { 602 if (originalNode instanceof Invoke) { 603 Invoke invoke = (Invoke) replacements.get(originalNode); 604 assert invoke.asNode().graph() == graph; 605 CallTargetNode call = invoke.callTarget(); 606 607 if (!call.targetMethod().equals(originalArraycopy)) { 608 throw new GraalError("unexpected invoke %s in snippet", call.targetMethod()); 609 } 610 // Here we need to fix the bci of the invoke 611 InvokeNode newInvoke = graph.add(new InvokeNode(invoke.callTarget(), arraycopy.getBci())); 612 if (arraycopy.stateDuring() != null) { 613 newInvoke.setStateDuring(arraycopy.stateDuring()); 614 } else { 615 assert arraycopy.stateAfter() != null; 616 newInvoke.setStateAfter(arraycopy.stateAfter()); 617 } 618 graph.replaceFixedWithFixed((InvokeNode) invoke.asNode(), newInvoke); 619 } else if (originalNode instanceof ArrayCopySlowPathNode) { 620 ArrayCopySlowPathNode slowPath = (ArrayCopySlowPathNode) replacements.get(originalNode); 621 assert arraycopy.stateAfter() != null; 622 slowPath.setStateAfter(arraycopy.stateAfter()); 623 slowPath.setBci(arraycopy.getBci()); 624 } 625 } 626 } 627 } 628 }