1 /* 2 * Copyright (c) 2011, 2018, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 */ 23 24 25 package org.graalvm.compiler.replacements.arraycopy; 26 27 import static jdk.vm.ci.services.Services.IS_BUILDING_NATIVE_IMAGE; 28 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FREQUENT_PROBABILITY; 29 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.LIKELY_PROBABILITY; 30 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.NOT_FREQUENT_PROBABILITY; 31 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY; 32 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability; 33 34 import java.util.EnumMap; 35 36 import jdk.internal.vm.compiler.collections.UnmodifiableEconomicMap; 37 import org.graalvm.compiler.api.directives.GraalDirectives; 38 import org.graalvm.compiler.api.replacements.Fold.InjectedParameter; 39 import org.graalvm.compiler.api.replacements.Snippet; 40 import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter; 41 import org.graalvm.compiler.api.replacements.SnippetReflectionProvider; 42 import org.graalvm.compiler.debug.DebugHandlersFactory; 43 import org.graalvm.compiler.debug.GraalError; 44 import org.graalvm.compiler.graph.Node; 45 import org.graalvm.compiler.nodes.CallTargetNode; 46 import org.graalvm.compiler.nodes.DeoptimizeNode; 47 import org.graalvm.compiler.nodes.InvokeNode; 48 import org.graalvm.compiler.nodes.InvokeWithExceptionNode; 49 import org.graalvm.compiler.nodes.NamedLocationIdentity; 50 import org.graalvm.compiler.nodes.NodeView; 51 import org.graalvm.compiler.nodes.PiNode; 52 import org.graalvm.compiler.nodes.SnippetAnchorNode; 53 import org.graalvm.compiler.nodes.StructuredGraph; 54 import org.graalvm.compiler.nodes.extended.GuardedUnsafeLoadNode; 55 import org.graalvm.compiler.nodes.extended.GuardingNode; 56 import org.graalvm.compiler.nodes.extended.RawStoreNode; 57 import org.graalvm.compiler.nodes.java.ArrayLengthNode; 58 import org.graalvm.compiler.nodes.spi.LoweringTool; 59 import org.graalvm.compiler.nodes.type.StampTool; 60 import org.graalvm.compiler.nodes.util.GraphUtil; 61 import org.graalvm.compiler.options.OptionValues; 62 import org.graalvm.compiler.phases.util.Providers; 63 import org.graalvm.compiler.replacements.ReplacementsUtil; 64 import org.graalvm.compiler.replacements.SnippetCounter; 65 import org.graalvm.compiler.replacements.SnippetCounter.Group; 66 import org.graalvm.compiler.replacements.SnippetCounter.Group.Factory; 67 import org.graalvm.compiler.replacements.SnippetIntegerHistogram; 68 import org.graalvm.compiler.replacements.SnippetTemplate; 69 import org.graalvm.compiler.replacements.SnippetTemplate.Arguments; 70 import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo; 71 import org.graalvm.compiler.replacements.Snippets; 72 import org.graalvm.compiler.replacements.nodes.BasicArrayCopyNode; 73 import org.graalvm.compiler.word.Word; 74 import jdk.internal.vm.compiler.word.LocationIdentity; 75 import jdk.internal.vm.compiler.word.Pointer; 76 77 import jdk.vm.ci.code.TargetDescription; 78 import jdk.vm.ci.meta.DeoptimizationAction; 79 import jdk.vm.ci.meta.DeoptimizationReason; 80 import jdk.vm.ci.meta.JavaKind; 81 import jdk.vm.ci.meta.MetaAccessProvider; 82 import jdk.vm.ci.meta.ResolvedJavaMethod; 83 import jdk.vm.ci.meta.ResolvedJavaType; 84 85 public abstract class ArrayCopySnippets implements Snippets { 86 87 private enum ArrayCopyTypeCheck { 88 UNDEFINED_ARRAY_TYPE_CHECK, 89 // either we know that both objects are arrays and have the same type, 90 // or we apply generic array copy snippet, which enforces type check 91 NO_ARRAY_TYPE_CHECK, 92 // can be used when we know that one of the objects is a primitive array 93 HUB_BASED_ARRAY_TYPE_CHECK, 94 // can be used when we know that one of the objects is an object array 95 LAYOUT_HELPER_BASED_ARRAY_TYPE_CHECK 96 } 97 98 /** Marker value for the {@link InjectedParameter} injected parameter. */ 99 static final MetaAccessProvider INJECTED_META_ACCESS = null; 100 101 public abstract Pointer loadHub(Object nonNullSrc); 102 103 public abstract Pointer getDestElemClass(Pointer destKlass); 104 105 public abstract Word getSuperCheckOffset(Pointer destElemKlass); 106 107 public abstract int getReadLayoutHelper(Pointer srcHub); 108 109 protected abstract int heapWordSize(); 110 111 @SuppressWarnings("unused") 112 @Snippet 113 public void arraycopyExactSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck, 114 @ConstantParameter SnippetInfo workSnippet, @ConstantParameter JavaKind elementKind, @ConstantParameter LocationIdentity locationIdentity, 115 @ConstantParameter SnippetCounter elementKindCounter, @ConstantParameter SnippetCounter elementKindCopiedCounter, @ConstantParameter Counters counters) { 116 Object nonNullSrc = GraalDirectives.guardingNonNull(src); 117 Object nonNullDest = GraalDirectives.guardingNonNull(dest); 118 checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck); 119 checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters); 120 incrementLengthCounter(length, counters); 121 122 elementKindCounter.inc(); 123 elementKindCopiedCounter.add(length); 124 125 ArrayCopyWithDelayedLoweringNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, workSnippet, elementKind); 126 } 127 128 @SuppressWarnings("unused") 129 @Snippet 130 public void arraycopyExactStubCallSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck, 131 @ConstantParameter SnippetInfo workSnippet, @ConstantParameter JavaKind elementKind, @ConstantParameter LocationIdentity locationIdentity, 132 @ConstantParameter SnippetCounter elementKindCounter, @ConstantParameter SnippetCounter elementKindCopiedCounter, @ConstantParameter Counters counters) { 133 Object nonNullSrc = GraalDirectives.guardingNonNull(src); 134 Object nonNullDest = GraalDirectives.guardingNonNull(dest); 135 checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck); 136 checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters); 137 incrementLengthCounter(length, counters); 138 139 elementKindCounter.inc(); 140 elementKindCopiedCounter.add(length); 141 142 ArrayCopyCallNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, elementKind, locationIdentity, heapWordSize()); 143 } 144 145 @Snippet 146 public void arraycopyCheckcastSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck, @ConstantParameter Counters counters, 147 @ConstantParameter SnippetInfo workSnippet, @ConstantParameter JavaKind elementKind) { 148 Object nonNullSrc = GraalDirectives.guardingNonNull(src); 149 Object nonNullDest = GraalDirectives.guardingNonNull(dest); 150 checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck); 151 checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters); 152 incrementLengthCounter(length, counters); 153 154 ArrayCopyWithDelayedLoweringNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, workSnippet, elementKind); 155 } 156 157 @Snippet 158 public void arraycopyGenericSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck, @ConstantParameter Counters counters, 159 @ConstantParameter SnippetInfo workSnippet, @ConstantParameter JavaKind elementKind) { 160 Object nonNullSrc = GraalDirectives.guardingNonNull(src); 161 Object nonNullDest = GraalDirectives.guardingNonNull(dest); 162 checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck); 163 checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters); 164 incrementLengthCounter(length, counters); 165 166 ArrayCopyWithDelayedLoweringNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, workSnippet, elementKind); 167 } 168 169 @Snippet 170 public static void arraycopyNativeSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter Counters counters) { 171 // all checks are done in the native method, so no need to emit additional checks here 172 incrementLengthCounter(length, counters); 173 counters.systemArraycopyCounter.inc(); 174 counters.systemArraycopyCopiedCounter.add(length); 175 176 System.arraycopy(src, srcPos, dest, destPos, length); 177 } 178 179 @SuppressWarnings("unused") 180 @Snippet(allowPartialIntrinsicArgumentMismatch = true) 181 public void exactArraycopyWithSlowPathWork(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter JavaKind elementKind, @ConstantParameter LocationIdentity arrayLocation, 182 @ConstantParameter Counters counters) { 183 int scale = ReplacementsUtil.arrayIndexScale(INJECTED_META_ACCESS, elementKind); 184 int arrayBaseOffset = ReplacementsUtil.getArrayBaseOffset(INJECTED_META_ACCESS, elementKind); 185 long sourceOffset = arrayBaseOffset + (long) srcPos * scale; 186 long destOffset = arrayBaseOffset + (long) destPos * scale; 187 188 GuardingNode anchor = SnippetAnchorNode.anchor(); 189 if (probability(NOT_FREQUENT_PROBABILITY, src == dest && srcPos < destPos)) { 190 // bad aliased case so we need to copy the array from back to front 191 for (int position = length - 1; position >= 0; position--) { 192 Object value = GuardedUnsafeLoadNode.guardedLoad(src, sourceOffset + ((long) position) * scale, elementKind, arrayLocation, anchor); 193 RawStoreNode.storeObject(dest, destOffset + ((long) position) * scale, value, elementKind, arrayLocation, true); 194 } 195 } else { 196 for (int position = 0; position < length; position++) { 197 Object value = GuardedUnsafeLoadNode.guardedLoad(src, sourceOffset + ((long) position) * scale, elementKind, arrayLocation, anchor); 198 RawStoreNode.storeObject(dest, destOffset + ((long) position) * scale, value, elementKind, arrayLocation, true); 199 } 200 } 201 } 202 203 @SuppressWarnings("unused") 204 @Snippet(allowPartialIntrinsicArgumentMismatch = true) 205 public void checkcastArraycopyWithSlowPathWork(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter JavaKind elementKind, 206 @ConstantParameter LocationIdentity arrayLocation, 207 @ConstantParameter Counters counters) { 208 if (probability(FREQUENT_PROBABILITY, length > 0)) { 209 Object nonNullSrc = PiNode.asNonNullObject(src); 210 Object nonNullDest = PiNode.asNonNullObject(dest); 211 Pointer srcKlass = loadHub(nonNullSrc); 212 Pointer destKlass = loadHub(nonNullDest); 213 if (probability(LIKELY_PROBABILITY, srcKlass == destKlass)) { 214 // no storecheck required. 215 counters.objectCheckcastSameTypeCounter.inc(); 216 counters.objectCheckcastSameTypeCopiedCounter.add(length); 217 ArrayCopyCallNode.arraycopyObjectKillsAny(nonNullSrc, srcPos, nonNullDest, destPos, length, heapWordSize()); 218 } else { 219 Pointer destElemKlass = getDestElemClass(destKlass); 220 Word superCheckOffset = getSuperCheckOffset(destElemKlass); 221 222 counters.objectCheckcastDifferentTypeCounter.inc(); 223 counters.objectCheckcastDifferentTypeCopiedCounter.add(length); 224 225 int copiedElements = CheckcastArrayCopyCallNode.checkcastArraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, superCheckOffset, destElemKlass, false); 226 if (probability(SLOW_PATH_PROBABILITY, copiedElements != 0)) { 227 /* 228 * the stub doesn't throw the ArrayStoreException, but returns the number of 229 * copied elements (xor'd with -1). 230 */ 231 copiedElements ^= -1; 232 System.arraycopy(nonNullSrc, srcPos + copiedElements, nonNullDest, destPos + copiedElements, length - copiedElements); 233 } 234 } 235 } 236 } 237 238 @SuppressWarnings("unused") 239 @Snippet(allowPartialIntrinsicArgumentMismatch = true) 240 public void genericArraycopyWithSlowPathWork(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter JavaKind elementKind, 241 @ConstantParameter LocationIdentity arrayLocation, 242 @ConstantParameter Counters counters) { 243 // The length > 0 check should not be placed here because generic array copy stub should 244 // enforce type check. This is fine performance-wise because this snippet is rarely used. 245 counters.genericArraycopyDifferentTypeCounter.inc(); 246 counters.genericArraycopyDifferentTypeCopiedCounter.add(length); 247 int copiedElements = GenericArrayCopyCallNode.genericArraycopy(src, srcPos, dest, destPos, length); 248 if (probability(SLOW_PATH_PROBABILITY, copiedElements != 0)) { 249 /* 250 * the stub doesn't throw the ArrayStoreException, but returns the number of copied 251 * elements (xor'd with -1). 252 */ 253 copiedElements ^= -1; 254 System.arraycopy(src, srcPos + copiedElements, dest, destPos + copiedElements, length - copiedElements); 255 } 256 } 257 258 private static void incrementLengthCounter(int length, Counters counters) { 259 if (!IS_BUILDING_NATIVE_IMAGE) { 260 counters.lengthHistogram.inc(length); 261 } 262 } 263 264 private static void checkLimits(Object src, int srcPos, Object dest, int destPos, int length, Counters counters) { 265 if (probability(SLOW_PATH_PROBABILITY, srcPos < 0) || 266 probability(SLOW_PATH_PROBABILITY, destPos < 0) || 267 probability(SLOW_PATH_PROBABILITY, length < 0) || 268 probability(SLOW_PATH_PROBABILITY, srcPos > ArrayLengthNode.arrayLength(src) - length) || 269 probability(SLOW_PATH_PROBABILITY, destPos > ArrayLengthNode.arrayLength(dest) - length)) { 270 counters.checkAIOOBECounter.inc(); 271 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); 272 } 273 counters.checkSuccessCounter.inc(); 274 } 275 276 private void checkArrayTypes(Object nonNullSrc, Object nonNullDest, ArrayCopyTypeCheck arrayTypeCheck) { 277 if (arrayTypeCheck == ArrayCopyTypeCheck.NO_ARRAY_TYPE_CHECK) { 278 // nothing to do 279 } else if (arrayTypeCheck == ArrayCopyTypeCheck.HUB_BASED_ARRAY_TYPE_CHECK) { 280 Pointer srcHub = loadHub(nonNullSrc); 281 Pointer destHub = loadHub(nonNullDest); 282 if (probability(SLOW_PATH_PROBABILITY, srcHub != destHub)) { 283 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); 284 } 285 } else if (arrayTypeCheck == ArrayCopyTypeCheck.LAYOUT_HELPER_BASED_ARRAY_TYPE_CHECK) { 286 Pointer srcHub = loadHub(nonNullSrc); 287 Pointer destHub = loadHub(nonNullDest); 288 if (probability(SLOW_PATH_PROBABILITY, getReadLayoutHelper(srcHub) != getReadLayoutHelper(destHub))) { 289 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); 290 } 291 } else { 292 ReplacementsUtil.staticAssert(false, "unknown array type check ", arrayTypeCheck); 293 } 294 } 295 296 static class Counters { 297 final SnippetCounter checkSuccessCounter; 298 final SnippetCounter checkAIOOBECounter; 299 300 final SnippetCounter zeroLengthStaticCounter; 301 final SnippetIntegerHistogram lengthHistogram; 302 303 final SnippetCounter systemArraycopyCounter; 304 final SnippetCounter systemArraycopyCopiedCounter; 305 306 final SnippetCounter genericArraycopyDifferentTypeCopiedCounter; 307 final SnippetCounter genericArraycopyDifferentTypeCounter; 308 309 final SnippetCounter objectCheckcastSameTypeCopiedCounter; 310 final SnippetCounter objectCheckcastSameTypeCounter; 311 final SnippetCounter objectCheckcastDifferentTypeCopiedCounter; 312 final SnippetCounter objectCheckcastDifferentTypeCounter; 313 314 final EnumMap<JavaKind, SnippetCounter> arraycopyCallCounters = new EnumMap<>(JavaKind.class); 315 final EnumMap<JavaKind, SnippetCounter> arraycopyCallCopiedCounters = new EnumMap<>(JavaKind.class); 316 317 Counters(SnippetCounter.Group.Factory factory) { 318 final Group checkCounters = factory.createSnippetCounterGroup("System.arraycopy checkInputs"); 319 final Group callCounters = factory.createSnippetCounterGroup("System.arraycopy calls"); 320 final Group copiedElementsCounters = factory.createSnippetCounterGroup("System.arraycopy copied elements"); 321 final Group lengthCounters = factory.createSnippetCounterGroup("System.arraycopy with 0-length"); 322 323 checkSuccessCounter = new SnippetCounter(checkCounters, "checkSuccess", "checkSuccess"); 324 checkAIOOBECounter = new SnippetCounter(checkCounters, "checkAIOOBE", "checkAIOOBE"); 325 326 zeroLengthStaticCounter = new SnippetCounter(lengthCounters, "0-length copy static", "calls where the length is statically 0"); 327 lengthHistogram = new SnippetIntegerHistogram(lengthCounters, 2, "length", "length"); 328 329 systemArraycopyCounter = new SnippetCounter(callCounters, "native System.arraycopy", "JNI-based System.arraycopy call"); 330 systemArraycopyCopiedCounter = new SnippetCounter(copiedElementsCounters, "native System.arraycopy", "JNI-based System.arraycopy call"); 331 332 genericArraycopyDifferentTypeCounter = new SnippetCounter(callCounters, "generic[] stub", "generic arraycopy stub"); 333 genericArraycopyDifferentTypeCopiedCounter = new SnippetCounter(copiedElementsCounters, "generic[] stub", "generic arraycopy stub"); 334 335 objectCheckcastSameTypeCounter = new SnippetCounter(callCounters, "checkcast object[] (same-type)", "checkcast object[] stub but src.klass == dest.klass Object[] arrays"); 336 objectCheckcastSameTypeCopiedCounter = new SnippetCounter(copiedElementsCounters, "checkcast object[] (same-type)", "checkcast object[] stub but src.klass == dest.klass Object[] arrays"); 337 objectCheckcastDifferentTypeCounter = new SnippetCounter(callCounters, "checkcast object[] (store-check)", "checkcast object[] stub with store check"); 338 objectCheckcastDifferentTypeCopiedCounter = new SnippetCounter(copiedElementsCounters, "checkcast object[] (store-check)", "checkcast object[] stub with store check"); 339 340 createArraycopyCounter(JavaKind.Byte, callCounters, copiedElementsCounters); 341 createArraycopyCounter(JavaKind.Boolean, callCounters, copiedElementsCounters); 342 createArraycopyCounter(JavaKind.Char, callCounters, copiedElementsCounters); 343 createArraycopyCounter(JavaKind.Short, callCounters, copiedElementsCounters); 344 createArraycopyCounter(JavaKind.Int, callCounters, copiedElementsCounters); 345 createArraycopyCounter(JavaKind.Long, callCounters, copiedElementsCounters); 346 createArraycopyCounter(JavaKind.Float, callCounters, copiedElementsCounters); 347 createArraycopyCounter(JavaKind.Double, callCounters, copiedElementsCounters); 348 createArraycopyCounter(JavaKind.Object, callCounters, copiedElementsCounters); 349 } 350 351 void createArraycopyCounter(JavaKind kind, Group counters, Group copiedCounters) { 352 arraycopyCallCounters.put(kind, new SnippetCounter(counters, kind + "[] stub", "arraycopy call for " + kind + "[] arrays")); 353 arraycopyCallCopiedCounters.put(kind, new SnippetCounter(copiedCounters, kind + "[] stub", "arraycopy call for " + kind + "[] arrays")); 354 } 355 } 356 357 public static class Templates extends SnippetTemplate.AbstractTemplates { 358 private final SnippetInfo arraycopyGenericSnippet; 359 private final SnippetInfo arraycopyExactSnippet; 360 private final SnippetInfo arraycopyExactStubCallSnippet; 361 private final SnippetInfo arraycopyCheckcastSnippet; 362 private final SnippetInfo arraycopyNativeSnippet; 363 private final SnippetInfo checkcastArraycopyWithSlowPathWork; 364 private final SnippetInfo genericArraycopyWithSlowPathWork; 365 private final SnippetInfo exactArraycopyWithSlowPathWork; 366 367 private ResolvedJavaMethod originalArraycopy; 368 private final Counters counters; 369 private boolean expandArraycopyLoop; 370 371 public Templates(ArrayCopySnippets receiver, OptionValues options, Iterable<DebugHandlersFactory> factories, Factory factory, Providers providers, 372 SnippetReflectionProvider snippetReflection, TargetDescription target) { 373 super(options, factories, providers, snippetReflection, target); 374 this.counters = new Counters(factory); 375 376 arraycopyGenericSnippet = snippet(receiver, "arraycopyGenericSnippet"); 377 arraycopyExactSnippet = snippet(receiver, "arraycopyExactSnippet"); 378 arraycopyExactStubCallSnippet = snippet(receiver, "arraycopyExactStubCallSnippet"); 379 arraycopyCheckcastSnippet = snippet(receiver, "arraycopyCheckcastSnippet"); 380 arraycopyNativeSnippet = snippet(null, "arraycopyNativeSnippet"); 381 checkcastArraycopyWithSlowPathWork = snippet(receiver, "checkcastArraycopyWithSlowPathWork"); 382 genericArraycopyWithSlowPathWork = snippet(receiver, "genericArraycopyWithSlowPathWork"); 383 exactArraycopyWithSlowPathWork = snippet(receiver, "exactArraycopyWithSlowPathWork"); 384 } 385 386 protected SnippetInfo snippet(ArrayCopySnippets receiver, String methodName) { 387 SnippetInfo info = snippet(ArrayCopySnippets.class, methodName, originalArraycopy(), receiver, LocationIdentity.any()); 388 return info; 389 } 390 391 public void lower(ArrayCopyNode arraycopy, LoweringTool tool) { 392 JavaKind elementKind = selectComponentKind(arraycopy); 393 SnippetInfo snippetInfo; 394 ArrayCopyTypeCheck arrayTypeCheck; 395 396 ResolvedJavaType srcType = StampTool.typeOrNull(arraycopy.getSource().stamp(NodeView.DEFAULT)); 397 ResolvedJavaType destType = StampTool.typeOrNull(arraycopy.getDestination().stamp(NodeView.DEFAULT)); 398 if (!canBeArray(srcType) || !canBeArray(destType)) { 399 // at least one of the objects is definitely not an array - use the native call 400 // right away as the copying will fail anyways 401 snippetInfo = arraycopyNativeSnippet; 402 arrayTypeCheck = ArrayCopyTypeCheck.UNDEFINED_ARRAY_TYPE_CHECK; 403 } else { 404 ResolvedJavaType srcComponentType = srcType == null ? null : srcType.getComponentType(); 405 ResolvedJavaType destComponentType = destType == null ? null : destType.getComponentType(); 406 407 if (arraycopy.isExact()) { 408 // there is a sufficient type match - we don't need any additional type checks 409 snippetInfo = arraycopyExactStubCallSnippet; 410 arrayTypeCheck = ArrayCopyTypeCheck.NO_ARRAY_TYPE_CHECK; 411 } else if (srcComponentType == null && destComponentType == null) { 412 // we don't know anything about the types - use the generic copying 413 snippetInfo = arraycopyGenericSnippet; 414 // no need for additional type check to avoid duplicated work 415 arrayTypeCheck = ArrayCopyTypeCheck.NO_ARRAY_TYPE_CHECK; 416 } else if (srcComponentType != null && destComponentType != null) { 417 if (!srcComponentType.isPrimitive() && !destComponentType.isPrimitive()) { 418 // it depends on the array content if the copy succeeds - we need 419 // a type check for every store 420 snippetInfo = arraycopyCheckcastSnippet; 421 arrayTypeCheck = ArrayCopyTypeCheck.NO_ARRAY_TYPE_CHECK; 422 } else { 423 // one object is an object array, the other one is a primitive array. 424 // this copy will always fail - use the native call right away 425 assert !srcComponentType.equals(destComponentType) : "must be handled by arraycopy.isExact()"; 426 snippetInfo = arraycopyNativeSnippet; 427 arrayTypeCheck = ArrayCopyTypeCheck.UNDEFINED_ARRAY_TYPE_CHECK; 428 } 429 } else { 430 ResolvedJavaType nonNullComponentType = srcComponentType != null ? srcComponentType : destComponentType; 431 if (nonNullComponentType.isPrimitive()) { 432 // one involved object is a primitive array - it is sufficient to directly 433 // compare the hub. 434 snippetInfo = arraycopyExactStubCallSnippet; 435 arrayTypeCheck = ArrayCopyTypeCheck.HUB_BASED_ARRAY_TYPE_CHECK; 436 elementKind = nonNullComponentType.getJavaKind(); 437 } else { 438 // one involved object is an object array - the other array's element type 439 // may be primitive or object, hence we compare the layout helper. 440 snippetInfo = arraycopyCheckcastSnippet; 441 arrayTypeCheck = ArrayCopyTypeCheck.LAYOUT_HELPER_BASED_ARRAY_TYPE_CHECK; 442 } 443 } 444 } 445 446 if (this.expandArraycopyLoop && snippetInfo == arraycopyExactStubCallSnippet) { 447 snippetInfo = arraycopyExactSnippet; 448 } 449 450 // create the snippet 451 Arguments args = new Arguments(snippetInfo, arraycopy.graph().getGuardsStage(), tool.getLoweringStage()); 452 args.add("src", arraycopy.getSource()); 453 args.add("srcPos", arraycopy.getSourcePosition()); 454 args.add("dest", arraycopy.getDestination()); 455 args.add("destPos", arraycopy.getDestinationPosition()); 456 args.add("length", arraycopy.getLength()); 457 if (snippetInfo != arraycopyNativeSnippet) { 458 assert arrayTypeCheck != ArrayCopyTypeCheck.UNDEFINED_ARRAY_TYPE_CHECK; 459 args.addConst("arrayTypeCheck", arrayTypeCheck); 460 } 461 Object locationIdentity = arraycopy.killsAnyLocation() ? LocationIdentity.any() : NamedLocationIdentity.getArrayLocation(elementKind); 462 if (snippetInfo == arraycopyExactStubCallSnippet || snippetInfo == arraycopyExactSnippet) { 463 assert elementKind != null; 464 args.addConst("workSnippet", exactArraycopyWithSlowPathWork); 465 args.addConst("elementKind", elementKind); 466 args.addConst("locationIdentity", locationIdentity); 467 args.addConst("elementKindCounter", counters.arraycopyCallCounters.get(elementKind)); 468 args.addConst("elementKindCopiedCounter", counters.arraycopyCallCopiedCounters.get(elementKind)); 469 } 470 args.addConst("counters", counters); 471 if (snippetInfo == arraycopyCheckcastSnippet) { 472 args.addConst("workSnippet", checkcastArraycopyWithSlowPathWork); 473 args.addConst("elementKind", JavaKind.Illegal); 474 } 475 if (snippetInfo == arraycopyGenericSnippet) { 476 args.addConst("workSnippet", genericArraycopyWithSlowPathWork); 477 args.addConst("elementKind", JavaKind.Illegal); 478 } 479 480 instantiate(args, arraycopy); 481 } 482 483 public void lower(ArrayCopyWithDelayedLoweringNode arraycopy, LoweringTool tool) { 484 StructuredGraph graph = arraycopy.graph(); 485 486 if (arraycopy.getSnippet() == exactArraycopyWithSlowPathWork && this.expandArraycopyLoop) { 487 if (!graph.getGuardsStage().areDeoptsFixed()) { 488 // Don't lower until floating guards are fixed. 489 return; 490 } 491 } else { 492 if (!graph.getGuardsStage().areFrameStatesAtDeopts()) { 493 // Don't lower until frame states are assigned to deoptimization points. 494 return; 495 } 496 } 497 498 SnippetInfo snippetInfo = arraycopy.getSnippet(); 499 Arguments args = new Arguments(snippetInfo, graph.getGuardsStage(), tool.getLoweringStage()); 500 args.add("src", arraycopy.getSource()); 501 args.add("srcPos", arraycopy.getSourcePosition()); 502 args.add("dest", arraycopy.getDestination()); 503 args.add("destPos", arraycopy.getDestinationPosition()); 504 args.add("length", arraycopy.getLength()); 505 506 JavaKind elementKind = arraycopy.getElementKind(); 507 args.addConst("elementKind", (elementKind == null) ? JavaKind.Illegal : elementKind); 508 509 Object locationIdentity = (elementKind == null) ? LocationIdentity.any() : NamedLocationIdentity.getArrayLocation(arraycopy.getElementKind()); 510 args.addConst("arrayLocation", locationIdentity); 511 args.addConst("counters", counters); 512 instantiate(args, arraycopy); 513 } 514 515 private static boolean canBeArray(ResolvedJavaType type) { 516 return type == null || type.isJavaLangObject() || type.isArray(); 517 } 518 519 public static JavaKind selectComponentKind(BasicArrayCopyNode arraycopy) { 520 ResolvedJavaType srcType = StampTool.typeOrNull(arraycopy.getSource().stamp(NodeView.DEFAULT)); 521 ResolvedJavaType destType = StampTool.typeOrNull(arraycopy.getDestination().stamp(NodeView.DEFAULT)); 522 523 if (srcType == null || !srcType.isArray() || destType == null || !destType.isArray()) { 524 return null; 525 } 526 if (!destType.getComponentType().isAssignableFrom(srcType.getComponentType())) { 527 return null; 528 } 529 if (!arraycopy.isExact()) { 530 return null; 531 } 532 return srcType.getComponentType().getJavaKind(); 533 } 534 535 /** 536 * Instantiate the snippet template and fix up the FrameState of any Invokes of 537 * System.arraycopy and propagate the captured bci in the ArrayCopySlowPathNode. 538 * 539 * @param args 540 * @param arraycopy 541 */ 542 private void instantiate(Arguments args, BasicArrayCopyNode arraycopy) { 543 StructuredGraph graph = arraycopy.graph(); 544 SnippetTemplate template = template(arraycopy, args); 545 UnmodifiableEconomicMap<Node, Node> replacements = template.instantiate(providers.getMetaAccess(), arraycopy, SnippetTemplate.DEFAULT_REPLACER, args, false); 546 for (Node originalNode : replacements.getKeys()) { 547 if (originalNode instanceof InvokeNode) { 548 InvokeNode invoke = (InvokeNode) replacements.get(originalNode); 549 assert invoke.asNode().graph() == graph; 550 CallTargetNode call = invoke.callTarget(); 551 552 if (!call.targetMethod().equals(originalArraycopy)) { 553 throw new GraalError("unexpected invoke %s in snippet", call.targetMethod()); 554 } 555 // Here we need to fix the bci of the invoke 556 invoke.replaceBci(arraycopy.getBci()); 557 invoke.setStateDuring(null); 558 invoke.setStateAfter(null); 559 if (arraycopy.stateDuring() != null) { 560 invoke.setStateDuring(arraycopy.stateDuring()); 561 } else { 562 assert arraycopy.stateAfter() != null : arraycopy; 563 invoke.setStateAfter(arraycopy.stateAfter()); 564 } 565 } else if (originalNode instanceof InvokeWithExceptionNode) { 566 throw new GraalError("unexpected invoke with exception %s in snippet", originalNode); 567 } else if (originalNode instanceof ArrayCopyWithDelayedLoweringNode) { 568 ArrayCopyWithDelayedLoweringNode slowPath = (ArrayCopyWithDelayedLoweringNode) replacements.get(originalNode); 569 assert arraycopy.stateAfter() != null : arraycopy; 570 assert slowPath.stateAfter() == arraycopy.stateAfter(); 571 slowPath.setBci(arraycopy.getBci()); 572 } 573 } 574 GraphUtil.killCFG(arraycopy); 575 } 576 577 private ResolvedJavaMethod originalArraycopy() throws GraalError { 578 if (originalArraycopy == null) { 579 try { 580 originalArraycopy = findMethod(providers.getMetaAccess(), System.class, "arraycopy"); 581 } catch (SecurityException e) { 582 throw new GraalError(e); 583 } 584 } 585 return originalArraycopy; 586 } 587 588 public void setExpandArraycopyLoop(boolean b) { 589 this.expandArraycopyLoop = b; 590 } 591 } 592 }