11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 */ 23 package org.graalvm.compiler.hotspot.replacements.arraycopy; 24 25 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFIG; 26 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.KLASS_SUPER_CHECK_OFFSET_LOCATION; 27 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.OBJ_ARRAY_KLASS_ELEMENT_KLASS_LOCATION; 28 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayBaseOffset; 29 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayClassElementOffset; 30 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayIndexScale; 31 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperElementTypePrimitiveInPlace; 32 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.loadHub; 33 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readLayoutHelper; 34 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.superCheckOffsetOffset; 35 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FAST_PATH_PROBABILITY; 36 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY; 37 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability; 38 39 import java.lang.reflect.Method; 40 import java.util.EnumMap; 41 42 import org.graalvm.compiler.api.directives.GraalDirectives; 43 import org.graalvm.compiler.api.replacements.Fold; 44 import org.graalvm.compiler.api.replacements.Snippet; 45 import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter; 46 import org.graalvm.compiler.debug.DebugHandlersFactory; 47 import org.graalvm.compiler.debug.GraalError; 48 import org.graalvm.compiler.graph.Node; 49 import org.graalvm.compiler.hotspot.meta.HotSpotProviders; 50 import org.graalvm.compiler.hotspot.nodes.type.KlassPointerStamp; 51 import org.graalvm.compiler.hotspot.word.KlassPointer; 52 import org.graalvm.compiler.nodes.CallTargetNode; 53 import org.graalvm.compiler.nodes.ConstantNode; 54 import org.graalvm.compiler.nodes.DeoptimizeNode; 55 import org.graalvm.compiler.nodes.Invoke; 56 import org.graalvm.compiler.nodes.InvokeNode; 57 import org.graalvm.compiler.nodes.NamedLocationIdentity; 58 import org.graalvm.compiler.nodes.PiNode; 59 import org.graalvm.compiler.nodes.StructuredGraph; 60 import org.graalvm.compiler.nodes.ValueNode; 61 import org.graalvm.compiler.nodes.extended.RawLoadNode; 62 import org.graalvm.compiler.nodes.extended.RawStoreNode; 63 import org.graalvm.compiler.nodes.java.ArrayLengthNode; 64 import org.graalvm.compiler.nodes.spi.LoweringTool; 65 import org.graalvm.compiler.nodes.type.StampTool; 66 import org.graalvm.compiler.nodes.util.GraphUtil; 67 import org.graalvm.compiler.options.OptionValues; 68 import org.graalvm.compiler.replacements.SnippetCounter; 69 import org.graalvm.compiler.replacements.SnippetCounter.Group; 70 import org.graalvm.compiler.replacements.SnippetTemplate; 71 import org.graalvm.compiler.replacements.SnippetTemplate.Arguments; 72 import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo; 73 import org.graalvm.compiler.replacements.Snippets; 74 import org.graalvm.compiler.replacements.nodes.BasicArrayCopyNode; 75 import org.graalvm.compiler.replacements.nodes.ExplodeLoopNode; 76 import org.graalvm.compiler.word.Word; 77 import org.graalvm.util.UnmodifiableEconomicMap; 78 import org.graalvm.word.LocationIdentity; 79 import org.graalvm.word.WordFactory; 80 81 import jdk.vm.ci.code.TargetDescription; 82 import jdk.vm.ci.hotspot.HotSpotResolvedObjectType; 83 import jdk.vm.ci.meta.DeoptimizationAction; 84 import jdk.vm.ci.meta.DeoptimizationReason; 85 import jdk.vm.ci.meta.JavaConstant; 86 import jdk.vm.ci.meta.JavaKind; 87 import jdk.vm.ci.meta.ResolvedJavaMethod; 88 import jdk.vm.ci.meta.ResolvedJavaType; 89 90 public class ArrayCopySnippets implements Snippets { 91 92 private static int checkArrayType(KlassPointer nonNullHub) { 93 int layoutHelper = readLayoutHelper(nonNullHub); 94 if (probability(SLOW_PATH_PROBABILITY, layoutHelper >= 0)) { 95 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); 96 } 97 return layoutHelper; 98 } 99 100 private static void checkLimits(Object src, int srcPos, Object dest, int destPos, int length, Counters counters) { 101 if (probability(SLOW_PATH_PROBABILITY, srcPos < 0)) { 102 counters.checkAIOOBECounter.inc(); 103 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); 104 } 105 if (probability(SLOW_PATH_PROBABILITY, destPos < 0)) { 106 counters.checkAIOOBECounter.inc(); 107 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); 108 } 109 if (probability(SLOW_PATH_PROBABILITY, length < 0)) { 110 counters.checkAIOOBECounter.inc(); 111 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); 112 } 113 if (probability(SLOW_PATH_PROBABILITY, srcPos > ArrayLengthNode.arrayLength(src) - length)) { 114 counters.checkAIOOBECounter.inc(); 115 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); 116 } 117 if (probability(SLOW_PATH_PROBABILITY, destPos > ArrayLengthNode.arrayLength(dest) - length)) { 118 counters.checkAIOOBECounter.inc(); 119 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); 120 } 121 counters.checkSuccessCounter.inc(); 122 } 123 124 @Snippet 125 public static void arraycopyZeroLengthIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter Counters counters) { 126 Object nonNullSrc = GraalDirectives.guardingNonNull(src); 127 Object nonNullDest = GraalDirectives.guardingNonNull(dest); 128 KlassPointer srcHub = loadHub(nonNullSrc); 129 KlassPointer destHub = loadHub(nonNullDest); 130 checkArrayType(srcHub); 131 checkArrayType(destHub); 132 checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters); 133 counters.zeroLengthStaticCounter.inc(); 134 } 135 136 @Snippet 137 public static void arraycopyExactIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter JavaKind elementKind, @ConstantParameter SnippetCounter counter, 138 @ConstantParameter SnippetCounter copiedCounter, @ConstantParameter Counters counters) { 139 Object nonNullSrc = GraalDirectives.guardingNonNull(src); 140 Object nonNullDest = GraalDirectives.guardingNonNull(dest); 141 checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters); 142 counter.inc(); 143 copiedCounter.add(length); 144 ArrayCopyCallNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, elementKind); 145 if (length == 0) { 146 counters.zeroLengthDynamicCounter.inc(); 147 } else { 148 counters.nonZeroLengthDynamicCounter.inc(); 149 counters.nonZeroLengthDynamicCopiedCounter.add(length); 150 } 151 } 152 153 /** 154 * This intrinsic is useful for the case where we know something statically about one of the 155 * inputs but not the other. 156 */ 157 @Snippet 158 public static void arraycopyPredictedExactIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter JavaKind elementKind, 159 @ConstantParameter SnippetCounter counter, @ConstantParameter SnippetCounter copiedCounter, @ConstantParameter Counters counters) { 160 Object nonNullSrc = GraalDirectives.guardingNonNull(src); 161 Object nonNullDest = GraalDirectives.guardingNonNull(dest); 162 KlassPointer srcHub = loadHub(nonNullSrc); 163 KlassPointer destHub = loadHub(nonNullDest); 164 if (probability(SLOW_PATH_PROBABILITY, srcHub != destHub)) { 165 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); 166 } 167 checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters); 168 counter.inc(); 169 copiedCounter.add(length); 170 ArrayCopyCallNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, elementKind); 171 if (length == 0) { 172 counters.zeroLengthDynamicCounter.inc(); 173 } else { 174 counters.nonZeroLengthDynamicCounter.inc(); 175 counters.nonZeroLengthDynamicCopiedCounter.add(length); 176 } 177 } 178 179 @Snippet 180 public static void arraycopyPredictedObjectWork(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, int length, KlassPointer objectArrayKlass, 181 @ConstantParameter SnippetCounter counter, @ConstantParameter SnippetCounter copiedCounter, @ConstantParameter Counters counters) { 182 if (length > 0) { 183 KlassPointer srcHub = loadHub(PiNode.asNonNullObject(nonNullSrc)); 184 KlassPointer destHub = loadHub(PiNode.asNonNullObject(nonNullDest)); 185 if (probability(FAST_PATH_PROBABILITY, srcHub == destHub || destHub == objectArrayKlass)) { 186 counter.inc(); 187 copiedCounter.add(length); 188 counters.predictedObjectArrayCopyFastPathCounter.inc(); 189 counters.predictedObjectArrayCopyFastPathCopiedCounter.add(length); 190 ArrayCopyCallNode.arraycopyObjectKillsAny(nonNullSrc, srcPos, nonNullDest, destPos, length); 191 } else { 192 counters.predictedObjectArrayCopySlowPathCounter.inc(); 193 counters.predictedObjectArrayCopySlowPathCopiedCounter.add(length); 194 System.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length); 195 } 196 } 197 } 198 199 /** 200 * This is the basic template for the full arraycopy checks, including a check that the 201 * underlying type is really an array type. 202 */ 203 @Snippet 204 public static void arraycopySlowPathIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, KlassPointer predictedKlass, @ConstantParameter JavaKind elementKind, 205 @ConstantParameter SnippetInfo slowPath, @ConstantParameter Object slowPathArgument, @ConstantParameter Counters counters) { 206 Object nonNullSrc = GraalDirectives.guardingNonNull(src); 207 Object nonNullDest = GraalDirectives.guardingNonNull(dest); 208 KlassPointer srcHub = loadHub(nonNullSrc); 209 KlassPointer destHub = loadHub(nonNullDest); 210 checkArrayType(srcHub); 211 checkArrayType(destHub); 212 checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters); 213 if (length == 0) { 214 counters.zeroLengthDynamicCounter.inc(); 215 } else { 216 counters.nonZeroLengthDynamicCounter.inc(); 217 counters.nonZeroLengthDynamicCopiedCounter.add(length); 218 } 219 ArrayCopySlowPathNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, predictedKlass, elementKind, slowPath, slowPathArgument); 220 } 221 222 /** 223 * Snippet for unrolled arraycopy. 224 */ 225 @Snippet 226 public static void arraycopyUnrolledIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter int unrolledLength, @ConstantParameter JavaKind elementKind, 227 @ConstantParameter Counters counters) { 228 Object nonNullSrc = GraalDirectives.guardingNonNull(src); 229 Object nonNullDest = GraalDirectives.guardingNonNull(dest); 230 checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters); 231 if (length == 0) { 232 counters.zeroLengthDynamicCounter.inc(); 233 } else { 234 counters.nonZeroLengthDynamicCounter.inc(); 235 counters.nonZeroLengthDynamicCopiedCounter.add(length); 236 } 237 ArrayCopyUnrollNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, unrolledLength, elementKind); 238 } 239 240 @Snippet 241 public static void checkcastArraycopyWork(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, int length, @ConstantParameter Counters counters) { 242 if (length > 0) { 243 KlassPointer destKlass = loadHub(nonNullDest); 244 KlassPointer srcKlass = loadHub(nonNullSrc); 245 if (probability(SLOW_PATH_PROBABILITY, srcKlass == destKlass)) { 246 // no storecheck required. 247 counters.objectCheckcastSameTypeCounter.inc(); 248 counters.objectCheckcastSameTypeCopiedCounter.add(length); 249 ArrayCopyCallNode.arraycopyObjectKillsAny(nonNullSrc, srcPos, nonNullDest, destPos, length); 250 } else { 251 KlassPointer destElemKlass = destKlass.readKlassPointer(arrayClassElementOffset(INJECTED_VMCONFIG), OBJ_ARRAY_KLASS_ELEMENT_KLASS_LOCATION); 252 Word superCheckOffset = WordFactory.signed(destElemKlass.readInt(superCheckOffsetOffset(INJECTED_VMCONFIG), KLASS_SUPER_CHECK_OFFSET_LOCATION)); 253 counters.objectCheckcastCounter.inc(); 254 counters.objectCheckcastCopiedCounter.add(length); 255 int copiedElements = CheckcastArrayCopyCallNode.checkcastArraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, superCheckOffset, destElemKlass, false); 256 if (copiedElements != 0) { 257 /* 258 * the checkcast stub doesn't throw the ArrayStoreException, but returns the 259 * number of copied elements (xor'd with -1). 260 */ 261 copiedElements ^= -1; 262 System.arraycopy(nonNullSrc, srcPos + copiedElements, nonNullDest, destPos + copiedElements, length - copiedElements); 263 } 264 } 265 } 266 } 267 268 @Snippet 269 public static void arraycopyGeneric(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter Counters counters) { 270 Object nonNullSrc = GraalDirectives.guardingNonNull(src); 271 Object nonNullDest = GraalDirectives.guardingNonNull(dest); 272 KlassPointer srcHub = loadHub(nonNullSrc); 273 KlassPointer destHub = loadHub(nonNullDest); 274 if (probability(FAST_PATH_PROBABILITY, srcHub.equal(destHub)) && probability(FAST_PATH_PROBABILITY, nonNullSrc != nonNullDest)) { 275 int layoutHelper = checkArrayType(srcHub); 276 final boolean isObjectArray = ((layoutHelper & layoutHelperElementTypePrimitiveInPlace(INJECTED_VMCONFIG)) == 0); 277 checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters); 278 if (probability(FAST_PATH_PROBABILITY, isObjectArray)) { 279 counters.genericObjectExactCallCounter.inc(); 280 counters.genericObjectExactCallCopiedCounter.add(length); 281 ArrayCopyCallNode.disjointArraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, JavaKind.Object); 282 } else { 283 counters.genericPrimitiveCallCounter.inc(); 284 counters.genericPrimitiveCallCopiedCounter.add(length); 285 UnsafeArrayCopyNode.arraycopyPrimitive(nonNullSrc, srcPos, nonNullDest, destPos, length, layoutHelper); 286 } 287 } else { 288 counters.systemArraycopyCounter.inc(); 289 counters.systemArraycopyCopiedCounter.add(length); 290 System.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length); 291 } 292 } 293 294 @Fold 295 static LocationIdentity getArrayLocation(JavaKind kind) { 296 return NamedLocationIdentity.getArrayLocation(kind); 297 } 298 299 @Snippet 300 public static void arraycopyUnrolledWork(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, @ConstantParameter int length, @ConstantParameter JavaKind elementKind) { 301 final int scale = arrayIndexScale(elementKind); 302 int arrayBaseOffset = arrayBaseOffset(elementKind); 303 LocationIdentity arrayLocation = getArrayLocation(elementKind); 304 if (nonNullSrc == nonNullDest && srcPos < destPos) { // bad aliased case 305 long start = (long) (length - 1) * scale; 306 long i = start; 307 ExplodeLoopNode.explodeLoop(); 308 for (int iteration = 0; iteration < length; iteration++) { 309 if (i >= 0) { 310 Object a = RawLoadNode.load(nonNullSrc, arrayBaseOffset + i + (long) srcPos * scale, elementKind, arrayLocation); 311 RawStoreNode.storeObject(nonNullDest, arrayBaseOffset + i + (long) destPos * scale, a, elementKind, arrayLocation, false); 312 i -= scale; 313 } 314 } 315 } else { 316 long end = (long) length * scale; 317 long i = 0; 318 ExplodeLoopNode.explodeLoop(); 319 for (int iteration = 0; iteration < length; iteration++) { 320 if (i < end) { 321 Object a = RawLoadNode.load(nonNullSrc, arrayBaseOffset + i + (long) srcPos * scale, elementKind, arrayLocation); 322 RawStoreNode.storeObject(nonNullDest, arrayBaseOffset + i + (long) destPos * scale, a, elementKind, arrayLocation, false); 323 i += scale; 324 } 325 } 326 } 327 } 328 329 static class Counters { 330 final SnippetCounter checkSuccessCounter; 331 final SnippetCounter checkAIOOBECounter; 332 333 final SnippetCounter objectCheckcastCounter; 334 final SnippetCounter objectCheckcastSameTypeCounter; 335 final SnippetCounter predictedObjectArrayCopySlowPathCounter; 336 final SnippetCounter predictedObjectArrayCopyFastPathCounter; 337 338 final SnippetCounter genericPrimitiveCallCounter; 339 final SnippetCounter genericObjectExactCallCounter; 340 final SnippetCounter systemArraycopyCounter; 341 342 final SnippetCounter zeroLengthStaticCounter; 343 final SnippetCounter zeroLengthDynamicCounter; 344 final SnippetCounter nonZeroLengthDynamicCounter; 345 346 final SnippetCounter nonZeroLengthDynamicCopiedCounter; 347 final SnippetCounter genericPrimitiveCallCopiedCounter; 348 final SnippetCounter genericObjectExactCallCopiedCounter; 349 final SnippetCounter systemArraycopyCopiedCounter; 350 351 final SnippetCounter objectCheckcastCopiedCounter; 352 final SnippetCounter objectCheckcastSameTypeCopiedCounter; 353 final SnippetCounter predictedObjectArrayCopySlowPathCopiedCounter; 354 final SnippetCounter predictedObjectArrayCopyFastPathCopiedCounter; 355 356 final EnumMap<JavaKind, SnippetCounter> arraycopyCallCounters = new EnumMap<>(JavaKind.class); 357 final EnumMap<JavaKind, SnippetCounter> arraycopyCounters = new EnumMap<>(JavaKind.class); 358 359 final EnumMap<JavaKind, SnippetCounter> arraycopyCallCopiedCounters = new EnumMap<>(JavaKind.class); 360 final EnumMap<JavaKind, SnippetCounter> arraycopyCopiedCounters = new EnumMap<>(JavaKind.class); 361 362 Counters(SnippetCounter.Group.Factory factory) { 363 final Group checkCounters = factory.createSnippetCounterGroup("System.arraycopy checkInputs"); 364 final Group counters = factory.createSnippetCounterGroup("System.arraycopy"); 365 final Group copiedCounters = factory.createSnippetCounterGroup("System.arraycopy copied elements"); 366 final Group lengthCounters = factory.createSnippetCounterGroup("System.arraycopy 0-length checks"); 367 368 checkSuccessCounter = new SnippetCounter(checkCounters, "checkSuccess", "checkSuccess"); 369 checkAIOOBECounter = new SnippetCounter(checkCounters, "checkAIOOBE", "checkAIOOBE"); 370 371 objectCheckcastCounter = new SnippetCounter(counters, "Object[]{non-exact}", "arraycopy for non-exact Object[] arrays"); 372 objectCheckcastSameTypeCounter = new SnippetCounter(counters, "Object[]{same-type}", "arraycopy call for src.klass == dest.klass Object[] arrays"); 373 predictedObjectArrayCopySlowPathCounter = new SnippetCounter(counters, "Object[]{slow-path}", "used System.arraycopy slow path for predicted Object[] arrays"); 374 predictedObjectArrayCopyFastPathCounter = new SnippetCounter(counters, "Object[]{fast-path}", "used oop_arraycopy for predicted Object[] arrays"); 375 genericPrimitiveCallCounter = new SnippetCounter(counters, "genericPrimitive", "generic arraycopy snippet for primitive arrays"); 376 genericObjectExactCallCounter = new SnippetCounter(counters, "genericObjectExact", "generic arraycopy snippet for special object arrays"); 377 systemArraycopyCounter = new SnippetCounter(counters, "genericObject", "call to System.arraycopy"); 378 379 zeroLengthStaticCounter = new SnippetCounter(lengthCounters, "0-lengthcopy static", "arraycopy where the length is statically 0"); 380 zeroLengthDynamicCounter = new SnippetCounter(lengthCounters, "0-lengthcopy dynamically", "arraycopy where the length is dynamically 0"); 381 nonZeroLengthDynamicCounter = new SnippetCounter(lengthCounters, "non-0-lengthcopy dynamically", "arraycopy where the length is dynamically not zero"); 382 383 nonZeroLengthDynamicCopiedCounter = new SnippetCounter(copiedCounters, "non-0-lengthcopy dynamically", "arraycopy where the length is dynamically not zero"); 384 genericPrimitiveCallCopiedCounter = new SnippetCounter(copiedCounters, "genericPrimitive", "generic arraycopy snippet for primitive arrays"); 385 genericObjectExactCallCopiedCounter = new SnippetCounter(copiedCounters, "genericObjectExact", "generic arraycopy snippet for special object arrays"); 386 systemArraycopyCopiedCounter = new SnippetCounter(copiedCounters, "genericObject", "call to System.arraycopy"); 387 388 objectCheckcastCopiedCounter = new SnippetCounter(copiedCounters, "Object[]{non-exact}", "arraycopy for non-exact Object[] arrays"); 389 objectCheckcastSameTypeCopiedCounter = new SnippetCounter(copiedCounters, "Object[]{same-type}", "arraycopy call for src.klass == dest.klass Object[] arrays"); 390 predictedObjectArrayCopySlowPathCopiedCounter = new SnippetCounter(copiedCounters, "Object[]{slow-path}", 391 "used System.arraycopy slow path for predicted Object[] arrays"); 392 predictedObjectArrayCopyFastPathCopiedCounter = new SnippetCounter(copiedCounters, "Object[]{fast-path}", "used oop_arraycopy for predicted Object[] arrays"); 393 createArraycopyCounter(JavaKind.Byte, counters, copiedCounters); 394 createArraycopyCounter(JavaKind.Boolean, counters, copiedCounters); 395 createArraycopyCounter(JavaKind.Char, counters, copiedCounters); 396 createArraycopyCounter(JavaKind.Short, counters, copiedCounters); 397 createArraycopyCounter(JavaKind.Int, counters, copiedCounters); 398 createArraycopyCounter(JavaKind.Long, counters, copiedCounters); 399 createArraycopyCounter(JavaKind.Float, counters, copiedCounters); 400 createArraycopyCounter(JavaKind.Double, counters, copiedCounters); 401 createArraycopyCounter(JavaKind.Object, counters, copiedCounters); 402 } 403 404 void createArraycopyCounter(JavaKind kind, Group counters, Group copiedCounters) { 405 arraycopyCallCounters.put(kind, new SnippetCounter(counters, kind + "[]{stub}", "arraycopy call for " + kind + "[] arrays")); 406 arraycopyCounters.put(kind, new SnippetCounter(counters, kind + "[]{inline}", "inline arraycopy for " + kind + "[] arrays")); 407 408 arraycopyCallCopiedCounters.put(kind, new SnippetCounter(copiedCounters, kind + "[]{stub}", "arraycopy call for " + kind + "[] arrays")); 409 arraycopyCopiedCounters.put(kind, new SnippetCounter(copiedCounters, kind + "[]{inline}", "inline arraycopy for " + kind + "[] arrays")); 410 } 411 } 412 413 public static class Templates extends SnippetTemplate.AbstractTemplates { 414 415 public Templates(OptionValues options, Iterable<DebugHandlersFactory> factories, SnippetCounter.Group.Factory factory, HotSpotProviders providers, TargetDescription target) { 416 super(options, factories, providers, providers.getSnippetReflection(), target); 417 this.counters = new Counters(factory); 418 } 419 420 private ResolvedJavaMethod originalArraycopy() throws GraalError { 421 if (originalArraycopy == null) { 422 Method method; 423 try { 424 method = System.class.getDeclaredMethod("arraycopy", Object.class, int.class, Object.class, int.class, int.class); 425 } catch (NoSuchMethodException | SecurityException e) { 426 throw new GraalError(e); 427 } 428 originalArraycopy = providers.getMetaAccess().lookupJavaMethod(method); 429 } 430 return originalArraycopy; 431 } 432 433 private ResolvedJavaMethod originalArraycopy; 434 435 private final SnippetInfo checkcastArraycopyWorkSnippet = snippet("checkcastArraycopyWork"); 436 private final SnippetInfo arraycopyGenericSnippet = snippet("arraycopyGeneric"); 437 438 private final SnippetInfo arraycopySlowPathIntrinsicSnippet = snippet("arraycopySlowPathIntrinsic"); 439 private final SnippetInfo arraycopyUnrolledIntrinsicSnippet = snippet("arraycopyUnrolledIntrinsic"); 440 private final SnippetInfo arraycopyExactIntrinsicSnippet = snippet("arraycopyExactIntrinsic"); 441 private final SnippetInfo arraycopyZeroLengthIntrinsicSnippet = snippet("arraycopyZeroLengthIntrinsic"); 442 private final SnippetInfo arraycopyPredictedExactIntrinsicSnippet = snippet("arraycopyPredictedExactIntrinsic"); 443 private final SnippetInfo arraycopyPredictedObjectWorkSnippet = snippet("arraycopyPredictedObjectWork"); 444 445 private final SnippetInfo arraycopyUnrolledWorkSnippet = snippet("arraycopyUnrolledWork"); 446 447 private final Counters counters; 448 449 protected SnippetInfo snippet(String methodName) { 450 SnippetInfo info = snippet(ArrayCopySnippets.class, methodName, LocationIdentity.any()); 451 info.setOriginalMethod(originalArraycopy()); 452 return info; 453 } 454 455 public static JavaKind selectComponentKind(BasicArrayCopyNode arraycopy) { 456 return selectComponentKind(arraycopy, true); 457 } 458 459 public static JavaKind selectComponentKind(BasicArrayCopyNode arraycopy, boolean exact) { 460 ResolvedJavaType srcType = StampTool.typeOrNull(arraycopy.getSource().stamp()); 461 ResolvedJavaType destType = StampTool.typeOrNull(arraycopy.getDestination().stamp()); 462 463 if (srcType == null || !srcType.isArray() || destType == null || !destType.isArray()) { 464 if (!exact) { 465 JavaKind component = getComponentKind(srcType); 466 if (component != null) { 467 return component; 468 } 469 return getComponentKind(destType); 470 } 471 return null; 472 } 473 if (exact) { 474 if (!destType.getComponentType().isAssignableFrom(srcType.getComponentType())) { 475 return null; 476 } 477 if (!arraycopy.isExact()) { 478 return null; 479 } 480 } 481 return srcType.getComponentType().getJavaKind(); 482 } 483 484 private static JavaKind getComponentKind(ResolvedJavaType type) { 485 if (type != null && type.isArray()) { 486 return type.getComponentType().getJavaKind(); 487 } 488 return null; 489 } 490 491 private static boolean shouldUnroll(ValueNode length) { 492 return length.isConstant() && length.asJavaConstant().asInt() <= 8 && length.asJavaConstant().asInt() != 0; 493 } 494 495 public void lower(ArrayCopyNode arraycopy, LoweringTool tool) { 496 JavaKind componentKind = selectComponentKind(arraycopy); 497 SnippetInfo snippetInfo = null; 498 SnippetInfo slowPathSnippetInfo = null; 499 Object slowPathArgument = null; 500 501 if (arraycopy.getLength().isConstant() && arraycopy.getLength().asJavaConstant().asLong() == 0) { 502 snippetInfo = arraycopyZeroLengthIntrinsicSnippet; 503 } else if (arraycopy.isExact()) { 504 snippetInfo = arraycopyExactIntrinsicSnippet; 505 if (shouldUnroll(arraycopy.getLength())) { 506 snippetInfo = arraycopyUnrolledIntrinsicSnippet; 507 } 508 } else { 509 if (componentKind == JavaKind.Object) { 510 ResolvedJavaType srcType = StampTool.typeOrNull(arraycopy.getSource().stamp()); 511 ResolvedJavaType destType = StampTool.typeOrNull(arraycopy.getDestination().stamp()); 512 ResolvedJavaType srcComponentType = srcType == null ? null : srcType.getComponentType(); 513 ResolvedJavaType destComponentType = destType == null ? null : destType.getComponentType(); 514 if (srcComponentType != null && destComponentType != null && !srcComponentType.isPrimitive() && !destComponentType.isPrimitive()) { 515 snippetInfo = arraycopySlowPathIntrinsicSnippet; 516 slowPathSnippetInfo = checkcastArraycopyWorkSnippet; 517 slowPathArgument = LocationIdentity.any(); 518 /* 519 * Because this snippet has to use Sysytem.arraycopy as a slow path, we must 520 * pretend to kill any() so clear the componentKind. 521 */ 522 componentKind = null; 523 } 524 } 525 if (componentKind == null && snippetInfo == null) { 526 JavaKind predictedKind = selectComponentKind(arraycopy, false); 527 if (predictedKind != null) { 528 /* 529 * At least one array is of a known type requiring no store checks, so 530 * assume the other is of the same type. Generally this is working around 531 * deficiencies in our propagation of type information. 532 */ 533 componentKind = predictedKind; 534 if (predictedKind == JavaKind.Object) { 535 snippetInfo = arraycopySlowPathIntrinsicSnippet; 536 slowPathSnippetInfo = arraycopyPredictedObjectWorkSnippet; 537 slowPathArgument = predictedKind; 538 componentKind = null; 539 } else { 540 snippetInfo = arraycopyPredictedExactIntrinsicSnippet; 541 } 542 } 543 } 544 if (snippetInfo == null) { 545 snippetInfo = arraycopyGenericSnippet; 546 } 547 } 548 Arguments args = new Arguments(snippetInfo, arraycopy.graph().getGuardsStage(), tool.getLoweringStage()); 549 args.add("src", arraycopy.getSource()); 550 args.add("srcPos", arraycopy.getSourcePosition()); 551 args.add("dest", arraycopy.getDestination()); 552 args.add("destPos", arraycopy.getDestinationPosition()); 553 args.add("length", arraycopy.getLength()); 554 if (snippetInfo == arraycopyUnrolledIntrinsicSnippet) { 555 args.addConst("unrolledLength", arraycopy.getLength().asJavaConstant().asInt()); 556 args.addConst("elementKind", componentKind != null ? componentKind : JavaKind.Illegal); 557 } else if (snippetInfo == arraycopySlowPathIntrinsicSnippet) { 558 ValueNode predictedKlass = null; 559 if (slowPathArgument == arraycopyPredictedObjectWorkSnippet) { 560 HotSpotResolvedObjectType arrayClass = (HotSpotResolvedObjectType) tool.getMetaAccess().lookupJavaType(Object[].class); 561 predictedKlass = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), arrayClass.klass(), tool.getMetaAccess(), arraycopy.graph()); 562 } else { 563 predictedKlass = ConstantNode.forConstant(KlassPointerStamp.klassAlwaysNull(), JavaConstant.NULL_POINTER, tool.getMetaAccess(), arraycopy.graph()); 564 } 565 args.add("predictedKlass", predictedKlass); 566 args.addConst("elementKind", componentKind != null ? componentKind : JavaKind.Illegal); 567 args.addConst("slowPath", slowPathSnippetInfo); 568 assert slowPathArgument != null; 569 args.addConst("slowPathArgument", slowPathArgument); 570 } else if (snippetInfo == arraycopyExactIntrinsicSnippet || snippetInfo == arraycopyPredictedExactIntrinsicSnippet) { 571 assert componentKind != null; 572 args.addConst("elementKind", componentKind); 573 args.addConst("counter", counters.arraycopyCallCounters.get(componentKind)); 574 args.addConst("copiedCounter", counters.arraycopyCallCopiedCounters.get(componentKind)); 575 } 576 args.addConst("counters", counters); 577 instantiate(args, arraycopy); 578 } 579 580 public void lower(ArrayCopySlowPathNode arraycopy, LoweringTool tool) { 581 StructuredGraph graph = arraycopy.graph(); 582 if (!graph.getGuardsStage().areFrameStatesAtDeopts()) { 583 // Can't be lowered yet 584 return; 585 } 586 SnippetInfo snippetInfo = arraycopy.getSnippet(); 587 Arguments args = new Arguments(snippetInfo, graph.getGuardsStage(), tool.getLoweringStage()); 588 args.add("nonNullSrc", arraycopy.getSource()); 589 args.add("srcPos", arraycopy.getSourcePosition()); 590 args.add("nonNullDest", arraycopy.getDestination()); 591 args.add("destPos", arraycopy.getDestinationPosition()); 592 if (snippetInfo == arraycopyUnrolledWorkSnippet) { 593 args.addConst("length", ((Integer) arraycopy.getArgument()).intValue()); 594 args.addConst("elementKind", arraycopy.getElementKind()); 595 } else { 596 args.add("length", arraycopy.getLength()); 597 } 598 if (snippetInfo == arraycopyPredictedObjectWorkSnippet) { 599 args.add("objectArrayKlass", arraycopy.getPredictedKlass()); 600 args.addConst("counter", counters.arraycopyCallCounters.get(JavaKind.Object)); 601 args.addConst("copiedCounter", counters.arraycopyCallCopiedCounters.get(JavaKind.Object)); 602 args.addConst("counters", counters); 603 } 604 instantiate(args, arraycopy); 605 } 606 607 public void lower(ArrayCopyUnrollNode arraycopy, LoweringTool tool) { 608 StructuredGraph graph = arraycopy.graph(); 609 if (!graph.getGuardsStage().areFrameStatesAtDeopts()) { 610 // Can't be lowered yet 611 return; 612 } 613 SnippetInfo snippetInfo = arraycopyUnrolledWorkSnippet; 614 Arguments args = new Arguments(snippetInfo, graph.getGuardsStage(), tool.getLoweringStage()); 615 args.add("nonNullSrc", arraycopy.getSource()); 616 args.add("srcPos", arraycopy.getSourcePosition()); 617 args.add("nonNullDest", arraycopy.getDestination()); 618 args.add("destPos", arraycopy.getDestinationPosition()); 619 args.addConst("length", arraycopy.getUnrollLength()); 620 args.addConst("elementKind", arraycopy.getElementKind()); 621 template(graph.getDebug(), args).instantiate(providers.getMetaAccess(), arraycopy, SnippetTemplate.DEFAULT_REPLACER, args); 622 } 623 624 /** 625 * Instantiate the snippet template and fix up the FrameState of any Invokes of 626 * System.arraycopy and propagate the captured bci in the ArrayCopySlowPathNode. 627 * 628 * @param args 629 * @param arraycopy 630 */ 631 private void instantiate(Arguments args, BasicArrayCopyNode arraycopy) { 632 StructuredGraph graph = arraycopy.graph(); 633 SnippetTemplate template = template(graph.getDebug(), args); 634 UnmodifiableEconomicMap<Node, Node> replacements = template.instantiate(providers.getMetaAccess(), arraycopy, SnippetTemplate.DEFAULT_REPLACER, args, false); 635 for (Node originalNode : replacements.getKeys()) { 636 if (originalNode instanceof Invoke) { 637 Invoke invoke = (Invoke) replacements.get(originalNode); 638 assert invoke.asNode().graph() == graph; 639 CallTargetNode call = invoke.callTarget(); 640 641 if (!call.targetMethod().equals(originalArraycopy)) { 642 throw new GraalError("unexpected invoke %s in snippet", call.targetMethod()); 643 } 644 // Here we need to fix the bci of the invoke 645 InvokeNode newInvoke = graph.add(new InvokeNode(invoke.callTarget(), arraycopy.getBci())); 646 if (arraycopy.stateDuring() != null) { 647 newInvoke.setStateDuring(arraycopy.stateDuring()); 648 } else { 649 assert arraycopy.stateAfter() != null : arraycopy; 650 newInvoke.setStateAfter(arraycopy.stateAfter()); 651 } 652 graph.replaceFixedWithFixed((InvokeNode) invoke.asNode(), newInvoke); 653 } else if (originalNode instanceof ArrayCopySlowPathNode) { 654 ArrayCopySlowPathNode slowPath = (ArrayCopySlowPathNode) replacements.get(originalNode); 655 assert arraycopy.stateAfter() != null : arraycopy; 656 assert slowPath.stateAfter() == arraycopy.stateAfter(); 657 slowPath.setBci(arraycopy.getBci()); 658 } 659 } 660 GraphUtil.killCFG(arraycopy); 661 } 662 } 663 } | 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 */ 23 package org.graalvm.compiler.hotspot.replacements.arraycopy; 24 25 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFIG; 26 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.KLASS_SUPER_CHECK_OFFSET_LOCATION; 27 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.OBJ_ARRAY_KLASS_ELEMENT_KLASS_LOCATION; 28 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayBaseOffset; 29 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayClassElementOffset; 30 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayIndexScale; 31 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.loadHub; 32 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readLayoutHelper; 33 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.superCheckOffsetOffset; 34 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FREQUENT_PROBABILITY; 35 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.LIKELY_PROBABILITY; 36 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.NOT_FREQUENT_PROBABILITY; 37 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY; 38 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability; 39 40 import java.lang.reflect.Method; 41 import java.util.EnumMap; 42 43 import org.graalvm.compiler.api.directives.GraalDirectives; 44 import org.graalvm.compiler.api.replacements.Fold; 45 import org.graalvm.compiler.api.replacements.Snippet; 46 import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter; 47 import org.graalvm.compiler.debug.DebugHandlersFactory; 48 import org.graalvm.compiler.debug.GraalError; 49 import org.graalvm.compiler.graph.Node; 50 import org.graalvm.compiler.hotspot.meta.HotSpotProviders; 51 import org.graalvm.compiler.hotspot.word.KlassPointer; 52 import org.graalvm.compiler.nodes.CallTargetNode; 53 import org.graalvm.compiler.nodes.DeoptimizeNode; 54 import org.graalvm.compiler.nodes.Invoke; 55 import org.graalvm.compiler.nodes.InvokeNode; 56 import org.graalvm.compiler.nodes.NamedLocationIdentity; 57 import org.graalvm.compiler.nodes.PiNode; 58 import org.graalvm.compiler.nodes.StructuredGraph; 59 import org.graalvm.compiler.nodes.ValueNode; 60 import org.graalvm.compiler.nodes.extended.RawLoadNode; 61 import org.graalvm.compiler.nodes.extended.RawStoreNode; 62 import org.graalvm.compiler.nodes.java.ArrayLengthNode; 63 import org.graalvm.compiler.nodes.spi.LoweringTool; 64 import org.graalvm.compiler.nodes.type.StampTool; 65 import org.graalvm.compiler.nodes.util.GraphUtil; 66 import org.graalvm.compiler.options.OptionValues; 67 import org.graalvm.compiler.replacements.ReplacementsUtil; 68 import org.graalvm.compiler.replacements.SnippetCounter; 69 import org.graalvm.compiler.replacements.SnippetCounter.Group; 70 import org.graalvm.compiler.replacements.SnippetIntegerHistogram; 71 import org.graalvm.compiler.replacements.SnippetTemplate; 72 import org.graalvm.compiler.replacements.SnippetTemplate.Arguments; 73 import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo; 74 import org.graalvm.compiler.replacements.Snippets; 75 import org.graalvm.compiler.replacements.nodes.BasicArrayCopyNode; 76 import org.graalvm.compiler.replacements.nodes.ExplodeLoopNode; 77 import org.graalvm.compiler.word.Word; 78 import org.graalvm.util.UnmodifiableEconomicMap; 79 import org.graalvm.word.LocationIdentity; 80 import org.graalvm.word.WordFactory; 81 82 import jdk.vm.ci.code.TargetDescription; 83 import jdk.vm.ci.meta.DeoptimizationAction; 84 import jdk.vm.ci.meta.DeoptimizationReason; 85 import jdk.vm.ci.meta.JavaKind; 86 import jdk.vm.ci.meta.ResolvedJavaMethod; 87 import jdk.vm.ci.meta.ResolvedJavaType; 88 89 public class ArrayCopySnippets implements Snippets { 90 91 private enum ArrayCopyTypeCheck { 92 UNDEFINED_ARRAY_TYPE_CHECK, 93 // we know that both objects are arrays and have the same type 94 NO_ARRAY_TYPE_CHECK, 95 // can be used when we know that one of the objects is a primitive array 96 HUB_BASED_ARRAY_TYPE_CHECK, 97 // must be used when we don't have sufficient information to use one of the others 98 LAYOUT_HELPER_BASED_ARRAY_TYPE_CHECK 99 } 100 101 @Snippet 102 public static void arraycopyZeroLengthSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck, 103 @ConstantParameter Counters counters) { 104 Object nonNullSrc = GraalDirectives.guardingNonNull(src); 105 Object nonNullDest = GraalDirectives.guardingNonNull(dest); 106 checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck); 107 checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters); 108 counters.zeroLengthStaticCounter.inc(); 109 } 110 111 @Snippet 112 public static void arraycopyExactSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck, 113 @ConstantParameter JavaKind elementKind, @ConstantParameter SnippetCounter elementKindCounter, @ConstantParameter SnippetCounter elementKindCopiedCounter, 114 @ConstantParameter Counters counters) { 115 Object nonNullSrc = GraalDirectives.guardingNonNull(src); 116 Object nonNullDest = GraalDirectives.guardingNonNull(dest); 117 checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck); 118 checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters); 119 incrementLengthCounter(length, counters); 120 121 elementKindCounter.inc(); 122 elementKindCopiedCounter.add(length); 123 ArrayCopyCallNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, elementKind); 124 } 125 126 @Snippet 127 public static void arraycopyUnrolledSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck, 128 @ConstantParameter JavaKind elementKind, @ConstantParameter int unrolledLength, @ConstantParameter Counters counters) { 129 Object nonNullSrc = GraalDirectives.guardingNonNull(src); 130 Object nonNullDest = GraalDirectives.guardingNonNull(dest); 131 checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck); 132 checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters); 133 incrementLengthCounter(length, counters); 134 135 unrolledArraycopyWork(nonNullSrc, srcPos, nonNullDest, destPos, unrolledLength, elementKind); 136 } 137 138 @Snippet 139 public static void arraycopyCheckcastSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck, 140 @ConstantParameter Counters counters, @ConstantParameter SnippetInfo workSnippet, @ConstantParameter JavaKind elementKind) { 141 Object nonNullSrc = GraalDirectives.guardingNonNull(src); 142 Object nonNullDest = GraalDirectives.guardingNonNull(dest); 143 checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck); 144 checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters); 145 incrementLengthCounter(length, counters); 146 147 ArrayCopyWithSlowPathNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, workSnippet, elementKind); 148 } 149 150 @Snippet 151 public static void arraycopyGenericSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck, @ConstantParameter Counters counters, 152 @ConstantParameter SnippetInfo workSnippet, @ConstantParameter JavaKind elementKind) { 153 Object nonNullSrc = GraalDirectives.guardingNonNull(src); 154 Object nonNullDest = GraalDirectives.guardingNonNull(dest); 155 checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck); 156 checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters); 157 incrementLengthCounter(length, counters); 158 159 ArrayCopyWithSlowPathNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, workSnippet, elementKind); 160 } 161 162 @Snippet 163 public static void arraycopyNativeSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter Counters counters) { 164 // all checks are done in the native method, so no need to emit additional checks here 165 incrementLengthCounter(length, counters); 166 counters.systemArraycopyCounter.inc(); 167 counters.systemArraycopyCopiedCounter.add(length); 168 169 System.arraycopy(src, srcPos, dest, destPos, length); 170 } 171 172 @Fold 173 static LocationIdentity getArrayLocation(JavaKind kind) { 174 return NamedLocationIdentity.getArrayLocation(kind); 175 } 176 177 private static void unrolledArraycopyWork(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, int length, JavaKind elementKind) { 178 int scale = arrayIndexScale(elementKind); 179 int arrayBaseOffset = arrayBaseOffset(elementKind); 180 LocationIdentity arrayLocation = getArrayLocation(elementKind); 181 182 long sourceOffset = arrayBaseOffset + (long) srcPos * scale; 183 long destOffset = arrayBaseOffset + (long) destPos * scale; 184 long position = 0; 185 long delta = scale; 186 if (probability(NOT_FREQUENT_PROBABILITY, nonNullSrc == nonNullDest && srcPos < destPos)) { 187 // bad aliased case so we need to copy the array from back to front 188 position = (long) (length - 1) * scale; 189 delta = -delta; 190 } 191 192 // the length was already checked before - we can emit unconditional instructions 193 ExplodeLoopNode.explodeLoop(); 194 for (int iteration = 0; iteration < length; iteration++) { 195 Object value = RawLoadNode.load(nonNullSrc, sourceOffset + position, elementKind, arrayLocation); 196 RawStoreNode.storeObject(nonNullDest, destOffset + position, value, elementKind, arrayLocation, false); 197 position += delta; 198 } 199 } 200 201 @Snippet(allowPartialIntrinsicArgumentMismatch = true) 202 public static void checkcastArraycopyWithSlowPathWork(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter Counters counters) { 203 if (probability(FREQUENT_PROBABILITY, length > 0)) { 204 Object nonNullSrc = PiNode.asNonNullObject(src); 205 Object nonNullDest = PiNode.asNonNullObject(dest); 206 KlassPointer srcKlass = loadHub(nonNullSrc); 207 KlassPointer destKlass = loadHub(nonNullDest); 208 if (probability(LIKELY_PROBABILITY, srcKlass == destKlass)) { 209 // no storecheck required. 210 counters.objectCheckcastSameTypeCounter.inc(); 211 counters.objectCheckcastSameTypeCopiedCounter.add(length); 212 ArrayCopyCallNode.arraycopyObjectKillsAny(nonNullSrc, srcPos, nonNullDest, destPos, length); 213 } else { 214 KlassPointer destElemKlass = destKlass.readKlassPointer(arrayClassElementOffset(INJECTED_VMCONFIG), OBJ_ARRAY_KLASS_ELEMENT_KLASS_LOCATION); 215 Word superCheckOffset = WordFactory.signed(destElemKlass.readInt(superCheckOffsetOffset(INJECTED_VMCONFIG), KLASS_SUPER_CHECK_OFFSET_LOCATION)); 216 217 counters.objectCheckcastDifferentTypeCounter.inc(); 218 counters.objectCheckcastDifferentTypeCopiedCounter.add(length); 219 220 int copiedElements = CheckcastArrayCopyCallNode.checkcastArraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, superCheckOffset, destElemKlass, false); 221 if (probability(SLOW_PATH_PROBABILITY, copiedElements != 0)) { 222 /* 223 * the stub doesn't throw the ArrayStoreException, but returns the number of 224 * copied elements (xor'd with -1). 225 */ 226 copiedElements ^= -1; 227 System.arraycopy(nonNullSrc, srcPos + copiedElements, nonNullDest, destPos + copiedElements, length - copiedElements); 228 } 229 } 230 } 231 } 232 233 @Snippet(allowPartialIntrinsicArgumentMismatch = true) 234 public static void genericArraycopyWithSlowPathWork(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter Counters counters) { 235 if (probability(FREQUENT_PROBABILITY, length > 0)) { 236 counters.genericArraycopyDifferentTypeCounter.inc(); 237 counters.genericArraycopyDifferentTypeCopiedCounter.add(length); 238 int copiedElements = GenericArrayCopyCallNode.genericArraycopy(src, srcPos, dest, destPos, length); 239 if (probability(SLOW_PATH_PROBABILITY, copiedElements != 0)) { 240 /* 241 * the stub doesn't throw the ArrayStoreException, but returns the number of copied 242 * elements (xor'd with -1). 243 */ 244 copiedElements ^= -1; 245 System.arraycopy(src, srcPos + copiedElements, dest, destPos + copiedElements, length - copiedElements); 246 } 247 } 248 } 249 250 private static void incrementLengthCounter(int length, Counters counters) { 251 counters.lengthHistogram.inc(length); 252 } 253 254 private static void checkLimits(Object src, int srcPos, Object dest, int destPos, int length, Counters counters) { 255 if (probability(SLOW_PATH_PROBABILITY, srcPos < 0) || 256 probability(SLOW_PATH_PROBABILITY, destPos < 0) || 257 probability(SLOW_PATH_PROBABILITY, length < 0) || 258 probability(SLOW_PATH_PROBABILITY, srcPos > ArrayLengthNode.arrayLength(src) - length) || 259 probability(SLOW_PATH_PROBABILITY, destPos > ArrayLengthNode.arrayLength(dest) - length)) { 260 counters.checkAIOOBECounter.inc(); 261 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); 262 } 263 counters.checkSuccessCounter.inc(); 264 } 265 266 private static void checkArrayTypes(Object nonNullSrc, Object nonNullDest, ArrayCopyTypeCheck arrayTypeCheck) { 267 if (arrayTypeCheck == ArrayCopyTypeCheck.NO_ARRAY_TYPE_CHECK) { 268 // nothing to do 269 } else if (arrayTypeCheck == ArrayCopyTypeCheck.HUB_BASED_ARRAY_TYPE_CHECK) { 270 KlassPointer srcHub = loadHub(nonNullSrc); 271 KlassPointer destHub = loadHub(nonNullDest); 272 if (probability(SLOW_PATH_PROBABILITY, srcHub != destHub)) { 273 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); 274 } 275 } else if (arrayTypeCheck == ArrayCopyTypeCheck.LAYOUT_HELPER_BASED_ARRAY_TYPE_CHECK) { 276 KlassPointer srcHub = loadHub(nonNullSrc); 277 KlassPointer destHub = loadHub(nonNullDest); 278 checkArrayType(srcHub); 279 checkArrayType(destHub); 280 } else { 281 ReplacementsUtil.staticAssert(false, "unknown array type check"); 282 } 283 } 284 285 private static int checkArrayType(KlassPointer nonNullHub) { 286 int layoutHelper = readLayoutHelper(nonNullHub); 287 if (probability(SLOW_PATH_PROBABILITY, layoutHelper >= 0)) { 288 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); 289 } 290 return layoutHelper; 291 } 292 293 static class Counters { 294 final SnippetCounter checkSuccessCounter; 295 final SnippetCounter checkAIOOBECounter; 296 297 final SnippetCounter zeroLengthStaticCounter; 298 final SnippetIntegerHistogram lengthHistogram; 299 300 final SnippetCounter systemArraycopyCounter; 301 final SnippetCounter systemArraycopyCopiedCounter; 302 303 final SnippetCounter genericArraycopyDifferentTypeCopiedCounter; 304 final SnippetCounter genericArraycopyDifferentTypeCounter; 305 306 final SnippetCounter objectCheckcastSameTypeCopiedCounter; 307 final SnippetCounter objectCheckcastSameTypeCounter; 308 final SnippetCounter objectCheckcastDifferentTypeCopiedCounter; 309 final SnippetCounter objectCheckcastDifferentTypeCounter; 310 311 final EnumMap<JavaKind, SnippetCounter> arraycopyCallCounters = new EnumMap<>(JavaKind.class); 312 final EnumMap<JavaKind, SnippetCounter> arraycopyCallCopiedCounters = new EnumMap<>(JavaKind.class); 313 314 Counters(SnippetCounter.Group.Factory factory) { 315 final Group checkCounters = factory.createSnippetCounterGroup("System.arraycopy checkInputs"); 316 final Group callCounters = factory.createSnippetCounterGroup("System.arraycopy calls"); 317 final Group copiedElementsCounters = factory.createSnippetCounterGroup("System.arraycopy copied elements"); 318 final Group lengthCounters = factory.createSnippetCounterGroup("System.arraycopy with 0-length"); 319 320 checkSuccessCounter = new SnippetCounter(checkCounters, "checkSuccess", "checkSuccess"); 321 checkAIOOBECounter = new SnippetCounter(checkCounters, "checkAIOOBE", "checkAIOOBE"); 322 323 zeroLengthStaticCounter = new SnippetCounter(lengthCounters, "0-length copy static", "calls where the length is statically 0"); 324 lengthHistogram = new SnippetIntegerHistogram(lengthCounters, 2, "length", "length"); 325 326 systemArraycopyCounter = new SnippetCounter(callCounters, "native System.arraycopy", "JNI-based System.arraycopy call"); 327 systemArraycopyCopiedCounter = new SnippetCounter(copiedElementsCounters, "native System.arraycopy", "JNI-based System.arraycopy call"); 328 329 genericArraycopyDifferentTypeCounter = new SnippetCounter(callCounters, "generic[] stub", "generic arraycopy stub"); 330 genericArraycopyDifferentTypeCopiedCounter = new SnippetCounter(copiedElementsCounters, "generic[] stub", "generic arraycopy stub"); 331 332 objectCheckcastSameTypeCounter = new SnippetCounter(callCounters, "checkcast object[] (same-type)", "checkcast object[] stub but src.klass == dest.klass Object[] arrays"); 333 objectCheckcastSameTypeCopiedCounter = new SnippetCounter(copiedElementsCounters, "checkcast object[] (same-type)", "checkcast object[] stub but src.klass == dest.klass Object[] arrays"); 334 objectCheckcastDifferentTypeCounter = new SnippetCounter(callCounters, "checkcast object[] (store-check)", "checkcast object[] stub with store check"); 335 objectCheckcastDifferentTypeCopiedCounter = new SnippetCounter(copiedElementsCounters, "checkcast object[] (store-check)", "checkcast object[] stub with store check"); 336 337 createArraycopyCounter(JavaKind.Byte, callCounters, copiedElementsCounters); 338 createArraycopyCounter(JavaKind.Boolean, callCounters, copiedElementsCounters); 339 createArraycopyCounter(JavaKind.Char, callCounters, copiedElementsCounters); 340 createArraycopyCounter(JavaKind.Short, callCounters, copiedElementsCounters); 341 createArraycopyCounter(JavaKind.Int, callCounters, copiedElementsCounters); 342 createArraycopyCounter(JavaKind.Long, callCounters, copiedElementsCounters); 343 createArraycopyCounter(JavaKind.Float, callCounters, copiedElementsCounters); 344 createArraycopyCounter(JavaKind.Double, callCounters, copiedElementsCounters); 345 createArraycopyCounter(JavaKind.Object, callCounters, copiedElementsCounters); 346 } 347 348 void createArraycopyCounter(JavaKind kind, Group counters, Group copiedCounters) { 349 arraycopyCallCounters.put(kind, new SnippetCounter(counters, kind + "[] stub", "arraycopy call for " + kind + "[] arrays")); 350 arraycopyCallCopiedCounters.put(kind, new SnippetCounter(copiedCounters, kind + "[] stub", "arraycopy call for " + kind + "[] arrays")); 351 } 352 } 353 354 public static class Templates extends SnippetTemplate.AbstractTemplates { 355 private final SnippetInfo arraycopyGenericSnippet = snippet("arraycopyGenericSnippet"); 356 private final SnippetInfo arraycopyUnrolledSnippet = snippet("arraycopyUnrolledSnippet"); 357 private final SnippetInfo arraycopyExactSnippet = snippet("arraycopyExactSnippet"); 358 private final SnippetInfo arraycopyZeroLengthSnippet = snippet("arraycopyZeroLengthSnippet"); 359 private final SnippetInfo arraycopyCheckcastSnippet = snippet("arraycopyCheckcastSnippet"); 360 private final SnippetInfo arraycopyNativeSnippet = snippet("arraycopyNativeSnippet"); 361 362 private final SnippetInfo checkcastArraycopyWithSlowPathWork = snippet("checkcastArraycopyWithSlowPathWork"); 363 private final SnippetInfo genericArraycopyWithSlowPathWork = snippet("genericArraycopyWithSlowPathWork"); 364 365 private ResolvedJavaMethod originalArraycopy; 366 private final Counters counters; 367 368 public Templates(OptionValues options, Iterable<DebugHandlersFactory> factories, SnippetCounter.Group.Factory factory, HotSpotProviders providers, TargetDescription target) { 369 super(options, factories, providers, providers.getSnippetReflection(), target); 370 this.counters = new Counters(factory); 371 } 372 373 protected SnippetInfo snippet(String methodName) { 374 SnippetInfo info = snippet(ArrayCopySnippets.class, methodName, LocationIdentity.any()); 375 info.setOriginalMethod(originalArraycopy()); 376 return info; 377 } 378 379 public void lower(ArrayCopyNode arraycopy, LoweringTool tool) { 380 JavaKind elementKind = selectComponentKind(arraycopy); 381 SnippetInfo snippetInfo; 382 ArrayCopyTypeCheck arrayTypeCheck; 383 384 ResolvedJavaType srcType = StampTool.typeOrNull(arraycopy.getSource().stamp()); 385 ResolvedJavaType destType = StampTool.typeOrNull(arraycopy.getDestination().stamp()); 386 if (!canBeArray(srcType) || !canBeArray(destType)) { 387 // at least one of the objects is definitely not an array - use the native call 388 // right away as the copying will fail anyways 389 snippetInfo = arraycopyNativeSnippet; 390 arrayTypeCheck = ArrayCopyTypeCheck.UNDEFINED_ARRAY_TYPE_CHECK; 391 } else { 392 ResolvedJavaType srcComponentType = srcType == null ? null : srcType.getComponentType(); 393 ResolvedJavaType destComponentType = destType == null ? null : destType.getComponentType(); 394 395 if (arraycopy.isExact()) { 396 // there is a sufficient type match - we don't need any additional type checks 397 snippetInfo = arraycopyExactSnippet; 398 arrayTypeCheck = ArrayCopyTypeCheck.NO_ARRAY_TYPE_CHECK; 399 } else if (srcComponentType == null && destComponentType == null) { 400 // we don't know anything about the types - use the generic copying 401 snippetInfo = arraycopyGenericSnippet; 402 arrayTypeCheck = ArrayCopyTypeCheck.LAYOUT_HELPER_BASED_ARRAY_TYPE_CHECK; 403 } else if (srcComponentType != null && destComponentType != null) { 404 if (!srcComponentType.isPrimitive() && !destComponentType.isPrimitive()) { 405 // it depends on the array content if the copy succeeds - we need 406 // a type check for every store 407 snippetInfo = arraycopyCheckcastSnippet; 408 arrayTypeCheck = ArrayCopyTypeCheck.NO_ARRAY_TYPE_CHECK; 409 } else { 410 // one object is an object array, the other one is a primitive array. 411 // this copy will always fail - use the native call right away 412 assert !srcComponentType.equals(destComponentType) : "must be handled by arraycopy.isExact()"; 413 snippetInfo = arraycopyNativeSnippet; 414 arrayTypeCheck = ArrayCopyTypeCheck.UNDEFINED_ARRAY_TYPE_CHECK; 415 } 416 } else { 417 ResolvedJavaType nonNullComponentType = srcComponentType != null ? srcComponentType : destComponentType; 418 if (nonNullComponentType.isPrimitive()) { 419 // one involved object is a primitive array - we can safely assume that we 420 // are copying primitive arrays 421 snippetInfo = arraycopyExactSnippet; 422 arrayTypeCheck = ArrayCopyTypeCheck.HUB_BASED_ARRAY_TYPE_CHECK; 423 elementKind = nonNullComponentType.getJavaKind(); 424 } else { 425 // one involved object is an object array - we can safely assume that we are 426 // copying object arrays that might require a store check 427 snippetInfo = arraycopyCheckcastSnippet; 428 arrayTypeCheck = ArrayCopyTypeCheck.LAYOUT_HELPER_BASED_ARRAY_TYPE_CHECK; 429 } 430 } 431 } 432 433 // a few special cases that are easier to handle when all other variables already have a 434 // value 435 if (arraycopy.getLength().isConstant() && arraycopy.getLength().asJavaConstant().asLong() == 0) { 436 snippetInfo = arraycopyZeroLengthSnippet; 437 } else if (snippetInfo == arraycopyExactSnippet && shouldUnroll(arraycopy.getLength())) { 438 snippetInfo = arraycopyUnrolledSnippet; 439 } 440 441 // create the snippet 442 Arguments args = new Arguments(snippetInfo, arraycopy.graph().getGuardsStage(), tool.getLoweringStage()); 443 args.add("src", arraycopy.getSource()); 444 args.add("srcPos", arraycopy.getSourcePosition()); 445 args.add("dest", arraycopy.getDestination()); 446 args.add("destPos", arraycopy.getDestinationPosition()); 447 args.add("length", arraycopy.getLength()); 448 if (snippetInfo != arraycopyNativeSnippet) { 449 assert arrayTypeCheck != ArrayCopyTypeCheck.UNDEFINED_ARRAY_TYPE_CHECK; 450 args.addConst("arrayTypeCheck", arrayTypeCheck); 451 } 452 if (snippetInfo == arraycopyUnrolledSnippet) { 453 args.addConst("elementKind", elementKind != null ? elementKind : JavaKind.Illegal); 454 args.addConst("unrolledLength", arraycopy.getLength().asJavaConstant().asInt()); 455 } 456 if (snippetInfo == arraycopyExactSnippet) { 457 assert elementKind != null; 458 args.addConst("elementKind", elementKind); 459 args.addConst("elementKindCounter", counters.arraycopyCallCounters.get(elementKind)); 460 args.addConst("elementKindCopiedCounter", counters.arraycopyCallCopiedCounters.get(elementKind)); 461 } 462 args.addConst("counters", counters); 463 if (snippetInfo == arraycopyCheckcastSnippet) { 464 args.addConst("workSnippet", checkcastArraycopyWithSlowPathWork); 465 args.addConst("elementKind", JavaKind.Illegal); 466 } 467 if (snippetInfo == arraycopyGenericSnippet) { 468 args.addConst("workSnippet", genericArraycopyWithSlowPathWork); 469 args.addConst("elementKind", JavaKind.Illegal); 470 } 471 472 instantiate(args, arraycopy); 473 } 474 475 public void lower(ArrayCopyWithSlowPathNode arraycopy, LoweringTool tool) { 476 StructuredGraph graph = arraycopy.graph(); 477 if (!graph.getGuardsStage().areFrameStatesAtDeopts()) { 478 // if an arraycopy contains a slow path, we can't lower it right away 479 return; 480 } 481 482 SnippetInfo snippetInfo = arraycopy.getSnippet(); 483 Arguments args = new Arguments(snippetInfo, graph.getGuardsStage(), tool.getLoweringStage()); 484 args.add("src", arraycopy.getSource()); 485 args.add("srcPos", arraycopy.getSourcePosition()); 486 args.add("dest", arraycopy.getDestination()); 487 args.add("destPos", arraycopy.getDestinationPosition()); 488 args.add("length", arraycopy.getLength()); 489 args.addConst("counters", counters); 490 instantiate(args, arraycopy); 491 } 492 493 private static boolean canBeArray(ResolvedJavaType type) { 494 return type == null || type.isJavaLangObject() || type.isArray(); 495 } 496 497 public static JavaKind selectComponentKind(BasicArrayCopyNode arraycopy) { 498 ResolvedJavaType srcType = StampTool.typeOrNull(arraycopy.getSource().stamp()); 499 ResolvedJavaType destType = StampTool.typeOrNull(arraycopy.getDestination().stamp()); 500 501 if (srcType == null || !srcType.isArray() || destType == null || !destType.isArray()) { 502 return null; 503 } 504 if (!destType.getComponentType().isAssignableFrom(srcType.getComponentType())) { 505 return null; 506 } 507 if (!arraycopy.isExact()) { 508 return null; 509 } 510 return srcType.getComponentType().getJavaKind(); 511 } 512 513 private static boolean shouldUnroll(ValueNode length) { 514 return length.isConstant() && length.asJavaConstant().asInt() <= 8 && length.asJavaConstant().asInt() != 0; 515 } 516 517 /** 518 * Instantiate the snippet template and fix up the FrameState of any Invokes of 519 * System.arraycopy and propagate the captured bci in the ArrayCopySlowPathNode. 520 * 521 * @param args 522 * @param arraycopy 523 */ 524 private void instantiate(Arguments args, BasicArrayCopyNode arraycopy) { 525 StructuredGraph graph = arraycopy.graph(); 526 SnippetTemplate template = template(graph.getDebug(), args); 527 UnmodifiableEconomicMap<Node, Node> replacements = template.instantiate(providers.getMetaAccess(), arraycopy, SnippetTemplate.DEFAULT_REPLACER, args, false); 528 for (Node originalNode : replacements.getKeys()) { 529 if (originalNode instanceof Invoke) { 530 Invoke invoke = (Invoke) replacements.get(originalNode); 531 assert invoke.asNode().graph() == graph; 532 CallTargetNode call = invoke.callTarget(); 533 534 if (!call.targetMethod().equals(originalArraycopy)) { 535 throw new GraalError("unexpected invoke %s in snippet", call.targetMethod()); 536 } 537 // Here we need to fix the bci of the invoke 538 InvokeNode newInvoke = graph.add(new InvokeNode(invoke.callTarget(), arraycopy.getBci())); 539 if (arraycopy.stateDuring() != null) { 540 newInvoke.setStateDuring(arraycopy.stateDuring()); 541 } else { 542 assert arraycopy.stateAfter() != null : arraycopy; 543 newInvoke.setStateAfter(arraycopy.stateAfter()); 544 } 545 graph.replaceFixedWithFixed((InvokeNode) invoke.asNode(), newInvoke); 546 } else if (originalNode instanceof ArrayCopyWithSlowPathNode) { 547 ArrayCopyWithSlowPathNode slowPath = (ArrayCopyWithSlowPathNode) replacements.get(originalNode); 548 assert arraycopy.stateAfter() != null : arraycopy; 549 assert slowPath.stateAfter() == arraycopy.stateAfter(); 550 slowPath.setBci(arraycopy.getBci()); 551 } 552 } 553 GraphUtil.killCFG(arraycopy); 554 } 555 556 private ResolvedJavaMethod originalArraycopy() throws GraalError { 557 if (originalArraycopy == null) { 558 Method method; 559 try { 560 method = System.class.getDeclaredMethod("arraycopy", Object.class, int.class, Object.class, int.class, int.class); 561 } catch (NoSuchMethodException | SecurityException e) { 562 throw new GraalError(e); 563 } 564 originalArraycopy = providers.getMetaAccess().lookupJavaMethod(method); 565 } 566 return originalArraycopy; 567 } 568 } 569 } |