1 /* 2 * Copyright (c) 2012, 2016, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 */ 23 package org.graalvm.compiler.hotspot.replacements; 24 25 import static jdk.vm.ci.hotspot.HotSpotJVMCIRuntimeProvider.getArrayBaseOffset; 26 import static jdk.vm.ci.hotspot.HotSpotMetaAccessProvider.computeArrayAllocationSize; 27 import static org.graalvm.compiler.core.common.GraalOptions.GeneratePIC; 28 import static org.graalvm.compiler.core.common.calc.UnsignedMath.belowThan; 29 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFIG; 30 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.CLASS_ARRAY_KLASS_LOCATION; 31 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.HUB_WRITE_LOCATION; 32 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.MARK_WORD_LOCATION; 33 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.PROTOTYPE_MARK_WORD_LOCATION; 34 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_END_LOCATION; 35 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_TOP_LOCATION; 36 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayKlassOffset; 37 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayLengthOffset; 38 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.config; 39 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.initializeObjectHeader; 40 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.instanceHeaderSize; 41 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.isInstanceKlassFullyInitialized; 42 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeMask; 43 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeShift; 44 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperLog2ElementSizeMask; 45 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperLog2ElementSizeShift; 46 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.loadKlassFromObject; 47 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.prototypeMarkWordOffset; 48 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readLayoutHelper; 49 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabEnd; 50 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabTop; 51 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.registerAsWord; 52 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useBiasedLocking; 53 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useTLAB; 54 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.verifyOop; 55 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.wordSize; 56 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.writeTlabTop; 57 import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.ProfileAllocations; 58 import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.ProfileAllocationsContext; 59 import static org.graalvm.compiler.nodes.PiArrayNode.piArrayCastToSnippetReplaceeStamp; 60 import static org.graalvm.compiler.nodes.PiNode.piCastToSnippetReplaceeStamp; 61 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FAST_PATH_PROBABILITY; 62 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FREQUENT_PROBABILITY; 63 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY; 64 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability; 65 import static org.graalvm.compiler.replacements.ReplacementsUtil.REPLACEMENTS_ASSERTIONS_ENABLED; 66 import static org.graalvm.compiler.replacements.ReplacementsUtil.staticAssert; 67 import static org.graalvm.compiler.replacements.SnippetTemplate.DEFAULT_REPLACER; 68 import static org.graalvm.compiler.replacements.nodes.CStringConstant.cstring; 69 import static org.graalvm.compiler.replacements.nodes.ExplodeLoopNode.explodeLoop; 70 71 import org.graalvm.compiler.api.replacements.Fold; 72 import org.graalvm.compiler.api.replacements.Snippet; 73 import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter; 74 import org.graalvm.compiler.api.replacements.Snippet.VarargsParameter; 75 import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor; 76 import org.graalvm.compiler.core.common.type.StampFactory; 77 import org.graalvm.compiler.debug.Debug; 78 import org.graalvm.compiler.debug.GraalError; 79 import org.graalvm.compiler.graph.Node.ConstantNodeParameter; 80 import org.graalvm.compiler.graph.Node.NodeIntrinsic; 81 import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig; 82 import org.graalvm.compiler.hotspot.HotSpotBackend; 83 import org.graalvm.compiler.hotspot.meta.HotSpotProviders; 84 import org.graalvm.compiler.hotspot.meta.HotSpotRegistersProvider; 85 import org.graalvm.compiler.hotspot.nodes.DimensionsNode; 86 import org.graalvm.compiler.hotspot.nodes.aot.LoadConstantIndirectlyFixedNode; 87 import org.graalvm.compiler.hotspot.nodes.aot.LoadConstantIndirectlyNode; 88 import org.graalvm.compiler.hotspot.nodes.type.KlassPointerStamp; 89 import org.graalvm.compiler.hotspot.replacements.aot.ResolveConstantSnippets; 90 import org.graalvm.compiler.hotspot.word.KlassPointer; 91 import org.graalvm.compiler.nodes.ConstantNode; 92 import org.graalvm.compiler.nodes.DeoptimizeNode; 93 import org.graalvm.compiler.nodes.PiNode; 94 import org.graalvm.compiler.nodes.PrefetchAllocateNode; 95 import org.graalvm.compiler.nodes.SnippetAnchorNode; 96 import org.graalvm.compiler.nodes.StructuredGraph; 97 import org.graalvm.compiler.nodes.ValueNode; 98 import org.graalvm.compiler.nodes.debug.DynamicCounterNode; 99 import org.graalvm.compiler.nodes.debug.VerifyHeapNode; 100 import org.graalvm.compiler.nodes.extended.ForeignCallNode; 101 import org.graalvm.compiler.nodes.extended.MembarNode; 102 import org.graalvm.compiler.nodes.java.DynamicNewArrayNode; 103 import org.graalvm.compiler.nodes.java.DynamicNewInstanceNode; 104 import org.graalvm.compiler.nodes.java.NewArrayNode; 105 import org.graalvm.compiler.nodes.java.NewInstanceNode; 106 import org.graalvm.compiler.nodes.java.NewMultiArrayNode; 107 import org.graalvm.compiler.nodes.memory.address.OffsetAddressNode; 108 import org.graalvm.compiler.nodes.spi.LoweringTool; 109 import org.graalvm.compiler.nodes.util.GraphUtil; 110 import org.graalvm.compiler.options.OptionValues; 111 import org.graalvm.compiler.replacements.ReplacementsUtil; 112 import org.graalvm.compiler.replacements.SnippetCounter; 113 import org.graalvm.compiler.replacements.SnippetCounter.Group; 114 import org.graalvm.compiler.replacements.SnippetTemplate; 115 import org.graalvm.compiler.replacements.SnippetTemplate.AbstractTemplates; 116 import org.graalvm.compiler.replacements.SnippetTemplate.Arguments; 117 import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo; 118 import org.graalvm.compiler.replacements.Snippets; 119 import org.graalvm.compiler.replacements.nodes.ExplodeLoopNode; 120 import org.graalvm.compiler.word.Word; 121 import org.graalvm.word.LocationIdentity; 122 import org.graalvm.word.WordFactory; 123 124 import jdk.vm.ci.code.CodeUtil; 125 import jdk.vm.ci.code.MemoryBarriers; 126 import jdk.vm.ci.code.Register; 127 import jdk.vm.ci.code.TargetDescription; 128 import jdk.vm.ci.hotspot.HotSpotJVMCIRuntimeProvider; 129 import jdk.vm.ci.hotspot.HotSpotResolvedObjectType; 130 import jdk.vm.ci.meta.DeoptimizationAction; 131 import jdk.vm.ci.meta.DeoptimizationReason; 132 import jdk.vm.ci.meta.JavaKind; 133 import jdk.vm.ci.meta.ResolvedJavaType; 134 135 /** 136 * Snippets used for implementing NEW, ANEWARRAY and NEWARRAY. 137 */ 138 public class NewObjectSnippets implements Snippets { 139 140 enum ProfileContext { 141 AllocatingMethod, 142 InstanceOrArray, 143 AllocatedType, 144 AllocatedTypesInMethod, 145 Total 146 } 147 148 @Fold 149 static String createName(String path, String typeContext, OptionValues options) { 150 switch (ProfileAllocationsContext.getValue(options)) { 151 case AllocatingMethod: 152 return ""; 153 case InstanceOrArray: 154 return path; 155 case AllocatedType: 156 case AllocatedTypesInMethod: 157 return typeContext; 158 case Total: 159 return "bytes"; 160 default: 161 throw GraalError.shouldNotReachHere(); 162 } 163 } 164 165 @Fold 166 static boolean doProfile(OptionValues options) { 167 return ProfileAllocations.getValue(options); 168 } 169 170 @Fold 171 static boolean withContext(OptionValues options) { 172 ProfileContext context = ProfileAllocationsContext.getValue(options); 173 return context == ProfileContext.AllocatingMethod || context == ProfileContext.AllocatedTypesInMethod; 174 } 175 176 protected static void profileAllocation(String path, long size, String typeContext, OptionValues options) { 177 if (doProfile(options)) { 178 String name = createName(path, typeContext, options); 179 180 boolean context = withContext(options); 181 DynamicCounterNode.counter(name, "number of bytes allocated", size, context); 182 DynamicCounterNode.counter(name, "number of allocations", 1, context); 183 } 184 } 185 186 public static void emitPrefetchAllocate(Word address, boolean isArray) { 187 GraalHotSpotVMConfig config = config(INJECTED_VMCONFIG); 188 if (config.allocatePrefetchStyle > 0) { 189 // Insert a prefetch for each allocation only on the fast-path 190 // Generate several prefetch instructions. 191 int lines = isArray ? config.allocatePrefetchLines : config.allocateInstancePrefetchLines; 192 int stepSize = config.allocatePrefetchStepSize; 193 int distance = config.allocatePrefetchDistance; 194 ExplodeLoopNode.explodeLoop(); 195 for (int i = 0; i < lines; i++) { 196 PrefetchAllocateNode.prefetch(OffsetAddressNode.address(address, distance)); 197 distance += stepSize; 198 } 199 } 200 } 201 202 @Snippet 203 public static Object allocateInstance(@ConstantParameter int size, KlassPointer hub, Word prototypeMarkWord, @ConstantParameter boolean fillContents, 204 @ConstantParameter Register threadRegister, @ConstantParameter boolean constantSize, @ConstantParameter String typeContext, @ConstantParameter OptionValues options, 205 @ConstantParameter Counters counters) { 206 return piCastToSnippetReplaceeStamp(allocateInstanceHelper(size, hub, prototypeMarkWord, fillContents, threadRegister, constantSize, typeContext, options, counters)); 207 } 208 209 public static Object allocateInstanceHelper(int size, KlassPointer hub, Word prototypeMarkWord, boolean fillContents, 210 Register threadRegister, boolean constantSize, String typeContext, OptionValues options, Counters counters) { 211 Object result; 212 Word thread = registerAsWord(threadRegister); 213 Word top = readTlabTop(thread); 214 Word end = readTlabEnd(thread); 215 Word newTop = top.add(size); 216 if (useTLAB(INJECTED_VMCONFIG) && probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) { 217 writeTlabTop(thread, newTop); 218 emitPrefetchAllocate(newTop, false); 219 result = formatObject(hub, size, top, prototypeMarkWord, fillContents, constantSize, counters); 220 } else { 221 if (counters != null && counters.stub != null) { 222 counters.stub.inc(); 223 } 224 result = newInstance(HotSpotBackend.NEW_INSTANCE, hub); 225 } 226 profileAllocation("instance", size, typeContext, options); 227 return verifyOop(result); 228 } 229 230 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true) 231 public static native Object newInstance(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub); 232 233 @Snippet 234 public static Object allocateInstancePIC(@ConstantParameter int size, KlassPointer hub, Word prototypeMarkWord, @ConstantParameter boolean fillContents, 235 @ConstantParameter Register threadRegister, @ConstantParameter boolean constantSize, @ConstantParameter String typeContext, @ConstantParameter OptionValues options, 236 @ConstantParameter Counters counters) { 237 // Klass must be initialized by the time the first instance is allocated, therefore we can 238 // just load it from the corresponding cell and avoid the resolution check. We have to use a 239 // fixed load though, to prevent it from floating above the initialization. 240 KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub); 241 return piCastToSnippetReplaceeStamp(allocateInstanceHelper(size, picHub, prototypeMarkWord, fillContents, threadRegister, constantSize, typeContext, options, counters)); 242 } 243 244 @Snippet 245 public static Object allocateInstanceDynamic(Class<?> type, Class<?> classClass, @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, 246 @ConstantParameter OptionValues options, @ConstantParameter Counters counters) { 247 if (probability(SLOW_PATH_PROBABILITY, type == null)) { 248 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); 249 } 250 Class<?> nonNullType = PiNode.piCastNonNullClass(type, SnippetAnchorNode.anchor()); 251 252 if (probability(SLOW_PATH_PROBABILITY, DynamicNewInstanceNode.throwsInstantiationException(type, classClass))) { 253 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); 254 } 255 256 return PiNode.piCastToSnippetReplaceeStamp(allocateInstanceDynamicHelper(type, fillContents, threadRegister, options, counters, nonNullType)); 257 } 258 259 private static Object allocateInstanceDynamicHelper(Class<?> type, boolean fillContents, Register threadRegister, OptionValues options, Counters counters, Class<?> nonNullType) { 260 KlassPointer hub = ClassGetHubNode.readClass(nonNullType); 261 if (probability(FAST_PATH_PROBABILITY, !hub.isNull())) { 262 KlassPointer nonNullHub = ClassGetHubNode.piCastNonNull(hub, SnippetAnchorNode.anchor()); 263 264 if (probability(FAST_PATH_PROBABILITY, isInstanceKlassFullyInitialized(nonNullHub))) { 265 int layoutHelper = readLayoutHelper(nonNullHub); 266 /* 267 * src/share/vm/oops/klass.hpp: For instances, layout helper is a positive number, 268 * the instance size. This size is already passed through align_object_size and 269 * scaled to bytes. The low order bit is set if instances of this class cannot be 270 * allocated using the fastpath. 271 */ 272 if (probability(FAST_PATH_PROBABILITY, (layoutHelper & 1) == 0)) { 273 Word prototypeMarkWord = nonNullHub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION); 274 /* 275 * FIXME(je,ds): we should actually pass typeContext instead of "" but late 276 * binding of parameters is not yet supported by the GraphBuilderPlugin system. 277 */ 278 return allocateInstanceHelper(layoutHelper, nonNullHub, prototypeMarkWord, fillContents, threadRegister, false, "", options, counters); 279 } 280 } 281 } 282 return dynamicNewInstanceStub(type); 283 } 284 285 /** 286 * Maximum array length for which fast path allocation is used. 287 */ 288 public static final int MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH = 0x00FFFFFF; 289 290 @Snippet 291 public static Object allocatePrimitiveArrayPIC(KlassPointer hub, int length, Word prototypeMarkWord, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize, 292 @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext, 293 @ConstantParameter OptionValues options, @ConstantParameter Counters counters) { 294 // Primitive array types are eagerly pre-resolved. We can use a floating load. 295 KlassPointer picHub = LoadConstantIndirectlyNode.loadKlass(hub); 296 return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false, options, counters); 297 } 298 299 @Snippet 300 public static Object allocateArrayPIC(KlassPointer hub, int length, Word prototypeMarkWord, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize, 301 @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext, 302 @ConstantParameter OptionValues options, @ConstantParameter Counters counters) { 303 // We need to resolve array type. While element type is guaranteed to be initialized, we 304 // cannot guarantee initialization of the array class if NewArrayInstance comes from 305 // canonicalization of DynamicNewArrayInstance. 306 KlassPointer picHub = ResolveConstantSnippets.pureInitializeKlass(hub); 307 return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false, options, counters); 308 } 309 310 @Snippet 311 public static Object allocateArray(KlassPointer hub, int length, Word prototypeMarkWord, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize, 312 @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext, 313 @ConstantParameter OptionValues options, @ConstantParameter Counters counters) { 314 Object result = allocateArrayImpl(hub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false, options, counters); 315 return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length); 316 } 317 318 private static Object allocateArrayImpl(KlassPointer hub, int length, Word prototypeMarkWord, int headerSize, int log2ElementSize, boolean fillContents, Register threadRegister, 319 boolean maybeUnroll, String typeContext, boolean skipNegativeCheck, OptionValues options, Counters counters) { 320 Object result; 321 int alignment = wordSize(); 322 int allocationSize = computeArrayAllocationSize(length, alignment, headerSize, log2ElementSize); 323 Word thread = registerAsWord(threadRegister); 324 Word top = readTlabTop(thread); 325 Word end = readTlabEnd(thread); 326 Word newTop = top.add(allocationSize); 327 if (probability(FREQUENT_PROBABILITY, skipNegativeCheck || belowThan(length, MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH)) && useTLAB(INJECTED_VMCONFIG) && 328 probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) { 329 writeTlabTop(thread, newTop); 330 emitPrefetchAllocate(newTop, true); 331 if (counters != null && counters.arrayLoopInit != null) { 332 counters.arrayLoopInit.inc(); 333 } 334 result = formatArray(hub, allocationSize, length, headerSize, top, prototypeMarkWord, fillContents, maybeUnroll, counters); 335 } else { 336 result = newArray(HotSpotBackend.NEW_ARRAY, hub, length, fillContents); 337 } 338 profileAllocation("array", allocationSize, typeContext, options); 339 return result; 340 } 341 342 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true) 343 public static native Object newArray(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int length, boolean fillContents); 344 345 public static final ForeignCallDescriptor DYNAMIC_NEW_ARRAY = new ForeignCallDescriptor("dynamic_new_array", Object.class, Class.class, int.class); 346 public static final ForeignCallDescriptor DYNAMIC_NEW_INSTANCE = new ForeignCallDescriptor("dynamic_new_instance", Object.class, Class.class); 347 348 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true) 349 public static native Object dynamicNewArrayStub(@ConstantNodeParameter ForeignCallDescriptor descriptor, Class<?> elementType, int length); 350 351 public static Object dynamicNewInstanceStub(Class<?> elementType) { 352 return dynamicNewInstanceStubCall(DYNAMIC_NEW_INSTANCE, elementType); 353 } 354 355 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true) 356 public static native Object dynamicNewInstanceStubCall(@ConstantNodeParameter ForeignCallDescriptor descriptor, Class<?> elementType); 357 358 @Snippet 359 public static Object allocateArrayDynamic(Class<?> elementType, Class<?> voidClass, int length, @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, 360 @ConstantParameter JavaKind knownElementKind, @ConstantParameter int knownLayoutHelper, Word prototypeMarkWord, @ConstantParameter OptionValues options, 361 @ConstantParameter Counters counters) { 362 Object result = allocateArrayDynamicImpl(elementType, voidClass, length, fillContents, threadRegister, knownElementKind, knownLayoutHelper, prototypeMarkWord, options, counters); 363 return result; 364 } 365 366 private static Object allocateArrayDynamicImpl(Class<?> elementType, Class<?> voidClass, int length, boolean fillContents, Register threadRegister, JavaKind knownElementKind, 367 int knownLayoutHelper, Word prototypeMarkWord, OptionValues options, Counters counters) { 368 /* 369 * We only need the dynamic check for void when we have no static information from 370 * knownElementKind. 371 */ 372 staticAssert(knownElementKind != JavaKind.Void, "unsupported knownElementKind"); 373 if (knownElementKind == JavaKind.Illegal && probability(SLOW_PATH_PROBABILITY, elementType == null || DynamicNewArrayNode.throwsIllegalArgumentException(elementType, voidClass))) { 374 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); 375 } 376 377 KlassPointer klass = loadKlassFromObject(elementType, arrayKlassOffset(INJECTED_VMCONFIG), CLASS_ARRAY_KLASS_LOCATION); 378 if (klass.isNull()) { 379 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); 380 } 381 KlassPointer nonNullKlass = ClassGetHubNode.piCastNonNull(klass, SnippetAnchorNode.anchor()); 382 383 if (length < 0) { 384 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); 385 } 386 int layoutHelper = knownElementKind != JavaKind.Illegal ? knownLayoutHelper : readLayoutHelper(nonNullKlass); 387 //@formatter:off 388 // from src/share/vm/oops/klass.hpp: 389 // 390 // For arrays, layout helper is a negative number, containing four 391 // distinct bytes, as follows: 392 // MSB:[tag, hsz, ebt, log2(esz)]:LSB 393 // where: 394 // tag is 0x80 if the elements are oops, 0xC0 if non-oops 395 // hsz is array header size in bytes (i.e., offset of first element) 396 // ebt is the BasicType of the elements 397 // esz is the element size in bytes 398 //@formatter:on 399 400 int headerSize = (layoutHelper >> layoutHelperHeaderSizeShift(INJECTED_VMCONFIG)) & layoutHelperHeaderSizeMask(INJECTED_VMCONFIG); 401 int log2ElementSize = (layoutHelper >> layoutHelperLog2ElementSizeShift(INJECTED_VMCONFIG)) & layoutHelperLog2ElementSizeMask(INJECTED_VMCONFIG); 402 403 Object result = allocateArrayImpl(nonNullKlass, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, false, "dynamic type", true, options, counters); 404 return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length); 405 } 406 407 /** 408 * Calls the runtime stub for implementing MULTIANEWARRAY. 409 */ 410 @Snippet 411 public static Object newmultiarray(KlassPointer hub, @ConstantParameter int rank, @VarargsParameter int[] dimensions) { 412 Word dims = DimensionsNode.allocaDimsArray(rank); 413 ExplodeLoopNode.explodeLoop(); 414 for (int i = 0; i < rank; i++) { 415 dims.writeInt(i * 4, dimensions[i], LocationIdentity.init()); 416 } 417 return newArrayCall(HotSpotBackend.NEW_MULTI_ARRAY, hub, rank, dims); 418 } 419 420 @Snippet 421 public static Object newmultiarrayPIC(KlassPointer hub, @ConstantParameter int rank, @VarargsParameter int[] dimensions) { 422 KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub); 423 return newmultiarray(picHub, rank, dimensions); 424 } 425 426 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true) 427 public static native Object newArrayCall(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int rank, Word dims); 428 429 /** 430 * Maximum number of long stores to emit when zeroing an object with a constant size. Larger 431 * objects have their bodies initialized in a loop. 432 */ 433 private static final int MAX_UNROLLED_OBJECT_ZEROING_STORES = 8; 434 435 /** 436 * Zero uninitialized memory in a newly allocated object, unrolling as necessary and ensuring 437 * that stores are aligned. 438 * 439 * @param size number of bytes to zero 440 * @param memory beginning of object which is being zeroed 441 * @param constantSize is {@code size} known to be constant in the snippet 442 * @param startOffset offset to begin zeroing. May not be word aligned. 443 * @param manualUnroll maximally unroll zeroing 444 */ 445 private static void zeroMemory(int size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) { 446 fillMemory(0, size, memory, constantSize, startOffset, manualUnroll, counters); 447 } 448 449 private static void fillMemory(long value, int size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) { 450 ReplacementsUtil.runtimeAssert((size & 0x7) == 0, "unaligned object size"); 451 int offset = startOffset; 452 if ((offset & 0x7) != 0) { 453 memory.writeInt(offset, (int) value, LocationIdentity.init()); 454 offset += 4; 455 } 456 ReplacementsUtil.runtimeAssert((offset & 0x7) == 0, "unaligned offset"); 457 if (manualUnroll && ((size - offset) / 8) <= MAX_UNROLLED_OBJECT_ZEROING_STORES) { 458 ReplacementsUtil.staticAssert(!constantSize, "size shouldn't be constant at instantiation time"); 459 // This case handles arrays of constant length. Instead of having a snippet variant for 460 // each length, generate a chain of stores of maximum length. Once it's inlined the 461 // break statement will trim excess stores. 462 if (counters != null && counters.instanceSeqInit != null) { 463 counters.instanceSeqInit.inc(); 464 } 465 466 explodeLoop(); 467 for (int i = 0; i < MAX_UNROLLED_OBJECT_ZEROING_STORES; i++, offset += 8) { 468 if (offset == size) { 469 break; 470 } 471 memory.initializeLong(offset, value, LocationIdentity.init()); 472 } 473 } else { 474 // Use Word instead of int to avoid extension to long in generated code 475 Word off = WordFactory.signed(offset); 476 if (constantSize && ((size - offset) / 8) <= MAX_UNROLLED_OBJECT_ZEROING_STORES) { 477 if (counters != null && counters.instanceSeqInit != null) { 478 counters.instanceSeqInit.inc(); 479 } 480 explodeLoop(); 481 } else { 482 if (counters != null && counters.instanceLoopInit != null) { 483 counters.instanceLoopInit.inc(); 484 } 485 } 486 for (; off.rawValue() < size; off = off.add(8)) { 487 memory.initializeLong(off, value, LocationIdentity.init()); 488 } 489 } 490 } 491 492 /** 493 * Fill uninitialized memory with garbage value in a newly allocated object, unrolling as 494 * necessary and ensuring that stores are aligned. 495 * 496 * @param size number of bytes to zero 497 * @param memory beginning of object which is being zeroed 498 * @param constantSize is {@code size} known to be constant in the snippet 499 * @param startOffset offset to begin zeroing. May not be word aligned. 500 * @param manualUnroll maximally unroll zeroing 501 */ 502 private static void fillWithGarbage(int size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) { 503 fillMemory(0xfefefefefefefefeL, size, memory, constantSize, startOffset, manualUnroll, counters); 504 } 505 506 /** 507 * Formats some allocated memory with an object header and zeroes out the rest. Disables asserts 508 * since they can't be compiled in stubs. 509 */ 510 public static Object formatObjectForStub(KlassPointer hub, int size, Word memory, Word compileTimePrototypeMarkWord) { 511 return formatObject(hub, size, memory, compileTimePrototypeMarkWord, true, false, null); 512 } 513 514 /** 515 * Formats some allocated memory with an object header and zeroes out the rest. 516 */ 517 protected static Object formatObject(KlassPointer hub, int size, Word memory, Word compileTimePrototypeMarkWord, boolean fillContents, boolean constantSize, Counters counters) { 518 Word prototypeMarkWord = useBiasedLocking(INJECTED_VMCONFIG) ? hub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION) : compileTimePrototypeMarkWord; 519 initializeObjectHeader(memory, prototypeMarkWord, hub); 520 if (fillContents) { 521 zeroMemory(size, memory, constantSize, instanceHeaderSize(INJECTED_VMCONFIG), false, counters); 522 } else if (REPLACEMENTS_ASSERTIONS_ENABLED) { 523 fillWithGarbage(size, memory, constantSize, instanceHeaderSize(INJECTED_VMCONFIG), false, counters); 524 } 525 MembarNode.memoryBarrier(MemoryBarriers.STORE_STORE, LocationIdentity.init()); 526 return memory.toObjectNonNull(); 527 } 528 529 @Snippet 530 protected static void verifyHeap(@ConstantParameter Register threadRegister) { 531 Word thread = registerAsWord(threadRegister); 532 Word topValue = readTlabTop(thread); 533 if (!topValue.equal(WordFactory.zero())) { 534 Word topValueContents = topValue.readWord(0, MARK_WORD_LOCATION); 535 if (topValueContents.equal(WordFactory.zero())) { 536 AssertionSnippets.vmMessageC(AssertionSnippets.ASSERTION_VM_MESSAGE_C, true, cstring("overzeroing of TLAB detected"), 0L, 0L, 0L); 537 } 538 } 539 } 540 541 /** 542 * Formats some allocated memory with an object header and zeroes out the rest. 543 */ 544 public static Object formatArray(KlassPointer hub, int allocationSize, int length, int headerSize, Word memory, Word prototypeMarkWord, boolean fillContents, boolean maybeUnroll, 545 Counters counters) { 546 memory.writeInt(arrayLengthOffset(INJECTED_VMCONFIG), length, LocationIdentity.init()); 547 /* 548 * store hub last as the concurrent garbage collectors assume length is valid if hub field 549 * is not null 550 */ 551 initializeObjectHeader(memory, prototypeMarkWord, hub); 552 if (fillContents) { 553 zeroMemory(allocationSize, memory, false, headerSize, maybeUnroll, counters); 554 } else if (REPLACEMENTS_ASSERTIONS_ENABLED) { 555 fillWithGarbage(allocationSize, memory, false, headerSize, maybeUnroll, counters); 556 } 557 MembarNode.memoryBarrier(MemoryBarriers.STORE_STORE, LocationIdentity.init()); 558 return memory.toObjectNonNull(); 559 } 560 561 static class Counters { 562 Counters(SnippetCounter.Group.Factory factory) { 563 Group newInstance = factory.createSnippetCounterGroup("NewInstance"); 564 Group newArray = factory.createSnippetCounterGroup("NewArray"); 565 instanceSeqInit = new SnippetCounter(newInstance, "tlabSeqInit", "TLAB alloc with unrolled zeroing"); 566 instanceLoopInit = new SnippetCounter(newInstance, "tlabLoopInit", "TLAB alloc with zeroing in a loop"); 567 arrayLoopInit = new SnippetCounter(newArray, "tlabLoopInit", "TLAB alloc with zeroing in a loop"); 568 stub = new SnippetCounter(newInstance, "stub", "alloc and zeroing via stub"); 569 } 570 571 final SnippetCounter instanceSeqInit; 572 final SnippetCounter instanceLoopInit; 573 final SnippetCounter arrayLoopInit; 574 final SnippetCounter stub; 575 } 576 577 public static class Templates extends AbstractTemplates { 578 579 private final SnippetInfo allocateInstance = snippet(NewObjectSnippets.class, "allocateInstance", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION); 580 private final SnippetInfo allocateInstancePIC = snippet(NewObjectSnippets.class, "allocateInstancePIC", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, 581 TLAB_END_LOCATION); 582 private final SnippetInfo allocateArray = snippet(NewObjectSnippets.class, "allocateArray", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION); 583 private final SnippetInfo allocateArrayPIC = snippet(NewObjectSnippets.class, "allocateArrayPIC", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION); 584 private final SnippetInfo allocatePrimitiveArrayPIC = snippet(NewObjectSnippets.class, "allocatePrimitiveArrayPIC", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, 585 TLAB_END_LOCATION); 586 private final SnippetInfo allocateArrayDynamic = snippet(NewObjectSnippets.class, "allocateArrayDynamic", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, 587 TLAB_END_LOCATION); 588 private final SnippetInfo allocateInstanceDynamic = snippet(NewObjectSnippets.class, "allocateInstanceDynamic", MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, 589 TLAB_END_LOCATION); 590 private final SnippetInfo newmultiarray = snippet(NewObjectSnippets.class, "newmultiarray", TLAB_TOP_LOCATION, TLAB_END_LOCATION); 591 private final SnippetInfo newmultiarrayPIC = snippet(NewObjectSnippets.class, "newmultiarrayPIC", TLAB_TOP_LOCATION, TLAB_END_LOCATION); 592 private final SnippetInfo verifyHeap = snippet(NewObjectSnippets.class, "verifyHeap"); 593 private final GraalHotSpotVMConfig config; 594 private final Counters counters; 595 596 public Templates(OptionValues options, SnippetCounter.Group.Factory factory, HotSpotProviders providers, TargetDescription target, GraalHotSpotVMConfig config) { 597 super(options, providers, providers.getSnippetReflection(), target); 598 this.config = config; 599 counters = new Counters(factory); 600 } 601 602 /** 603 * Lowers a {@link NewInstanceNode}. 604 */ 605 public void lower(NewInstanceNode newInstanceNode, HotSpotRegistersProvider registers, LoweringTool tool) { 606 StructuredGraph graph = newInstanceNode.graph(); 607 HotSpotResolvedObjectType type = (HotSpotResolvedObjectType) newInstanceNode.instanceClass(); 608 assert !type.isArray(); 609 ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), type.klass(), providers.getMetaAccess(), graph); 610 int size = instanceSize(type); 611 612 OptionValues localOptions = graph.getOptions(); 613 SnippetInfo snippet = GeneratePIC.getValue(localOptions) ? allocateInstancePIC : allocateInstance; 614 Arguments args = new Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage()); 615 args.addConst("size", size); 616 args.add("hub", hub); 617 args.add("prototypeMarkWord", type.prototypeMarkWord()); 618 args.addConst("fillContents", newInstanceNode.fillContents()); 619 args.addConst("threadRegister", registers.getThreadRegister()); 620 args.addConst("constantSize", true); 621 args.addConst("typeContext", ProfileAllocations.getValue(localOptions) ? type.toJavaName(false) : ""); 622 args.addConst("options", localOptions); 623 args.addConst("counters", counters); 624 625 SnippetTemplate template = template(args); 626 Debug.log("Lowering allocateInstance in %s: node=%s, template=%s, arguments=%s", graph, newInstanceNode, template, args); 627 template.instantiate(providers.getMetaAccess(), newInstanceNode, DEFAULT_REPLACER, args); 628 } 629 630 /** 631 * Lowers a {@link NewArrayNode}. 632 */ 633 public void lower(NewArrayNode newArrayNode, HotSpotRegistersProvider registers, LoweringTool tool) { 634 StructuredGraph graph = newArrayNode.graph(); 635 ResolvedJavaType elementType = newArrayNode.elementType(); 636 HotSpotResolvedObjectType arrayType = (HotSpotResolvedObjectType) elementType.getArrayClass(); 637 JavaKind elementKind = elementType.getJavaKind(); 638 ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), arrayType.klass(), providers.getMetaAccess(), graph); 639 final int headerSize = getArrayBaseOffset(elementKind); 640 int log2ElementSize = CodeUtil.log2(HotSpotJVMCIRuntimeProvider.getArrayIndexScale(elementKind)); 641 642 OptionValues localOptions = graph.getOptions(); 643 SnippetInfo snippet; 644 if (GeneratePIC.getValue(localOptions)) { 645 if (elementType.isPrimitive()) { 646 snippet = allocatePrimitiveArrayPIC; 647 } else { 648 snippet = allocateArrayPIC; 649 } 650 } else { 651 snippet = allocateArray; 652 } 653 654 Arguments args = new Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage()); 655 args.add("hub", hub); 656 ValueNode length = newArrayNode.length(); 657 args.add("length", length.isAlive() ? length : graph.addOrUniqueWithInputs(length)); 658 assert arrayType.prototypeMarkWord() == lookupArrayClass(tool, JavaKind.Object).prototypeMarkWord() : "all array types are assumed to have the same prototypeMarkWord"; 659 args.add("prototypeMarkWord", arrayType.prototypeMarkWord()); 660 args.addConst("headerSize", headerSize); 661 args.addConst("log2ElementSize", log2ElementSize); 662 args.addConst("fillContents", newArrayNode.fillContents()); 663 args.addConst("threadRegister", registers.getThreadRegister()); 664 args.addConst("maybeUnroll", length.isConstant()); 665 args.addConst("typeContext", ProfileAllocations.getValue(localOptions) ? arrayType.toJavaName(false) : ""); 666 args.addConst("options", localOptions); 667 args.addConst("counters", counters); 668 SnippetTemplate template = template(args); 669 Debug.log("Lowering allocateArray in %s: node=%s, template=%s, arguments=%s", graph, newArrayNode, template, args); 670 template.instantiate(providers.getMetaAccess(), newArrayNode, DEFAULT_REPLACER, args); 671 } 672 673 public void lower(DynamicNewInstanceNode newInstanceNode, HotSpotRegistersProvider registers, LoweringTool tool) { 674 Arguments args = new Arguments(allocateInstanceDynamic, newInstanceNode.graph().getGuardsStage(), tool.getLoweringStage()); 675 OptionValues localOptions = newInstanceNode.getOptions(); 676 args.add("type", newInstanceNode.getInstanceType()); 677 ValueNode classClass = newInstanceNode.getClassClass(); 678 assert classClass != null; 679 args.add("classClass", classClass); 680 args.addConst("fillContents", newInstanceNode.fillContents()); 681 args.addConst("threadRegister", registers.getThreadRegister()); 682 args.addConst("options", localOptions); 683 args.addConst("counters", counters); 684 685 SnippetTemplate template = template(args); 686 template.instantiate(providers.getMetaAccess(), newInstanceNode, DEFAULT_REPLACER, args); 687 } 688 689 public void lower(DynamicNewArrayNode newArrayNode, HotSpotRegistersProvider registers, LoweringTool tool) { 690 StructuredGraph graph = newArrayNode.graph(); 691 OptionValues localOptions = graph.getOptions(); 692 Arguments args = new Arguments(allocateArrayDynamic, newArrayNode.graph().getGuardsStage(), tool.getLoweringStage()); 693 args.add("elementType", newArrayNode.getElementType()); 694 ValueNode voidClass = newArrayNode.getVoidClass(); 695 assert voidClass != null; 696 args.add("voidClass", voidClass); 697 ValueNode length = newArrayNode.length(); 698 args.add("length", length.isAlive() ? length : graph.addOrUniqueWithInputs(length)); 699 args.addConst("fillContents", newArrayNode.fillContents()); 700 args.addConst("threadRegister", registers.getThreadRegister()); 701 /* 702 * We use Kind.Illegal as a marker value instead of null because constant snippet 703 * parameters cannot be null. 704 */ 705 args.addConst("knownElementKind", newArrayNode.getKnownElementKind() == null ? JavaKind.Illegal : newArrayNode.getKnownElementKind()); 706 if (newArrayNode.getKnownElementKind() != null) { 707 args.addConst("knownLayoutHelper", lookupArrayClass(tool, newArrayNode.getKnownElementKind()).layoutHelper()); 708 } else { 709 args.addConst("knownLayoutHelper", 0); 710 } 711 args.add("prototypeMarkWord", lookupArrayClass(tool, JavaKind.Object).prototypeMarkWord()); 712 args.addConst("options", localOptions); 713 args.addConst("counters", counters); 714 SnippetTemplate template = template(args); 715 template.instantiate(providers.getMetaAccess(), newArrayNode, DEFAULT_REPLACER, args); 716 } 717 718 private static HotSpotResolvedObjectType lookupArrayClass(LoweringTool tool, JavaKind kind) { 719 return (HotSpotResolvedObjectType) tool.getMetaAccess().lookupJavaType(kind == JavaKind.Object ? Object.class : kind.toJavaClass()).getArrayClass(); 720 } 721 722 public void lower(NewMultiArrayNode newmultiarrayNode, LoweringTool tool) { 723 StructuredGraph graph = newmultiarrayNode.graph(); 724 OptionValues localOptions = graph.getOptions(); 725 int rank = newmultiarrayNode.dimensionCount(); 726 ValueNode[] dims = new ValueNode[rank]; 727 for (int i = 0; i < newmultiarrayNode.dimensionCount(); i++) { 728 dims[i] = newmultiarrayNode.dimension(i); 729 } 730 HotSpotResolvedObjectType type = (HotSpotResolvedObjectType) newmultiarrayNode.type(); 731 ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), type.klass(), providers.getMetaAccess(), graph); 732 733 SnippetInfo snippet = GeneratePIC.getValue(localOptions) ? newmultiarrayPIC : newmultiarray; 734 Arguments args = new Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage()); 735 args.add("hub", hub); 736 args.addConst("rank", rank); 737 args.addVarargs("dimensions", int.class, StampFactory.forKind(JavaKind.Int), dims); 738 template(args).instantiate(providers.getMetaAccess(), newmultiarrayNode, DEFAULT_REPLACER, args); 739 } 740 741 private static int instanceSize(HotSpotResolvedObjectType type) { 742 int size = type.instanceSize(); 743 assert size >= 0; 744 return size; 745 } 746 747 public void lower(VerifyHeapNode verifyHeapNode, HotSpotRegistersProvider registers, LoweringTool tool) { 748 if (config.cAssertions) { 749 Arguments args = new Arguments(verifyHeap, verifyHeapNode.graph().getGuardsStage(), tool.getLoweringStage()); 750 args.addConst("threadRegister", registers.getThreadRegister()); 751 752 SnippetTemplate template = template(args); 753 template.instantiate(providers.getMetaAccess(), verifyHeapNode, DEFAULT_REPLACER, args); 754 } else { 755 GraphUtil.removeFixedWithUnusedInputs(verifyHeapNode); 756 } 757 } 758 } 759 }