1 /* 2 * Copyright (c) 2012, 2016, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 */ 23 package org.graalvm.compiler.hotspot.replacements; 24 25 import static org.graalvm.compiler.core.common.GraalOptions.GeneratePIC; 26 import static org.graalvm.compiler.core.common.GraalOptions.SnippetCounters; 27 import static org.graalvm.compiler.core.common.calc.UnsignedMath.belowThan; 28 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFIG; 29 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.CLASS_ARRAY_KLASS_LOCATION; 30 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.HUB_WRITE_LOCATION; 31 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.MARK_WORD_LOCATION; 32 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.PROTOTYPE_MARK_WORD_LOCATION; 33 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_END_LOCATION; 34 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_TOP_LOCATION; 35 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayKlassOffset; 36 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayLengthOffset; 37 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.config; 38 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.initializeObjectHeader; 39 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.instanceHeaderSize; 40 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.isInstanceKlassFullyInitialized; 41 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeMask; 42 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeShift; 43 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperLog2ElementSizeMask; 44 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperLog2ElementSizeShift; 45 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.loadKlassFromObject; 46 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.prototypeMarkWordOffset; 47 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readLayoutHelper; 48 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabEnd; 49 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabTop; 50 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.registerAsWord; 51 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useBiasedLocking; 52 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useTLAB; 53 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.verifyOop; 54 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.wordSize; 55 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.writeTlabTop; 56 import static org.graalvm.compiler.nodes.PiArrayNode.piArrayCast; 57 import static org.graalvm.compiler.nodes.PiNode.piCast; 58 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FAST_PATH_PROBABILITY; 59 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FREQUENT_PROBABILITY; 60 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY; 61 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability; 62 import static org.graalvm.compiler.replacements.ReplacementsUtil.REPLACEMENTS_ASSERTIONS_ENABLED; 63 import static org.graalvm.compiler.replacements.ReplacementsUtil.staticAssert; 64 import static org.graalvm.compiler.replacements.SnippetTemplate.DEFAULT_REPLACER; 65 import static org.graalvm.compiler.replacements.nodes.CStringConstant.cstring; 66 import static org.graalvm.compiler.replacements.nodes.ExplodeLoopNode.explodeLoop; 67 import static jdk.vm.ci.hotspot.HotSpotJVMCIRuntimeProvider.getArrayBaseOffset; 68 import static jdk.vm.ci.hotspot.HotSpotMetaAccessProvider.computeArrayAllocationSize; 69 70 import org.graalvm.compiler.api.replacements.Fold; 71 import org.graalvm.compiler.api.replacements.Snippet; 72 import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter; 73 import org.graalvm.compiler.api.replacements.Snippet.VarargsParameter; 74 import org.graalvm.compiler.core.common.LocationIdentity; 75 import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor; 76 import org.graalvm.compiler.core.common.type.StampFactory; 77 import org.graalvm.compiler.debug.Debug; 78 import org.graalvm.compiler.debug.GraalError; 79 import org.graalvm.compiler.graph.Node.ConstantNodeParameter; 80 import org.graalvm.compiler.graph.Node.NodeIntrinsic; 81 import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig; 82 import org.graalvm.compiler.hotspot.HotSpotBackend; 83 import org.graalvm.compiler.hotspot.meta.HotSpotProviders; 84 import org.graalvm.compiler.hotspot.meta.HotSpotRegistersProvider; 85 import org.graalvm.compiler.hotspot.nodes.DimensionsNode; 86 import org.graalvm.compiler.hotspot.nodes.aot.LoadConstantIndirectlyFixedNode; 87 import org.graalvm.compiler.hotspot.nodes.aot.LoadConstantIndirectlyNode; 88 import org.graalvm.compiler.hotspot.nodes.type.KlassPointerStamp; 89 import org.graalvm.compiler.hotspot.replacements.aot.ResolveConstantSnippets; 90 import org.graalvm.compiler.hotspot.word.KlassPointer; 91 import org.graalvm.compiler.nodes.ConstantNode; 92 import org.graalvm.compiler.nodes.DeoptimizeNode; 93 import org.graalvm.compiler.nodes.NamedLocationIdentity; 94 import org.graalvm.compiler.nodes.PrefetchAllocateNode; 95 import org.graalvm.compiler.nodes.StructuredGraph; 96 import org.graalvm.compiler.nodes.ValueNode; 97 import org.graalvm.compiler.nodes.debug.DynamicCounterNode; 98 import org.graalvm.compiler.nodes.debug.VerifyHeapNode; 99 import org.graalvm.compiler.nodes.extended.BranchProbabilityNode; 100 import org.graalvm.compiler.nodes.extended.ForeignCallNode; 101 import org.graalvm.compiler.nodes.extended.MembarNode; 102 import org.graalvm.compiler.nodes.java.DynamicNewArrayNode; 103 import org.graalvm.compiler.nodes.java.DynamicNewInstanceNode; 104 import org.graalvm.compiler.nodes.java.NewArrayNode; 105 import org.graalvm.compiler.nodes.java.NewInstanceNode; 106 import org.graalvm.compiler.nodes.java.NewMultiArrayNode; 107 import org.graalvm.compiler.nodes.memory.address.OffsetAddressNode; 108 import org.graalvm.compiler.nodes.spi.LoweringTool; 109 import org.graalvm.compiler.nodes.util.GraphUtil; 110 import org.graalvm.compiler.replacements.ReplacementsUtil; 111 import org.graalvm.compiler.replacements.SnippetCounter; 112 import org.graalvm.compiler.replacements.SnippetTemplate; 113 import org.graalvm.compiler.replacements.SnippetTemplate.AbstractTemplates; 114 import org.graalvm.compiler.replacements.SnippetTemplate.Arguments; 115 import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo; 116 import org.graalvm.compiler.replacements.Snippets; 117 import org.graalvm.compiler.replacements.nodes.ExplodeLoopNode; 118 import org.graalvm.compiler.word.Word; 119 120 import jdk.vm.ci.code.CodeUtil; 121 import jdk.vm.ci.code.MemoryBarriers; 122 import jdk.vm.ci.code.Register; 123 import jdk.vm.ci.code.TargetDescription; 124 import jdk.vm.ci.hotspot.HotSpotJVMCIRuntimeProvider; 125 import jdk.vm.ci.hotspot.HotSpotResolvedObjectType; 126 import jdk.vm.ci.meta.DeoptimizationAction; 127 import jdk.vm.ci.meta.DeoptimizationReason; 128 import jdk.vm.ci.meta.JavaKind; 129 import jdk.vm.ci.meta.ResolvedJavaType; 130 131 /** 132 * Snippets used for implementing NEW, ANEWARRAY and NEWARRAY. 133 */ 134 public class NewObjectSnippets implements Snippets { 135 136 public static final LocationIdentity INIT_LOCATION = NamedLocationIdentity.mutable("Initialization"); 137 138 enum ProfileContext { 139 AllocatingMethod, 140 InstanceOrArray, 141 AllocatedType, 142 AllocatedTypesInMethod, 143 Total 144 } 145 146 @Fold 147 static String createName(String path, String typeContext) { 148 switch (HotspotSnippetsOptions.ProfileAllocationsContext.getValue()) { 149 case AllocatingMethod: 150 return ""; 151 case InstanceOrArray: 152 return path; 153 case AllocatedType: 154 case AllocatedTypesInMethod: 155 return typeContext; 156 case Total: 157 return "bytes"; 158 default: 159 throw GraalError.shouldNotReachHere(); 160 } 161 } 162 163 @Fold 164 static boolean doProfile() { 165 return HotspotSnippetsOptions.ProfileAllocations.getValue(); 166 } 167 168 @Fold 169 static boolean withContext() { 170 ProfileContext context = HotspotSnippetsOptions.ProfileAllocationsContext.getValue(); 171 return context == ProfileContext.AllocatingMethod || context == ProfileContext.AllocatedTypesInMethod; 172 } 173 174 protected static void profileAllocation(String path, long size, String typeContext) { 175 if (doProfile()) { 176 String name = createName(path, typeContext); 177 178 boolean context = withContext(); 179 DynamicCounterNode.counter(name, "number of bytes allocated", size, context); 180 DynamicCounterNode.counter(name, "number of allocations", 1, context); 181 } 182 } 183 184 public static void emitPrefetchAllocate(Word address, boolean isArray) { 185 GraalHotSpotVMConfig config = config(INJECTED_VMCONFIG); 186 if (config.allocatePrefetchStyle > 0) { 187 // Insert a prefetch for each allocation only on the fast-path 188 // Generate several prefetch instructions. 189 int lines = isArray ? config.allocatePrefetchLines : config.allocateInstancePrefetchLines; 190 int stepSize = config.allocatePrefetchStepSize; 191 int distance = config.allocatePrefetchDistance; 192 ExplodeLoopNode.explodeLoop(); 193 for (int i = 0; i < lines; i++) { 194 PrefetchAllocateNode.prefetch(OffsetAddressNode.address(address, distance)); 195 distance += stepSize; 196 } 197 } 198 } 199 200 @Snippet 201 public static Object allocateInstance(@ConstantParameter int size, KlassPointer hub, Word prototypeMarkWord, @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, 202 @ConstantParameter boolean constantSize, @ConstantParameter String typeContext) { 203 Object result; 204 Word thread = registerAsWord(threadRegister); 205 Word top = readTlabTop(thread); 206 Word end = readTlabEnd(thread); 207 Word newTop = top.add(size); 208 if (useTLAB(INJECTED_VMCONFIG) && probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) { 209 writeTlabTop(thread, newTop); 210 emitPrefetchAllocate(newTop, false); 211 result = formatObject(hub, size, top, prototypeMarkWord, fillContents, constantSize, true); 212 } else { 213 new_stub.inc(); 214 result = newInstance(HotSpotBackend.NEW_INSTANCE, hub); 215 } 216 profileAllocation("instance", size, typeContext); 217 return piCast(verifyOop(result), StampFactory.forNodeIntrinsic()); 218 } 219 220 @NodeIntrinsic(value = ForeignCallNode.class, returnStampIsNonNull = true) 221 public static native Object newInstance(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub); 222 223 @Snippet 224 public static Object allocateInstancePIC(@ConstantParameter int size, KlassPointer hub, Word prototypeMarkWord, @ConstantParameter boolean fillContents, 225 @ConstantParameter Register threadRegister, 226 @ConstantParameter boolean constantSize, @ConstantParameter String typeContext) { 227 // Klass must be initialized by the time the first instance is allocated, therefore we can 228 // just load it from the corresponding cell and avoid the resolution check. We have to use a 229 // fixed load though, to prevent it from floating above the initialization. 230 KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub); 231 return allocateInstance(size, picHub, prototypeMarkWord, fillContents, threadRegister, constantSize, typeContext); 232 } 233 234 @Snippet 235 public static Object allocateInstanceDynamic(Class<?> type, Class<?> classClass, @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister) { 236 if (probability(SLOW_PATH_PROBABILITY, type == null || DynamicNewInstanceNode.throwsInstantiationException(type, classClass))) { 237 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); 238 } 239 240 KlassPointer hub = ClassGetHubNode.readClass(type); 241 if (probability(FAST_PATH_PROBABILITY, !hub.isNull())) { 242 if (probability(FAST_PATH_PROBABILITY, isInstanceKlassFullyInitialized(hub))) { 243 int layoutHelper = readLayoutHelper(hub); 244 /* 245 * src/share/vm/oops/klass.hpp: For instances, layout helper is a positive number, 246 * the instance size. This size is already passed through align_object_size and 247 * scaled to bytes. The low order bit is set if instances of this class cannot be 248 * allocated using the fastpath. 249 */ 250 if (probability(FAST_PATH_PROBABILITY, (layoutHelper & 1) == 0)) { 251 Word prototypeMarkWord = hub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION); 252 /* 253 * FIXME(je,ds): we should actually pass typeContext instead of "" but late 254 * binding of parameters is not yet supported by the GraphBuilderPlugin system. 255 */ 256 return allocateInstance(layoutHelper, hub, prototypeMarkWord, fillContents, threadRegister, false, ""); 257 } 258 } 259 } 260 return dynamicNewInstanceStub(type); 261 } 262 263 /** 264 * Maximum array length for which fast path allocation is used. 265 */ 266 public static final int MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH = 0x00FFFFFF; 267 268 @Snippet 269 public static Object allocatePrimitiveArrayPIC(KlassPointer hub, int length, Word prototypeMarkWord, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize, 270 @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext) { 271 KlassPointer picHub = LoadConstantIndirectlyNode.loadKlass(hub); 272 return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false); 273 } 274 275 @Snippet 276 public static Object allocateArrayPIC(KlassPointer hub, int length, Word prototypeMarkWord, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize, 277 @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext) { 278 KlassPointer picHub = ResolveConstantSnippets.resolveKlassConstant(hub); 279 return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false); 280 } 281 282 @Snippet 283 public static Object allocateArray(KlassPointer hub, int length, Word prototypeMarkWord, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize, 284 @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext) { 285 Object result = allocateArrayImpl(hub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false); 286 return piArrayCast(verifyOop(result), length, StampFactory.forNodeIntrinsic()); 287 } 288 289 private static Object allocateArrayImpl(KlassPointer hub, int length, Word prototypeMarkWord, int headerSize, int log2ElementSize, boolean fillContents, 290 @ConstantParameter Register threadRegister, 291 @ConstantParameter boolean maybeUnroll, String typeContext, boolean skipNegativeCheck) { 292 Object result; 293 int alignment = wordSize(); 294 int allocationSize = computeArrayAllocationSize(length, alignment, headerSize, log2ElementSize); 295 Word thread = registerAsWord(threadRegister); 296 Word top = readTlabTop(thread); 297 Word end = readTlabEnd(thread); 298 Word newTop = top.add(allocationSize); 299 if (probability(FREQUENT_PROBABILITY, skipNegativeCheck || belowThan(length, MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH)) && useTLAB(INJECTED_VMCONFIG) && 300 probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) { 301 writeTlabTop(thread, newTop); 302 emitPrefetchAllocate(newTop, true); 303 newarray_loopInit.inc(); 304 result = formatArray(hub, allocationSize, length, headerSize, top, prototypeMarkWord, fillContents, maybeUnroll, true); 305 } else { 306 result = newArray(HotSpotBackend.NEW_ARRAY, hub, length, fillContents); 307 } 308 profileAllocation("array", allocationSize, typeContext); 309 return result; 310 } 311 312 @NodeIntrinsic(value = ForeignCallNode.class, returnStampIsNonNull = true) 313 public static native Object newArray(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int length, boolean fillContents); 314 315 public static final ForeignCallDescriptor DYNAMIC_NEW_ARRAY = new ForeignCallDescriptor("dynamic_new_array", Object.class, Class.class, int.class); 316 public static final ForeignCallDescriptor DYNAMIC_NEW_INSTANCE = new ForeignCallDescriptor("dynamic_new_instance", Object.class, Class.class); 317 318 @NodeIntrinsic(value = ForeignCallNode.class, returnStampIsNonNull = true) 319 public static native Object dynamicNewArrayStub(@ConstantNodeParameter ForeignCallDescriptor descriptor, Class<?> elementType, int length); 320 321 public static Object dynamicNewInstanceStub(Class<?> elementType) { 322 return dynamicNewInstanceStubCall(DYNAMIC_NEW_INSTANCE, elementType); 323 } 324 325 @NodeIntrinsic(value = ForeignCallNode.class, returnStampIsNonNull = true) 326 public static native Object dynamicNewInstanceStubCall(@ConstantNodeParameter ForeignCallDescriptor descriptor, Class<?> elementType); 327 328 @Snippet 329 public static Object allocateArrayDynamic(Class<?> elementType, Class<?> voidClass, int length, @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, 330 @ConstantParameter JavaKind knownElementKind, @ConstantParameter int knownLayoutHelper, Word prototypeMarkWord) { 331 Object result = allocateArrayDynamicImpl(elementType, voidClass, length, fillContents, threadRegister, knownElementKind, knownLayoutHelper, prototypeMarkWord); 332 return result; 333 } 334 335 private static Object allocateArrayDynamicImpl(Class<?> elementType, Class<?> voidClass, int length, boolean fillContents, Register threadRegister, JavaKind knownElementKind, 336 int knownLayoutHelper, Word prototypeMarkWord) { 337 /* 338 * We only need the dynamic check for void when we have no static information from 339 * knownElementKind. 340 */ 341 staticAssert(knownElementKind != JavaKind.Void, "unsupported knownElementKind"); 342 if (knownElementKind == JavaKind.Illegal && probability(SLOW_PATH_PROBABILITY, elementType == null || DynamicNewArrayNode.throwsIllegalArgumentException(elementType, voidClass))) { 343 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); 344 } 345 346 KlassPointer klass = loadKlassFromObject(elementType, arrayKlassOffset(INJECTED_VMCONFIG), CLASS_ARRAY_KLASS_LOCATION); 347 if (probability(BranchProbabilityNode.NOT_FREQUENT_PROBABILITY, klass.isNull() || length < 0)) { 348 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); 349 } 350 int layoutHelper = knownElementKind != JavaKind.Illegal ? knownLayoutHelper : readLayoutHelper(klass); 351 //@formatter:off 352 // from src/share/vm/oops/klass.hpp: 353 // 354 // For arrays, layout helper is a negative number, containing four 355 // distinct bytes, as follows: 356 // MSB:[tag, hsz, ebt, log2(esz)]:LSB 357 // where: 358 // tag is 0x80 if the elements are oops, 0xC0 if non-oops 359 // hsz is array header size in bytes (i.e., offset of first element) 360 // ebt is the BasicType of the elements 361 // esz is the element size in bytes 362 //@formatter:on 363 364 int headerSize = (layoutHelper >> layoutHelperHeaderSizeShift(INJECTED_VMCONFIG)) & layoutHelperHeaderSizeMask(INJECTED_VMCONFIG); 365 int log2ElementSize = (layoutHelper >> layoutHelperLog2ElementSizeShift(INJECTED_VMCONFIG)) & layoutHelperLog2ElementSizeMask(INJECTED_VMCONFIG); 366 367 Object result = allocateArrayImpl(klass, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, false, "dynamic type", true); 368 return piArrayCast(verifyOop(result), length, StampFactory.forNodeIntrinsic()); 369 } 370 371 /** 372 * Calls the runtime stub for implementing MULTIANEWARRAY. 373 */ 374 @Snippet 375 public static Object newmultiarray(KlassPointer hub, @ConstantParameter int rank, @VarargsParameter int[] dimensions) { 376 Word dims = DimensionsNode.allocaDimsArray(rank); 377 ExplodeLoopNode.explodeLoop(); 378 for (int i = 0; i < rank; i++) { 379 dims.writeInt(i * 4, dimensions[i], INIT_LOCATION); 380 } 381 return newArrayCall(HotSpotBackend.NEW_MULTI_ARRAY, hub, rank, dims); 382 } 383 384 @Snippet 385 public static Object newmultiarrayPIC(KlassPointer hub, @ConstantParameter int rank, @VarargsParameter int[] dimensions) { 386 KlassPointer hubPIC = ResolveConstantSnippets.resolveKlassConstant(hub); 387 return newmultiarray(hubPIC, rank, dimensions); 388 } 389 390 @NodeIntrinsic(value = ForeignCallNode.class, returnStampIsNonNull = true) 391 public static native Object newArrayCall(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub, int rank, Word dims); 392 393 /** 394 * Maximum number of long stores to emit when zeroing an object with a constant size. Larger 395 * objects have their bodies initialized in a loop. 396 */ 397 private static final int MAX_UNROLLED_OBJECT_ZEROING_STORES = 8; 398 399 /** 400 * Zero uninitialized memory in a newly allocated object, unrolling as necessary and ensuring 401 * that stores are aligned. 402 * 403 * @param size number of bytes to zero 404 * @param memory beginning of object which is being zeroed 405 * @param constantSize is {@code size} known to be constant in the snippet 406 * @param startOffset offset to begin zeroing. May not be word aligned. 407 * @param manualUnroll maximally unroll zeroing 408 */ 409 private static void zeroMemory(int size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, boolean useSnippetCounters) { 410 fillMemory(0, size, memory, constantSize, startOffset, manualUnroll, useSnippetCounters); 411 } 412 413 private static void fillMemory(long value, int size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, boolean useSnippetCounters) { 414 ReplacementsUtil.runtimeAssert((size & 0x7) == 0, "unaligned object size"); 415 int offset = startOffset; 416 if ((offset & 0x7) != 0) { 417 memory.writeInt(offset, (int) value, INIT_LOCATION); 418 offset += 4; 419 } 420 ReplacementsUtil.runtimeAssert((offset & 0x7) == 0, "unaligned offset"); 421 if (manualUnroll && ((size - offset) / 8) <= MAX_UNROLLED_OBJECT_ZEROING_STORES) { 422 ReplacementsUtil.staticAssert(!constantSize, "size shouldn't be constant at instantiation time"); 423 // This case handles arrays of constant length. Instead of having a snippet variant for 424 // each length, generate a chain of stores of maximum length. Once it's inlined the 425 // break statement will trim excess stores. 426 if (useSnippetCounters) { 427 new_seqInit.inc(); 428 } 429 explodeLoop(); 430 for (int i = 0; i < MAX_UNROLLED_OBJECT_ZEROING_STORES; i++, offset += 8) { 431 if (offset == size) { 432 break; 433 } 434 memory.initializeLong(offset, value, INIT_LOCATION); 435 } 436 } else { 437 // Use Word instead of int to avoid extension to long in generated code 438 Word off = Word.signed(offset); 439 if (constantSize && ((size - offset) / 8) <= MAX_UNROLLED_OBJECT_ZEROING_STORES) { 440 if (useSnippetCounters) { 441 new_seqInit.inc(); 442 } 443 explodeLoop(); 444 } else { 445 if (useSnippetCounters) { 446 new_loopInit.inc(); 447 } 448 } 449 for (; off.rawValue() < size; off = off.add(8)) { 450 memory.initializeLong(off, value, INIT_LOCATION); 451 } 452 } 453 } 454 455 /** 456 * Fill uninitialized memory with garbage value in a newly allocated object, unrolling as 457 * necessary and ensuring that stores are aligned. 458 * 459 * @param size number of bytes to zero 460 * @param memory beginning of object which is being zeroed 461 * @param constantSize is {@code size} known to be constant in the snippet 462 * @param startOffset offset to begin zeroing. May not be word aligned. 463 * @param manualUnroll maximally unroll zeroing 464 */ 465 private static void fillWithGarbage(int size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, boolean useSnippetCounters) { 466 fillMemory(0xfefefefefefefefeL, size, memory, constantSize, startOffset, manualUnroll, useSnippetCounters); 467 } 468 469 /** 470 * Formats some allocated memory with an object header and zeroes out the rest. Disables asserts 471 * since they can't be compiled in stubs. 472 */ 473 public static Object formatObjectForStub(KlassPointer hub, int size, Word memory, Word compileTimePrototypeMarkWord) { 474 return formatObject(hub, size, memory, compileTimePrototypeMarkWord, true, false, false); 475 } 476 477 /** 478 * Formats some allocated memory with an object header and zeroes out the rest. 479 */ 480 protected static Object formatObject(KlassPointer hub, int size, Word memory, Word compileTimePrototypeMarkWord, boolean fillContents, boolean constantSize, boolean useSnippetCounters) { 481 Word prototypeMarkWord = useBiasedLocking(INJECTED_VMCONFIG) ? hub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION) : compileTimePrototypeMarkWord; 482 initializeObjectHeader(memory, prototypeMarkWord, hub); 483 if (fillContents) { 484 zeroMemory(size, memory, constantSize, instanceHeaderSize(INJECTED_VMCONFIG), false, useSnippetCounters); 485 } else if (REPLACEMENTS_ASSERTIONS_ENABLED) { 486 fillWithGarbage(size, memory, constantSize, instanceHeaderSize(INJECTED_VMCONFIG), false, useSnippetCounters); 487 } 488 MembarNode.memoryBarrier(MemoryBarriers.STORE_STORE, INIT_LOCATION); 489 return memory.toObject(); 490 } 491 492 @Snippet 493 protected static void verifyHeap(@ConstantParameter Register threadRegister) { 494 Word thread = registerAsWord(threadRegister); 495 Word topValue = readTlabTop(thread); 496 if (!topValue.equal(Word.zero())) { 497 Word topValueContents = topValue.readWord(0, MARK_WORD_LOCATION); 498 if (topValueContents.equal(Word.zero())) { 499 AssertionSnippets.vmMessageC(AssertionSnippets.ASSERTION_VM_MESSAGE_C, true, cstring("overzeroing of TLAB detected"), 0L, 0L, 0L); 500 } 501 } 502 } 503 504 /** 505 * Formats some allocated memory with an object header and zeroes out the rest. 506 */ 507 public static Object formatArray(KlassPointer hub, int allocationSize, int length, int headerSize, Word memory, Word prototypeMarkWord, boolean fillContents, boolean maybeUnroll, 508 boolean useSnippetCounters) { 509 memory.writeInt(arrayLengthOffset(INJECTED_VMCONFIG), length, INIT_LOCATION); 510 /* 511 * store hub last as the concurrent garbage collectors assume length is valid if hub field 512 * is not null 513 */ 514 initializeObjectHeader(memory, prototypeMarkWord, hub); 515 if (fillContents) { 516 zeroMemory(allocationSize, memory, false, headerSize, maybeUnroll, useSnippetCounters); 517 } else if (REPLACEMENTS_ASSERTIONS_ENABLED) { 518 fillWithGarbage(allocationSize, memory, false, headerSize, maybeUnroll, useSnippetCounters); 519 } 520 MembarNode.memoryBarrier(MemoryBarriers.STORE_STORE, INIT_LOCATION); 521 return memory.toObject(); 522 } 523 524 public static class Templates extends AbstractTemplates { 525 526 private final SnippetInfo allocateInstance = snippet(NewObjectSnippets.class, "allocateInstance", INIT_LOCATION, MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION); 527 private final SnippetInfo allocateInstancePIC = snippet(NewObjectSnippets.class, "allocateInstancePIC", INIT_LOCATION, MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, 528 TLAB_END_LOCATION); 529 private final SnippetInfo allocateArray = snippet(NewObjectSnippets.class, "allocateArray", INIT_LOCATION, MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION); 530 private final SnippetInfo allocateArrayPIC = snippet(NewObjectSnippets.class, "allocateArrayPIC", INIT_LOCATION, MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION); 531 private final SnippetInfo allocatePrimitiveArrayPIC = snippet(NewObjectSnippets.class, "allocatePrimitiveArrayPIC", INIT_LOCATION, MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, 532 TLAB_END_LOCATION); 533 private final SnippetInfo allocateArrayDynamic = snippet(NewObjectSnippets.class, "allocateArrayDynamic", INIT_LOCATION, MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, 534 TLAB_END_LOCATION); 535 private final SnippetInfo allocateInstanceDynamic = snippet(NewObjectSnippets.class, "allocateInstanceDynamic", INIT_LOCATION, MARK_WORD_LOCATION, HUB_WRITE_LOCATION, TLAB_TOP_LOCATION, 536 TLAB_END_LOCATION); 537 private final SnippetInfo newmultiarray = snippet(NewObjectSnippets.class, "newmultiarray", INIT_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION); 538 private final SnippetInfo newmultiarrayPIC = snippet(NewObjectSnippets.class, "newmultiarrayPIC", INIT_LOCATION, TLAB_TOP_LOCATION, TLAB_END_LOCATION); 539 private final SnippetInfo verifyHeap = snippet(NewObjectSnippets.class, "verifyHeap"); 540 private final GraalHotSpotVMConfig config; 541 542 public Templates(HotSpotProviders providers, TargetDescription target, GraalHotSpotVMConfig config) { 543 super(providers, providers.getSnippetReflection(), target); 544 this.config = config; 545 } 546 547 /** 548 * Lowers a {@link NewInstanceNode}. 549 */ 550 public void lower(NewInstanceNode newInstanceNode, HotSpotRegistersProvider registers, LoweringTool tool) { 551 StructuredGraph graph = newInstanceNode.graph(); 552 HotSpotResolvedObjectType type = (HotSpotResolvedObjectType) newInstanceNode.instanceClass(); 553 assert !type.isArray(); 554 ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), type.klass(), providers.getMetaAccess(), graph); 555 int size = instanceSize(type); 556 557 SnippetInfo snippet = GeneratePIC.getValue() ? allocateInstancePIC : allocateInstance; 558 Arguments args = new Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage()); 559 args.addConst("size", size); 560 args.add("hub", hub); 561 args.add("prototypeMarkWord", type.prototypeMarkWord()); 562 args.addConst("fillContents", newInstanceNode.fillContents()); 563 args.addConst("threadRegister", registers.getThreadRegister()); 564 args.addConst("constantSize", true); 565 args.addConst("typeContext", HotspotSnippetsOptions.ProfileAllocations.getValue() ? type.toJavaName(false) : ""); 566 567 SnippetTemplate template = template(args); 568 Debug.log("Lowering allocateInstance in %s: node=%s, template=%s, arguments=%s", graph, newInstanceNode, template, args); 569 template.instantiate(providers.getMetaAccess(), newInstanceNode, DEFAULT_REPLACER, args); 570 } 571 572 /** 573 * Lowers a {@link NewArrayNode}. 574 */ 575 public void lower(NewArrayNode newArrayNode, HotSpotRegistersProvider registers, LoweringTool tool) { 576 StructuredGraph graph = newArrayNode.graph(); 577 ResolvedJavaType elementType = newArrayNode.elementType(); 578 HotSpotResolvedObjectType arrayType = (HotSpotResolvedObjectType) elementType.getArrayClass(); 579 JavaKind elementKind = elementType.getJavaKind(); 580 ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), arrayType.klass(), providers.getMetaAccess(), graph); 581 final int headerSize = getArrayBaseOffset(elementKind); 582 int log2ElementSize = CodeUtil.log2(HotSpotJVMCIRuntimeProvider.getArrayIndexScale(elementKind)); 583 584 SnippetInfo snippet; 585 if (GeneratePIC.getValue()) { 586 if (elementType.isPrimitive()) { 587 snippet = allocatePrimitiveArrayPIC; 588 } else { 589 snippet = allocateArrayPIC; 590 } 591 } else { 592 snippet = allocateArray; 593 } 594 595 Arguments args = new Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage()); 596 args.add("hub", hub); 597 ValueNode length = newArrayNode.length(); 598 args.add("length", length.isAlive() ? length : graph.addOrUniqueWithInputs(length)); 599 assert arrayType.prototypeMarkWord() == lookupArrayClass(tool, JavaKind.Object).prototypeMarkWord() : "all array types are assumed to have the same prototypeMarkWord"; 600 args.add("prototypeMarkWord", arrayType.prototypeMarkWord()); 601 args.addConst("headerSize", headerSize); 602 args.addConst("log2ElementSize", log2ElementSize); 603 args.addConst("fillContents", newArrayNode.fillContents()); 604 args.addConst("threadRegister", registers.getThreadRegister()); 605 args.addConst("maybeUnroll", length.isConstant()); 606 args.addConst("typeContext", HotspotSnippetsOptions.ProfileAllocations.getValue() ? arrayType.toJavaName(false) : ""); 607 SnippetTemplate template = template(args); 608 Debug.log("Lowering allocateArray in %s: node=%s, template=%s, arguments=%s", graph, newArrayNode, template, args); 609 template.instantiate(providers.getMetaAccess(), newArrayNode, DEFAULT_REPLACER, args); 610 } 611 612 public void lower(DynamicNewInstanceNode newInstanceNode, HotSpotRegistersProvider registers, LoweringTool tool) { 613 Arguments args = new Arguments(allocateInstanceDynamic, newInstanceNode.graph().getGuardsStage(), tool.getLoweringStage()); 614 args.add("type", newInstanceNode.getInstanceType()); 615 ValueNode classClass = newInstanceNode.getClassClass(); 616 assert classClass != null; 617 args.add("classClass", classClass); 618 args.addConst("fillContents", newInstanceNode.fillContents()); 619 args.addConst("threadRegister", registers.getThreadRegister()); 620 621 SnippetTemplate template = template(args); 622 template.instantiate(providers.getMetaAccess(), newInstanceNode, DEFAULT_REPLACER, args); 623 } 624 625 public void lower(DynamicNewArrayNode newArrayNode, HotSpotRegistersProvider registers, LoweringTool tool) { 626 StructuredGraph graph = newArrayNode.graph(); 627 Arguments args = new Arguments(allocateArrayDynamic, newArrayNode.graph().getGuardsStage(), tool.getLoweringStage()); 628 args.add("elementType", newArrayNode.getElementType()); 629 ValueNode voidClass = newArrayNode.getVoidClass(); 630 assert voidClass != null; 631 args.add("voidClass", voidClass); 632 ValueNode length = newArrayNode.length(); 633 args.add("length", length.isAlive() ? length : graph.addOrUniqueWithInputs(length)); 634 args.addConst("fillContents", newArrayNode.fillContents()); 635 args.addConst("threadRegister", registers.getThreadRegister()); 636 /* 637 * We use Kind.Illegal as a marker value instead of null because constant snippet 638 * parameters cannot be null. 639 */ 640 args.addConst("knownElementKind", newArrayNode.getKnownElementKind() == null ? JavaKind.Illegal : newArrayNode.getKnownElementKind()); 641 if (newArrayNode.getKnownElementKind() != null) { 642 args.addConst("knownLayoutHelper", lookupArrayClass(tool, newArrayNode.getKnownElementKind()).layoutHelper()); 643 } else { 644 args.addConst("knownLayoutHelper", 0); 645 } 646 args.add("prototypeMarkWord", lookupArrayClass(tool, JavaKind.Object).prototypeMarkWord()); 647 SnippetTemplate template = template(args); 648 template.instantiate(providers.getMetaAccess(), newArrayNode, DEFAULT_REPLACER, args); 649 } 650 651 private static HotSpotResolvedObjectType lookupArrayClass(LoweringTool tool, JavaKind kind) { 652 return (HotSpotResolvedObjectType) tool.getMetaAccess().lookupJavaType(kind == JavaKind.Object ? Object.class : kind.toJavaClass()).getArrayClass(); 653 } 654 655 public void lower(NewMultiArrayNode newmultiarrayNode, LoweringTool tool) { 656 StructuredGraph graph = newmultiarrayNode.graph(); 657 int rank = newmultiarrayNode.dimensionCount(); 658 ValueNode[] dims = new ValueNode[rank]; 659 for (int i = 0; i < newmultiarrayNode.dimensionCount(); i++) { 660 dims[i] = newmultiarrayNode.dimension(i); 661 } 662 HotSpotResolvedObjectType type = (HotSpotResolvedObjectType) newmultiarrayNode.type(); 663 ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), type.klass(), providers.getMetaAccess(), graph); 664 665 SnippetInfo snippet = GeneratePIC.getValue() ? newmultiarrayPIC : newmultiarray; 666 Arguments args = new Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage()); 667 args.add("hub", hub); 668 args.addConst("rank", rank); 669 args.addVarargs("dimensions", int.class, StampFactory.forKind(JavaKind.Int), dims); 670 template(args).instantiate(providers.getMetaAccess(), newmultiarrayNode, DEFAULT_REPLACER, args); 671 } 672 673 private static int instanceSize(HotSpotResolvedObjectType type) { 674 int size = type.instanceSize(); 675 assert size >= 0; 676 return size; 677 } 678 679 public void lower(VerifyHeapNode verifyHeapNode, HotSpotRegistersProvider registers, LoweringTool tool) { 680 if (config.cAssertions) { 681 Arguments args = new Arguments(verifyHeap, verifyHeapNode.graph().getGuardsStage(), tool.getLoweringStage()); 682 args.addConst("threadRegister", registers.getThreadRegister()); 683 684 SnippetTemplate template = template(args); 685 template.instantiate(providers.getMetaAccess(), verifyHeapNode, DEFAULT_REPLACER, args); 686 } else { 687 GraphUtil.removeFixedWithUnusedInputs(verifyHeapNode); 688 } 689 } 690 } 691 692 private static final SnippetCounter.Group countersNew = SnippetCounters.getValue() ? new SnippetCounter.Group("NewInstance") : null; 693 private static final SnippetCounter new_seqInit = new SnippetCounter(countersNew, "tlabSeqInit", "TLAB alloc with unrolled zeroing"); 694 private static final SnippetCounter new_loopInit = new SnippetCounter(countersNew, "tlabLoopInit", "TLAB alloc with zeroing in a loop"); 695 private static final SnippetCounter new_stub = new SnippetCounter(countersNew, "stub", "alloc and zeroing via stub"); 696 697 private static final SnippetCounter.Group countersNewArray = SnippetCounters.getValue() ? new SnippetCounter.Group("NewArray") : null; 698 private static final SnippetCounter newarray_loopInit = new SnippetCounter(countersNewArray, "tlabLoopInit", "TLAB alloc with zeroing in a loop"); 699 private static final SnippetCounter newarray_stub = new SnippetCounter(countersNewArray, "stub", "alloc and zeroing via stub"); 700 }