6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23 package org.graalvm.compiler.hotspot.replacements;
24
25 import static jdk.vm.ci.hotspot.HotSpotJVMCIRuntimeProvider.getArrayBaseOffset;
26 import static jdk.vm.ci.hotspot.HotSpotMetaAccessProvider.computeArrayAllocationSize;
27 import static org.graalvm.compiler.core.common.GraalOptions.GeneratePIC;
28 import static org.graalvm.compiler.core.common.calc.UnsignedMath.belowThan;
29 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFIG;
30 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.CLASS_ARRAY_KLASS_LOCATION;
31 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.HUB_WRITE_LOCATION;
32 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.MARK_WORD_LOCATION;
33 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.PROTOTYPE_MARK_WORD_LOCATION;
34 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_END_LOCATION;
35 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_TOP_LOCATION;
36 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayKlassOffset;
37 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayLengthOffset;
38 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.config;
39 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.initializeObjectHeader;
40 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.instanceHeaderSize;
41 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.isInstanceKlassFullyInitialized;
42 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeMask;
43 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeShift;
44 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperLog2ElementSizeMask;
45 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperLog2ElementSizeShift;
46 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.loadKlassFromObject;
47 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.prototypeMarkWordOffset;
48 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readLayoutHelper;
49 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabEnd;
50 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabTop;
51 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.registerAsWord;
52 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useBiasedLocking;
53 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useTLAB;
54 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.verifyOop;
55 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.wordSize;
56 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.writeTlabTop;
57 import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.ProfileAllocations;
58 import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.ProfileAllocationsContext;
59 import static org.graalvm.compiler.nodes.PiArrayNode.piArrayCastToSnippetReplaceeStamp;
60 import static org.graalvm.compiler.nodes.PiNode.piCastToSnippetReplaceeStamp;
61 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FAST_PATH_PROBABILITY;
62 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FREQUENT_PROBABILITY;
63 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY;
64 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;
65 import static org.graalvm.compiler.replacements.ReplacementsUtil.REPLACEMENTS_ASSERTIONS_ENABLED;
66 import static org.graalvm.compiler.replacements.ReplacementsUtil.staticAssert;
67 import static org.graalvm.compiler.replacements.SnippetTemplate.DEFAULT_REPLACER;
68 import static org.graalvm.compiler.replacements.nodes.CStringConstant.cstring;
69 import static org.graalvm.compiler.replacements.nodes.ExplodeLoopNode.explodeLoop;
70
71 import org.graalvm.compiler.api.replacements.Fold;
72 import org.graalvm.compiler.api.replacements.Snippet;
73 import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter;
74 import org.graalvm.compiler.api.replacements.Snippet.VarargsParameter;
75 import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor;
298 @Snippet
299 public static Object allocateArrayPIC(KlassPointer hub, int length, Word prototypeMarkWord, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize,
300 @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext,
301 @ConstantParameter OptionValues options, @ConstantParameter Counters counters) {
302 // Array type would be resolved by dominating resolution.
303 KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
304 return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false, options, counters);
305 }
306
307 @Snippet
308 public static Object allocateArray(KlassPointer hub, int length, Word prototypeMarkWord, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize,
309 @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext,
310 @ConstantParameter OptionValues options, @ConstantParameter Counters counters) {
311 Object result = allocateArrayImpl(hub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false, options, counters);
312 return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length);
313 }
314
315 private static Object allocateArrayImpl(KlassPointer hub, int length, Word prototypeMarkWord, int headerSize, int log2ElementSize, boolean fillContents, Register threadRegister,
316 boolean maybeUnroll, String typeContext, boolean skipNegativeCheck, OptionValues options, Counters counters) {
317 Object result;
318 int alignment = wordSize();
319 int allocationSize = computeArrayAllocationSize(length, alignment, headerSize, log2ElementSize);
320 Word thread = registerAsWord(threadRegister);
321 Word top = readTlabTop(thread);
322 Word end = readTlabEnd(thread);
323 Word newTop = top.add(allocationSize);
324 if (probability(FREQUENT_PROBABILITY, skipNegativeCheck || belowThan(length, MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH)) && useTLAB(INJECTED_VMCONFIG) &&
325 probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
326 writeTlabTop(thread, newTop);
327 emitPrefetchAllocate(newTop, true);
328 if (counters != null && counters.arrayLoopInit != null) {
329 counters.arrayLoopInit.inc();
330 }
331 result = formatArray(hub, allocationSize, length, headerSize, top, prototypeMarkWord, fillContents, maybeUnroll, counters);
332 } else {
333 result = newArray(HotSpotBackend.NEW_ARRAY, hub, length, fillContents);
334 }
335 profileAllocation("array", allocationSize, typeContext, options);
336 return result;
337 }
338
339 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
|
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23 package org.graalvm.compiler.hotspot.replacements;
24
25 import static jdk.vm.ci.hotspot.HotSpotJVMCIRuntimeProvider.getArrayBaseOffset;
26 import static org.graalvm.compiler.core.common.GraalOptions.GeneratePIC;
27 import static org.graalvm.compiler.core.common.calc.UnsignedMath.belowThan;
28 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFIG;
29 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.CLASS_ARRAY_KLASS_LOCATION;
30 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.HUB_WRITE_LOCATION;
31 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.MARK_WORD_LOCATION;
32 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.PROTOTYPE_MARK_WORD_LOCATION;
33 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_END_LOCATION;
34 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_TOP_LOCATION;
35 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayAllocationSize;
36 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayKlassOffset;
37 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayLengthOffset;
38 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.config;
39 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.initializeObjectHeader;
40 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.instanceHeaderSize;
41 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.isInstanceKlassFullyInitialized;
42 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeMask;
43 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeShift;
44 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperLog2ElementSizeMask;
45 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperLog2ElementSizeShift;
46 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.loadKlassFromObject;
47 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.prototypeMarkWordOffset;
48 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readLayoutHelper;
49 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabEnd;
50 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabTop;
51 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.registerAsWord;
52 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useBiasedLocking;
53 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useTLAB;
54 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.verifyOop;
55 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.writeTlabTop;
56 import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.ProfileAllocations;
57 import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.ProfileAllocationsContext;
58 import static org.graalvm.compiler.nodes.PiArrayNode.piArrayCastToSnippetReplaceeStamp;
59 import static org.graalvm.compiler.nodes.PiNode.piCastToSnippetReplaceeStamp;
60 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FAST_PATH_PROBABILITY;
61 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FREQUENT_PROBABILITY;
62 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY;
63 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;
64 import static org.graalvm.compiler.replacements.ReplacementsUtil.REPLACEMENTS_ASSERTIONS_ENABLED;
65 import static org.graalvm.compiler.replacements.ReplacementsUtil.staticAssert;
66 import static org.graalvm.compiler.replacements.SnippetTemplate.DEFAULT_REPLACER;
67 import static org.graalvm.compiler.replacements.nodes.CStringConstant.cstring;
68 import static org.graalvm.compiler.replacements.nodes.ExplodeLoopNode.explodeLoop;
69
70 import org.graalvm.compiler.api.replacements.Fold;
71 import org.graalvm.compiler.api.replacements.Snippet;
72 import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter;
73 import org.graalvm.compiler.api.replacements.Snippet.VarargsParameter;
74 import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor;
297 @Snippet
298 public static Object allocateArrayPIC(KlassPointer hub, int length, Word prototypeMarkWord, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize,
299 @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext,
300 @ConstantParameter OptionValues options, @ConstantParameter Counters counters) {
301 // Array type would be resolved by dominating resolution.
302 KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
303 return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false, options, counters);
304 }
305
306 @Snippet
307 public static Object allocateArray(KlassPointer hub, int length, Word prototypeMarkWord, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize,
308 @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean maybeUnroll, @ConstantParameter String typeContext,
309 @ConstantParameter OptionValues options, @ConstantParameter Counters counters) {
310 Object result = allocateArrayImpl(hub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false, options, counters);
311 return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length);
312 }
313
314 private static Object allocateArrayImpl(KlassPointer hub, int length, Word prototypeMarkWord, int headerSize, int log2ElementSize, boolean fillContents, Register threadRegister,
315 boolean maybeUnroll, String typeContext, boolean skipNegativeCheck, OptionValues options, Counters counters) {
316 Object result;
317 int allocationSize = arrayAllocationSize(length, headerSize, log2ElementSize);
318 Word thread = registerAsWord(threadRegister);
319 Word top = readTlabTop(thread);
320 Word end = readTlabEnd(thread);
321 Word newTop = top.add(allocationSize);
322 if (probability(FREQUENT_PROBABILITY, skipNegativeCheck || belowThan(length, MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH)) && useTLAB(INJECTED_VMCONFIG) &&
323 probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
324 writeTlabTop(thread, newTop);
325 emitPrefetchAllocate(newTop, true);
326 if (counters != null && counters.arrayLoopInit != null) {
327 counters.arrayLoopInit.inc();
328 }
329 result = formatArray(hub, allocationSize, length, headerSize, top, prototypeMarkWord, fillContents, maybeUnroll, counters);
330 } else {
331 result = newArray(HotSpotBackend.NEW_ARRAY, hub, length, fillContents);
332 }
333 profileAllocation("array", allocationSize, typeContext, options);
334 return result;
335 }
336
337 @NodeIntrinsic(value = ForeignCallNode.class, injectedStampIsNonNull = true)
|