1 /*
2 * Copyright (c) 2012, 2015, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23 package org.graalvm.compiler.hotspot.stubs;
24
25 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFIG;
26 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.HEAP_END_LOCATION;
27 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.HEAP_TOP_LOCATION;
28 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.PROTOTYPE_MARK_WORD_LOCATION;
29 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_FAST_REFILL_WASTE_LOCATION;
30 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_NOF_REFILLS_LOCATION;
31 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_REFILL_WASTE_LIMIT_LOCATION;
32 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_SIZE_LOCATION;
33 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_SLOW_ALLOCATIONS_LOCATION;
34 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_THREAD_ALLOCATED_BYTES_LOCATION;
35 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayBaseOffset;
36 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.getAndClearObjectResult;
37 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.initializeTlab;
38 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.isInstanceKlassFullyInitialized;
39 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readLayoutHelper;
40 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.log2WordSize;
41 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.prototypeMarkWordOffset;
42 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabEnd;
43 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabStart;
44 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabTop;
45 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.registerAsWord;
46 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.threadAllocatedBytesOffset;
47 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.threadTlabSizeOffset;
48 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabAlignmentReserveInHeapWords;
49 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabFastRefillWasteOffset;
50 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabIntArrayMarkWord;
51 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabNumberOfRefillsOffset;
52 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabRefillWasteIncrement;
53 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabRefillWasteLimitOffset;
54 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabSlowAllocationsOffset;
55 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabStats;
56 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useG1GC;
57 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useTLAB;
58 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.wordSize;
59 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.writeTlabTop;
60 import static org.graalvm.compiler.hotspot.stubs.StubUtil.handlePendingException;
61 import static org.graalvm.compiler.hotspot.stubs.StubUtil.newDescriptor;
62 import static org.graalvm.compiler.hotspot.stubs.StubUtil.printf;
63 import static org.graalvm.compiler.hotspot.stubs.StubUtil.verifyObject;
64 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FAST_PATH_PROBABILITY;
65 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;
66
67 import org.graalvm.compiler.api.replacements.Fold;
68 import org.graalvm.compiler.api.replacements.Snippet;
69 import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter;
70 import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor;
71 import org.graalvm.compiler.graph.Node.ConstantNodeParameter;
72 import org.graalvm.compiler.graph.Node.NodeIntrinsic;
73 import org.graalvm.compiler.hotspot.HotSpotForeignCallLinkage;
74 import org.graalvm.compiler.hotspot.meta.HotSpotProviders;
75 import org.graalvm.compiler.hotspot.nodes.GraalHotSpotVMConfigNode;
76 import org.graalvm.compiler.hotspot.nodes.StubForeignCallNode;
77 import org.graalvm.compiler.hotspot.nodes.type.KlassPointerStamp;
78 import org.graalvm.compiler.hotspot.replacements.NewObjectSnippets;
79 import org.graalvm.compiler.hotspot.word.KlassPointer;
80 import org.graalvm.compiler.nodes.ConstantNode;
81 import org.graalvm.compiler.options.OptionValues;
82 import org.graalvm.compiler.word.Word;
83 import org.graalvm.word.WordFactory;
84
85 import jdk.vm.ci.code.Register;
86 import jdk.vm.ci.hotspot.HotSpotResolvedObjectType;
87 import jdk.vm.ci.meta.JavaKind;
88
89 /**
90 * Stub implementing the fast path for TLAB refill during instance class allocation. This stub is
91 * called from the {@linkplain NewObjectSnippets inline} allocation code when TLAB allocation fails.
92 * If this stub fails to refill the TLAB or allocate the object, it calls out to the HotSpot C++
93 * runtime for to complete the allocation.
94 */
95 public class NewInstanceStub extends SnippetStub {
96
97 public NewInstanceStub(OptionValues options, HotSpotProviders providers, HotSpotForeignCallLinkage linkage) {
98 super("newInstance", options, providers, linkage);
99 }
100
101 @Override
102 protected Object[] makeConstArgs() {
103 HotSpotResolvedObjectType intArrayType = (HotSpotResolvedObjectType) providers.getMetaAccess().lookupJavaType(int[].class);
104 int count = method.getSignature().getParameterCount(false);
105 Object[] args = new Object[count];
106 assert checkConstArg(1, "intArrayHub");
107 assert checkConstArg(2, "threadRegister");
108 assert checkConstArg(3, "options");
109 args[1] = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), intArrayType.klass(), null);
110 args[2] = providers.getRegisters().getThreadRegister();
111 args[3] = options;
112 return args;
113 }
114
115 private static Word allocate(Word thread, int size) {
116 Word top = readTlabTop(thread);
117 Word end = readTlabEnd(thread);
118 Word newTop = top.add(size);
119 /*
120 * this check might lead to problems if the TLAB is within 16GB of the address space end
121 * (checked in c++ code)
122 */
123 if (probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
124 writeTlabTop(thread, newTop);
125 return top;
126 }
127 return WordFactory.zero();
128 }
129
130 @Fold
131 static boolean logging(OptionValues options) {
132 return StubOptions.TraceNewInstanceStub.getValue(options);
133 }
134
135 /**
136 * Re-attempts allocation after an initial TLAB allocation failed or was skipped (e.g., due to
137 * -XX:-UseTLAB).
138 *
139 * @param hub the hub of the object to be allocated
140 * @param intArrayHub the hub for {@code int[].class}
141 */
142 @Snippet
143 private static Object newInstance(KlassPointer hub, @ConstantParameter KlassPointer intArrayHub, @ConstantParameter Register threadRegister, @ConstantParameter OptionValues options) {
144 /*
145 * The type is known to be an instance so Klass::_layout_helper is the instance size as a
146 * raw number
147 */
148 Word thread = registerAsWord(threadRegister);
149 boolean inlineContiguousAllocationSupported = GraalHotSpotVMConfigNode.inlineContiguousAllocationSupported();
150 if (!forceSlowPath(options) && inlineContiguousAllocationSupported) {
151 if (isInstanceKlassFullyInitialized(hub)) {
152 int sizeInBytes = readLayoutHelper(hub);
153 Word memory = refillAllocate(thread, intArrayHub, sizeInBytes, logging(options));
154 if (memory.notEqual(0)) {
155 Word prototypeMarkWord = hub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION);
156 NewObjectSnippets.formatObjectForStub(hub, sizeInBytes, memory, prototypeMarkWord);
157 return verifyObject(memory.toObject());
158 }
159 }
160 }
161
162 if (logging(options)) {
163 printf("newInstance: calling new_instance_c\n");
164 }
165
166 newInstanceC(NEW_INSTANCE_C, thread, hub);
167 handlePendingException(thread, true);
168 return verifyObject(getAndClearObjectResult(thread));
169 }
170
171 /**
172 * Attempts to refill the current thread's TLAB and retries the allocation.
173 *
174 * @param intArrayHub the hub for {@code int[].class}
175 * @param sizeInBytes the size of the allocation
176 * @param log specifies if logging is enabled
177 *
178 * @return the newly allocated, uninitialized chunk of memory, or {@link WordFactory#zero()} if
179 * the operation was unsuccessful
180 */
181 static Word refillAllocate(Word thread, KlassPointer intArrayHub, int sizeInBytes, boolean log) {
182 // If G1 is enabled, the "eden" allocation space is not the same always
183 // and therefore we have to go to slowpath to allocate a new TLAB.
184 if (useG1GC(INJECTED_VMCONFIG)) {
185 return WordFactory.zero();
186 }
187 if (!useTLAB(INJECTED_VMCONFIG)) {
188 return edenAllocate(WordFactory.unsigned(sizeInBytes), log);
189 }
190 Word intArrayMarkWord = WordFactory.unsigned(tlabIntArrayMarkWord(INJECTED_VMCONFIG));
191 int alignmentReserveInBytes = tlabAlignmentReserveInHeapWords(INJECTED_VMCONFIG) * wordSize();
192
193 Word top = readTlabTop(thread);
194 Word end = readTlabEnd(thread);
195
196 // calculate amount of free space
197 long tlabFreeSpaceInBytes = end.subtract(top).rawValue();
198
199 if (log) {
200 printf("refillTLAB: thread=%p\n", thread.rawValue());
201 printf("refillTLAB: top=%p\n", top.rawValue());
202 printf("refillTLAB: end=%p\n", end.rawValue());
203 printf("refillTLAB: tlabFreeSpaceInBytes=%ld\n", tlabFreeSpaceInBytes);
204 }
205
206 long tlabFreeSpaceInWords = tlabFreeSpaceInBytes >>> log2WordSize();
207
208 // Retain TLAB and allocate object in shared space if
209 // the amount free in the TLAB is too large to discard.
210 Word refillWasteLimit = thread.readWord(tlabRefillWasteLimitOffset(INJECTED_VMCONFIG), TLAB_REFILL_WASTE_LIMIT_LOCATION);
211 if (tlabFreeSpaceInWords <= refillWasteLimit.rawValue()) {
212 if (tlabStats(INJECTED_VMCONFIG)) {
213 // increment number of refills
214 thread.writeInt(tlabNumberOfRefillsOffset(INJECTED_VMCONFIG), thread.readInt(tlabNumberOfRefillsOffset(INJECTED_VMCONFIG), TLAB_NOF_REFILLS_LOCATION) + 1, TLAB_NOF_REFILLS_LOCATION);
215 if (log) {
216 printf("thread: %p -- number_of_refills %d\n", thread.rawValue(), thread.readInt(tlabNumberOfRefillsOffset(INJECTED_VMCONFIG), TLAB_NOF_REFILLS_LOCATION));
217 }
218 // accumulate wastage
219 int wastage = thread.readInt(tlabFastRefillWasteOffset(INJECTED_VMCONFIG), TLAB_FAST_REFILL_WASTE_LOCATION) + (int) tlabFreeSpaceInWords;
220 if (log) {
221 printf("thread: %p -- accumulated wastage %d\n", thread.rawValue(), wastage);
222 }
223 thread.writeInt(tlabFastRefillWasteOffset(INJECTED_VMCONFIG), wastage, TLAB_FAST_REFILL_WASTE_LOCATION);
224 }
225
226 // if TLAB is currently allocated (top or end != null) then
227 // fill [top, end + alignment_reserve) with array object
228 if (top.notEqual(0)) {
229 int headerSize = arrayBaseOffset(JavaKind.Int);
230 // just like the HotSpot assembler stubs, assumes that tlabFreeSpaceInInts fits in
231 // an int
232 int tlabFreeSpaceInInts = (int) tlabFreeSpaceInBytes >>> 2;
233 int length = ((alignmentReserveInBytes - headerSize) >>> 2) + tlabFreeSpaceInInts;
234 NewObjectSnippets.formatArray(intArrayHub, 0, length, headerSize, top, intArrayMarkWord, false, false, null);
235
236 long allocated = thread.readLong(threadAllocatedBytesOffset(INJECTED_VMCONFIG), TLAB_THREAD_ALLOCATED_BYTES_LOCATION);
237 allocated = allocated + top.subtract(readTlabStart(thread)).rawValue();
238 thread.writeLong(threadAllocatedBytesOffset(INJECTED_VMCONFIG), allocated, TLAB_THREAD_ALLOCATED_BYTES_LOCATION);
239 }
240
241 // refill the TLAB with an eden allocation
242 Word tlabRefillSizeInWords = thread.readWord(threadTlabSizeOffset(INJECTED_VMCONFIG), TLAB_SIZE_LOCATION);
243 Word tlabRefillSizeInBytes = tlabRefillSizeInWords.multiply(wordSize());
244 // allocate new TLAB, address returned in top
245 top = edenAllocate(tlabRefillSizeInBytes, log);
246 if (top.notEqual(0)) {
247 end = top.add(tlabRefillSizeInBytes.subtract(alignmentReserveInBytes));
248 initializeTlab(thread, top, end);
249
250 return NewInstanceStub.allocate(thread, sizeInBytes);
251 } else {
252 return WordFactory.zero();
253 }
254 } else {
255 // Retain TLAB
256 Word newRefillWasteLimit = refillWasteLimit.add(tlabRefillWasteIncrement(INJECTED_VMCONFIG));
257 thread.writeWord(tlabRefillWasteLimitOffset(INJECTED_VMCONFIG), newRefillWasteLimit, TLAB_REFILL_WASTE_LIMIT_LOCATION);
258 if (log) {
259 printf("refillTLAB: retaining TLAB - newRefillWasteLimit=%p\n", newRefillWasteLimit.rawValue());
260 }
261
262 if (tlabStats(INJECTED_VMCONFIG)) {
263 thread.writeInt(tlabSlowAllocationsOffset(INJECTED_VMCONFIG), thread.readInt(tlabSlowAllocationsOffset(INJECTED_VMCONFIG), TLAB_SLOW_ALLOCATIONS_LOCATION) + 1,
264 TLAB_SLOW_ALLOCATIONS_LOCATION);
265 }
266
267 return edenAllocate(WordFactory.unsigned(sizeInBytes), log);
268 }
269 }
270
271 /**
272 * Attempts to allocate a chunk of memory from Eden space.
273 *
274 * @param sizeInBytes the size of the chunk to allocate
275 * @param log specifies if logging is enabled
276 * @return the allocated chunk or {@link WordFactory#zero()} if allocation fails
277 */
278 public static Word edenAllocate(Word sizeInBytes, boolean log) {
279 final long heapTopRawAddress = GraalHotSpotVMConfigNode.heapTopAddress();
280 final long heapEndRawAddress = GraalHotSpotVMConfigNode.heapEndAddress();
281
282 Word heapTopAddress = WordFactory.unsigned(heapTopRawAddress);
283 Word heapEndAddress = WordFactory.unsigned(heapEndRawAddress);
284
285 while (true) {
286 Word heapTop = heapTopAddress.readWord(0, HEAP_TOP_LOCATION);
287 Word newHeapTop = heapTop.add(sizeInBytes);
288 if (newHeapTop.belowOrEqual(heapTop)) {
289 return WordFactory.zero();
290 }
291
292 Word heapEnd = heapEndAddress.readWord(0, HEAP_END_LOCATION);
293 if (newHeapTop.aboveThan(heapEnd)) {
294 return WordFactory.zero();
295 }
296 if (heapTopAddress.logicCompareAndSwapWord(0, heapTop, newHeapTop, HEAP_TOP_LOCATION)) {
297 return heapTop;
298 }
299 }
300 }
301
302 @Fold
303 static boolean forceSlowPath(OptionValues options) {
304 return StubOptions.ForceUseOfNewInstanceStub.getValue(options);
305 }
306
307 public static final ForeignCallDescriptor NEW_INSTANCE_C = newDescriptor(NewInstanceStub.class, "newInstanceC", void.class, Word.class, KlassPointer.class);
308
309 @NodeIntrinsic(StubForeignCallNode.class)
310 public static native void newInstanceC(@ConstantNodeParameter ForeignCallDescriptor newInstanceC, Word thread, KlassPointer hub);
311 }