1 /*
2 * Copyright (c) 2012, 2015, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23 package org.graalvm.compiler.hotspot.stubs;
24
25 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFIG;
26 import static org.graalvm.compiler.hotspot.nodes.DirectCompareAndSwapNode.compareAndSwap;
27 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.HEAP_END_LOCATION;
28 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.HEAP_TOP_LOCATION;
29 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.PROTOTYPE_MARK_WORD_LOCATION;
30 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_FAST_REFILL_WASTE_LOCATION;
31 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_NOF_REFILLS_LOCATION;
32 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_REFILL_WASTE_LIMIT_LOCATION;
33 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_SIZE_LOCATION;
34 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_SLOW_ALLOCATIONS_LOCATION;
35 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_THREAD_ALLOCATED_BYTES_LOCATION;
36 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayBaseOffset;
37 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.getAndClearObjectResult;
38 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.initializeTlab;
39 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.isInstanceKlassFullyInitialized;
40 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.loadKlassLayoutHelperIntrinsic;
41 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.log2WordSize;
42 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.prototypeMarkWordOffset;
43 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabEnd;
44 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabStart;
45 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabTop;
46 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.registerAsWord;
47 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.threadAllocatedBytesOffset;
48 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.threadTlabSizeOffset;
49 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabAlignmentReserveInHeapWords;
50 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabFastRefillWasteOffset;
51 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabIntArrayMarkWord;
52 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabNumberOfRefillsOffset;
53 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabRefillWasteIncrement;
54 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabRefillWasteLimitOffset;
55 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabSlowAllocationsOffset;
56 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabStats;
57 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useG1GC;
58 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useTLAB;
59 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.wordSize;
60 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.writeTlabTop;
61 import static org.graalvm.compiler.hotspot.stubs.StubUtil.handlePendingException;
62 import static org.graalvm.compiler.hotspot.stubs.StubUtil.newDescriptor;
63 import static org.graalvm.compiler.hotspot.stubs.StubUtil.printf;
64 import static org.graalvm.compiler.hotspot.stubs.StubUtil.verifyObject;
65 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FAST_PATH_PROBABILITY;
66 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;
67
68 import org.graalvm.compiler.api.replacements.Fold;
69 import org.graalvm.compiler.api.replacements.Snippet;
70 import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter;
71 import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor;
72 import org.graalvm.compiler.graph.Node.ConstantNodeParameter;
73 import org.graalvm.compiler.graph.Node.NodeIntrinsic;
74 import org.graalvm.compiler.hotspot.HotSpotForeignCallLinkage;
75 import org.graalvm.compiler.hotspot.meta.HotSpotProviders;
76 import org.graalvm.compiler.hotspot.nodes.GraalHotSpotVMConfigNode;
77 import org.graalvm.compiler.hotspot.nodes.StubForeignCallNode;
78 import org.graalvm.compiler.hotspot.nodes.type.KlassPointerStamp;
79 import org.graalvm.compiler.hotspot.replacements.NewObjectSnippets;
80 import org.graalvm.compiler.hotspot.word.KlassPointer;
81 import org.graalvm.compiler.nodes.ConstantNode;
82 import org.graalvm.compiler.nodes.memory.address.RawAddressNode;
83 import org.graalvm.compiler.word.Word;
84
85 import jdk.vm.ci.code.Register;
86 import jdk.vm.ci.hotspot.HotSpotResolvedObjectType;
87 import jdk.vm.ci.meta.JavaKind;
88
89 /**
90 * Stub implementing the fast path for TLAB refill during instance class allocation. This stub is
91 * called from the {@linkplain NewObjectSnippets inline} allocation code when TLAB allocation fails.
92 * If this stub fails to refill the TLAB or allocate the object, it calls out to the HotSpot C++
93 * runtime for to complete the allocation.
94 */
95 public class NewInstanceStub extends SnippetStub {
96
97 public NewInstanceStub(HotSpotProviders providers, HotSpotForeignCallLinkage linkage) {
98 super("newInstance", providers, linkage);
99 }
100
101 @Override
102 protected Object[] makeConstArgs() {
103 HotSpotResolvedObjectType intArrayType = (HotSpotResolvedObjectType) providers.getMetaAccess().lookupJavaType(int[].class);
104 int count = method.getSignature().getParameterCount(false);
105 Object[] args = new Object[count];
106 assert checkConstArg(1, "intArrayHub");
107 assert checkConstArg(2, "threadRegister");
108 args[1] = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), intArrayType.klass(), null);
109 args[2] = providers.getRegisters().getThreadRegister();
110 return args;
111 }
112
113 private static Word allocate(Word thread, int size) {
114 Word top = readTlabTop(thread);
115 Word end = readTlabEnd(thread);
116 Word newTop = top.add(size);
117 /*
118 * this check might lead to problems if the TLAB is within 16GB of the address space end
119 * (checked in c++ code)
120 */
121 if (probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
122 writeTlabTop(thread, newTop);
123 return top;
124 }
125 return Word.zero();
126 }
127
128 @Fold
129 static boolean logging() {
130 return StubOptions.TraceNewInstanceStub.getValue();
131 }
132
133 /**
134 * Re-attempts allocation after an initial TLAB allocation failed or was skipped (e.g., due to
135 * -XX:-UseTLAB).
136 *
137 * @param hub the hub of the object to be allocated
138 * @param intArrayHub the hub for {@code int[].class}
139 */
140 @Snippet
141 private static Object newInstance(KlassPointer hub, @ConstantParameter KlassPointer intArrayHub, @ConstantParameter Register threadRegister) {
142 /*
143 * The type is known to be an instance so Klass::_layout_helper is the instance size as a
144 * raw number
145 */
146 int sizeInBytes = loadKlassLayoutHelperIntrinsic(hub);
147 Word thread = registerAsWord(threadRegister);
148 boolean inlineContiguousAllocationSupported = GraalHotSpotVMConfigNode.inlineContiguousAllocationSupported();
149 if (!forceSlowPath() && inlineContiguousAllocationSupported) {
150 if (isInstanceKlassFullyInitialized(hub)) {
151 Word memory = refillAllocate(thread, intArrayHub, sizeInBytes, logging());
152 if (memory.notEqual(0)) {
153 Word prototypeMarkWord = hub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION);
154 NewObjectSnippets.formatObjectForStub(hub, sizeInBytes, memory, prototypeMarkWord);
155 return verifyObject(memory.toObject());
156 }
157 }
158 }
159
160 if (logging()) {
161 printf("newInstance: calling new_instance_c\n");
162 }
163
164 newInstanceC(NEW_INSTANCE_C, thread, hub);
165 handlePendingException(thread, true);
166 return verifyObject(getAndClearObjectResult(thread));
167 }
168
169 /**
170 * Attempts to refill the current thread's TLAB and retries the allocation.
171 *
172 * @param intArrayHub the hub for {@code int[].class}
173 * @param sizeInBytes the size of the allocation
174 * @param log specifies if logging is enabled
175 *
176 * @return the newly allocated, uninitialized chunk of memory, or {@link Word#zero()} if the
177 * operation was unsuccessful
178 */
179 static Word refillAllocate(Word thread, KlassPointer intArrayHub, int sizeInBytes, boolean log) {
180 // If G1 is enabled, the "eden" allocation space is not the same always
181 // and therefore we have to go to slowpath to allocate a new TLAB.
182 if (useG1GC(INJECTED_VMCONFIG)) {
183 return Word.zero();
184 }
185 if (!useTLAB(INJECTED_VMCONFIG)) {
186 return edenAllocate(Word.unsigned(sizeInBytes), log);
187 }
188 Word intArrayMarkWord = Word.unsigned(tlabIntArrayMarkWord(INJECTED_VMCONFIG));
189 int alignmentReserveInBytes = tlabAlignmentReserveInHeapWords(INJECTED_VMCONFIG) * wordSize();
190
191 Word top = readTlabTop(thread);
192 Word end = readTlabEnd(thread);
193
194 // calculate amount of free space
195 long tlabFreeSpaceInBytes = end.subtract(top).rawValue();
196
197 if (log) {
198 printf("refillTLAB: thread=%p\n", thread.rawValue());
199 printf("refillTLAB: top=%p\n", top.rawValue());
200 printf("refillTLAB: end=%p\n", end.rawValue());
201 printf("refillTLAB: tlabFreeSpaceInBytes=%ld\n", tlabFreeSpaceInBytes);
202 }
203
204 long tlabFreeSpaceInWords = tlabFreeSpaceInBytes >>> log2WordSize();
205
206 // Retain TLAB and allocate object in shared space if
207 // the amount free in the TLAB is too large to discard.
208 Word refillWasteLimit = thread.readWord(tlabRefillWasteLimitOffset(INJECTED_VMCONFIG), TLAB_REFILL_WASTE_LIMIT_LOCATION);
209 if (tlabFreeSpaceInWords <= refillWasteLimit.rawValue()) {
210 if (tlabStats(INJECTED_VMCONFIG)) {
211 // increment number of refills
212 thread.writeInt(tlabNumberOfRefillsOffset(INJECTED_VMCONFIG), thread.readInt(tlabNumberOfRefillsOffset(INJECTED_VMCONFIG), TLAB_NOF_REFILLS_LOCATION) + 1, TLAB_NOF_REFILLS_LOCATION);
213 if (log) {
214 printf("thread: %p -- number_of_refills %d\n", thread.rawValue(), thread.readInt(tlabNumberOfRefillsOffset(INJECTED_VMCONFIG), TLAB_NOF_REFILLS_LOCATION));
215 }
216 // accumulate wastage
217 int wastage = thread.readInt(tlabFastRefillWasteOffset(INJECTED_VMCONFIG), TLAB_FAST_REFILL_WASTE_LOCATION) + (int) tlabFreeSpaceInWords;
218 if (log) {
219 printf("thread: %p -- accumulated wastage %d\n", thread.rawValue(), wastage);
220 }
221 thread.writeInt(tlabFastRefillWasteOffset(INJECTED_VMCONFIG), wastage, TLAB_FAST_REFILL_WASTE_LOCATION);
222 }
223
224 // if TLAB is currently allocated (top or end != null) then
225 // fill [top, end + alignment_reserve) with array object
226 if (top.notEqual(0)) {
227 int headerSize = arrayBaseOffset(JavaKind.Int);
228 // just like the HotSpot assembler stubs, assumes that tlabFreeSpaceInInts fits in
229 // an int
230 int tlabFreeSpaceInInts = (int) tlabFreeSpaceInBytes >>> 2;
231 int length = ((alignmentReserveInBytes - headerSize) >>> 2) + tlabFreeSpaceInInts;
232 NewObjectSnippets.formatArray(intArrayHub, 0, length, headerSize, top, intArrayMarkWord, false, false, false);
233
234 long allocated = thread.readLong(threadAllocatedBytesOffset(INJECTED_VMCONFIG), TLAB_THREAD_ALLOCATED_BYTES_LOCATION);
235 allocated = allocated + top.subtract(readTlabStart(thread)).rawValue();
236 thread.writeLong(threadAllocatedBytesOffset(INJECTED_VMCONFIG), allocated, TLAB_THREAD_ALLOCATED_BYTES_LOCATION);
237 }
238
239 // refill the TLAB with an eden allocation
240 Word tlabRefillSizeInWords = thread.readWord(threadTlabSizeOffset(INJECTED_VMCONFIG), TLAB_SIZE_LOCATION);
241 Word tlabRefillSizeInBytes = tlabRefillSizeInWords.multiply(wordSize());
242 // allocate new TLAB, address returned in top
243 top = edenAllocate(tlabRefillSizeInBytes, log);
244 if (top.notEqual(0)) {
245 end = top.add(tlabRefillSizeInBytes.subtract(alignmentReserveInBytes));
246 initializeTlab(thread, top, end);
247
248 return NewInstanceStub.allocate(thread, sizeInBytes);
249 } else {
250 return Word.zero();
251 }
252 } else {
253 // Retain TLAB
254 Word newRefillWasteLimit = refillWasteLimit.add(tlabRefillWasteIncrement(INJECTED_VMCONFIG));
255 thread.writeWord(tlabRefillWasteLimitOffset(INJECTED_VMCONFIG), newRefillWasteLimit, TLAB_REFILL_WASTE_LIMIT_LOCATION);
256 if (log) {
257 printf("refillTLAB: retaining TLAB - newRefillWasteLimit=%p\n", newRefillWasteLimit.rawValue());
258 }
259
260 if (tlabStats(INJECTED_VMCONFIG)) {
261 thread.writeInt(tlabSlowAllocationsOffset(INJECTED_VMCONFIG), thread.readInt(tlabSlowAllocationsOffset(INJECTED_VMCONFIG), TLAB_SLOW_ALLOCATIONS_LOCATION) + 1,
262 TLAB_SLOW_ALLOCATIONS_LOCATION);
263 }
264
265 return edenAllocate(Word.unsigned(sizeInBytes), log);
266 }
267 }
268
269 /**
270 * Attempts to allocate a chunk of memory from Eden space.
271 *
272 * @param sizeInBytes the size of the chunk to allocate
273 * @param log specifies if logging is enabled
274 * @return the allocated chunk or {@link Word#zero()} if allocation fails
275 */
276 public static Word edenAllocate(Word sizeInBytes, boolean log) {
277 final long heapTopRawAddress = GraalHotSpotVMConfigNode.heapTopAddress();
278 final long heapEndRawAddress = GraalHotSpotVMConfigNode.heapEndAddress();
279
280 Word heapTopAddress = Word.unsigned(heapTopRawAddress);
281 Word heapEndAddress = Word.unsigned(heapEndRawAddress);
282
283 while (true) {
284 Word heapTop = heapTopAddress.readWord(0, HEAP_TOP_LOCATION);
285 Word newHeapTop = heapTop.add(sizeInBytes);
286 if (newHeapTop.belowOrEqual(heapTop)) {
287 return Word.zero();
288 }
289
290 Word heapEnd = heapEndAddress.readWord(0, HEAP_END_LOCATION);
291 if (newHeapTop.aboveThan(heapEnd)) {
292 return Word.zero();
293 }
294
295 if (compareAndSwap(RawAddressNode.address(heapTopAddress), heapTop, newHeapTop, HEAP_TOP_LOCATION).equal(heapTop)) {
296 return heapTop;
297 }
298 }
299 }
300
301 @Fold
302 static boolean forceSlowPath() {
303 return StubOptions.ForceUseOfNewInstanceStub.getValue();
304 }
305
306 public static final ForeignCallDescriptor NEW_INSTANCE_C = newDescriptor(NewInstanceStub.class, "newInstanceC", void.class, Word.class, KlassPointer.class);
307
308 @NodeIntrinsic(StubForeignCallNode.class)
309 public static native void newInstanceC(@ConstantNodeParameter ForeignCallDescriptor newInstanceC, Word thread, KlassPointer hub);
310 }