22 */
23 package org.graalvm.compiler.hotspot.stubs;
24
25 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFIG;
26 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.HEAP_END_LOCATION;
27 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.HEAP_TOP_LOCATION;
28 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.PROTOTYPE_MARK_WORD_LOCATION;
29 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_FAST_REFILL_WASTE_LOCATION;
30 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_NOF_REFILLS_LOCATION;
31 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_REFILL_WASTE_LIMIT_LOCATION;
32 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_SIZE_LOCATION;
33 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_SLOW_ALLOCATIONS_LOCATION;
34 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_THREAD_ALLOCATED_BYTES_LOCATION;
35 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayBaseOffset;
36 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.getAndClearObjectResult;
37 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.initializeTlab;
38 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.isInstanceKlassFullyInitialized;
39 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readLayoutHelper;
40 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.log2WordSize;
41 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.prototypeMarkWordOffset;
42 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabEnd;
43 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabStart;
44 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabTop;
45 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.registerAsWord;
46 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.threadAllocatedBytesOffset;
47 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.threadTlabSizeOffset;
48 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabAlignmentReserveInHeapWords;
49 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabFastRefillWasteOffset;
50 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabIntArrayMarkWord;
51 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabNumberOfRefillsOffset;
52 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabRefillWasteIncrement;
53 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabRefillWasteLimitOffset;
54 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabSlowAllocationsOffset;
55 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabStats;
56 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useCMSIncrementalMode;
57 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useG1GC;
58 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useTLAB;
59 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.wordSize;
60 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.writeTlabTop;
61 import static org.graalvm.compiler.hotspot.stubs.StubUtil.handlePendingException;
62 import static org.graalvm.compiler.hotspot.stubs.StubUtil.newDescriptor;
98 public NewInstanceStub(OptionValues options, HotSpotProviders providers, HotSpotForeignCallLinkage linkage) {
99 super("newInstance", options, providers, linkage);
100 }
101
102 @Override
103 protected Object[] makeConstArgs() {
104 HotSpotResolvedObjectType intArrayType = (HotSpotResolvedObjectType) providers.getMetaAccess().lookupJavaType(int[].class);
105 int count = method.getSignature().getParameterCount(false);
106 Object[] args = new Object[count];
107 assert checkConstArg(1, "intArrayHub");
108 assert checkConstArg(2, "threadRegister");
109 assert checkConstArg(3, "options");
110 args[1] = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), intArrayType.klass(), null);
111 args[2] = providers.getRegisters().getThreadRegister();
112 args[3] = options;
113 return args;
114 }
115
116 private static Word allocate(Word thread, int size) {
117 Word top = readTlabTop(thread);
118 Word end = readTlabEnd(thread);
119 Word newTop = top.add(size);
120 /*
121 * this check might lead to problems if the TLAB is within 16GB of the address space end
122 * (checked in c++ code)
123 */
124 if (probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
125 writeTlabTop(thread, newTop);
126 return top;
127 }
128 return WordFactory.zero();
129 }
130
131 @Fold
132 static boolean logging(OptionValues options) {
133 return StubOptions.TraceNewInstanceStub.getValue(options);
134 }
135
136 /**
137 * Re-attempts allocation after an initial TLAB allocation failed or was skipped (e.g., due to
138 * -XX:-UseTLAB).
175 * @param intArrayHub the hub for {@code int[].class}
176 * @param sizeInBytes the size of the allocation
177 * @param log specifies if logging is enabled
178 *
179 * @return the newly allocated, uninitialized chunk of memory, or {@link WordFactory#zero()} if
180 * the operation was unsuccessful
181 */
182 static Word refillAllocate(Word thread, KlassPointer intArrayHub, int sizeInBytes, boolean log) {
183 // If G1 is enabled, the "eden" allocation space is not the same always
184 // and therefore we have to go to slowpath to allocate a new TLAB.
185 if (useG1GC(INJECTED_VMCONFIG)) {
186 return WordFactory.zero();
187 }
188 if (!useTLAB(INJECTED_VMCONFIG)) {
189 return edenAllocate(WordFactory.unsigned(sizeInBytes), log);
190 }
191 Word intArrayMarkWord = WordFactory.unsigned(tlabIntArrayMarkWord(INJECTED_VMCONFIG));
192 int alignmentReserveInBytes = tlabAlignmentReserveInHeapWords(INJECTED_VMCONFIG) * wordSize();
193
194 Word top = readTlabTop(thread);
195 Word end = readTlabEnd(thread);
196
197 // calculate amount of free space
198 long tlabFreeSpaceInBytes = end.subtract(top).rawValue();
199
200 if (log) {
201 printf("refillTLAB: thread=%p\n", thread.rawValue());
202 printf("refillTLAB: top=%p\n", top.rawValue());
203 printf("refillTLAB: end=%p\n", end.rawValue());
204 printf("refillTLAB: tlabFreeSpaceInBytes=%ld\n", tlabFreeSpaceInBytes);
205 }
206
207 long tlabFreeSpaceInWords = tlabFreeSpaceInBytes >>> log2WordSize();
208
209 // Retain TLAB and allocate object in shared space if
210 // the amount free in the TLAB is too large to discard.
211 Word refillWasteLimit = thread.readWord(tlabRefillWasteLimitOffset(INJECTED_VMCONFIG), TLAB_REFILL_WASTE_LIMIT_LOCATION);
212 if (tlabFreeSpaceInWords <= refillWasteLimit.rawValue()) {
213 if (tlabStats(INJECTED_VMCONFIG)) {
214 // increment number of refills
215 thread.writeInt(tlabNumberOfRefillsOffset(INJECTED_VMCONFIG), thread.readInt(tlabNumberOfRefillsOffset(INJECTED_VMCONFIG), TLAB_NOF_REFILLS_LOCATION) + 1, TLAB_NOF_REFILLS_LOCATION);
|
22 */
23 package org.graalvm.compiler.hotspot.stubs;
24
25 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFIG;
26 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.HEAP_END_LOCATION;
27 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.HEAP_TOP_LOCATION;
28 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.PROTOTYPE_MARK_WORD_LOCATION;
29 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_FAST_REFILL_WASTE_LOCATION;
30 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_NOF_REFILLS_LOCATION;
31 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_REFILL_WASTE_LIMIT_LOCATION;
32 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_SIZE_LOCATION;
33 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_SLOW_ALLOCATIONS_LOCATION;
34 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_THREAD_ALLOCATED_BYTES_LOCATION;
35 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayBaseOffset;
36 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.getAndClearObjectResult;
37 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.initializeTlab;
38 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.isInstanceKlassFullyInitialized;
39 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readLayoutHelper;
40 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.log2WordSize;
41 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.prototypeMarkWordOffset;
42 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabFastPathEnd;
43 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabStart;
44 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabTop;
45 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.registerAsWord;
46 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.threadAllocatedBytesOffset;
47 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.threadTlabSizeOffset;
48 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabAlignmentReserveInHeapWords;
49 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabFastRefillWasteOffset;
50 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabIntArrayMarkWord;
51 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabNumberOfRefillsOffset;
52 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabRefillWasteIncrement;
53 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabRefillWasteLimitOffset;
54 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabSlowAllocationsOffset;
55 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabStats;
56 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useCMSIncrementalMode;
57 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useG1GC;
58 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useTLAB;
59 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.wordSize;
60 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.writeTlabTop;
61 import static org.graalvm.compiler.hotspot.stubs.StubUtil.handlePendingException;
62 import static org.graalvm.compiler.hotspot.stubs.StubUtil.newDescriptor;
98 public NewInstanceStub(OptionValues options, HotSpotProviders providers, HotSpotForeignCallLinkage linkage) {
99 super("newInstance", options, providers, linkage);
100 }
101
102 @Override
103 protected Object[] makeConstArgs() {
104 HotSpotResolvedObjectType intArrayType = (HotSpotResolvedObjectType) providers.getMetaAccess().lookupJavaType(int[].class);
105 int count = method.getSignature().getParameterCount(false);
106 Object[] args = new Object[count];
107 assert checkConstArg(1, "intArrayHub");
108 assert checkConstArg(2, "threadRegister");
109 assert checkConstArg(3, "options");
110 args[1] = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), intArrayType.klass(), null);
111 args[2] = providers.getRegisters().getThreadRegister();
112 args[3] = options;
113 return args;
114 }
115
116 private static Word allocate(Word thread, int size) {
117 Word top = readTlabTop(thread);
118 Word end = readTlabFastPathEnd(thread);
119 Word newTop = top.add(size);
120 /*
121 * this check might lead to problems if the TLAB is within 16GB of the address space end
122 * (checked in c++ code)
123 */
124 if (probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
125 writeTlabTop(thread, newTop);
126 return top;
127 }
128 return WordFactory.zero();
129 }
130
131 @Fold
132 static boolean logging(OptionValues options) {
133 return StubOptions.TraceNewInstanceStub.getValue(options);
134 }
135
136 /**
137 * Re-attempts allocation after an initial TLAB allocation failed or was skipped (e.g., due to
138 * -XX:-UseTLAB).
175 * @param intArrayHub the hub for {@code int[].class}
176 * @param sizeInBytes the size of the allocation
177 * @param log specifies if logging is enabled
178 *
179 * @return the newly allocated, uninitialized chunk of memory, or {@link WordFactory#zero()} if
180 * the operation was unsuccessful
181 */
182 static Word refillAllocate(Word thread, KlassPointer intArrayHub, int sizeInBytes, boolean log) {
183 // If G1 is enabled, the "eden" allocation space is not the same always
184 // and therefore we have to go to slowpath to allocate a new TLAB.
185 if (useG1GC(INJECTED_VMCONFIG)) {
186 return WordFactory.zero();
187 }
188 if (!useTLAB(INJECTED_VMCONFIG)) {
189 return edenAllocate(WordFactory.unsigned(sizeInBytes), log);
190 }
191 Word intArrayMarkWord = WordFactory.unsigned(tlabIntArrayMarkWord(INJECTED_VMCONFIG));
192 int alignmentReserveInBytes = tlabAlignmentReserveInHeapWords(INJECTED_VMCONFIG) * wordSize();
193
194 Word top = readTlabTop(thread);
195 Word end = readTlabFastPathEnd(thread);
196
197 // calculate amount of free space
198 long tlabFreeSpaceInBytes = end.subtract(top).rawValue();
199
200 if (log) {
201 printf("refillTLAB: thread=%p\n", thread.rawValue());
202 printf("refillTLAB: top=%p\n", top.rawValue());
203 printf("refillTLAB: end=%p\n", end.rawValue());
204 printf("refillTLAB: tlabFreeSpaceInBytes=%ld\n", tlabFreeSpaceInBytes);
205 }
206
207 long tlabFreeSpaceInWords = tlabFreeSpaceInBytes >>> log2WordSize();
208
209 // Retain TLAB and allocate object in shared space if
210 // the amount free in the TLAB is too large to discard.
211 Word refillWasteLimit = thread.readWord(tlabRefillWasteLimitOffset(INJECTED_VMCONFIG), TLAB_REFILL_WASTE_LIMIT_LOCATION);
212 if (tlabFreeSpaceInWords <= refillWasteLimit.rawValue()) {
213 if (tlabStats(INJECTED_VMCONFIG)) {
214 // increment number of refills
215 thread.writeInt(tlabNumberOfRefillsOffset(INJECTED_VMCONFIG), thread.readInt(tlabNumberOfRefillsOffset(INJECTED_VMCONFIG), TLAB_NOF_REFILLS_LOCATION) + 1, TLAB_NOF_REFILLS_LOCATION);
|