1 /*
2 * Copyright (c) 2012, 2016, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23 package org.graalvm.compiler.hotspot.replacements;
24
25 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFIG;
26 import static org.graalvm.compiler.hotspot.meta.HotSpotForeignCallsProviderImpl.VERIFY_OOP;
27 import static org.graalvm.compiler.hotspot.replacements.UnsafeAccess.UNSAFE;
28 import static jdk.vm.ci.hotspot.HotSpotJVMCIRuntimeProvider.getArrayBaseOffset;
29 import static jdk.vm.ci.hotspot.HotSpotJVMCIRuntimeProvider.getArrayIndexScale;
30
31 import org.graalvm.compiler.api.replacements.Fold;
32 import org.graalvm.compiler.api.replacements.Fold.InjectedParameter;
33 import org.graalvm.compiler.core.common.LocationIdentity;
34 import org.graalvm.compiler.core.common.SuppressFBWarnings;
35 import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor;
36 import org.graalvm.compiler.core.common.type.ObjectStamp;
37 import org.graalvm.compiler.core.common.type.TypeReference;
38 import org.graalvm.compiler.debug.GraalError;
39 import org.graalvm.compiler.graph.Node.ConstantNodeParameter;
40 import org.graalvm.compiler.graph.Node.NodeIntrinsic;
41 import org.graalvm.compiler.graph.spi.CanonicalizerTool;
42 import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig;
43 import org.graalvm.compiler.hotspot.nodes.CompressionNode;
44 import org.graalvm.compiler.hotspot.nodes.ComputeObjectAddressNode;
45 import org.graalvm.compiler.hotspot.nodes.SnippetAnchorNode;
46 import org.graalvm.compiler.hotspot.word.KlassPointer;
47 import org.graalvm.compiler.nodes.CanonicalizableLocation;
48 import org.graalvm.compiler.nodes.ConstantNode;
49 import org.graalvm.compiler.nodes.NamedLocationIdentity;
50 import org.graalvm.compiler.nodes.ValueNode;
51 import org.graalvm.compiler.nodes.extended.ForeignCallNode;
52 import org.graalvm.compiler.nodes.extended.GuardingNode;
53 import org.graalvm.compiler.nodes.extended.LoadHubNode;
54 import org.graalvm.compiler.nodes.extended.StoreHubNode;
55 import org.graalvm.compiler.nodes.extended.UnsafeLoadNode;
56 import org.graalvm.compiler.nodes.memory.Access;
57 import org.graalvm.compiler.nodes.memory.address.AddressNode;
58 import org.graalvm.compiler.nodes.memory.address.OffsetAddressNode;
59 import org.graalvm.compiler.nodes.type.StampTool;
60 import org.graalvm.compiler.replacements.ReplacementsUtil;
61 import org.graalvm.compiler.replacements.nodes.ReadRegisterNode;
62 import org.graalvm.compiler.replacements.nodes.WriteRegisterNode;
63 import org.graalvm.compiler.word.Word;
64
65 import jdk.vm.ci.code.CodeUtil;
66 import jdk.vm.ci.code.Register;
67 import jdk.vm.ci.hotspot.HotSpotJVMCIRuntime;
68 import jdk.vm.ci.hotspot.HotSpotJVMCIRuntimeProvider;
69 import jdk.vm.ci.hotspot.HotSpotMetaspaceConstant;
70 import jdk.vm.ci.hotspot.HotSpotResolvedObjectType;
71 import jdk.vm.ci.meta.Assumptions;
72 import jdk.vm.ci.meta.Assumptions.AssumptionResult;
73 import jdk.vm.ci.meta.JavaKind;
74 import jdk.vm.ci.meta.ResolvedJavaType;
75
76 //JaCoCo Exclude
77
78 /**
79 * A collection of methods used in HotSpot snippets, substitutions and stubs.
80 */
81 public class HotSpotReplacementsUtil {
82
83 abstract static class HotSpotOptimizingLocationIdentity extends NamedLocationIdentity implements CanonicalizableLocation {
84
85 HotSpotOptimizingLocationIdentity(String name) {
86 super(name, true);
87 }
88
89 @Override
90 public abstract ValueNode canonicalizeRead(ValueNode read, AddressNode location, ValueNode object, CanonicalizerTool tool);
91
92 protected ValueNode findReadHub(ValueNode object) {
93 ValueNode base = object;
94 if (base instanceof CompressionNode) {
95 base = ((CompressionNode) base).getValue();
96 }
97 if (base instanceof Access) {
98 Access access = (Access) base;
99 if (access.getLocationIdentity().equals(HUB_LOCATION) || access.getLocationIdentity().equals(COMPRESSED_HUB_LOCATION)) {
100 AddressNode address = access.getAddress();
101 if (address instanceof OffsetAddressNode) {
102 OffsetAddressNode offset = (OffsetAddressNode) address;
103 return offset.getBase();
104 }
105 }
106 } else if (base instanceof LoadHubNode) {
107 LoadHubNode loadhub = (LoadHubNode) base;
108 return loadhub.getValue();
109 }
110 return null;
111 }
112
113 /**
114 * Fold reads that convert from Class -> Hub -> Class or vice versa.
115 *
116 * @param read
117 * @param object
118 * @param otherLocation
119 * @return an earlier read or the original {@code read}
120 */
121 protected static ValueNode foldIndirection(ValueNode read, ValueNode object, LocationIdentity otherLocation) {
122 if (object instanceof Access) {
123 Access access = (Access) object;
124 if (access.getLocationIdentity().equals(otherLocation)) {
125 AddressNode address = access.getAddress();
126 if (address instanceof OffsetAddressNode) {
127 OffsetAddressNode offset = (OffsetAddressNode) address;
128 assert offset.getBase().stamp().isCompatible(read.stamp());
129 return offset.getBase();
130 }
131 }
132 }
133 return read;
134 }
135 }
136
137 public static HotSpotJVMCIRuntimeProvider runtime() {
138 return HotSpotJVMCIRuntime.runtime();
139 }
140
141 @Fold
142 public static GraalHotSpotVMConfig config(@InjectedParameter GraalHotSpotVMConfig config) {
143 assert config != null;
144 return config;
145 }
146
147 @Fold
148 public static boolean useTLAB(@InjectedParameter GraalHotSpotVMConfig config) {
149 return config.useTLAB;
150 }
151
152 @Fold
153 public static boolean verifyOops(@InjectedParameter GraalHotSpotVMConfig config) {
154 return config.verifyOops;
155 }
156
157 public static final LocationIdentity EXCEPTION_OOP_LOCATION = NamedLocationIdentity.mutable("ExceptionOop");
158
159 /**
160 * @see GraalHotSpotVMConfig#threadExceptionOopOffset
161 */
162 @Fold
163 public static int threadExceptionOopOffset(@InjectedParameter GraalHotSpotVMConfig config) {
164 return config.threadExceptionOopOffset;
165 }
166
167 public static final LocationIdentity EXCEPTION_PC_LOCATION = NamedLocationIdentity.mutable("ExceptionPc");
168
169 @Fold
170 public static int threadExceptionPcOffset(@InjectedParameter GraalHotSpotVMConfig config) {
171 return config.threadExceptionPcOffset;
172 }
173
174 public static final LocationIdentity TLAB_TOP_LOCATION = NamedLocationIdentity.mutable("TlabTop");
175
176 @Fold
177 public static int threadTlabTopOffset(@InjectedParameter GraalHotSpotVMConfig config) {
178 return config.threadTlabTopOffset();
179 }
180
181 public static final LocationIdentity TLAB_END_LOCATION = NamedLocationIdentity.mutable("TlabEnd");
182
183 @Fold
184 static int threadTlabEndOffset(@InjectedParameter GraalHotSpotVMConfig config) {
185 return config.threadTlabEndOffset();
186 }
187
188 public static final LocationIdentity TLAB_START_LOCATION = NamedLocationIdentity.mutable("TlabStart");
189
190 @Fold
191 static int threadTlabStartOffset(@InjectedParameter GraalHotSpotVMConfig config) {
192 return config.threadTlabStartOffset();
193 }
194
195 public static final LocationIdentity PENDING_EXCEPTION_LOCATION = NamedLocationIdentity.mutable("PendingException");
196
197 /**
198 * @see GraalHotSpotVMConfig#pendingExceptionOffset
199 */
200 @Fold
201 static int threadPendingExceptionOffset(@InjectedParameter GraalHotSpotVMConfig config) {
202 return config.pendingExceptionOffset;
203 }
204
205 public static final LocationIdentity PENDING_DEOPTIMIZATION_LOCATION = NamedLocationIdentity.mutable("PendingDeoptimization");
206
207 /**
208 * @see GraalHotSpotVMConfig#pendingDeoptimizationOffset
209 */
210 @Fold
211 static int threadPendingDeoptimizationOffset(@InjectedParameter GraalHotSpotVMConfig config) {
212 return config.pendingDeoptimizationOffset;
213 }
214
215 public static final LocationIdentity OBJECT_RESULT_LOCATION = NamedLocationIdentity.mutable("ObjectResult");
216
217 @Fold
218 static int objectResultOffset(@InjectedParameter GraalHotSpotVMConfig config) {
219 return config.threadObjectResultOffset;
220 }
221
222 /**
223 * @see GraalHotSpotVMConfig#threadExceptionOopOffset
224 */
225 public static Object readExceptionOop(Word thread) {
226 return thread.readObject(threadExceptionOopOffset(INJECTED_VMCONFIG), EXCEPTION_OOP_LOCATION);
227 }
228
229 public static Word readExceptionPc(Word thread) {
230 return thread.readWord(threadExceptionPcOffset(INJECTED_VMCONFIG), EXCEPTION_PC_LOCATION);
231 }
232
233 /**
234 * @see GraalHotSpotVMConfig#threadExceptionOopOffset
235 */
236 public static void writeExceptionOop(Word thread, Object value) {
237 thread.writeObject(threadExceptionOopOffset(INJECTED_VMCONFIG), value, EXCEPTION_OOP_LOCATION);
238 }
239
240 public static void writeExceptionPc(Word thread, Word value) {
241 thread.writeWord(threadExceptionPcOffset(INJECTED_VMCONFIG), value, EXCEPTION_PC_LOCATION);
242 }
243
244 public static Word readTlabTop(Word thread) {
245 return thread.readWord(threadTlabTopOffset(INJECTED_VMCONFIG), TLAB_TOP_LOCATION);
246 }
247
248 public static Word readTlabEnd(Word thread) {
249 return thread.readWord(threadTlabEndOffset(INJECTED_VMCONFIG), TLAB_END_LOCATION);
250 }
251
252 public static Word readTlabStart(Word thread) {
253 return thread.readWord(threadTlabStartOffset(INJECTED_VMCONFIG), TLAB_START_LOCATION);
254 }
255
256 public static void writeTlabTop(Word thread, Word top) {
257 thread.writeWord(threadTlabTopOffset(INJECTED_VMCONFIG), top, TLAB_TOP_LOCATION);
258 }
259
260 @SuppressFBWarnings(value = "NP_NULL_PARAM_DEREF_NONVIRTUAL", justification = "foldable method parameters are injected")
261 public static void initializeTlab(Word thread, Word start, Word end) {
262 thread.writeWord(threadTlabStartOffset(INJECTED_VMCONFIG), start, TLAB_START_LOCATION);
263 thread.writeWord(threadTlabTopOffset(INJECTED_VMCONFIG), start, TLAB_TOP_LOCATION);
264 thread.writeWord(threadTlabEndOffset(INJECTED_VMCONFIG), end, TLAB_END_LOCATION);
265 }
266
267 /**
268 * Clears the pending exception for the given thread.
269 *
270 * @return the pending exception, or null if there was none
271 */
272 @SuppressFBWarnings(value = "NP_NULL_PARAM_DEREF_NONVIRTUAL", justification = "foldable method parameters are injected")
273 public static Object clearPendingException(Word thread) {
274 Object result = thread.readObject(threadPendingExceptionOffset(INJECTED_VMCONFIG), PENDING_EXCEPTION_LOCATION);
275 thread.writeObject(threadPendingExceptionOffset(INJECTED_VMCONFIG), null, PENDING_EXCEPTION_LOCATION);
276 return result;
277 }
278
279 /**
280 * Reads the pending deoptimization value for the given thread.
281 *
282 * @return {@code true} if there was a pending deoptimization
283 */
284 public static int readPendingDeoptimization(Word thread) {
285 return thread.readInt(threadPendingDeoptimizationOffset(INJECTED_VMCONFIG), PENDING_DEOPTIMIZATION_LOCATION);
286 }
287
288 /**
289 * Writes the pending deoptimization value for the given thread.
290 */
291 public static void writePendingDeoptimization(Word thread, int value) {
292 thread.writeInt(threadPendingDeoptimizationOffset(INJECTED_VMCONFIG), value, PENDING_DEOPTIMIZATION_LOCATION);
293 }
294
295 /**
296 * Gets and clears the object result from a runtime call stored in a thread local.
297 *
298 * @return the object that was in the thread local
299 */
300 public static Object getAndClearObjectResult(Word thread) {
301 Object result = thread.readObject(objectResultOffset(INJECTED_VMCONFIG), OBJECT_RESULT_LOCATION);
302 thread.writeObject(objectResultOffset(INJECTED_VMCONFIG), null, OBJECT_RESULT_LOCATION);
303 return result;
304 }
305
306 public static final LocationIdentity JAVA_THREAD_THREAD_OBJECT_LOCATION = NamedLocationIdentity.mutable("JavaThread::_threadObj");
307
308 @Fold
309 public static int threadObjectOffset(@InjectedParameter GraalHotSpotVMConfig config) {
310 return config.threadObjectOffset;
311 }
312
313 public static final LocationIdentity JAVA_THREAD_OSTHREAD_LOCATION = NamedLocationIdentity.mutable("JavaThread::_osthread");
314
315 @Fold
316 public static int osThreadOffset(@InjectedParameter GraalHotSpotVMConfig config) {
317 return config.osThreadOffset;
318 }
319
320 @Fold
321 public static int osThreadInterruptedOffset(@InjectedParameter GraalHotSpotVMConfig config) {
322 return config.osThreadInterruptedOffset;
323 }
324
325 @Fold
326 public static JavaKind getWordKind() {
327 return runtime().getHostJVMCIBackend().getCodeCache().getTarget().wordJavaKind;
328 }
329
330 @Fold
331 public static int wordSize() {
332 return runtime().getHostJVMCIBackend().getCodeCache().getTarget().wordSize;
333 }
334
335 @Fold
336 public static int pageSize() {
337 return UNSAFE.pageSize();
338 }
339
340 @Fold
341 public static int heapWordSize(@InjectedParameter GraalHotSpotVMConfig config) {
342 return config.heapWordSize;
343 }
344
345 public static final LocationIdentity PROTOTYPE_MARK_WORD_LOCATION = NamedLocationIdentity.mutable("PrototypeMarkWord");
346
347 @Fold
348 public static int prototypeMarkWordOffset(@InjectedParameter GraalHotSpotVMConfig config) {
349 return config.prototypeMarkWordOffset;
350 }
351
352 @Fold
353 public static long arrayPrototypeMarkWord(@InjectedParameter GraalHotSpotVMConfig config) {
354 return config.arrayPrototypeMarkWord();
355 }
356
357 public static final LocationIdentity KLASS_ACCESS_FLAGS_LOCATION = NamedLocationIdentity.immutable("Klass::_access_flags");
358
359 @Fold
360 public static int klassAccessFlagsOffset(@InjectedParameter GraalHotSpotVMConfig config) {
361 return config.klassAccessFlagsOffset;
362 }
363
364 @Fold
365 public static int jvmAccWrittenFlags(@InjectedParameter GraalHotSpotVMConfig config) {
366 return config.jvmAccWrittenFlags;
367 }
368
369 public static final LocationIdentity KLASS_LAYOUT_HELPER_LOCATION = new HotSpotOptimizingLocationIdentity("Klass::_layout_helper") {
370 @Override
371 public ValueNode canonicalizeRead(ValueNode read, AddressNode location, ValueNode object, CanonicalizerTool tool) {
372 ValueNode javaObject = findReadHub(object);
373 if (javaObject != null) {
374 if (javaObject.stamp() instanceof ObjectStamp) {
375 ObjectStamp stamp = (ObjectStamp) javaObject.stamp();
376 HotSpotResolvedObjectType type = (HotSpotResolvedObjectType) stamp.javaType(tool.getMetaAccess());
377 if (type.isArray() && !type.getComponentType().isPrimitive()) {
378 int layout = type.layoutHelper();
379 return ConstantNode.forInt(layout);
380 }
381 }
382 }
383 return read;
384 }
385 };
386
387 @Fold
388 public static int klassLayoutHelperOffset(@InjectedParameter GraalHotSpotVMConfig config) {
389 return config.klassLayoutHelperOffset;
390 }
391
392 public static int readLayoutHelper(KlassPointer hub) {
393 // return hub.readInt(klassLayoutHelperOffset(), KLASS_LAYOUT_HELPER_LOCATION);
394 GuardingNode anchorNode = SnippetAnchorNode.anchor();
395 return loadKlassLayoutHelperIntrinsic(hub, anchorNode);
396 }
397
398 @NodeIntrinsic(value = KlassLayoutHelperNode.class)
399 public static native int loadKlassLayoutHelperIntrinsic(KlassPointer object, GuardingNode anchor);
400
401 @NodeIntrinsic(value = KlassLayoutHelperNode.class)
402 public static native int loadKlassLayoutHelperIntrinsic(KlassPointer object);
403
404 /**
405 * Checks if class {@code klass} is an array.
406 *
407 * See: Klass::layout_helper_is_array
408 *
409 * @param klass the class to be checked
410 * @return true if klass is an array, false otherwise
411 */
412 public static boolean klassIsArray(KlassPointer klass) {
413 /*
414 * The less-than check only works if both values are ints. We use local variables to make
415 * sure these are still ints and haven't changed.
416 */
417 final int layoutHelper = readLayoutHelper(klass);
418 final int layoutHelperNeutralValue = config(INJECTED_VMCONFIG).klassLayoutHelperNeutralValue;
419 return (layoutHelper < layoutHelperNeutralValue);
420 }
421
422 public static final LocationIdentity ARRAY_KLASS_COMPONENT_MIRROR = NamedLocationIdentity.immutable("ArrayKlass::_component_mirror");
423
424 @Fold
425 public static int arrayKlassComponentMirrorOffset(@InjectedParameter GraalHotSpotVMConfig config) {
426 return config.getFieldOffset("ArrayKlass::_component_mirror", Integer.class, "oop");
427 }
428
429 public static final LocationIdentity KLASS_SUPER_KLASS_LOCATION = NamedLocationIdentity.immutable("Klass::_super");
430
431 @Fold
432 public static int klassSuperKlassOffset(@InjectedParameter GraalHotSpotVMConfig config) {
433 return config.klassSuperKlassOffset;
434 }
435
436 public static final LocationIdentity MARK_WORD_LOCATION = NamedLocationIdentity.mutable("MarkWord");
437
438 @Fold
439 public static int markOffset(@InjectedParameter GraalHotSpotVMConfig config) {
440 return config.markOffset;
441 }
442
443 public static final LocationIdentity HUB_WRITE_LOCATION = NamedLocationIdentity.mutable("Hub:write");
444
445 public static final LocationIdentity HUB_LOCATION = new HotSpotOptimizingLocationIdentity("Hub") {
446 @Override
447 public ValueNode canonicalizeRead(ValueNode read, AddressNode location, ValueNode object, CanonicalizerTool tool) {
448 TypeReference constantType = StampTool.typeReferenceOrNull(object);
449 if (constantType != null && constantType.isExact()) {
450 return ConstantNode.forConstant(read.stamp(), tool.getConstantReflection().asObjectHub(constantType.getType()), tool.getMetaAccess());
451 }
452 return read;
453 }
454 };
455
456 public static final LocationIdentity COMPRESSED_HUB_LOCATION = new HotSpotOptimizingLocationIdentity("CompressedHub") {
457 @Override
458 public ValueNode canonicalizeRead(ValueNode read, AddressNode location, ValueNode object, CanonicalizerTool tool) {
459 TypeReference constantType = StampTool.typeReferenceOrNull(object);
460 if (constantType != null && constantType.isExact()) {
461 return ConstantNode.forConstant(read.stamp(), ((HotSpotMetaspaceConstant) tool.getConstantReflection().asObjectHub(constantType.getType())).compress(), tool.getMetaAccess());
462 }
463 return read;
464 }
465 };
466
467 @Fold
468 static int hubOffset(@InjectedParameter GraalHotSpotVMConfig config) {
469 return config.hubOffset;
470 }
471
472 public static void initializeObjectHeader(Word memory, Word markWord, KlassPointer hub) {
473 memory.writeWord(markOffset(INJECTED_VMCONFIG), markWord, MARK_WORD_LOCATION);
474 StoreHubNode.write(memory, hub);
475 }
476
477 @Fold
478 public static int unlockedMask(@InjectedParameter GraalHotSpotVMConfig config) {
479 return config.unlockedMask;
480 }
481
482 /**
483 * Mask for a biasable, locked or unlocked mark word.
484 *
485 * <pre>
486 * +----------------------------------+-+-+
487 * | 1|1|1|
488 * +----------------------------------+-+-+
489 * </pre>
490 *
491 */
492 @Fold
493 public static int biasedLockMaskInPlace(@InjectedParameter GraalHotSpotVMConfig config) {
494 return config.biasedLockMaskInPlace;
495 }
496
497 @Fold
498 public static int epochMaskInPlace(@InjectedParameter GraalHotSpotVMConfig config) {
499 return config.epochMaskInPlace;
500 }
501
502 /**
503 * Pattern for a biasable, unlocked mark word.
504 *
505 * <pre>
506 * +----------------------------------+-+-+
507 * | 1|0|1|
508 * +----------------------------------+-+-+
509 * </pre>
510 *
511 */
512 @Fold
513 public static int biasedLockPattern(@InjectedParameter GraalHotSpotVMConfig config) {
514 return config.biasedLockPattern;
515 }
516
517 @Fold
518 public static int ageMaskInPlace(@InjectedParameter GraalHotSpotVMConfig config) {
519 return config.ageMaskInPlace;
520 }
521
522 @Fold
523 public static int metaspaceArrayLengthOffset(@InjectedParameter GraalHotSpotVMConfig config) {
524 return config.metaspaceArrayLengthOffset;
525 }
526
527 @Fold
528 public static int metaspaceArrayBaseOffset(@InjectedParameter GraalHotSpotVMConfig config) {
529 return config.metaspaceArrayBaseOffset;
530 }
531
532 @Fold
533 public static int arrayLengthOffset(@InjectedParameter GraalHotSpotVMConfig config) {
534 return config.arrayOopDescLengthOffset();
535 }
536
537 @Fold
538 public static int arrayBaseOffset(JavaKind elementKind) {
539 return getArrayBaseOffset(elementKind);
540 }
541
542 @Fold
543 public static int arrayIndexScale(JavaKind elementKind) {
544 return getArrayIndexScale(elementKind);
545 }
546
547 public static Word arrayStart(int[] a) {
548 return Word.unsigned(ComputeObjectAddressNode.get(a, getArrayBaseOffset(JavaKind.Int)));
549 }
550
551 @Fold
552 public static int instanceHeaderSize(@InjectedParameter GraalHotSpotVMConfig config) {
553 return config.useCompressedClassPointers ? (2 * wordSize()) - 4 : 2 * wordSize();
554 }
555
556 @Fold
557 public static byte dirtyCardValue(@InjectedParameter GraalHotSpotVMConfig config) {
558 return config.dirtyCardValue;
559 }
560
561 @Fold
562 public static byte g1YoungCardValue(@InjectedParameter GraalHotSpotVMConfig config) {
563 return config.g1YoungCardValue;
564 }
565
566 @Fold
567 public static int cardTableShift(@InjectedParameter GraalHotSpotVMConfig config) {
568 return config.cardtableShift;
569 }
570
571 @Fold
572 public static long cardTableStart(@InjectedParameter GraalHotSpotVMConfig config) {
573 return config.cardtableStartAddress;
574 }
575
576 @Fold
577 public static int g1CardQueueIndexOffset(@InjectedParameter GraalHotSpotVMConfig config) {
578 return config.g1CardQueueIndexOffset();
579 }
580
581 @Fold
582 public static int g1CardQueueBufferOffset(@InjectedParameter GraalHotSpotVMConfig config) {
583 return config.g1CardQueueBufferOffset();
584 }
585
586 @Fold
587 public static int logOfHeapRegionGrainBytes(@InjectedParameter GraalHotSpotVMConfig config) {
588 return config.logOfHRGrainBytes;
589 }
590
591 @Fold
592 public static int g1SATBQueueMarkingOffset(@InjectedParameter GraalHotSpotVMConfig config) {
593 return config.g1SATBQueueMarkingOffset();
594 }
595
596 @Fold
597 public static int g1SATBQueueIndexOffset(@InjectedParameter GraalHotSpotVMConfig config) {
598 return config.g1SATBQueueIndexOffset();
599 }
600
601 @Fold
602 public static int g1SATBQueueBufferOffset(@InjectedParameter GraalHotSpotVMConfig config) {
603 return config.g1SATBQueueBufferOffset();
604 }
605
606 public static final LocationIdentity KLASS_SUPER_CHECK_OFFSET_LOCATION = NamedLocationIdentity.immutable("Klass::_super_check_offset");
607
608 @Fold
609 public static int superCheckOffsetOffset(@InjectedParameter GraalHotSpotVMConfig config) {
610 return config.superCheckOffsetOffset;
611 }
612
613 public static final LocationIdentity SECONDARY_SUPER_CACHE_LOCATION = NamedLocationIdentity.mutable("SecondarySuperCache");
614
615 @Fold
616 public static int secondarySuperCacheOffset(@InjectedParameter GraalHotSpotVMConfig config) {
617 return config.secondarySuperCacheOffset;
618 }
619
620 public static final LocationIdentity SECONDARY_SUPERS_LOCATION = NamedLocationIdentity.immutable("SecondarySupers");
621
622 @Fold
623 public static int secondarySupersOffset(@InjectedParameter GraalHotSpotVMConfig config) {
624 return config.secondarySupersOffset;
625 }
626
627 public static final LocationIdentity DISPLACED_MARK_WORD_LOCATION = NamedLocationIdentity.mutable("DisplacedMarkWord");
628
629 @Fold
630 public static int lockDisplacedMarkOffset(@InjectedParameter GraalHotSpotVMConfig config) {
631 return config.basicLockDisplacedHeaderOffset;
632 }
633
634 @Fold
635 public static boolean useBiasedLocking(@InjectedParameter GraalHotSpotVMConfig config) {
636 return config.useBiasedLocking;
637 }
638
639 @Fold
640 public static boolean useDeferredInitBarriers(@InjectedParameter GraalHotSpotVMConfig config) {
641 return config.useDeferredInitBarriers;
642 }
643
644 @Fold
645 public static boolean useG1GC(@InjectedParameter GraalHotSpotVMConfig config) {
646 return config.useG1GC;
647 }
648
649 @Fold
650 public static boolean useCompressedOops(@InjectedParameter GraalHotSpotVMConfig config) {
651 return config.useCompressedOops;
652 }
653
654 @Fold
655 static int uninitializedIdentityHashCodeValue(@InjectedParameter GraalHotSpotVMConfig config) {
656 return config.uninitializedIdentityHashCodeValue;
657 }
658
659 @Fold
660 static int identityHashCodeShift(@InjectedParameter GraalHotSpotVMConfig config) {
661 return config.identityHashCodeShift;
662 }
663
664 /**
665 * Loads the hub of an object (without null checking it first).
666 */
667 public static KlassPointer loadHub(Object object) {
668 return loadHubIntrinsic(object);
669 }
670
671 public static Object verifyOop(Object object) {
672 if (verifyOops(INJECTED_VMCONFIG)) {
673 verifyOopStub(VERIFY_OOP, object);
674 }
675 return object;
676 }
677
678 @NodeIntrinsic(ForeignCallNode.class)
679 private static native Object verifyOopStub(@ConstantNodeParameter ForeignCallDescriptor descriptor, Object object);
680
681 public static Word loadWordFromObject(Object object, int offset) {
682 ReplacementsUtil.staticAssert(offset != hubOffset(INJECTED_VMCONFIG), "Use loadHubIntrinsic instead of loadWordFromObject");
683 return loadWordFromObjectIntrinsic(object, offset, getWordKind(), LocationIdentity.any());
684 }
685
686 public static Word loadWordFromObject(Object object, int offset, LocationIdentity identity) {
687 ReplacementsUtil.staticAssert(offset != hubOffset(INJECTED_VMCONFIG), "Use loadHubIntrinsic instead of loadWordFromObject");
688 return loadWordFromObjectIntrinsic(object, offset, getWordKind(), identity);
689 }
690
691 public static KlassPointer loadKlassFromObject(Object object, int offset, LocationIdentity identity) {
692 ReplacementsUtil.staticAssert(offset != hubOffset(INJECTED_VMCONFIG), "Use loadHubIntrinsic instead of loadWordFromObject");
693 return loadKlassFromObjectIntrinsic(object, offset, getWordKind(), identity);
694 }
695
696 /**
697 * Reads the value of a given register.
698 *
699 * @param register a register which must not be available to the register allocator
700 * @return the value of {@code register} as a word
701 */
702 public static Word registerAsWord(@ConstantNodeParameter Register register) {
703 return registerAsWord(register, true, false);
704 }
705
706 @NodeIntrinsic(value = ReadRegisterNode.class, setStampFromReturnType = true)
707 public static native Word registerAsWord(@ConstantNodeParameter Register register, @ConstantNodeParameter boolean directUse, @ConstantNodeParameter boolean incoming);
708
709 @NodeIntrinsic(value = WriteRegisterNode.class, setStampFromReturnType = true)
710 public static native void writeRegisterAsWord(@ConstantNodeParameter Register register, Word value);
711
712 @NodeIntrinsic(value = UnsafeLoadNode.class, setStampFromReturnType = true)
713 private static native Word loadWordFromObjectIntrinsic(Object object, long offset, @ConstantNodeParameter JavaKind wordKind, @ConstantNodeParameter LocationIdentity locationIdentity);
714
715 @NodeIntrinsic(value = UnsafeLoadNode.class, setStampFromReturnType = true)
716 private static native KlassPointer loadKlassFromObjectIntrinsic(Object object, long offset, @ConstantNodeParameter JavaKind wordKind, @ConstantNodeParameter LocationIdentity locationIdentity);
717
718 @NodeIntrinsic(value = LoadHubNode.class)
719 public static native KlassPointer loadHubIntrinsic(Object object);
720
721 @Fold
722 public static int log2WordSize() {
723 return CodeUtil.log2(wordSize());
724 }
725
726 public static final LocationIdentity CLASS_STATE_LOCATION = NamedLocationIdentity.mutable("ClassState");
727
728 @Fold
729 public static int instanceKlassInitStateOffset(@InjectedParameter GraalHotSpotVMConfig config) {
730 return config.instanceKlassInitStateOffset;
731 }
732
733 @Fold
734 public static int instanceKlassStateFullyInitialized(@InjectedParameter GraalHotSpotVMConfig config) {
735 return config.instanceKlassStateFullyInitialized;
736 }
737
738 /**
739 *
740 * @param hub the hub of an InstanceKlass
741 * @return true is the InstanceKlass represented by hub is fully initialized
742 */
743 public static boolean isInstanceKlassFullyInitialized(KlassPointer hub) {
744 return readInstanceKlassState(hub) == instanceKlassStateFullyInitialized(INJECTED_VMCONFIG);
745 }
746
747 private static byte readInstanceKlassState(KlassPointer hub) {
748 return hub.readByte(instanceKlassInitStateOffset(INJECTED_VMCONFIG), CLASS_STATE_LOCATION);
749 }
750
751 public static final LocationIdentity KLASS_MODIFIER_FLAGS_LOCATION = NamedLocationIdentity.immutable("Klass::_modifier_flags");
752
753 @Fold
754 public static int klassModifierFlagsOffset(@InjectedParameter GraalHotSpotVMConfig config) {
755 return config.klassModifierFlagsOffset;
756 }
757
758 public static final LocationIdentity CLASS_KLASS_LOCATION = new HotSpotOptimizingLocationIdentity("Class._klass") {
759 @Override
760 public ValueNode canonicalizeRead(ValueNode read, AddressNode location, ValueNode object, CanonicalizerTool tool) {
761 return foldIndirection(read, object, CLASS_MIRROR_LOCATION);
762 }
763 };
764
765 @Fold
766 public static int klassOffset(@InjectedParameter GraalHotSpotVMConfig config) {
767 return config.klassOffset;
768 }
769
770 public static final LocationIdentity CLASS_ARRAY_KLASS_LOCATION = new HotSpotOptimizingLocationIdentity("Class._array_klass") {
771 @Override
772 public ValueNode canonicalizeRead(ValueNode read, AddressNode location, ValueNode object, CanonicalizerTool tool) {
773 return foldIndirection(read, object, ARRAY_KLASS_COMPONENT_MIRROR);
774 }
775 };
776
777 @Fold
778 public static int arrayKlassOffset(@InjectedParameter GraalHotSpotVMConfig config) {
779 return config.arrayKlassOffset;
780 }
781
782 public static final LocationIdentity CLASS_MIRROR_LOCATION = NamedLocationIdentity.immutable("Klass::_java_mirror");
783
784 public static final LocationIdentity HEAP_TOP_LOCATION = NamedLocationIdentity.mutable("HeapTop");
785
786 @Fold
787 public static long heapTopAddress(@InjectedParameter GraalHotSpotVMConfig config) {
788 return config.heapTopAddress;
789 }
790
791 public static final LocationIdentity HEAP_END_LOCATION = NamedLocationIdentity.mutable("HeapEnd");
792
793 @Fold
794 public static long heapEndAddress(@InjectedParameter GraalHotSpotVMConfig config) {
795 return config.heapEndAddress;
796 }
797
798 @Fold
799 public static long tlabIntArrayMarkWord(@InjectedParameter GraalHotSpotVMConfig config) {
800 return config.tlabIntArrayMarkWord();
801 }
802
803 @Fold
804 public static boolean inlineContiguousAllocationSupported(@InjectedParameter GraalHotSpotVMConfig config) {
805 return config.inlineContiguousAllocationSupported;
806 }
807
808 @Fold
809 public static int tlabAlignmentReserveInHeapWords(@InjectedParameter GraalHotSpotVMConfig config) {
810 return config.tlabAlignmentReserve;
811 }
812
813 public static final LocationIdentity TLAB_SIZE_LOCATION = NamedLocationIdentity.mutable("TlabSize");
814
815 @Fold
816 public static int threadTlabSizeOffset(@InjectedParameter GraalHotSpotVMConfig config) {
817 return config.threadTlabSizeOffset();
818 }
819
820 public static final LocationIdentity TLAB_THREAD_ALLOCATED_BYTES_LOCATION = NamedLocationIdentity.mutable("TlabThreadAllocatedBytes");
821
822 @Fold
823 public static int threadAllocatedBytesOffset(@InjectedParameter GraalHotSpotVMConfig config) {
824 return config.threadAllocatedBytesOffset;
825 }
826
827 public static final LocationIdentity TLAB_REFILL_WASTE_LIMIT_LOCATION = NamedLocationIdentity.mutable("RefillWasteLimit");
828
829 @Fold
830 public static int tlabRefillWasteLimitOffset(@InjectedParameter GraalHotSpotVMConfig config) {
831 return config.tlabRefillWasteLimitOffset();
832 }
833
834 public static final LocationIdentity TLAB_NOF_REFILLS_LOCATION = NamedLocationIdentity.mutable("TlabNOfRefills");
835
836 @Fold
837 public static int tlabNumberOfRefillsOffset(@InjectedParameter GraalHotSpotVMConfig config) {
838 return config.tlabNumberOfRefillsOffset();
839 }
840
841 public static final LocationIdentity TLAB_FAST_REFILL_WASTE_LOCATION = NamedLocationIdentity.mutable("TlabFastRefillWaste");
842
843 @Fold
844 public static int tlabFastRefillWasteOffset(@InjectedParameter GraalHotSpotVMConfig config) {
845 return config.tlabFastRefillWasteOffset();
846 }
847
848 public static final LocationIdentity TLAB_SLOW_ALLOCATIONS_LOCATION = NamedLocationIdentity.mutable("TlabSlowAllocations");
849
850 @Fold
851 public static int tlabSlowAllocationsOffset(@InjectedParameter GraalHotSpotVMConfig config) {
852 return config.tlabSlowAllocationsOffset();
853 }
854
855 @Fold
856 public static int tlabRefillWasteIncrement(@InjectedParameter GraalHotSpotVMConfig config) {
857 return config.tlabRefillWasteIncrement;
858 }
859
860 @Fold
861 public static boolean tlabStats(@InjectedParameter GraalHotSpotVMConfig config) {
862 return config.tlabStats;
863 }
864
865 @Fold
866 public static int layoutHelperHeaderSizeShift(@InjectedParameter GraalHotSpotVMConfig config) {
867 return config.layoutHelperHeaderSizeShift;
868 }
869
870 @Fold
871 public static int layoutHelperHeaderSizeMask(@InjectedParameter GraalHotSpotVMConfig config) {
872 return config.layoutHelperHeaderSizeMask;
873 }
874
875 @Fold
876 public static int layoutHelperLog2ElementSizeShift(@InjectedParameter GraalHotSpotVMConfig config) {
877 return config.layoutHelperLog2ElementSizeShift;
878 }
879
880 @Fold
881 public static int layoutHelperLog2ElementSizeMask(@InjectedParameter GraalHotSpotVMConfig config) {
882 return config.layoutHelperLog2ElementSizeMask;
883 }
884
885 @Fold
886 public static int layoutHelperElementTypeShift(@InjectedParameter GraalHotSpotVMConfig config) {
887 return config.layoutHelperElementTypeShift;
888 }
889
890 @Fold
891 public static int layoutHelperElementTypeMask(@InjectedParameter GraalHotSpotVMConfig config) {
892 return config.layoutHelperElementTypeMask;
893 }
894
895 @Fold
896 public static int layoutHelperElementTypePrimitiveInPlace(@InjectedParameter GraalHotSpotVMConfig config) {
897 return config.layoutHelperElementTypePrimitiveInPlace();
898 }
899
900 @NodeIntrinsic(ForeignCallNode.class)
901 public static native int identityHashCode(@ConstantNodeParameter ForeignCallDescriptor descriptor, Object object);
902
903 @Fold
904 public static int verifiedEntryPointOffset(@InjectedParameter GraalHotSpotVMConfig config) {
905 return config.nmethodEntryOffset;
906 }
907
908 @Fold
909 public static long gcTotalCollectionsAddress(@InjectedParameter GraalHotSpotVMConfig config) {
910 return config.gcTotalCollectionsAddress();
911 }
912
913 @Fold
914 public static long referentOffset() {
915 try {
916 return UNSAFE.objectFieldOffset(java.lang.ref.Reference.class.getDeclaredField("referent"));
917 } catch (Exception e) {
918 throw new GraalError(e);
919 }
920 }
921
922 public static final LocationIdentity OBJ_ARRAY_KLASS_ELEMENT_KLASS_LOCATION = new HotSpotOptimizingLocationIdentity("ObjArrayKlass::_element_klass") {
923 @Override
924 public ValueNode canonicalizeRead(ValueNode read, AddressNode location, ValueNode object, CanonicalizerTool tool) {
925 ValueNode javaObject = findReadHub(object);
926 if (javaObject != null) {
927 ResolvedJavaType type = StampTool.typeOrNull(javaObject);
928 if (type != null && type.isArray()) {
929 ResolvedJavaType element = type.getComponentType();
930 if (element != null && !element.isPrimitive() && !element.getElementalType().isInterface()) {
931 Assumptions assumptions = object.graph().getAssumptions();
932 AssumptionResult<ResolvedJavaType> leafType = element.findLeafConcreteSubtype();
933 if (leafType != null && leafType.canRecordTo(assumptions)) {
934 leafType.recordTo(assumptions);
935 return ConstantNode.forConstant(read.stamp(), tool.getConstantReflection().asObjectHub(leafType.getResult()), tool.getMetaAccess());
936 }
937 }
938 }
939 }
940 return read;
941 }
942 };
943
944 @Fold
945 public static int arrayClassElementOffset(@InjectedParameter GraalHotSpotVMConfig config) {
946 return config.arrayClassElementOffset;
947 }
948
949 public static final LocationIdentity PRIMARY_SUPERS_LOCATION = NamedLocationIdentity.immutable("PrimarySupers");
950
951 public static final LocationIdentity METASPACE_ARRAY_LENGTH_LOCATION = NamedLocationIdentity.immutable("MetaspaceArrayLength");
952
953 public static final LocationIdentity SECONDARY_SUPERS_ELEMENT_LOCATION = NamedLocationIdentity.immutable("SecondarySupersElement");
954 }