1 /*
2 * Copyright (c) 2012, 2016, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23 package org.graalvm.compiler.hotspot.replacements;
24
25 import static jdk.vm.ci.hotspot.HotSpotJVMCIRuntimeProvider.getArrayBaseOffset;
26 import static jdk.vm.ci.hotspot.HotSpotJVMCIRuntimeProvider.getArrayIndexScale;
27 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFIG;
28 import static org.graalvm.compiler.hotspot.meta.HotSpotForeignCallsProviderImpl.VERIFY_OOP;
29 import static org.graalvm.compiler.hotspot.replacements.UnsafeAccess.UNSAFE;
30
31 import org.graalvm.compiler.api.replacements.Fold;
32 import org.graalvm.compiler.api.replacements.Fold.InjectedParameter;
33 import org.graalvm.compiler.core.common.SuppressFBWarnings;
34 import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor;
35 import org.graalvm.compiler.core.common.type.ObjectStamp;
36 import org.graalvm.compiler.core.common.type.TypeReference;
37 import org.graalvm.compiler.debug.GraalError;
38 import org.graalvm.compiler.graph.Node.ConstantNodeParameter;
39 import org.graalvm.compiler.graph.Node.NodeIntrinsic;
40 import org.graalvm.compiler.graph.spi.CanonicalizerTool;
41 import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig;
42 import org.graalvm.compiler.hotspot.nodes.ComputeObjectAddressNode;
43 import org.graalvm.compiler.hotspot.word.KlassPointer;
44 import org.graalvm.compiler.nodes.CanonicalizableLocation;
45 import org.graalvm.compiler.nodes.CompressionNode;
46 import org.graalvm.compiler.nodes.ConstantNode;
47 import org.graalvm.compiler.nodes.NamedLocationIdentity;
48 import org.graalvm.compiler.nodes.ValueNode;
49 import org.graalvm.compiler.nodes.extended.ForeignCallNode;
50 import org.graalvm.compiler.nodes.extended.LoadHubNode;
51 import org.graalvm.compiler.nodes.extended.RawLoadNode;
52 import org.graalvm.compiler.nodes.extended.StoreHubNode;
53 import org.graalvm.compiler.nodes.memory.Access;
54 import org.graalvm.compiler.nodes.memory.address.AddressNode;
55 import org.graalvm.compiler.nodes.memory.address.OffsetAddressNode;
56 import org.graalvm.compiler.nodes.type.StampTool;
57 import org.graalvm.compiler.replacements.ReplacementsUtil;
58 import org.graalvm.compiler.replacements.nodes.ReadRegisterNode;
59 import org.graalvm.compiler.replacements.nodes.WriteRegisterNode;
60 import org.graalvm.compiler.word.Word;
61 import org.graalvm.word.LocationIdentity;
62 import org.graalvm.word.WordFactory;
63
64 import jdk.vm.ci.code.CodeUtil;
65 import jdk.vm.ci.code.Register;
66 import jdk.vm.ci.hotspot.HotSpotJVMCIRuntime;
67 import jdk.vm.ci.hotspot.HotSpotJVMCIRuntimeProvider;
68 import jdk.vm.ci.hotspot.HotSpotMetaspaceConstant;
69 import jdk.vm.ci.hotspot.HotSpotResolvedObjectType;
70 import jdk.vm.ci.meta.Assumptions;
71 import jdk.vm.ci.meta.Assumptions.AssumptionResult;
72 import jdk.vm.ci.meta.JavaKind;
73 import jdk.vm.ci.meta.ResolvedJavaType;
74
75 //JaCoCo Exclude
76
77 /**
78 * A collection of methods used in HotSpot snippets, substitutions and stubs.
79 */
80 public class HotSpotReplacementsUtil {
81
82 abstract static class HotSpotOptimizingLocationIdentity extends NamedLocationIdentity implements CanonicalizableLocation {
83
84 HotSpotOptimizingLocationIdentity(String name) {
85 super(name, true);
86 }
87
88 @Override
89 public abstract ValueNode canonicalizeRead(ValueNode read, AddressNode location, ValueNode object, CanonicalizerTool tool);
90
91 protected ValueNode findReadHub(ValueNode object) {
92 ValueNode base = object;
93 if (base instanceof CompressionNode) {
94 base = ((CompressionNode) base).getValue();
95 }
96 if (base instanceof Access) {
97 Access access = (Access) base;
98 if (access.getLocationIdentity().equals(HUB_LOCATION) || access.getLocationIdentity().equals(COMPRESSED_HUB_LOCATION)) {
99 AddressNode address = access.getAddress();
100 if (address instanceof OffsetAddressNode) {
101 OffsetAddressNode offset = (OffsetAddressNode) address;
102 return offset.getBase();
103 }
104 }
105 } else if (base instanceof LoadHubNode) {
106 LoadHubNode loadhub = (LoadHubNode) base;
107 return loadhub.getValue();
108 }
109 return null;
110 }
111
112 /**
113 * Fold reads that convert from Class -> Hub -> Class or vice versa.
114 *
115 * @param read
116 * @param object
117 * @param otherLocation
118 * @return an earlier read or the original {@code read}
119 */
120 protected static ValueNode foldIndirection(ValueNode read, ValueNode object, LocationIdentity otherLocation) {
121 if (object instanceof Access) {
122 Access access = (Access) object;
123 if (access.getLocationIdentity().equals(otherLocation)) {
124 AddressNode address = access.getAddress();
125 if (address instanceof OffsetAddressNode) {
126 OffsetAddressNode offset = (OffsetAddressNode) address;
127 assert offset.getBase().stamp().isCompatible(read.stamp());
128 return offset.getBase();
129 }
130 }
131 }
132 return read;
133 }
134 }
135
136 public static HotSpotJVMCIRuntimeProvider runtime() {
137 return HotSpotJVMCIRuntime.runtime();
138 }
139
140 @Fold
141 public static GraalHotSpotVMConfig config(@InjectedParameter GraalHotSpotVMConfig config) {
142 assert config != null;
143 return config;
144 }
145
146 @Fold
147 public static boolean useTLAB(@InjectedParameter GraalHotSpotVMConfig config) {
148 return config.useTLAB;
149 }
150
151 @Fold
152 public static boolean verifyOops(@InjectedParameter GraalHotSpotVMConfig config) {
153 return config.verifyOops;
154 }
155
156 public static final LocationIdentity EXCEPTION_OOP_LOCATION = NamedLocationIdentity.mutable("ExceptionOop");
157
158 /**
159 * @see GraalHotSpotVMConfig#threadExceptionOopOffset
160 */
161 @Fold
162 public static int threadExceptionOopOffset(@InjectedParameter GraalHotSpotVMConfig config) {
163 return config.threadExceptionOopOffset;
164 }
165
166 public static final LocationIdentity EXCEPTION_PC_LOCATION = NamedLocationIdentity.mutable("ExceptionPc");
167
168 @Fold
169 public static int threadExceptionPcOffset(@InjectedParameter GraalHotSpotVMConfig config) {
170 return config.threadExceptionPcOffset;
171 }
172
173 public static final LocationIdentity TLAB_TOP_LOCATION = NamedLocationIdentity.mutable("TlabTop");
174
175 @Fold
176 public static int threadTlabTopOffset(@InjectedParameter GraalHotSpotVMConfig config) {
177 return config.threadTlabTopOffset();
178 }
179
180 public static final LocationIdentity TLAB_END_LOCATION = NamedLocationIdentity.mutable("TlabEnd");
181
182 @Fold
183 static int threadTlabEndOffset(@InjectedParameter GraalHotSpotVMConfig config) {
184 return config.threadTlabEndOffset();
185 }
186
187 public static final LocationIdentity TLAB_START_LOCATION = NamedLocationIdentity.mutable("TlabStart");
188
189 @Fold
190 static int threadTlabStartOffset(@InjectedParameter GraalHotSpotVMConfig config) {
191 return config.threadTlabStartOffset();
192 }
193
194 public static final LocationIdentity PENDING_EXCEPTION_LOCATION = NamedLocationIdentity.mutable("PendingException");
195
196 /**
197 * @see GraalHotSpotVMConfig#pendingExceptionOffset
198 */
199 @Fold
200 static int threadPendingExceptionOffset(@InjectedParameter GraalHotSpotVMConfig config) {
201 return config.pendingExceptionOffset;
202 }
203
204 public static final LocationIdentity PENDING_DEOPTIMIZATION_LOCATION = NamedLocationIdentity.mutable("PendingDeoptimization");
205
206 /**
207 * @see GraalHotSpotVMConfig#pendingDeoptimizationOffset
208 */
209 @Fold
210 static int threadPendingDeoptimizationOffset(@InjectedParameter GraalHotSpotVMConfig config) {
211 return config.pendingDeoptimizationOffset;
212 }
213
214 public static final LocationIdentity OBJECT_RESULT_LOCATION = NamedLocationIdentity.mutable("ObjectResult");
215
216 @Fold
217 static int objectResultOffset(@InjectedParameter GraalHotSpotVMConfig config) {
218 return config.threadObjectResultOffset;
219 }
220
221 /**
222 * @see GraalHotSpotVMConfig#threadExceptionOopOffset
223 */
224 public static Object readExceptionOop(Word thread) {
225 return thread.readObject(threadExceptionOopOffset(INJECTED_VMCONFIG), EXCEPTION_OOP_LOCATION);
226 }
227
228 public static Word readExceptionPc(Word thread) {
229 return thread.readWord(threadExceptionPcOffset(INJECTED_VMCONFIG), EXCEPTION_PC_LOCATION);
230 }
231
232 /**
233 * @see GraalHotSpotVMConfig#threadExceptionOopOffset
234 */
235 public static void writeExceptionOop(Word thread, Object value) {
236 thread.writeObject(threadExceptionOopOffset(INJECTED_VMCONFIG), value, EXCEPTION_OOP_LOCATION);
237 }
238
239 public static void writeExceptionPc(Word thread, Word value) {
240 thread.writeWord(threadExceptionPcOffset(INJECTED_VMCONFIG), value, EXCEPTION_PC_LOCATION);
241 }
242
243 public static Word readTlabTop(Word thread) {
244 return thread.readWord(threadTlabTopOffset(INJECTED_VMCONFIG), TLAB_TOP_LOCATION);
245 }
246
247 public static Word readTlabEnd(Word thread) {
248 return thread.readWord(threadTlabEndOffset(INJECTED_VMCONFIG), TLAB_END_LOCATION);
249 }
250
251 public static Word readTlabStart(Word thread) {
252 return thread.readWord(threadTlabStartOffset(INJECTED_VMCONFIG), TLAB_START_LOCATION);
253 }
254
255 public static void writeTlabTop(Word thread, Word top) {
256 thread.writeWord(threadTlabTopOffset(INJECTED_VMCONFIG), top, TLAB_TOP_LOCATION);
257 }
258
259 @SuppressFBWarnings(value = "NP_NULL_PARAM_DEREF_NONVIRTUAL", justification = "foldable method parameters are injected")
260 public static void initializeTlab(Word thread, Word start, Word end) {
261 thread.writeWord(threadTlabStartOffset(INJECTED_VMCONFIG), start, TLAB_START_LOCATION);
262 thread.writeWord(threadTlabTopOffset(INJECTED_VMCONFIG), start, TLAB_TOP_LOCATION);
263 thread.writeWord(threadTlabEndOffset(INJECTED_VMCONFIG), end, TLAB_END_LOCATION);
264 }
265
266 /**
267 * Clears the pending exception for the given thread.
268 *
269 * @return the pending exception, or null if there was none
270 */
271 @SuppressFBWarnings(value = "NP_NULL_PARAM_DEREF_NONVIRTUAL", justification = "foldable method parameters are injected")
272 public static Object clearPendingException(Word thread) {
273 Object result = thread.readObject(threadPendingExceptionOffset(INJECTED_VMCONFIG), PENDING_EXCEPTION_LOCATION);
274 thread.writeObject(threadPendingExceptionOffset(INJECTED_VMCONFIG), null, PENDING_EXCEPTION_LOCATION);
275 return result;
276 }
277
278 /**
279 * Reads the pending deoptimization value for the given thread.
280 *
281 * @return {@code true} if there was a pending deoptimization
282 */
283 public static int readPendingDeoptimization(Word thread) {
284 return thread.readInt(threadPendingDeoptimizationOffset(INJECTED_VMCONFIG), PENDING_DEOPTIMIZATION_LOCATION);
285 }
286
287 /**
288 * Writes the pending deoptimization value for the given thread.
289 */
290 public static void writePendingDeoptimization(Word thread, int value) {
291 thread.writeInt(threadPendingDeoptimizationOffset(INJECTED_VMCONFIG), value, PENDING_DEOPTIMIZATION_LOCATION);
292 }
293
294 /**
295 * Gets and clears the object result from a runtime call stored in a thread local.
296 *
297 * @return the object that was in the thread local
298 */
299 public static Object getAndClearObjectResult(Word thread) {
300 Object result = thread.readObject(objectResultOffset(INJECTED_VMCONFIG), OBJECT_RESULT_LOCATION);
301 thread.writeObject(objectResultOffset(INJECTED_VMCONFIG), null, OBJECT_RESULT_LOCATION);
302 return result;
303 }
304
305 public static final LocationIdentity JAVA_THREAD_THREAD_OBJECT_LOCATION = NamedLocationIdentity.mutable("JavaThread::_threadObj");
306
307 @Fold
308 public static int threadObjectOffset(@InjectedParameter GraalHotSpotVMConfig config) {
309 return config.threadObjectOffset;
310 }
311
312 public static final LocationIdentity JAVA_THREAD_OSTHREAD_LOCATION = NamedLocationIdentity.mutable("JavaThread::_osthread");
313
314 @Fold
315 public static int osThreadOffset(@InjectedParameter GraalHotSpotVMConfig config) {
316 return config.osThreadOffset;
317 }
318
319 @Fold
320 public static int osThreadInterruptedOffset(@InjectedParameter GraalHotSpotVMConfig config) {
321 return config.osThreadInterruptedOffset;
322 }
323
324 @Fold
325 public static JavaKind getWordKind() {
326 return runtime().getHostJVMCIBackend().getCodeCache().getTarget().wordJavaKind;
327 }
328
329 @Fold
330 public static int wordSize() {
331 return runtime().getHostJVMCIBackend().getCodeCache().getTarget().wordSize;
332 }
333
334 @Fold
335 public static int pageSize() {
336 return UNSAFE.pageSize();
337 }
338
339 @Fold
340 public static int heapWordSize(@InjectedParameter GraalHotSpotVMConfig config) {
341 return config.heapWordSize;
342 }
343
344 public static final LocationIdentity PROTOTYPE_MARK_WORD_LOCATION = NamedLocationIdentity.mutable("PrototypeMarkWord");
345
346 @Fold
347 public static int prototypeMarkWordOffset(@InjectedParameter GraalHotSpotVMConfig config) {
348 return config.prototypeMarkWordOffset;
349 }
350
351 @Fold
352 public static long arrayPrototypeMarkWord(@InjectedParameter GraalHotSpotVMConfig config) {
353 return config.arrayPrototypeMarkWord();
354 }
355
356 public static final LocationIdentity KLASS_ACCESS_FLAGS_LOCATION = NamedLocationIdentity.immutable("Klass::_access_flags");
357
358 @Fold
359 public static int klassAccessFlagsOffset(@InjectedParameter GraalHotSpotVMConfig config) {
360 return config.klassAccessFlagsOffset;
361 }
362
363 @Fold
364 public static int jvmAccWrittenFlags(@InjectedParameter GraalHotSpotVMConfig config) {
365 return config.jvmAccWrittenFlags;
366 }
367
368 public static final LocationIdentity KLASS_LAYOUT_HELPER_LOCATION = new HotSpotOptimizingLocationIdentity("Klass::_layout_helper") {
369 @Override
370 public ValueNode canonicalizeRead(ValueNode read, AddressNode location, ValueNode object, CanonicalizerTool tool) {
371 ValueNode javaObject = findReadHub(object);
372 if (javaObject != null) {
373 if (javaObject.stamp() instanceof ObjectStamp) {
374 ObjectStamp stamp = (ObjectStamp) javaObject.stamp();
375 HotSpotResolvedObjectType type = (HotSpotResolvedObjectType) stamp.javaType(tool.getMetaAccess());
376 if (type.isArray() && !type.getComponentType().isPrimitive()) {
377 int layout = type.layoutHelper();
378 return ConstantNode.forInt(layout);
379 }
380 }
381 }
382 return read;
383 }
384 };
385
386 @Fold
387 public static int klassLayoutHelperOffset(@InjectedParameter GraalHotSpotVMConfig config) {
388 return config.klassLayoutHelperOffset;
389 }
390
391 @NodeIntrinsic(value = KlassLayoutHelperNode.class)
392 public static native int readLayoutHelper(KlassPointer object);
393
394 /**
395 * Checks if class {@code klass} is an array.
396 *
397 * See: Klass::layout_helper_is_array
398 *
399 * @param klassNonNull the class to be checked
400 * @return true if klassNonNull is an array, false otherwise
401 */
402 public static boolean klassIsArray(KlassPointer klassNonNull) {
403 /*
404 * The less-than check only works if both values are ints. We use local variables to make
405 * sure these are still ints and haven't changed.
406 */
407 final int layoutHelper = readLayoutHelper(klassNonNull);
408 final int layoutHelperNeutralValue = config(INJECTED_VMCONFIG).klassLayoutHelperNeutralValue;
409 return (layoutHelper < layoutHelperNeutralValue);
410 }
411
412 public static final LocationIdentity ARRAY_KLASS_COMPONENT_MIRROR = NamedLocationIdentity.immutable("ArrayKlass::_component_mirror");
413
414 @Fold
415 public static int arrayKlassComponentMirrorOffset(@InjectedParameter GraalHotSpotVMConfig config) {
416 return config.getFieldOffset("ArrayKlass::_component_mirror", Integer.class, "oop");
417 }
418
419 public static final LocationIdentity KLASS_SUPER_KLASS_LOCATION = NamedLocationIdentity.immutable("Klass::_super");
420
421 @Fold
422 public static int klassSuperKlassOffset(@InjectedParameter GraalHotSpotVMConfig config) {
423 return config.klassSuperKlassOffset;
424 }
425
426 public static final LocationIdentity MARK_WORD_LOCATION = NamedLocationIdentity.mutable("MarkWord");
427
428 @Fold
429 public static int markOffset(@InjectedParameter GraalHotSpotVMConfig config) {
430 return config.markOffset;
431 }
432
433 public static final LocationIdentity HUB_WRITE_LOCATION = NamedLocationIdentity.mutable("Hub:write");
434
435 public static final LocationIdentity HUB_LOCATION = new HotSpotOptimizingLocationIdentity("Hub") {
436 @Override
437 public ValueNode canonicalizeRead(ValueNode read, AddressNode location, ValueNode object, CanonicalizerTool tool) {
438 TypeReference constantType = StampTool.typeReferenceOrNull(object);
439 if (constantType != null && constantType.isExact()) {
440 return ConstantNode.forConstant(read.stamp(), tool.getConstantReflection().asObjectHub(constantType.getType()), tool.getMetaAccess());
441 }
442 return read;
443 }
444 };
445
446 public static final LocationIdentity COMPRESSED_HUB_LOCATION = new HotSpotOptimizingLocationIdentity("CompressedHub") {
447 @Override
448 public ValueNode canonicalizeRead(ValueNode read, AddressNode location, ValueNode object, CanonicalizerTool tool) {
449 TypeReference constantType = StampTool.typeReferenceOrNull(object);
450 if (constantType != null && constantType.isExact()) {
451 return ConstantNode.forConstant(read.stamp(), ((HotSpotMetaspaceConstant) tool.getConstantReflection().asObjectHub(constantType.getType())).compress(), tool.getMetaAccess());
452 }
453 return read;
454 }
455 };
456
457 @Fold
458 static int hubOffset(@InjectedParameter GraalHotSpotVMConfig config) {
459 return config.hubOffset;
460 }
461
462 public static void initializeObjectHeader(Word memory, Word markWord, KlassPointer hub) {
463 memory.writeWord(markOffset(INJECTED_VMCONFIG), markWord, MARK_WORD_LOCATION);
464 StoreHubNode.write(memory, hub);
465 }
466
467 @Fold
468 public static int unlockedMask(@InjectedParameter GraalHotSpotVMConfig config) {
469 return config.unlockedMask;
470 }
471
472 @Fold
473 public static int monitorMask(@InjectedParameter GraalHotSpotVMConfig config) {
474 return config.monitorMask;
475 }
476
477 @Fold
478 public static int objectMonitorOwnerOffset(@InjectedParameter GraalHotSpotVMConfig config) {
479 return config.objectMonitorOwner;
480 }
481
482 @Fold
483 public static int objectMonitorRecursionsOffset(@InjectedParameter GraalHotSpotVMConfig config) {
484 return config.objectMonitorRecursions;
485 }
486
487 @Fold
488 public static int objectMonitorCxqOffset(@InjectedParameter GraalHotSpotVMConfig config) {
489 return config.objectMonitorCxq;
490 }
491
492 @Fold
493 public static int objectMonitorEntryListOffset(@InjectedParameter GraalHotSpotVMConfig config) {
494 return config.objectMonitorEntryList;
495 }
496
497 /**
498 * Mask for a biasable, locked or unlocked mark word.
499 *
500 * <pre>
501 * +----------------------------------+-+-+
502 * | 1|1|1|
503 * +----------------------------------+-+-+
504 * </pre>
505 *
506 */
507 @Fold
508 public static int biasedLockMaskInPlace(@InjectedParameter GraalHotSpotVMConfig config) {
509 return config.biasedLockMaskInPlace;
510 }
511
512 @Fold
513 public static int epochMaskInPlace(@InjectedParameter GraalHotSpotVMConfig config) {
514 return config.epochMaskInPlace;
515 }
516
517 /**
518 * Pattern for a biasable, unlocked mark word.
519 *
520 * <pre>
521 * +----------------------------------+-+-+
522 * | 1|0|1|
523 * +----------------------------------+-+-+
524 * </pre>
525 *
526 */
527 @Fold
528 public static int biasedLockPattern(@InjectedParameter GraalHotSpotVMConfig config) {
529 return config.biasedLockPattern;
530 }
531
532 @Fold
533 public static int ageMaskInPlace(@InjectedParameter GraalHotSpotVMConfig config) {
534 return config.ageMaskInPlace;
535 }
536
537 @Fold
538 public static int metaspaceArrayLengthOffset(@InjectedParameter GraalHotSpotVMConfig config) {
539 return config.metaspaceArrayLengthOffset;
540 }
541
542 @Fold
543 public static int metaspaceArrayBaseOffset(@InjectedParameter GraalHotSpotVMConfig config) {
544 return config.metaspaceArrayBaseOffset;
545 }
546
547 @Fold
548 public static int arrayLengthOffset(@InjectedParameter GraalHotSpotVMConfig config) {
549 return config.arrayOopDescLengthOffset();
550 }
551
552 @Fold
553 public static int arrayBaseOffset(JavaKind elementKind) {
554 return getArrayBaseOffset(elementKind);
555 }
556
557 @Fold
558 public static int arrayIndexScale(JavaKind elementKind) {
559 return getArrayIndexScale(elementKind);
560 }
561
562 public static Word arrayStart(int[] a) {
563 return WordFactory.unsigned(ComputeObjectAddressNode.get(a, getArrayBaseOffset(JavaKind.Int)));
564 }
565
566 @Fold
567 public static int instanceHeaderSize(@InjectedParameter GraalHotSpotVMConfig config) {
568 return config.useCompressedClassPointers ? (2 * wordSize()) - 4 : 2 * wordSize();
569 }
570
571 @Fold
572 public static byte dirtyCardValue(@InjectedParameter GraalHotSpotVMConfig config) {
573 return config.dirtyCardValue;
574 }
575
576 @Fold
577 public static byte g1YoungCardValue(@InjectedParameter GraalHotSpotVMConfig config) {
578 return config.g1YoungCardValue;
579 }
580
581 @Fold
582 public static int cardTableShift(@InjectedParameter GraalHotSpotVMConfig config) {
583 return config.cardtableShift;
584 }
585
586 @Fold
587 public static long cardTableStart(@InjectedParameter GraalHotSpotVMConfig config) {
588 return config.cardtableStartAddress;
589 }
590
591 @Fold
592 public static int g1CardQueueIndexOffset(@InjectedParameter GraalHotSpotVMConfig config) {
593 return config.g1CardQueueIndexOffset();
594 }
595
596 @Fold
597 public static int g1CardQueueBufferOffset(@InjectedParameter GraalHotSpotVMConfig config) {
598 return config.g1CardQueueBufferOffset();
599 }
600
601 @Fold
602 public static int logOfHeapRegionGrainBytes(@InjectedParameter GraalHotSpotVMConfig config) {
603 return config.logOfHRGrainBytes;
604 }
605
606 @Fold
607 public static int g1SATBQueueMarkingOffset(@InjectedParameter GraalHotSpotVMConfig config) {
608 return config.g1SATBQueueMarkingOffset();
609 }
610
611 @Fold
612 public static int g1SATBQueueIndexOffset(@InjectedParameter GraalHotSpotVMConfig config) {
613 return config.g1SATBQueueIndexOffset();
614 }
615
616 @Fold
617 public static int g1SATBQueueBufferOffset(@InjectedParameter GraalHotSpotVMConfig config) {
618 return config.g1SATBQueueBufferOffset();
619 }
620
621 public static final LocationIdentity KLASS_SUPER_CHECK_OFFSET_LOCATION = NamedLocationIdentity.immutable("Klass::_super_check_offset");
622
623 @Fold
624 public static int superCheckOffsetOffset(@InjectedParameter GraalHotSpotVMConfig config) {
625 return config.superCheckOffsetOffset;
626 }
627
628 public static final LocationIdentity SECONDARY_SUPER_CACHE_LOCATION = NamedLocationIdentity.mutable("SecondarySuperCache");
629
630 @Fold
631 public static int secondarySuperCacheOffset(@InjectedParameter GraalHotSpotVMConfig config) {
632 return config.secondarySuperCacheOffset;
633 }
634
635 public static final LocationIdentity SECONDARY_SUPERS_LOCATION = NamedLocationIdentity.immutable("SecondarySupers");
636
637 @Fold
638 public static int secondarySupersOffset(@InjectedParameter GraalHotSpotVMConfig config) {
639 return config.secondarySupersOffset;
640 }
641
642 public static final LocationIdentity DISPLACED_MARK_WORD_LOCATION = NamedLocationIdentity.mutable("DisplacedMarkWord");
643
644 public static final LocationIdentity OBJECT_MONITOR_OWNER_LOCATION = NamedLocationIdentity.mutable("ObjectMonitor::_owner");
645
646 public static final LocationIdentity OBJECT_MONITOR_RECURSION_LOCATION = NamedLocationIdentity.mutable("ObjectMonitor::_recursions");
647
648 public static final LocationIdentity OBJECT_MONITOR_CXQ_LOCATION = NamedLocationIdentity.mutable("ObjectMonitor::_cxq");
649
650 public static final LocationIdentity OBJECT_MONITOR_ENTRY_LIST_LOCATION = NamedLocationIdentity.mutable("ObjectMonitor::_EntryList");
651
652 @Fold
653 public static int lockDisplacedMarkOffset(@InjectedParameter GraalHotSpotVMConfig config) {
654 return config.basicLockDisplacedHeaderOffset;
655 }
656
657 @Fold
658 public static boolean useBiasedLocking(@InjectedParameter GraalHotSpotVMConfig config) {
659 return config.useBiasedLocking;
660 }
661
662 @Fold
663 public static boolean useDeferredInitBarriers(@InjectedParameter GraalHotSpotVMConfig config) {
664 return config.useDeferredInitBarriers;
665 }
666
667 @Fold
668 public static boolean useG1GC(@InjectedParameter GraalHotSpotVMConfig config) {
669 return config.useG1GC;
670 }
671
672 @Fold
673 public static boolean useCMSIncrementalMode(@InjectedParameter GraalHotSpotVMConfig config) {
674 return config.cmsIncrementalMode;
675 }
676
677 @Fold
678 public static boolean useCompressedOops(@InjectedParameter GraalHotSpotVMConfig config) {
679 return config.useCompressedOops;
680 }
681
682 @Fold
683 static int uninitializedIdentityHashCodeValue(@InjectedParameter GraalHotSpotVMConfig config) {
684 return config.uninitializedIdentityHashCodeValue;
685 }
686
687 @Fold
688 static int identityHashCodeShift(@InjectedParameter GraalHotSpotVMConfig config) {
689 return config.identityHashCodeShift;
690 }
691
692 /**
693 * Loads the hub of an object (without null checking it first).
694 */
695 public static KlassPointer loadHub(Object object) {
696 return loadHubIntrinsic(object);
697 }
698
699 public static Object verifyOop(Object object) {
700 if (verifyOops(INJECTED_VMCONFIG)) {
701 verifyOopStub(VERIFY_OOP, object);
702 }
703 return object;
704 }
705
706 @NodeIntrinsic(ForeignCallNode.class)
707 private static native Object verifyOopStub(@ConstantNodeParameter ForeignCallDescriptor descriptor, Object object);
708
709 public static Word loadWordFromObject(Object object, int offset) {
710 ReplacementsUtil.staticAssert(offset != hubOffset(INJECTED_VMCONFIG), "Use loadHubIntrinsic instead of loadWordFromObject");
711 return loadWordFromObjectIntrinsic(object, offset, LocationIdentity.any(), getWordKind());
712 }
713
714 public static Word loadWordFromObject(Object object, int offset, LocationIdentity identity) {
715 ReplacementsUtil.staticAssert(offset != hubOffset(INJECTED_VMCONFIG), "Use loadHubIntrinsic instead of loadWordFromObject");
716 return loadWordFromObjectIntrinsic(object, offset, identity, getWordKind());
717 }
718
719 public static KlassPointer loadKlassFromObject(Object object, int offset, LocationIdentity identity) {
720 ReplacementsUtil.staticAssert(offset != hubOffset(INJECTED_VMCONFIG), "Use loadHubIntrinsic instead of loadWordFromObject");
721 return loadKlassFromObjectIntrinsic(object, offset, identity, getWordKind());
722 }
723
724 /**
725 * Reads the value of a given register.
726 *
727 * @param register a register which must not be available to the register allocator
728 * @return the value of {@code register} as a word
729 */
730 public static Word registerAsWord(@ConstantNodeParameter Register register) {
731 return registerAsWord(register, true, false);
732 }
733
734 @NodeIntrinsic(value = ReadRegisterNode.class)
735 public static native Word registerAsWord(@ConstantNodeParameter Register register, @ConstantNodeParameter boolean directUse, @ConstantNodeParameter boolean incoming);
736
737 @NodeIntrinsic(value = WriteRegisterNode.class)
738 public static native void writeRegisterAsWord(@ConstantNodeParameter Register register, Word value);
739
740 @NodeIntrinsic(value = RawLoadNode.class)
741 private static native Word loadWordFromObjectIntrinsic(Object object, long offset, @ConstantNodeParameter LocationIdentity locationIdentity, @ConstantNodeParameter JavaKind wordKind);
742
743 @NodeIntrinsic(value = RawLoadNode.class)
744 private static native KlassPointer loadKlassFromObjectIntrinsic(Object object, long offset, @ConstantNodeParameter LocationIdentity locationIdentity, @ConstantNodeParameter JavaKind wordKind);
745
746 @NodeIntrinsic(value = LoadHubNode.class)
747 public static native KlassPointer loadHubIntrinsic(Object object);
748
749 @Fold
750 public static int log2WordSize() {
751 return CodeUtil.log2(wordSize());
752 }
753
754 public static final LocationIdentity CLASS_STATE_LOCATION = NamedLocationIdentity.mutable("ClassState");
755
756 @Fold
757 public static int instanceKlassInitStateOffset(@InjectedParameter GraalHotSpotVMConfig config) {
758 return config.instanceKlassInitStateOffset;
759 }
760
761 @Fold
762 public static int instanceKlassStateFullyInitialized(@InjectedParameter GraalHotSpotVMConfig config) {
763 return config.instanceKlassStateFullyInitialized;
764 }
765
766 /**
767 *
768 * @param hub the hub of an InstanceKlass
769 * @return true is the InstanceKlass represented by hub is fully initialized
770 */
771 public static boolean isInstanceKlassFullyInitialized(KlassPointer hub) {
772 return readInstanceKlassState(hub) == instanceKlassStateFullyInitialized(INJECTED_VMCONFIG);
773 }
774
775 private static byte readInstanceKlassState(KlassPointer hub) {
776 return hub.readByte(instanceKlassInitStateOffset(INJECTED_VMCONFIG), CLASS_STATE_LOCATION);
777 }
778
779 public static final LocationIdentity KLASS_MODIFIER_FLAGS_LOCATION = NamedLocationIdentity.immutable("Klass::_modifier_flags");
780
781 @Fold
782 public static int klassModifierFlagsOffset(@InjectedParameter GraalHotSpotVMConfig config) {
783 return config.klassModifierFlagsOffset;
784 }
785
786 public static final LocationIdentity CLASS_KLASS_LOCATION = new HotSpotOptimizingLocationIdentity("Class._klass") {
787 @Override
788 public ValueNode canonicalizeRead(ValueNode read, AddressNode location, ValueNode object, CanonicalizerTool tool) {
789 return foldIndirection(read, object, CLASS_MIRROR_LOCATION);
790 }
791 };
792
793 @Fold
794 public static int klassOffset(@InjectedParameter GraalHotSpotVMConfig config) {
795 return config.klassOffset;
796 }
797
798 public static final LocationIdentity CLASS_ARRAY_KLASS_LOCATION = new HotSpotOptimizingLocationIdentity("Class._array_klass") {
799 @Override
800 public ValueNode canonicalizeRead(ValueNode read, AddressNode location, ValueNode object, CanonicalizerTool tool) {
801 return foldIndirection(read, object, ARRAY_KLASS_COMPONENT_MIRROR);
802 }
803 };
804
805 @Fold
806 public static int arrayKlassOffset(@InjectedParameter GraalHotSpotVMConfig config) {
807 return config.arrayKlassOffset;
808 }
809
810 public static final LocationIdentity CLASS_MIRROR_LOCATION = NamedLocationIdentity.immutable("Klass::_java_mirror");
811
812 public static final LocationIdentity CLASS_MIRROR_HANDLE_LOCATION = NamedLocationIdentity.immutable("Klass::_java_mirror handle");
813
814 public static final LocationIdentity HEAP_TOP_LOCATION = NamedLocationIdentity.mutable("HeapTop");
815
816 @Fold
817 public static long heapTopAddress(@InjectedParameter GraalHotSpotVMConfig config) {
818 return config.heapTopAddress;
819 }
820
821 public static final LocationIdentity HEAP_END_LOCATION = NamedLocationIdentity.mutable("HeapEnd");
822
823 @Fold
824 public static long heapEndAddress(@InjectedParameter GraalHotSpotVMConfig config) {
825 return config.heapEndAddress;
826 }
827
828 @Fold
829 public static long tlabIntArrayMarkWord(@InjectedParameter GraalHotSpotVMConfig config) {
830 return config.tlabIntArrayMarkWord();
831 }
832
833 @Fold
834 public static boolean inlineContiguousAllocationSupported(@InjectedParameter GraalHotSpotVMConfig config) {
835 return config.inlineContiguousAllocationSupported;
836 }
837
838 @Fold
839 public static int tlabAlignmentReserveInHeapWords(@InjectedParameter GraalHotSpotVMConfig config) {
840 return config.tlabAlignmentReserve;
841 }
842
843 public static final LocationIdentity TLAB_SIZE_LOCATION = NamedLocationIdentity.mutable("TlabSize");
844
845 @Fold
846 public static int threadTlabSizeOffset(@InjectedParameter GraalHotSpotVMConfig config) {
847 return config.threadTlabSizeOffset();
848 }
849
850 public static final LocationIdentity TLAB_THREAD_ALLOCATED_BYTES_LOCATION = NamedLocationIdentity.mutable("TlabThreadAllocatedBytes");
851
852 @Fold
853 public static int threadAllocatedBytesOffset(@InjectedParameter GraalHotSpotVMConfig config) {
854 return config.threadAllocatedBytesOffset;
855 }
856
857 public static final LocationIdentity TLAB_REFILL_WASTE_LIMIT_LOCATION = NamedLocationIdentity.mutable("RefillWasteLimit");
858
859 @Fold
860 public static int tlabRefillWasteLimitOffset(@InjectedParameter GraalHotSpotVMConfig config) {
861 return config.tlabRefillWasteLimitOffset();
862 }
863
864 public static final LocationIdentity TLAB_NOF_REFILLS_LOCATION = NamedLocationIdentity.mutable("TlabNOfRefills");
865
866 @Fold
867 public static int tlabNumberOfRefillsOffset(@InjectedParameter GraalHotSpotVMConfig config) {
868 return config.tlabNumberOfRefillsOffset();
869 }
870
871 public static final LocationIdentity TLAB_FAST_REFILL_WASTE_LOCATION = NamedLocationIdentity.mutable("TlabFastRefillWaste");
872
873 @Fold
874 public static int tlabFastRefillWasteOffset(@InjectedParameter GraalHotSpotVMConfig config) {
875 return config.tlabFastRefillWasteOffset();
876 }
877
878 public static final LocationIdentity TLAB_SLOW_ALLOCATIONS_LOCATION = NamedLocationIdentity.mutable("TlabSlowAllocations");
879
880 @Fold
881 public static int tlabSlowAllocationsOffset(@InjectedParameter GraalHotSpotVMConfig config) {
882 return config.tlabSlowAllocationsOffset();
883 }
884
885 @Fold
886 public static int tlabRefillWasteIncrement(@InjectedParameter GraalHotSpotVMConfig config) {
887 return config.tlabRefillWasteIncrement;
888 }
889
890 @Fold
891 public static boolean tlabStats(@InjectedParameter GraalHotSpotVMConfig config) {
892 return config.tlabStats;
893 }
894
895 @Fold
896 public static int layoutHelperHeaderSizeShift(@InjectedParameter GraalHotSpotVMConfig config) {
897 return config.layoutHelperHeaderSizeShift;
898 }
899
900 @Fold
901 public static int layoutHelperHeaderSizeMask(@InjectedParameter GraalHotSpotVMConfig config) {
902 return config.layoutHelperHeaderSizeMask;
903 }
904
905 @Fold
906 public static int layoutHelperLog2ElementSizeShift(@InjectedParameter GraalHotSpotVMConfig config) {
907 return config.layoutHelperLog2ElementSizeShift;
908 }
909
910 @Fold
911 public static int layoutHelperLog2ElementSizeMask(@InjectedParameter GraalHotSpotVMConfig config) {
912 return config.layoutHelperLog2ElementSizeMask;
913 }
914
915 @Fold
916 public static int layoutHelperElementTypeShift(@InjectedParameter GraalHotSpotVMConfig config) {
917 return config.layoutHelperElementTypeShift;
918 }
919
920 @Fold
921 public static int layoutHelperElementTypeMask(@InjectedParameter GraalHotSpotVMConfig config) {
922 return config.layoutHelperElementTypeMask;
923 }
924
925 @Fold
926 public static int layoutHelperElementTypePrimitiveInPlace(@InjectedParameter GraalHotSpotVMConfig config) {
927 return config.layoutHelperElementTypePrimitiveInPlace();
928 }
929
930 @NodeIntrinsic(ForeignCallNode.class)
931 public static native int identityHashCode(@ConstantNodeParameter ForeignCallDescriptor descriptor, Object object);
932
933 @Fold
934 public static int verifiedEntryPointOffset(@InjectedParameter GraalHotSpotVMConfig config) {
935 return config.nmethodEntryOffset;
936 }
937
938 @Fold
939 public static long gcTotalCollectionsAddress(@InjectedParameter GraalHotSpotVMConfig config) {
940 return config.gcTotalCollectionsAddress();
941 }
942
943 @Fold
944 public static long referentOffset() {
945 try {
946 return UNSAFE.objectFieldOffset(java.lang.ref.Reference.class.getDeclaredField("referent"));
947 } catch (Exception e) {
948 throw new GraalError(e);
949 }
950 }
951
952 public static final LocationIdentity OBJ_ARRAY_KLASS_ELEMENT_KLASS_LOCATION = new HotSpotOptimizingLocationIdentity("ObjArrayKlass::_element_klass") {
953 @Override
954 public ValueNode canonicalizeRead(ValueNode read, AddressNode location, ValueNode object, CanonicalizerTool tool) {
955 ValueNode javaObject = findReadHub(object);
956 if (javaObject != null) {
957 ResolvedJavaType type = StampTool.typeOrNull(javaObject);
958 if (type != null && type.isArray()) {
959 ResolvedJavaType element = type.getComponentType();
960 if (element != null && !element.isPrimitive() && !element.getElementalType().isInterface()) {
961 Assumptions assumptions = object.graph().getAssumptions();
962 AssumptionResult<ResolvedJavaType> leafType = element.findLeafConcreteSubtype();
963 if (leafType != null && leafType.canRecordTo(assumptions)) {
964 leafType.recordTo(assumptions);
965 return ConstantNode.forConstant(read.stamp(), tool.getConstantReflection().asObjectHub(leafType.getResult()), tool.getMetaAccess());
966 }
967 }
968 }
969 }
970 return read;
971 }
972 };
973
974 @Fold
975 public static int arrayClassElementOffset(@InjectedParameter GraalHotSpotVMConfig config) {
976 return config.arrayClassElementOffset;
977 }
978
979 public static final LocationIdentity PRIMARY_SUPERS_LOCATION = NamedLocationIdentity.immutable("PrimarySupers");
980
981 public static final LocationIdentity METASPACE_ARRAY_LENGTH_LOCATION = NamedLocationIdentity.immutable("MetaspaceArrayLength");
982
983 public static final LocationIdentity SECONDARY_SUPERS_ELEMENT_LOCATION = NamedLocationIdentity.immutable("SecondarySupersElement");
984 }