1 /*
   2  * Copyright (c) 2012, 2016, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  */
  23 package org.graalvm.compiler.hotspot.replacements;
  24 
  25 import static jdk.vm.ci.hotspot.HotSpotJVMCIRuntimeProvider.getArrayBaseOffset;
  26 import static jdk.vm.ci.hotspot.HotSpotJVMCIRuntimeProvider.getArrayIndexScale;
  27 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFIG;
  28 import static org.graalvm.compiler.hotspot.meta.HotSpotForeignCallsProviderImpl.VERIFY_OOP;
  29 import static org.graalvm.compiler.hotspot.replacements.UnsafeAccess.UNSAFE;
  30 
  31 import org.graalvm.compiler.api.replacements.Fold;
  32 import org.graalvm.compiler.api.replacements.Fold.InjectedParameter;
  33 import org.graalvm.compiler.core.common.SuppressFBWarnings;
  34 import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor;
  35 import org.graalvm.compiler.core.common.type.ObjectStamp;
  36 import org.graalvm.compiler.core.common.type.TypeReference;
  37 import org.graalvm.compiler.debug.GraalError;
  38 import org.graalvm.compiler.graph.Node.ConstantNodeParameter;
  39 import org.graalvm.compiler.graph.Node.NodeIntrinsic;
  40 import org.graalvm.compiler.graph.spi.CanonicalizerTool;
  41 import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig;
  42 import org.graalvm.compiler.hotspot.nodes.ComputeObjectAddressNode;
  43 import org.graalvm.compiler.hotspot.word.KlassPointer;
  44 import org.graalvm.compiler.nodes.CanonicalizableLocation;
  45 import org.graalvm.compiler.nodes.CompressionNode;
  46 import org.graalvm.compiler.nodes.ConstantNode;
  47 import org.graalvm.compiler.nodes.NamedLocationIdentity;
  48 import org.graalvm.compiler.nodes.NodeView;
  49 import org.graalvm.compiler.nodes.ValueNode;
  50 import org.graalvm.compiler.nodes.extended.ForeignCallNode;
  51 import org.graalvm.compiler.nodes.extended.LoadHubNode;
  52 import org.graalvm.compiler.nodes.extended.RawLoadNode;
  53 import org.graalvm.compiler.nodes.extended.StoreHubNode;
  54 import org.graalvm.compiler.nodes.memory.Access;
  55 import org.graalvm.compiler.nodes.memory.address.AddressNode;
  56 import org.graalvm.compiler.nodes.memory.address.OffsetAddressNode;
  57 import org.graalvm.compiler.nodes.type.StampTool;
  58 import org.graalvm.compiler.replacements.ReplacementsUtil;
  59 import org.graalvm.compiler.replacements.nodes.ReadRegisterNode;
  60 import org.graalvm.compiler.replacements.nodes.WriteRegisterNode;
  61 import org.graalvm.compiler.word.Word;
  62 import org.graalvm.word.LocationIdentity;
  63 import org.graalvm.word.WordFactory;
  64 
  65 import jdk.vm.ci.code.CodeUtil;
  66 import jdk.vm.ci.code.Register;
  67 import jdk.vm.ci.hotspot.HotSpotJVMCIRuntime;
  68 import jdk.vm.ci.hotspot.HotSpotJVMCIRuntimeProvider;
  69 import jdk.vm.ci.hotspot.HotSpotMetaspaceConstant;
  70 import jdk.vm.ci.hotspot.HotSpotResolvedObjectType;
  71 import jdk.vm.ci.meta.Assumptions;
  72 import jdk.vm.ci.meta.Assumptions.AssumptionResult;
  73 import jdk.vm.ci.meta.JavaKind;
  74 import jdk.vm.ci.meta.ResolvedJavaType;
  75 
  76 //JaCoCo Exclude
  77 
  78 /**
  79  * A collection of methods used in HotSpot snippets, substitutions and stubs.
  80  */
  81 public class HotSpotReplacementsUtil {
  82 
  83     abstract static class HotSpotOptimizingLocationIdentity extends NamedLocationIdentity implements CanonicalizableLocation {
  84 
  85         HotSpotOptimizingLocationIdentity(String name) {
  86             super(name, true);
  87         }
  88 
  89         @Override
  90         public abstract ValueNode canonicalizeRead(ValueNode read, AddressNode location, ValueNode object, CanonicalizerTool tool);
  91 
  92         protected ValueNode findReadHub(ValueNode object) {
  93             ValueNode base = object;
  94             if (base instanceof CompressionNode) {
  95                 base = ((CompressionNode) base).getValue();
  96             }
  97             if (base instanceof Access) {
  98                 Access access = (Access) base;
  99                 if (access.getLocationIdentity().equals(HUB_LOCATION) || access.getLocationIdentity().equals(COMPRESSED_HUB_LOCATION)) {
 100                     AddressNode address = access.getAddress();
 101                     if (address instanceof OffsetAddressNode) {
 102                         OffsetAddressNode offset = (OffsetAddressNode) address;
 103                         return offset.getBase();
 104                     }
 105                 }
 106             } else if (base instanceof LoadHubNode) {
 107                 LoadHubNode loadhub = (LoadHubNode) base;
 108                 return loadhub.getValue();
 109             }
 110             return null;
 111         }
 112 
 113         /**
 114          * Fold reads that convert from Class -> Hub -> Class or vice versa.
 115          *
 116          * @param read
 117          * @param object
 118          * @param otherLocation
 119          * @return an earlier read or the original {@code read}
 120          */
 121         protected static ValueNode foldIndirection(ValueNode read, ValueNode object, LocationIdentity otherLocation) {
 122             if (object instanceof Access) {
 123                 Access access = (Access) object;
 124                 if (access.getLocationIdentity().equals(otherLocation)) {
 125                     AddressNode address = access.getAddress();
 126                     if (address instanceof OffsetAddressNode) {
 127                         OffsetAddressNode offset = (OffsetAddressNode) address;
 128                         assert offset.getBase().stamp(NodeView.DEFAULT).isCompatible(read.stamp(NodeView.DEFAULT));
 129                         return offset.getBase();
 130                     }
 131                 }
 132             }
 133             return read;
 134         }
 135     }
 136 
 137     public static HotSpotJVMCIRuntimeProvider runtime() {
 138         return HotSpotJVMCIRuntime.runtime();
 139     }
 140 
 141     @Fold
 142     public static GraalHotSpotVMConfig config(@InjectedParameter GraalHotSpotVMConfig config) {
 143         assert config != null;
 144         return config;
 145     }
 146 
 147     @Fold
 148     public static boolean useTLAB(@InjectedParameter GraalHotSpotVMConfig config) {
 149         return config.useTLAB;
 150     }
 151 
 152     @Fold
 153     public static boolean verifyOops(@InjectedParameter GraalHotSpotVMConfig config) {
 154         return config.verifyOops;
 155     }
 156 
 157     public static final LocationIdentity EXCEPTION_OOP_LOCATION = NamedLocationIdentity.mutable("ExceptionOop");
 158 
 159     /**
 160      * @see GraalHotSpotVMConfig#threadExceptionOopOffset
 161      */
 162     @Fold
 163     public static int threadExceptionOopOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 164         return config.threadExceptionOopOffset;
 165     }
 166 
 167     public static final LocationIdentity EXCEPTION_PC_LOCATION = NamedLocationIdentity.mutable("ExceptionPc");
 168 
 169     @Fold
 170     public static int threadExceptionPcOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 171         return config.threadExceptionPcOffset;
 172     }
 173 
 174     public static final LocationIdentity TLAB_TOP_LOCATION = NamedLocationIdentity.mutable("TlabTop");
 175 
 176     @Fold
 177     public static int threadTlabTopOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 178         return config.threadTlabTopOffset();
 179     }
 180 
 181     public static final LocationIdentity TLAB_END_LOCATION = NamedLocationIdentity.mutable("TlabEnd");
 182 
 183     @Fold
 184     static int threadTlabEndOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 185         return config.threadTlabEndOffset();
 186     }
 187 
 188     public static final LocationIdentity TLAB_START_LOCATION = NamedLocationIdentity.mutable("TlabStart");
 189 
 190     @Fold
 191     static int threadTlabStartOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 192         return config.threadTlabStartOffset();
 193     }
 194 
 195     public static final LocationIdentity PENDING_EXCEPTION_LOCATION = NamedLocationIdentity.mutable("PendingException");
 196 
 197     /**
 198      * @see GraalHotSpotVMConfig#pendingExceptionOffset
 199      */
 200     @Fold
 201     static int threadPendingExceptionOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 202         return config.pendingExceptionOffset;
 203     }
 204 
 205     public static final LocationIdentity PENDING_DEOPTIMIZATION_LOCATION = NamedLocationIdentity.mutable("PendingDeoptimization");
 206 
 207     /**
 208      * @see GraalHotSpotVMConfig#pendingDeoptimizationOffset
 209      */
 210     @Fold
 211     static int threadPendingDeoptimizationOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 212         return config.pendingDeoptimizationOffset;
 213     }
 214 
 215     public static final LocationIdentity OBJECT_RESULT_LOCATION = NamedLocationIdentity.mutable("ObjectResult");
 216 
 217     @Fold
 218     static int objectResultOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 219         return config.threadObjectResultOffset;
 220     }
 221 
 222     /**
 223      * @see GraalHotSpotVMConfig#threadExceptionOopOffset
 224      */
 225     public static Object readExceptionOop(Word thread) {
 226         return thread.readObject(threadExceptionOopOffset(INJECTED_VMCONFIG), EXCEPTION_OOP_LOCATION);
 227     }
 228 
 229     public static Word readExceptionPc(Word thread) {
 230         return thread.readWord(threadExceptionPcOffset(INJECTED_VMCONFIG), EXCEPTION_PC_LOCATION);
 231     }
 232 
 233     /**
 234      * @see GraalHotSpotVMConfig#threadExceptionOopOffset
 235      */
 236     public static void writeExceptionOop(Word thread, Object value) {
 237         thread.writeObject(threadExceptionOopOffset(INJECTED_VMCONFIG), value, EXCEPTION_OOP_LOCATION);
 238     }
 239 
 240     public static void writeExceptionPc(Word thread, Word value) {
 241         thread.writeWord(threadExceptionPcOffset(INJECTED_VMCONFIG), value, EXCEPTION_PC_LOCATION);
 242     }
 243 
 244     public static Word readTlabTop(Word thread) {
 245         return thread.readWord(threadTlabTopOffset(INJECTED_VMCONFIG), TLAB_TOP_LOCATION);
 246     }
 247 
 248     public static Word readTlabEnd(Word thread) {
 249         return thread.readWord(threadTlabEndOffset(INJECTED_VMCONFIG), TLAB_END_LOCATION);
 250     }
 251 
 252     public static Word readTlabStart(Word thread) {
 253         return thread.readWord(threadTlabStartOffset(INJECTED_VMCONFIG), TLAB_START_LOCATION);
 254     }
 255 
 256     public static void writeTlabTop(Word thread, Word top) {
 257         thread.writeWord(threadTlabTopOffset(INJECTED_VMCONFIG), top, TLAB_TOP_LOCATION);
 258     }
 259 
 260     @SuppressFBWarnings(value = "NP_NULL_PARAM_DEREF_NONVIRTUAL", justification = "foldable method parameters are injected")
 261     public static void initializeTlab(Word thread, Word start, Word end) {
 262         thread.writeWord(threadTlabStartOffset(INJECTED_VMCONFIG), start, TLAB_START_LOCATION);
 263         thread.writeWord(threadTlabTopOffset(INJECTED_VMCONFIG), start, TLAB_TOP_LOCATION);
 264         thread.writeWord(threadTlabEndOffset(INJECTED_VMCONFIG), end, TLAB_END_LOCATION);
 265     }
 266 
 267     /**
 268      * Clears the pending exception for the given thread.
 269      *
 270      * @return the pending exception, or null if there was none
 271      */
 272     @SuppressFBWarnings(value = "NP_NULL_PARAM_DEREF_NONVIRTUAL", justification = "foldable method parameters are injected")
 273     public static Object clearPendingException(Word thread) {
 274         Object result = thread.readObject(threadPendingExceptionOffset(INJECTED_VMCONFIG), PENDING_EXCEPTION_LOCATION);
 275         thread.writeObject(threadPendingExceptionOffset(INJECTED_VMCONFIG), null, PENDING_EXCEPTION_LOCATION);
 276         return result;
 277     }
 278 
 279     /**
 280      * Reads the pending deoptimization value for the given thread.
 281      *
 282      * @return {@code true} if there was a pending deoptimization
 283      */
 284     public static int readPendingDeoptimization(Word thread) {
 285         return thread.readInt(threadPendingDeoptimizationOffset(INJECTED_VMCONFIG), PENDING_DEOPTIMIZATION_LOCATION);
 286     }
 287 
 288     /**
 289      * Writes the pending deoptimization value for the given thread.
 290      */
 291     public static void writePendingDeoptimization(Word thread, int value) {
 292         thread.writeInt(threadPendingDeoptimizationOffset(INJECTED_VMCONFIG), value, PENDING_DEOPTIMIZATION_LOCATION);
 293     }
 294 
 295     /**
 296      * Gets and clears the object result from a runtime call stored in a thread local.
 297      *
 298      * @return the object that was in the thread local
 299      */
 300     public static Object getAndClearObjectResult(Word thread) {
 301         Object result = thread.readObject(objectResultOffset(INJECTED_VMCONFIG), OBJECT_RESULT_LOCATION);
 302         thread.writeObject(objectResultOffset(INJECTED_VMCONFIG), null, OBJECT_RESULT_LOCATION);
 303         return result;
 304     }
 305 
 306     /*
 307      * As far as Java code is concerned this can be considered immutable: it is set just after the
 308      * JavaThread is created, before it is published. After that, it is never changed.
 309      */
 310     public static final LocationIdentity JAVA_THREAD_THREAD_OBJECT_LOCATION = NamedLocationIdentity.immutable("JavaThread::_threadObj");
 311 
 312     @Fold
 313     public static int threadObjectOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 314         return config.threadObjectOffset;
 315     }
 316 
 317     public static final LocationIdentity JAVA_THREAD_OSTHREAD_LOCATION = NamedLocationIdentity.mutable("JavaThread::_osthread");
 318 
 319     @Fold
 320     public static int osThreadOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 321         return config.osThreadOffset;
 322     }
 323 
 324     @Fold
 325     public static int osThreadInterruptedOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 326         return config.osThreadInterruptedOffset;
 327     }
 328 
 329     @Fold
 330     public static JavaKind getWordKind() {
 331         return runtime().getHostJVMCIBackend().getCodeCache().getTarget().wordJavaKind;
 332     }
 333 
 334     @Fold
 335     public static int wordSize() {
 336         return runtime().getHostJVMCIBackend().getCodeCache().getTarget().wordSize;
 337     }
 338 
 339     @Fold
 340     public static int pageSize() {
 341         return UNSAFE.pageSize();
 342     }
 343 
 344     @Fold
 345     public static int heapWordSize(@InjectedParameter GraalHotSpotVMConfig config) {
 346         return config.heapWordSize;
 347     }
 348 
 349     public static final LocationIdentity PROTOTYPE_MARK_WORD_LOCATION = NamedLocationIdentity.mutable("PrototypeMarkWord");
 350 
 351     @Fold
 352     public static int prototypeMarkWordOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 353         return config.prototypeMarkWordOffset;
 354     }
 355 
 356     @Fold
 357     public static long arrayPrototypeMarkWord(@InjectedParameter GraalHotSpotVMConfig config) {
 358         return config.arrayPrototypeMarkWord();
 359     }
 360 
 361     public static final LocationIdentity KLASS_ACCESS_FLAGS_LOCATION = NamedLocationIdentity.immutable("Klass::_access_flags");
 362 
 363     @Fold
 364     public static int klassAccessFlagsOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 365         return config.klassAccessFlagsOffset;
 366     }
 367 
 368     @Fold
 369     public static int jvmAccWrittenFlags(@InjectedParameter GraalHotSpotVMConfig config) {
 370         return config.jvmAccWrittenFlags;
 371     }
 372 
 373     public static final LocationIdentity KLASS_LAYOUT_HELPER_LOCATION = new HotSpotOptimizingLocationIdentity("Klass::_layout_helper") {
 374         @Override
 375         public ValueNode canonicalizeRead(ValueNode read, AddressNode location, ValueNode object, CanonicalizerTool tool) {
 376             ValueNode javaObject = findReadHub(object);
 377             if (javaObject != null) {
 378                 if (javaObject.stamp(NodeView.DEFAULT) instanceof ObjectStamp) {
 379                     ObjectStamp stamp = (ObjectStamp) javaObject.stamp(NodeView.DEFAULT);
 380                     HotSpotResolvedObjectType type = (HotSpotResolvedObjectType) stamp.javaType(tool.getMetaAccess());
 381                     if (type.isArray() && !type.getComponentType().isPrimitive()) {
 382                         int layout = type.layoutHelper();
 383                         return ConstantNode.forInt(layout);
 384                     }
 385                 }
 386             }
 387             return read;
 388         }
 389     };
 390 
 391     @Fold
 392     public static int klassLayoutHelperOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 393         return config.klassLayoutHelperOffset;
 394     }
 395 
 396     @NodeIntrinsic(value = KlassLayoutHelperNode.class)
 397     public static native int readLayoutHelper(KlassPointer object);
 398 
 399     /**
 400      * Checks if class {@code klass} is an array.
 401      *
 402      * See: Klass::layout_helper_is_array
 403      *
 404      * @param klassNonNull the class to be checked
 405      * @return true if klassNonNull is an array, false otherwise
 406      */
 407     public static boolean klassIsArray(KlassPointer klassNonNull) {
 408         /*
 409          * The less-than check only works if both values are ints. We use local variables to make
 410          * sure these are still ints and haven't changed.
 411          */
 412         final int layoutHelper = readLayoutHelper(klassNonNull);
 413         final int layoutHelperNeutralValue = config(INJECTED_VMCONFIG).klassLayoutHelperNeutralValue;
 414         return (layoutHelper < layoutHelperNeutralValue);
 415     }
 416 
 417     public static final LocationIdentity ARRAY_KLASS_COMPONENT_MIRROR = NamedLocationIdentity.immutable("ArrayKlass::_component_mirror");
 418 
 419     @Fold
 420     public static int arrayKlassComponentMirrorOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 421         return config.getFieldOffset("ArrayKlass::_component_mirror", Integer.class, "oop");
 422     }
 423 
 424     public static final LocationIdentity KLASS_SUPER_KLASS_LOCATION = NamedLocationIdentity.immutable("Klass::_super");
 425 
 426     @Fold
 427     public static int klassSuperKlassOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 428         return config.klassSuperKlassOffset;
 429     }
 430 
 431     public static final LocationIdentity MARK_WORD_LOCATION = NamedLocationIdentity.mutable("MarkWord");
 432 
 433     @Fold
 434     public static int markOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 435         return config.markOffset;
 436     }
 437 
 438     public static final LocationIdentity HUB_WRITE_LOCATION = NamedLocationIdentity.mutable("Hub:write");
 439 
 440     public static final LocationIdentity HUB_LOCATION = new HotSpotOptimizingLocationIdentity("Hub") {
 441         @Override
 442         public ValueNode canonicalizeRead(ValueNode read, AddressNode location, ValueNode object, CanonicalizerTool tool) {
 443             TypeReference constantType = StampTool.typeReferenceOrNull(object);
 444             if (constantType != null && constantType.isExact()) {
 445                 return ConstantNode.forConstant(read.stamp(NodeView.DEFAULT), tool.getConstantReflection().asObjectHub(constantType.getType()), tool.getMetaAccess());
 446             }
 447             return read;
 448         }
 449     };
 450 
 451     public static final LocationIdentity COMPRESSED_HUB_LOCATION = new HotSpotOptimizingLocationIdentity("CompressedHub") {
 452         @Override
 453         public ValueNode canonicalizeRead(ValueNode read, AddressNode location, ValueNode object, CanonicalizerTool tool) {
 454             TypeReference constantType = StampTool.typeReferenceOrNull(object);
 455             if (constantType != null && constantType.isExact()) {
 456                 return ConstantNode.forConstant(read.stamp(NodeView.DEFAULT), ((HotSpotMetaspaceConstant) tool.getConstantReflection().asObjectHub(constantType.getType())).compress(),
 457                                 tool.getMetaAccess());
 458             }
 459             return read;
 460         }
 461     };
 462 
 463     @Fold
 464     static int hubOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 465         return config.hubOffset;
 466     }
 467 
 468     public static void initializeObjectHeader(Word memory, Word markWord, KlassPointer hub) {
 469         memory.writeWord(markOffset(INJECTED_VMCONFIG), markWord, MARK_WORD_LOCATION);
 470         StoreHubNode.write(memory, hub);
 471     }
 472 
 473     @Fold
 474     public static int unlockedMask(@InjectedParameter GraalHotSpotVMConfig config) {
 475         return config.unlockedMask;
 476     }
 477 
 478     @Fold
 479     public static int monitorMask(@InjectedParameter GraalHotSpotVMConfig config) {
 480         return config.monitorMask;
 481     }
 482 
 483     @Fold
 484     public static int objectMonitorOwnerOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 485         return config.objectMonitorOwner;
 486     }
 487 
 488     @Fold
 489     public static int objectMonitorRecursionsOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 490         return config.objectMonitorRecursions;
 491     }
 492 
 493     @Fold
 494     public static int objectMonitorCxqOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 495         return config.objectMonitorCxq;
 496     }
 497 
 498     @Fold
 499     public static int objectMonitorEntryListOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 500         return config.objectMonitorEntryList;
 501     }
 502 
 503     /**
 504      * Mask for a biasable, locked or unlocked mark word.
 505      *
 506      * <pre>
 507      * +----------------------------------+-+-+
 508      * |                                 1|1|1|
 509      * +----------------------------------+-+-+
 510      * </pre>
 511      *
 512      */
 513     @Fold
 514     public static int biasedLockMaskInPlace(@InjectedParameter GraalHotSpotVMConfig config) {
 515         return config.biasedLockMaskInPlace;
 516     }
 517 
 518     @Fold
 519     public static int epochMaskInPlace(@InjectedParameter GraalHotSpotVMConfig config) {
 520         return config.epochMaskInPlace;
 521     }
 522 
 523     /**
 524      * Pattern for a biasable, unlocked mark word.
 525      *
 526      * <pre>
 527      * +----------------------------------+-+-+
 528      * |                                 1|0|1|
 529      * +----------------------------------+-+-+
 530      * </pre>
 531      *
 532      */
 533     @Fold
 534     public static int biasedLockPattern(@InjectedParameter GraalHotSpotVMConfig config) {
 535         return config.biasedLockPattern;
 536     }
 537 
 538     @Fold
 539     public static int ageMaskInPlace(@InjectedParameter GraalHotSpotVMConfig config) {
 540         return config.ageMaskInPlace;
 541     }
 542 
 543     @Fold
 544     public static int metaspaceArrayLengthOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 545         return config.metaspaceArrayLengthOffset;
 546     }
 547 
 548     @Fold
 549     public static int metaspaceArrayBaseOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 550         return config.metaspaceArrayBaseOffset;
 551     }
 552 
 553     @Fold
 554     public static int arrayLengthOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 555         return config.arrayOopDescLengthOffset();
 556     }
 557 
 558     @Fold
 559     public static int arrayBaseOffset(JavaKind elementKind) {
 560         return getArrayBaseOffset(elementKind);
 561     }
 562 
 563     @Fold
 564     public static int arrayIndexScale(JavaKind elementKind) {
 565         return getArrayIndexScale(elementKind);
 566     }
 567 
 568     public static Word arrayStart(int[] a) {
 569         return WordFactory.unsigned(ComputeObjectAddressNode.get(a, getArrayBaseOffset(JavaKind.Int)));
 570     }
 571 
 572     /**
 573      * Idiom for making {@link GraalHotSpotVMConfig} a constant.
 574      */
 575     @Fold
 576     public static GraalHotSpotVMConfig getConfig(@InjectedParameter GraalHotSpotVMConfig config) {
 577         return config;
 578     }
 579 
 580     /**
 581      * Calls {@link #arrayAllocationSize(int, int, int, GraalHotSpotVMConfig)} using an injected VM
 582      * configuration object.
 583      */
 584     public static int arrayAllocationSize(int length, int headerSize, int log2ElementSize) {
 585         return arrayAllocationSize(length, headerSize, log2ElementSize, getConfig(INJECTED_VMCONFIG));
 586     }
 587 
 588     /**
 589      * Computes the size of the memory chunk allocated for an array. This size accounts for the
 590      * array header size, body size and any padding after the last element to satisfy object
 591      * alignment requirements.
 592      *
 593      * @param length the number of elements in the array
 594      * @param headerSize the size of the array header
 595      * @param log2ElementSize log2 of the size of an element in the array
 596      * @param config the VM configuration providing the
 597      *            {@linkplain GraalHotSpotVMConfig#objectAlignment object alignment requirement}
 598      * @return the size of the memory chunk
 599      */
 600     public static int arrayAllocationSize(int length, int headerSize, int log2ElementSize, GraalHotSpotVMConfig config) {
 601         int alignment = config.objectAlignment;
 602         int size = (length << log2ElementSize) + headerSize + (alignment - 1);
 603         int mask = ~(alignment - 1);
 604         return size & mask;
 605     }
 606 
 607     @Fold
 608     public static int instanceHeaderSize(@InjectedParameter GraalHotSpotVMConfig config) {
 609         return config.useCompressedClassPointers ? (2 * wordSize()) - 4 : 2 * wordSize();
 610     }
 611 
 612     @Fold
 613     public static byte dirtyCardValue(@InjectedParameter GraalHotSpotVMConfig config) {
 614         return config.dirtyCardValue;
 615     }
 616 
 617     @Fold
 618     public static byte g1YoungCardValue(@InjectedParameter GraalHotSpotVMConfig config) {
 619         return config.g1YoungCardValue;
 620     }
 621 
 622     @Fold
 623     public static int cardTableShift(@InjectedParameter GraalHotSpotVMConfig config) {
 624         return config.cardtableShift;
 625     }
 626 
 627     @Fold
 628     public static long cardTableStart(@InjectedParameter GraalHotSpotVMConfig config) {
 629         return config.cardtableStartAddress;
 630     }
 631 
 632     @Fold
 633     public static int g1CardQueueIndexOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 634         return config.g1CardQueueIndexOffset();
 635     }
 636 
 637     @Fold
 638     public static int g1CardQueueBufferOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 639         return config.g1CardQueueBufferOffset();
 640     }
 641 
 642     @Fold
 643     public static int logOfHeapRegionGrainBytes(@InjectedParameter GraalHotSpotVMConfig config) {
 644         return config.logOfHRGrainBytes;
 645     }
 646 
 647     @Fold
 648     public static int g1SATBQueueMarkingOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 649         return config.g1SATBQueueMarkingOffset();
 650     }
 651 
 652     @Fold
 653     public static int g1SATBQueueIndexOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 654         return config.g1SATBQueueIndexOffset();
 655     }
 656 
 657     @Fold
 658     public static int g1SATBQueueBufferOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 659         return config.g1SATBQueueBufferOffset();
 660     }
 661 
 662     public static final LocationIdentity KLASS_SUPER_CHECK_OFFSET_LOCATION = NamedLocationIdentity.immutable("Klass::_super_check_offset");
 663 
 664     @Fold
 665     public static int superCheckOffsetOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 666         return config.superCheckOffsetOffset;
 667     }
 668 
 669     public static final LocationIdentity SECONDARY_SUPER_CACHE_LOCATION = NamedLocationIdentity.mutable("SecondarySuperCache");
 670 
 671     @Fold
 672     public static int secondarySuperCacheOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 673         return config.secondarySuperCacheOffset;
 674     }
 675 
 676     public static final LocationIdentity SECONDARY_SUPERS_LOCATION = NamedLocationIdentity.immutable("SecondarySupers");
 677 
 678     @Fold
 679     public static int secondarySupersOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 680         return config.secondarySupersOffset;
 681     }
 682 
 683     public static final LocationIdentity DISPLACED_MARK_WORD_LOCATION = NamedLocationIdentity.mutable("DisplacedMarkWord");
 684 
 685     public static final LocationIdentity OBJECT_MONITOR_OWNER_LOCATION = NamedLocationIdentity.mutable("ObjectMonitor::_owner");
 686 
 687     public static final LocationIdentity OBJECT_MONITOR_RECURSION_LOCATION = NamedLocationIdentity.mutable("ObjectMonitor::_recursions");
 688 
 689     public static final LocationIdentity OBJECT_MONITOR_CXQ_LOCATION = NamedLocationIdentity.mutable("ObjectMonitor::_cxq");
 690 
 691     public static final LocationIdentity OBJECT_MONITOR_ENTRY_LIST_LOCATION = NamedLocationIdentity.mutable("ObjectMonitor::_EntryList");
 692 
 693     @Fold
 694     public static int lockDisplacedMarkOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 695         return config.basicLockDisplacedHeaderOffset;
 696     }
 697 
 698     @Fold
 699     public static boolean useBiasedLocking(@InjectedParameter GraalHotSpotVMConfig config) {
 700         return config.useBiasedLocking;
 701     }
 702 
 703     @Fold
 704     public static boolean useDeferredInitBarriers(@InjectedParameter GraalHotSpotVMConfig config) {
 705         return config.useDeferredInitBarriers;
 706     }
 707 
 708     @Fold
 709     public static boolean useG1GC(@InjectedParameter GraalHotSpotVMConfig config) {
 710         return config.useG1GC;
 711     }
 712 
 713     @Fold
 714     public static boolean useCMSIncrementalMode(@InjectedParameter GraalHotSpotVMConfig config) {
 715         return config.cmsIncrementalMode;
 716     }
 717 
 718     @Fold
 719     public static boolean useCompressedOops(@InjectedParameter GraalHotSpotVMConfig config) {
 720         return config.useCompressedOops;
 721     }
 722 
 723     @Fold
 724     static int uninitializedIdentityHashCodeValue(@InjectedParameter GraalHotSpotVMConfig config) {
 725         return config.uninitializedIdentityHashCodeValue;
 726     }
 727 
 728     @Fold
 729     static int identityHashCodeShift(@InjectedParameter GraalHotSpotVMConfig config) {
 730         return config.identityHashCodeShift;
 731     }
 732 
 733     /**
 734      * Loads the hub of an object (without null checking it first).
 735      */
 736     public static KlassPointer loadHub(Object object) {
 737         return loadHubIntrinsic(object);
 738     }
 739 
 740     public static Object verifyOop(Object object) {
 741         if (verifyOops(INJECTED_VMCONFIG)) {
 742             verifyOopStub(VERIFY_OOP, object);
 743         }
 744         return object;
 745     }
 746 
 747     @NodeIntrinsic(ForeignCallNode.class)
 748     private static native Object verifyOopStub(@ConstantNodeParameter ForeignCallDescriptor descriptor, Object object);
 749 
 750     public static Word loadWordFromObject(Object object, int offset) {
 751         ReplacementsUtil.staticAssert(offset != hubOffset(INJECTED_VMCONFIG), "Use loadHubIntrinsic instead of loadWordFromObject");
 752         return loadWordFromObjectIntrinsic(object, offset, LocationIdentity.any(), getWordKind());
 753     }
 754 
 755     public static Word loadWordFromObject(Object object, int offset, LocationIdentity identity) {
 756         ReplacementsUtil.staticAssert(offset != hubOffset(INJECTED_VMCONFIG), "Use loadHubIntrinsic instead of loadWordFromObject");
 757         return loadWordFromObjectIntrinsic(object, offset, identity, getWordKind());
 758     }
 759 
 760     public static KlassPointer loadKlassFromObject(Object object, int offset, LocationIdentity identity) {
 761         ReplacementsUtil.staticAssert(offset != hubOffset(INJECTED_VMCONFIG), "Use loadHubIntrinsic instead of loadWordFromObject");
 762         return loadKlassFromObjectIntrinsic(object, offset, identity, getWordKind());
 763     }
 764 
 765     /**
 766      * Reads the value of a given register.
 767      *
 768      * @param register a register which must not be available to the register allocator
 769      * @return the value of {@code register} as a word
 770      */
 771     public static Word registerAsWord(@ConstantNodeParameter Register register) {
 772         return registerAsWord(register, true, false);
 773     }
 774 
 775     @NodeIntrinsic(value = ReadRegisterNode.class)
 776     public static native Word registerAsWord(@ConstantNodeParameter Register register, @ConstantNodeParameter boolean directUse, @ConstantNodeParameter boolean incoming);
 777 
 778     @NodeIntrinsic(value = WriteRegisterNode.class)
 779     public static native void writeRegisterAsWord(@ConstantNodeParameter Register register, Word value);
 780 
 781     @NodeIntrinsic(value = RawLoadNode.class)
 782     private static native Word loadWordFromObjectIntrinsic(Object object, long offset, @ConstantNodeParameter LocationIdentity locationIdentity, @ConstantNodeParameter JavaKind wordKind);
 783 
 784     @NodeIntrinsic(value = RawLoadNode.class)
 785     private static native KlassPointer loadKlassFromObjectIntrinsic(Object object, long offset, @ConstantNodeParameter LocationIdentity locationIdentity, @ConstantNodeParameter JavaKind wordKind);
 786 
 787     @NodeIntrinsic(value = LoadHubNode.class)
 788     public static native KlassPointer loadHubIntrinsic(Object object);
 789 
 790     @Fold
 791     public static int log2WordSize() {
 792         return CodeUtil.log2(wordSize());
 793     }
 794 
 795     public static final LocationIdentity CLASS_STATE_LOCATION = NamedLocationIdentity.mutable("ClassState");
 796 
 797     @Fold
 798     public static int instanceKlassInitStateOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 799         return config.instanceKlassInitStateOffset;
 800     }
 801 
 802     @Fold
 803     public static int instanceKlassStateFullyInitialized(@InjectedParameter GraalHotSpotVMConfig config) {
 804         return config.instanceKlassStateFullyInitialized;
 805     }
 806 
 807     /**
 808      *
 809      * @param hub the hub of an InstanceKlass
 810      * @return true is the InstanceKlass represented by hub is fully initialized
 811      */
 812     public static boolean isInstanceKlassFullyInitialized(KlassPointer hub) {
 813         return readInstanceKlassState(hub) == instanceKlassStateFullyInitialized(INJECTED_VMCONFIG);
 814     }
 815 
 816     private static byte readInstanceKlassState(KlassPointer hub) {
 817         return hub.readByte(instanceKlassInitStateOffset(INJECTED_VMCONFIG), CLASS_STATE_LOCATION);
 818     }
 819 
 820     public static final LocationIdentity KLASS_MODIFIER_FLAGS_LOCATION = NamedLocationIdentity.immutable("Klass::_modifier_flags");
 821 
 822     @Fold
 823     public static int klassModifierFlagsOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 824         return config.klassModifierFlagsOffset;
 825     }
 826 
 827     public static final LocationIdentity CLASS_KLASS_LOCATION = new HotSpotOptimizingLocationIdentity("Class._klass") {
 828         @Override
 829         public ValueNode canonicalizeRead(ValueNode read, AddressNode location, ValueNode object, CanonicalizerTool tool) {
 830             return foldIndirection(read, object, CLASS_MIRROR_LOCATION);
 831         }
 832     };
 833 
 834     @Fold
 835     public static int klassOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 836         return config.klassOffset;
 837     }
 838 
 839     public static final LocationIdentity CLASS_ARRAY_KLASS_LOCATION = new HotSpotOptimizingLocationIdentity("Class._array_klass") {
 840         @Override
 841         public ValueNode canonicalizeRead(ValueNode read, AddressNode location, ValueNode object, CanonicalizerTool tool) {
 842             return foldIndirection(read, object, ARRAY_KLASS_COMPONENT_MIRROR);
 843         }
 844     };
 845 
 846     @Fold
 847     public static int arrayKlassOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 848         return config.arrayKlassOffset;
 849     }
 850 
 851     public static final LocationIdentity CLASS_MIRROR_LOCATION = NamedLocationIdentity.immutable("Klass::_java_mirror");
 852 
 853     public static final LocationIdentity CLASS_MIRROR_HANDLE_LOCATION = NamedLocationIdentity.immutable("Klass::_java_mirror handle");
 854 
 855     public static final LocationIdentity HEAP_TOP_LOCATION = NamedLocationIdentity.mutable("HeapTop");
 856 
 857     @Fold
 858     public static long heapTopAddress(@InjectedParameter GraalHotSpotVMConfig config) {
 859         return config.heapTopAddress;
 860     }
 861 
 862     public static final LocationIdentity HEAP_END_LOCATION = NamedLocationIdentity.mutable("HeapEnd");
 863 
 864     @Fold
 865     public static long heapEndAddress(@InjectedParameter GraalHotSpotVMConfig config) {
 866         return config.heapEndAddress;
 867     }
 868 
 869     @Fold
 870     public static long tlabIntArrayMarkWord(@InjectedParameter GraalHotSpotVMConfig config) {
 871         return config.tlabIntArrayMarkWord();
 872     }
 873 
 874     @Fold
 875     public static boolean inlineContiguousAllocationSupported(@InjectedParameter GraalHotSpotVMConfig config) {
 876         return config.inlineContiguousAllocationSupported;
 877     }
 878 
 879     @Fold
 880     public static int tlabAlignmentReserveInHeapWords(@InjectedParameter GraalHotSpotVMConfig config) {
 881         return config.tlabAlignmentReserve;
 882     }
 883 
 884     public static final LocationIdentity TLAB_SIZE_LOCATION = NamedLocationIdentity.mutable("TlabSize");
 885 
 886     @Fold
 887     public static int threadTlabSizeOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 888         return config.threadTlabSizeOffset();
 889     }
 890 
 891     public static final LocationIdentity TLAB_THREAD_ALLOCATED_BYTES_LOCATION = NamedLocationIdentity.mutable("TlabThreadAllocatedBytes");
 892 
 893     @Fold
 894     public static int threadAllocatedBytesOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 895         return config.threadAllocatedBytesOffset;
 896     }
 897 
 898     public static final LocationIdentity TLAB_REFILL_WASTE_LIMIT_LOCATION = NamedLocationIdentity.mutable("RefillWasteLimit");
 899 
 900     @Fold
 901     public static int tlabRefillWasteLimitOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 902         return config.tlabRefillWasteLimitOffset();
 903     }
 904 
 905     public static final LocationIdentity TLAB_NOF_REFILLS_LOCATION = NamedLocationIdentity.mutable("TlabNOfRefills");
 906 
 907     @Fold
 908     public static int tlabNumberOfRefillsOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 909         return config.tlabNumberOfRefillsOffset();
 910     }
 911 
 912     public static final LocationIdentity TLAB_FAST_REFILL_WASTE_LOCATION = NamedLocationIdentity.mutable("TlabFastRefillWaste");
 913 
 914     @Fold
 915     public static int tlabFastRefillWasteOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 916         return config.tlabFastRefillWasteOffset();
 917     }
 918 
 919     public static final LocationIdentity TLAB_SLOW_ALLOCATIONS_LOCATION = NamedLocationIdentity.mutable("TlabSlowAllocations");
 920 
 921     @Fold
 922     public static int tlabSlowAllocationsOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 923         return config.tlabSlowAllocationsOffset();
 924     }
 925 
 926     @Fold
 927     public static int tlabRefillWasteIncrement(@InjectedParameter GraalHotSpotVMConfig config) {
 928         return config.tlabRefillWasteIncrement;
 929     }
 930 
 931     @Fold
 932     public static boolean tlabStats(@InjectedParameter GraalHotSpotVMConfig config) {
 933         return config.tlabStats;
 934     }
 935 
 936     @Fold
 937     public static int layoutHelperHeaderSizeShift(@InjectedParameter GraalHotSpotVMConfig config) {
 938         return config.layoutHelperHeaderSizeShift;
 939     }
 940 
 941     @Fold
 942     public static int layoutHelperHeaderSizeMask(@InjectedParameter GraalHotSpotVMConfig config) {
 943         return config.layoutHelperHeaderSizeMask;
 944     }
 945 
 946     @Fold
 947     public static int layoutHelperLog2ElementSizeShift(@InjectedParameter GraalHotSpotVMConfig config) {
 948         return config.layoutHelperLog2ElementSizeShift;
 949     }
 950 
 951     @Fold
 952     public static int layoutHelperLog2ElementSizeMask(@InjectedParameter GraalHotSpotVMConfig config) {
 953         return config.layoutHelperLog2ElementSizeMask;
 954     }
 955 
 956     @Fold
 957     public static int layoutHelperElementTypeShift(@InjectedParameter GraalHotSpotVMConfig config) {
 958         return config.layoutHelperElementTypeShift;
 959     }
 960 
 961     @Fold
 962     public static int layoutHelperElementTypeMask(@InjectedParameter GraalHotSpotVMConfig config) {
 963         return config.layoutHelperElementTypeMask;
 964     }
 965 
 966     @Fold
 967     public static int layoutHelperElementTypePrimitiveInPlace(@InjectedParameter GraalHotSpotVMConfig config) {
 968         return config.layoutHelperElementTypePrimitiveInPlace();
 969     }
 970 
 971     @NodeIntrinsic(ForeignCallNode.class)
 972     public static native int identityHashCode(@ConstantNodeParameter ForeignCallDescriptor descriptor, Object object);
 973 
 974     @Fold
 975     public static int verifiedEntryPointOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 976         return config.nmethodEntryOffset;
 977     }
 978 
 979     @Fold
 980     public static long gcTotalCollectionsAddress(@InjectedParameter GraalHotSpotVMConfig config) {
 981         return config.gcTotalCollectionsAddress();
 982     }
 983 
 984     @Fold
 985     public static long referentOffset() {
 986         try {
 987             return UNSAFE.objectFieldOffset(java.lang.ref.Reference.class.getDeclaredField("referent"));
 988         } catch (Exception e) {
 989             throw new GraalError(e);
 990         }
 991     }
 992 
 993     public static final LocationIdentity OBJ_ARRAY_KLASS_ELEMENT_KLASS_LOCATION = new HotSpotOptimizingLocationIdentity("ObjArrayKlass::_element_klass") {
 994         @Override
 995         public ValueNode canonicalizeRead(ValueNode read, AddressNode location, ValueNode object, CanonicalizerTool tool) {
 996             ValueNode javaObject = findReadHub(object);
 997             if (javaObject != null) {
 998                 ResolvedJavaType type = StampTool.typeOrNull(javaObject);
 999                 if (type != null && type.isArray()) {
1000                     ResolvedJavaType element = type.getComponentType();
1001                     if (element != null && !element.isPrimitive() && !element.getElementalType().isInterface()) {
1002                         Assumptions assumptions = object.graph().getAssumptions();
1003                         AssumptionResult<ResolvedJavaType> leafType = element.findLeafConcreteSubtype();
1004                         if (leafType != null && leafType.canRecordTo(assumptions)) {
1005                             leafType.recordTo(assumptions);
1006                             return ConstantNode.forConstant(read.stamp(NodeView.DEFAULT), tool.getConstantReflection().asObjectHub(leafType.getResult()), tool.getMetaAccess());
1007                         }
1008                     }
1009                 }
1010             }
1011             return read;
1012         }
1013     };
1014 
1015     @Fold
1016     public static int arrayClassElementOffset(@InjectedParameter GraalHotSpotVMConfig config) {
1017         return config.arrayClassElementOffset;
1018     }
1019 
1020     public static final LocationIdentity PRIMARY_SUPERS_LOCATION = NamedLocationIdentity.immutable("PrimarySupers");
1021 
1022     public static final LocationIdentity METASPACE_ARRAY_LENGTH_LOCATION = NamedLocationIdentity.immutable("MetaspaceArrayLength");
1023 
1024     public static final LocationIdentity SECONDARY_SUPERS_ELEMENT_LOCATION = NamedLocationIdentity.immutable("SecondarySupersElement");
1025 }