1 /*
   2  * Copyright (c) 2012, 2019, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  */
  23 
  24 
  25 package org.graalvm.compiler.hotspot.replacements;
  26 
  27 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfigBase.INJECTED_METAACCESS;
  28 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfigBase.INJECTED_VMCONFIG;
  29 import static org.graalvm.compiler.hotspot.meta.HotSpotForeignCallsProviderImpl.VERIFY_OOP;
  30 
  31 import java.lang.ref.Reference;
  32 
  33 import org.graalvm.compiler.api.replacements.Fold;
  34 import org.graalvm.compiler.api.replacements.Fold.InjectedParameter;
  35 import org.graalvm.compiler.core.common.SuppressFBWarnings;
  36 import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor;
  37 import org.graalvm.compiler.core.common.type.ObjectStamp;
  38 import org.graalvm.compiler.core.common.type.TypeReference;
  39 import org.graalvm.compiler.debug.GraalError;
  40 import org.graalvm.compiler.graph.Node.ConstantNodeParameter;
  41 import org.graalvm.compiler.graph.Node.NodeIntrinsic;
  42 import org.graalvm.compiler.graph.spi.CanonicalizerTool;
  43 import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig;
  44 import org.graalvm.compiler.hotspot.word.KlassPointer;
  45 import org.graalvm.compiler.nodes.CanonicalizableLocation;
  46 import org.graalvm.compiler.nodes.CompressionNode;
  47 import org.graalvm.compiler.nodes.ComputeObjectAddressNode;
  48 import org.graalvm.compiler.nodes.ConstantNode;
  49 import org.graalvm.compiler.nodes.NamedLocationIdentity;
  50 import org.graalvm.compiler.nodes.NodeView;
  51 import org.graalvm.compiler.nodes.ValueNode;
  52 import org.graalvm.compiler.nodes.extended.ForeignCallNode;
  53 import org.graalvm.compiler.nodes.extended.LoadHubNode;
  54 import org.graalvm.compiler.nodes.extended.RawLoadNode;
  55 import org.graalvm.compiler.nodes.extended.StoreHubNode;
  56 import org.graalvm.compiler.nodes.graphbuilderconf.IntrinsicContext;
  57 import org.graalvm.compiler.nodes.memory.Access;
  58 import org.graalvm.compiler.nodes.memory.address.AddressNode;
  59 import org.graalvm.compiler.nodes.memory.address.OffsetAddressNode;
  60 import org.graalvm.compiler.nodes.type.StampTool;
  61 import org.graalvm.compiler.replacements.ReplacementsUtil;
  62 import org.graalvm.compiler.replacements.nodes.ReadRegisterNode;
  63 import org.graalvm.compiler.replacements.nodes.WriteRegisterNode;
  64 import org.graalvm.compiler.word.Word;
  65 import jdk.internal.vm.compiler.word.LocationIdentity;
  66 import jdk.internal.vm.compiler.word.WordFactory;
  67 
  68 import jdk.vm.ci.code.Register;
  69 import jdk.vm.ci.hotspot.HotSpotJVMCIRuntime;
  70 import jdk.vm.ci.hotspot.HotSpotMetaspaceConstant;
  71 import jdk.vm.ci.hotspot.HotSpotResolvedObjectType;
  72 import jdk.vm.ci.meta.Assumptions;
  73 import jdk.vm.ci.meta.Assumptions.AssumptionResult;
  74 import jdk.vm.ci.meta.JavaKind;
  75 import jdk.vm.ci.meta.MetaAccessProvider;
  76 import jdk.vm.ci.meta.ResolvedJavaField;
  77 import jdk.vm.ci.meta.ResolvedJavaType;
  78 import jdk.vm.ci.meta.UnresolvedJavaType;
  79 
  80 //JaCoCo Exclude
  81 
  82 /**
  83  * A collection of methods used in HotSpot snippets, substitutions and stubs.
  84  */
  85 public class HotSpotReplacementsUtil {
  86 
  87     abstract static class HotSpotOptimizingLocationIdentity extends NamedLocationIdentity implements CanonicalizableLocation {
  88 
  89         HotSpotOptimizingLocationIdentity(String name) {
  90             super(name, true);
  91         }
  92 
  93         @Override
  94         public abstract ValueNode canonicalizeRead(ValueNode read, AddressNode location, ValueNode object, CanonicalizerTool tool);
  95 
  96         protected ValueNode findReadHub(ValueNode object) {
  97             ValueNode base = object;
  98             if (base instanceof CompressionNode) {
  99                 base = ((CompressionNode) base).getValue();
 100             }
 101             if (base instanceof Access) {
 102                 Access access = (Access) base;
 103                 if (access.getLocationIdentity().equals(HUB_LOCATION) || access.getLocationIdentity().equals(COMPRESSED_HUB_LOCATION)) {
 104                     AddressNode address = access.getAddress();
 105                     if (address instanceof OffsetAddressNode) {
 106                         OffsetAddressNode offset = (OffsetAddressNode) address;
 107                         return offset.getBase();
 108                     }
 109                 }
 110             } else if (base instanceof LoadHubNode) {
 111                 LoadHubNode loadhub = (LoadHubNode) base;
 112                 return loadhub.getValue();
 113             }
 114             return null;
 115         }
 116 
 117         /**
 118          * Fold reads that convert from Class -> Hub -> Class or vice versa.
 119          *
 120          * @param read
 121          * @param object
 122          * @param otherLocation
 123          * @return an earlier read or the original {@code read}
 124          */
 125         protected static ValueNode foldIndirection(ValueNode read, ValueNode object, LocationIdentity otherLocation) {
 126             if (object instanceof Access) {
 127                 Access access = (Access) object;
 128                 if (access.getLocationIdentity().equals(otherLocation)) {
 129                     AddressNode address = access.getAddress();
 130                     if (address instanceof OffsetAddressNode) {
 131                         OffsetAddressNode offset = (OffsetAddressNode) address;
 132                         assert offset.getBase().stamp(NodeView.DEFAULT).isCompatible(read.stamp(NodeView.DEFAULT));
 133                         return offset.getBase();
 134                     }
 135                 }
 136             }
 137             return read;
 138         }
 139     }
 140 
 141     @Fold
 142     public static ResolvedJavaType methodHolderClass(@InjectedParameter IntrinsicContext context) {
 143         return context.getOriginalMethod().getDeclaringClass();
 144     }
 145 
 146     @Fold
 147     static ResolvedJavaType getType(@Fold.InjectedParameter IntrinsicContext context, String typeName) {
 148         try {
 149             UnresolvedJavaType unresolved = UnresolvedJavaType.create(typeName);
 150             return unresolved.resolve(methodHolderClass(context));
 151         } catch (LinkageError e) {
 152             throw new GraalError(e);
 153         }
 154     }
 155 
 156     @Fold
 157     static int getFieldOffset(ResolvedJavaType type, String fieldName) {
 158         for (ResolvedJavaField field : type.getInstanceFields(true)) {
 159             if (field.getName().equals(fieldName)) {
 160                 return field.getOffset();
 161             }
 162         }
 163         throw new GraalError("missing field " + fieldName + " in type " + type);
 164     }
 165 
 166     public static HotSpotJVMCIRuntime runtime() {
 167         return HotSpotJVMCIRuntime.runtime();
 168     }
 169 
 170     @Fold
 171     public static int getHeapWordSize(@InjectedParameter GraalHotSpotVMConfig injectedVMConfig) {
 172         return injectedVMConfig.heapWordSize;
 173     }
 174 
 175     @Fold
 176     public static int klassLayoutHelperNeutralValue(@InjectedParameter GraalHotSpotVMConfig config) {
 177         return config.klassLayoutHelperNeutralValue;
 178     }
 179 
 180     @Fold
 181     public static boolean useTLAB(@InjectedParameter GraalHotSpotVMConfig config) {
 182         return config.useTLAB;
 183     }
 184 
 185     @Fold
 186     public static boolean verifyOops(@InjectedParameter GraalHotSpotVMConfig config) {
 187         return config.verifyOops;
 188     }
 189 
 190     public static final LocationIdentity EXCEPTION_OOP_LOCATION = NamedLocationIdentity.mutable("ExceptionOop");
 191 
 192     /**
 193      * @see GraalHotSpotVMConfig#threadExceptionOopOffset
 194      */
 195     @Fold
 196     public static int threadExceptionOopOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 197         return config.threadExceptionOopOffset;
 198     }
 199 
 200     public static final LocationIdentity EXCEPTION_PC_LOCATION = NamedLocationIdentity.mutable("ExceptionPc");
 201 
 202     @Fold
 203     public static int threadExceptionPcOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 204         return config.threadExceptionPcOffset;
 205     }
 206 
 207     public static final LocationIdentity TLAB_TOP_LOCATION = NamedLocationIdentity.mutable("TlabTop");
 208 
 209     @Fold
 210     public static int threadTlabTopOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 211         return config.threadTlabTopOffset();
 212     }
 213 
 214     public static final LocationIdentity TLAB_END_LOCATION = NamedLocationIdentity.mutable("TlabEnd");
 215 
 216     @Fold
 217     static int threadTlabEndOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 218         return config.threadTlabEndOffset();
 219     }
 220 
 221     public static final LocationIdentity PENDING_EXCEPTION_LOCATION = NamedLocationIdentity.mutable("PendingException");
 222 
 223     /**
 224      * @see GraalHotSpotVMConfig#pendingExceptionOffset
 225      */
 226     @Fold
 227     static int threadPendingExceptionOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 228         return config.pendingExceptionOffset;
 229     }
 230 
 231     /**
 232      * @see GraalHotSpotVMConfig#threadExceptionOopOffset
 233      */
 234     public static Object readExceptionOop(Word thread) {
 235         return thread.readObject(threadExceptionOopOffset(INJECTED_VMCONFIG), EXCEPTION_OOP_LOCATION);
 236     }
 237 
 238     public static Word readExceptionPc(Word thread) {
 239         return thread.readWord(threadExceptionPcOffset(INJECTED_VMCONFIG), EXCEPTION_PC_LOCATION);
 240     }
 241 
 242     /**
 243      * @see GraalHotSpotVMConfig#threadExceptionOopOffset
 244      */
 245     public static void writeExceptionOop(Word thread, Object value) {
 246         thread.writeObject(threadExceptionOopOffset(INJECTED_VMCONFIG), value, EXCEPTION_OOP_LOCATION);
 247     }
 248 
 249     public static void writeExceptionPc(Word thread, Word value) {
 250         thread.writeWord(threadExceptionPcOffset(INJECTED_VMCONFIG), value, EXCEPTION_PC_LOCATION);
 251     }
 252 
 253     public static Word readTlabTop(Word thread) {
 254         return thread.readWord(threadTlabTopOffset(INJECTED_VMCONFIG), TLAB_TOP_LOCATION);
 255     }
 256 
 257     public static Word readTlabEnd(Word thread) {
 258         return thread.readWord(threadTlabEndOffset(INJECTED_VMCONFIG), TLAB_END_LOCATION);
 259     }
 260 
 261     public static void writeTlabTop(Word thread, Word top) {
 262         thread.writeWord(threadTlabTopOffset(INJECTED_VMCONFIG), top, TLAB_TOP_LOCATION);
 263     }
 264 
 265     /**
 266      * Clears the pending exception for the given thread.
 267      *
 268      * @return the pending exception, or null if there was none
 269      */
 270     @SuppressFBWarnings(value = "NP_NULL_PARAM_DEREF_NONVIRTUAL", justification = "foldable method parameters are injected")
 271     public static Object clearPendingException(Word thread) {
 272         Object result = thread.readObject(threadPendingExceptionOffset(INJECTED_VMCONFIG), PENDING_EXCEPTION_LOCATION);
 273         thread.writeObject(threadPendingExceptionOffset(INJECTED_VMCONFIG), null, PENDING_EXCEPTION_LOCATION);
 274         return result;
 275     }
 276 
 277     /**
 278      * Gets the pending exception for the given thread.
 279      *
 280      * @return the pending exception, or null if there was none
 281      */
 282     @SuppressFBWarnings(value = "NP_NULL_PARAM_DEREF_NONVIRTUAL", justification = "foldable method parameters are injected")
 283     public static Object getPendingException(Word thread) {
 284         return thread.readObject(threadPendingExceptionOffset(INJECTED_VMCONFIG), PENDING_EXCEPTION_LOCATION);
 285     }
 286 
 287     /*
 288      * As far as Java code is concerned this can be considered immutable: it is set just after the
 289      * JavaThread is created, before it is published. After that, it is never changed.
 290      */
 291     public static final LocationIdentity JAVA_THREAD_THREAD_OBJECT_LOCATION = NamedLocationIdentity.immutable("JavaThread::_threadObj");
 292 
 293     @Fold
 294     public static int threadObjectOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 295         return config.threadObjectOffset;
 296     }
 297 
 298     public static final LocationIdentity JAVA_THREAD_OSTHREAD_LOCATION = NamedLocationIdentity.mutable("JavaThread::_osthread");
 299 
 300     @Fold
 301     public static int osThreadOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 302         return config.osThreadOffset;
 303     }
 304 
 305     @Fold
 306     public static int osThreadInterruptedOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 307         return config.osThreadInterruptedOffset;
 308     }
 309 
 310     @Fold
 311     public static JavaKind getWordKind() {
 312         return runtime().getHostJVMCIBackend().getCodeCache().getTarget().wordJavaKind;
 313     }
 314 
 315     @Fold
 316     public static int wordSize() {
 317         return runtime().getHostJVMCIBackend().getCodeCache().getTarget().wordSize;
 318     }
 319 
 320     @Fold
 321     public static int pageSize(@InjectedParameter GraalHotSpotVMConfig config) {
 322         return config.vmPageSize;
 323     }
 324 
 325     public static final LocationIdentity PROTOTYPE_MARK_WORD_LOCATION = NamedLocationIdentity.mutable("PrototypeMarkWord");
 326 
 327     @Fold
 328     public static int prototypeMarkWordOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 329         return config.prototypeMarkWordOffset;
 330     }
 331 
 332     public static final LocationIdentity KLASS_ACCESS_FLAGS_LOCATION = NamedLocationIdentity.immutable("Klass::_access_flags");
 333 
 334     @Fold
 335     public static int klassAccessFlagsOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 336         return config.klassAccessFlagsOffset;
 337     }
 338 
 339     @Fold
 340     public static int jvmAccWrittenFlags(@InjectedParameter GraalHotSpotVMConfig config) {
 341         return config.jvmAccWrittenFlags;
 342     }
 343 
 344     public static final LocationIdentity KLASS_LAYOUT_HELPER_LOCATION = new HotSpotOptimizingLocationIdentity("Klass::_layout_helper") {
 345         @Override
 346         public ValueNode canonicalizeRead(ValueNode read, AddressNode location, ValueNode object, CanonicalizerTool tool) {
 347             ValueNode javaObject = findReadHub(object);
 348             if (javaObject != null) {
 349                 if (javaObject.stamp(NodeView.DEFAULT) instanceof ObjectStamp) {
 350                     ObjectStamp stamp = (ObjectStamp) javaObject.stamp(NodeView.DEFAULT);
 351                     HotSpotResolvedObjectType type = (HotSpotResolvedObjectType) stamp.javaType(tool.getMetaAccess());
 352                     if (type.isArray() && !type.getComponentType().isPrimitive()) {
 353                         int layout = type.layoutHelper();
 354                         return ConstantNode.forInt(layout);
 355                     }
 356                 }
 357             }
 358             return read;
 359         }
 360     };
 361 
 362     @Fold
 363     public static int allocatePrefetchStyle(@InjectedParameter GraalHotSpotVMConfig config) {
 364         return config.allocatePrefetchStyle;
 365     }
 366 
 367     @Fold
 368     public static int allocatePrefetchLines(@InjectedParameter GraalHotSpotVMConfig config) {
 369         return config.allocatePrefetchLines;
 370     }
 371 
 372     @Fold
 373     public static int allocatePrefetchDistance(@InjectedParameter GraalHotSpotVMConfig config) {
 374         return config.allocatePrefetchDistance;
 375     }
 376 
 377     @Fold
 378     public static int allocateInstancePrefetchLines(@InjectedParameter GraalHotSpotVMConfig config) {
 379         return config.allocateInstancePrefetchLines;
 380     }
 381 
 382     @Fold
 383     public static int allocatePrefetchStepSize(@InjectedParameter GraalHotSpotVMConfig config) {
 384         return config.allocatePrefetchStepSize;
 385     }
 386 
 387     @Fold
 388     public static int invocationCounterIncrement(@InjectedParameter GraalHotSpotVMConfig config) {
 389         return config.invocationCounterIncrement;
 390     }
 391 
 392     @Fold
 393     public static int invocationCounterOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 394         return config.invocationCounterOffset;
 395     }
 396 
 397     @Fold
 398     public static int backedgeCounterOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 399         return config.backedgeCounterOffset;
 400     }
 401 
 402     @Fold
 403     public static int invocationCounterShift(@InjectedParameter GraalHotSpotVMConfig config) {
 404         return config.invocationCounterShift;
 405     }
 406 
 407     @Fold
 408     public static int stackBias(@InjectedParameter GraalHotSpotVMConfig config) {
 409         return config.stackBias;
 410     }
 411 
 412     @NodeIntrinsic(value = KlassLayoutHelperNode.class)
 413     public static native int readLayoutHelper(KlassPointer object);
 414 
 415     /**
 416      * Checks if class {@code klass} is an array.
 417      *
 418      * See: Klass::layout_helper_is_array
 419      *
 420      * @param klassNonNull the class to be checked
 421      * @return true if klassNonNull is an array, false otherwise
 422      */
 423     public static boolean klassIsArray(KlassPointer klassNonNull) {
 424         /*
 425          * The less-than check only works if both values are ints. We use local variables to make
 426          * sure these are still ints and haven't changed.
 427          */
 428         final int layoutHelper = readLayoutHelper(klassNonNull);
 429         final int layoutHelperNeutralValue = klassLayoutHelperNeutralValue(INJECTED_VMCONFIG);
 430         return (layoutHelper < layoutHelperNeutralValue);
 431     }
 432 
 433     public static final LocationIdentity ARRAY_KLASS_COMPONENT_MIRROR = NamedLocationIdentity.immutable("ArrayKlass::_component_mirror");
 434 
 435     @Fold
 436     public static int arrayKlassComponentMirrorOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 437         return config.getFieldOffset("ArrayKlass::_component_mirror", Integer.class, "oop");
 438     }
 439 
 440     public static final LocationIdentity KLASS_SUPER_KLASS_LOCATION = NamedLocationIdentity.immutable("Klass::_super");
 441 
 442     @Fold
 443     public static int klassSuperKlassOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 444         return config.klassSuperKlassOffset;
 445     }
 446 
 447     public static final LocationIdentity MARK_WORD_LOCATION = NamedLocationIdentity.mutable("MarkWord");
 448 
 449     @Fold
 450     public static int markOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 451         return config.markOffset;
 452     }
 453 
 454     public static final LocationIdentity HUB_WRITE_LOCATION = NamedLocationIdentity.mutable("Hub:write");
 455 
 456     public static final LocationIdentity HUB_LOCATION = new HotSpotOptimizingLocationIdentity("Hub") {
 457         @Override
 458         public ValueNode canonicalizeRead(ValueNode read, AddressNode location, ValueNode object, CanonicalizerTool tool) {
 459             TypeReference constantType = StampTool.typeReferenceOrNull(object);
 460             if (constantType != null && constantType.isExact()) {
 461                 return ConstantNode.forConstant(read.stamp(NodeView.DEFAULT), tool.getConstantReflection().asObjectHub(constantType.getType()), tool.getMetaAccess());
 462             }
 463             return read;
 464         }
 465     };
 466 
 467     public static final LocationIdentity COMPRESSED_HUB_LOCATION = new HotSpotOptimizingLocationIdentity("CompressedHub") {
 468         @Override
 469         public ValueNode canonicalizeRead(ValueNode read, AddressNode location, ValueNode object, CanonicalizerTool tool) {
 470             TypeReference constantType = StampTool.typeReferenceOrNull(object);
 471             if (constantType != null && constantType.isExact()) {
 472                 return ConstantNode.forConstant(read.stamp(NodeView.DEFAULT), ((HotSpotMetaspaceConstant) tool.getConstantReflection().asObjectHub(constantType.getType())).compress(),
 473                                 tool.getMetaAccess());
 474             }
 475             return read;
 476         }
 477     };
 478 
 479     @Fold
 480     static int hubOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 481         return config.hubOffset;
 482     }
 483 
 484     public static void initializeObjectHeader(Word memory, Word markWord, KlassPointer hub) {
 485         memory.writeWord(markOffset(INJECTED_VMCONFIG), markWord, MARK_WORD_LOCATION);
 486         StoreHubNode.write(memory, hub);
 487     }
 488 
 489     @Fold
 490     public static int unlockedMask(@InjectedParameter GraalHotSpotVMConfig config) {
 491         return config.unlockedMask;
 492     }
 493 
 494     @Fold
 495     public static int monitorMask(@InjectedParameter GraalHotSpotVMConfig config) {
 496         return config.monitorMask;
 497     }
 498 
 499     @Fold
 500     public static int objectMonitorOwnerOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 501         return config.objectMonitorOwner;
 502     }
 503 
 504     @Fold
 505     public static int objectMonitorRecursionsOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 506         return config.objectMonitorRecursions;
 507     }
 508 
 509     @Fold
 510     public static int objectMonitorCxqOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 511         return config.objectMonitorCxq;
 512     }
 513 
 514     @Fold
 515     public static int objectMonitorEntryListOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 516         return config.objectMonitorEntryList;
 517     }
 518 
 519     @Fold
 520     public static int objectMonitorSuccOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 521         return config.objectMonitorSucc;
 522     }
 523 
 524     /**
 525      * Mask for a biasable, locked or unlocked mark word.
 526      *
 527      * <pre>
 528      * +----------------------------------+-+-+
 529      * |                                 1|1|1|
 530      * +----------------------------------+-+-+
 531      * </pre>
 532      *
 533      */
 534     @Fold
 535     public static int biasedLockMaskInPlace(@InjectedParameter GraalHotSpotVMConfig config) {
 536         return config.biasedLockMaskInPlace;
 537     }
 538 
 539     @Fold
 540     public static int epochMaskInPlace(@InjectedParameter GraalHotSpotVMConfig config) {
 541         return config.epochMaskInPlace;
 542     }
 543 
 544     /**
 545      * Pattern for a biasable, unlocked mark word.
 546      *
 547      * <pre>
 548      * +----------------------------------+-+-+
 549      * |                                 1|0|1|
 550      * +----------------------------------+-+-+
 551      * </pre>
 552      *
 553      */
 554     @Fold
 555     public static int biasedLockPattern(@InjectedParameter GraalHotSpotVMConfig config) {
 556         return config.biasedLockPattern;
 557     }
 558 
 559     @Fold
 560     public static int ageMaskInPlace(@InjectedParameter GraalHotSpotVMConfig config) {
 561         return config.ageMaskInPlace;
 562     }
 563 
 564     @Fold
 565     public static int metaspaceArrayLengthOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 566         return config.metaspaceArrayLengthOffset;
 567     }
 568 
 569     @Fold
 570     public static int metaspaceArrayBaseOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 571         return config.metaspaceArrayBaseOffset;
 572     }
 573 
 574     @Fold
 575     public static int arrayLengthOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 576         return config.arrayOopDescLengthOffset();
 577     }
 578 
 579     public static Word arrayStart(int[] a) {
 580         return WordFactory.unsigned(ComputeObjectAddressNode.get(a, ReplacementsUtil.getArrayBaseOffset(INJECTED_METAACCESS, JavaKind.Int)));
 581     }
 582 
 583     /**
 584      * Idiom for making {@link GraalHotSpotVMConfig} a constant.
 585      */
 586     @Fold
 587     public static int objectAlignment(@InjectedParameter GraalHotSpotVMConfig config) {
 588         return config.objectAlignment;
 589     }
 590 
 591     /**
 592      * Calls {@link #arrayAllocationSize(int, int, int, int)} using an injected VM configuration
 593      * object.
 594      */
 595     public static long arrayAllocationSize(int length, int headerSize, int log2ElementSize) {
 596         return arrayAllocationSize(length, headerSize, log2ElementSize, objectAlignment(INJECTED_VMCONFIG));
 597     }
 598 
 599     /**
 600      * Computes the size of the memory chunk allocated for an array. This size accounts for the
 601      * array header size, body size and any padding after the last element to satisfy object
 602      * alignment requirements.
 603      *
 604      * @param length the number of elements in the array
 605      * @param headerSize the size of the array header
 606      * @param log2ElementSize log2 of the size of an element in the array
 607      * @param alignment the {@linkplain GraalHotSpotVMConfig#objectAlignment object alignment
 608      *            requirement}
 609      * @return the size of the memory chunk
 610      */
 611     public static long arrayAllocationSize(int length, int headerSize, int log2ElementSize, int alignment) {
 612         long size = ((long) length << log2ElementSize) + headerSize + (alignment - 1);
 613         long mask = ~(alignment - 1);
 614         return size & mask;
 615     }
 616 
 617     @Fold
 618     public static int instanceHeaderSize(@InjectedParameter GraalHotSpotVMConfig config) {
 619         return config.useCompressedClassPointers ? (2 * wordSize()) - 4 : 2 * wordSize();
 620     }
 621 
 622     @Fold
 623     public static byte dirtyCardValue(@InjectedParameter GraalHotSpotVMConfig config) {
 624         return config.dirtyCardValue;
 625     }
 626 
 627     @Fold
 628     public static byte g1YoungCardValue(@InjectedParameter GraalHotSpotVMConfig config) {
 629         return config.g1YoungCardValue;
 630     }
 631 
 632     @Fold
 633     public static int cardTableShift(@InjectedParameter GraalHotSpotVMConfig config) {
 634         return config.cardtableShift;
 635     }
 636 
 637     @Fold
 638     public static int g1CardQueueIndexOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 639         return config.g1CardQueueIndexOffset;
 640     }
 641 
 642     @Fold
 643     public static int g1CardQueueBufferOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 644         return config.g1CardQueueBufferOffset;
 645     }
 646 
 647     @Fold
 648     public static int g1SATBQueueMarkingOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 649         return config.g1SATBQueueMarkingOffset;
 650     }
 651 
 652     @Fold
 653     public static int g1SATBQueueIndexOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 654         return config.g1SATBQueueIndexOffset;
 655     }
 656 
 657     @Fold
 658     public static int g1SATBQueueBufferOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 659         return config.g1SATBQueueBufferOffset;
 660     }
 661 
 662     public static final LocationIdentity KLASS_SUPER_CHECK_OFFSET_LOCATION = NamedLocationIdentity.immutable("Klass::_super_check_offset");
 663 
 664     @Fold
 665     public static int superCheckOffsetOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 666         return config.superCheckOffsetOffset;
 667     }
 668 
 669     public static final LocationIdentity SECONDARY_SUPER_CACHE_LOCATION = NamedLocationIdentity.mutable("SecondarySuperCache");
 670 
 671     @Fold
 672     public static int secondarySuperCacheOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 673         return config.secondarySuperCacheOffset;
 674     }
 675 
 676     public static final LocationIdentity SECONDARY_SUPERS_LOCATION = NamedLocationIdentity.immutable("SecondarySupers");
 677 
 678     @Fold
 679     public static int secondarySupersOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 680         return config.secondarySupersOffset;
 681     }
 682 
 683     public static final LocationIdentity DISPLACED_MARK_WORD_LOCATION = NamedLocationIdentity.mutable("DisplacedMarkWord");
 684 
 685     public static final LocationIdentity OBJECT_MONITOR_OWNER_LOCATION = NamedLocationIdentity.mutable("ObjectMonitor::_owner");
 686 
 687     public static final LocationIdentity OBJECT_MONITOR_RECURSION_LOCATION = NamedLocationIdentity.mutable("ObjectMonitor::_recursions");
 688 
 689     public static final LocationIdentity OBJECT_MONITOR_CXQ_LOCATION = NamedLocationIdentity.mutable("ObjectMonitor::_cxq");
 690 
 691     public static final LocationIdentity OBJECT_MONITOR_ENTRY_LIST_LOCATION = NamedLocationIdentity.mutable("ObjectMonitor::_EntryList");
 692 
 693     public static final LocationIdentity OBJECT_MONITOR_SUCC_LOCATION = NamedLocationIdentity.mutable("ObjectMonitor::_succ");
 694 
 695     @Fold
 696     public static int lockDisplacedMarkOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 697         return config.basicLockDisplacedHeaderOffset;
 698     }
 699 
 700     @Fold
 701     public static boolean useBiasedLocking(@InjectedParameter GraalHotSpotVMConfig config) {
 702         return config.useBiasedLocking;
 703     }
 704 
 705     @Fold
 706     static int uninitializedIdentityHashCodeValue(@InjectedParameter GraalHotSpotVMConfig config) {
 707         return config.uninitializedIdentityHashCodeValue;
 708     }
 709 
 710     @Fold
 711     static int identityHashCodeShift(@InjectedParameter GraalHotSpotVMConfig config) {
 712         return config.identityHashCodeShift;
 713     }
 714 
 715     /**
 716      * Loads the hub of an object (without null checking it first).
 717      */
 718     public static KlassPointer loadHub(Object object) {
 719         return loadHubIntrinsic(object);
 720     }
 721 
 722     public static Object verifyOop(Object object) {
 723         if (verifyOops(INJECTED_VMCONFIG)) {
 724             verifyOopStub(VERIFY_OOP, object);
 725         }
 726         return object;
 727     }
 728 
 729     @NodeIntrinsic(ForeignCallNode.class)
 730     private static native Object verifyOopStub(@ConstantNodeParameter ForeignCallDescriptor descriptor, Object object);
 731 
 732     public static Word loadWordFromObject(Object object, int offset) {
 733         ReplacementsUtil.staticAssert(offset != hubOffset(INJECTED_VMCONFIG), "Use loadHubIntrinsic instead of loadWordFromObject");
 734         return loadWordFromObjectIntrinsic(object, offset, LocationIdentity.any(), getWordKind());
 735     }
 736 
 737     public static Word loadWordFromObject(Object object, int offset, LocationIdentity identity) {
 738         ReplacementsUtil.staticAssert(offset != hubOffset(INJECTED_VMCONFIG), "Use loadHubIntrinsic instead of loadWordFromObject");
 739         return loadWordFromObjectIntrinsic(object, offset, identity, getWordKind());
 740     }
 741 
 742     public static KlassPointer loadKlassFromObject(Object object, int offset, LocationIdentity identity) {
 743         ReplacementsUtil.staticAssert(offset != hubOffset(INJECTED_VMCONFIG), "Use loadHubIntrinsic instead of loadKlassFromObject");
 744         return loadKlassFromObjectIntrinsic(object, offset, identity, getWordKind());
 745     }
 746 
 747     /**
 748      * Reads the value of a given register.
 749      *
 750      * @param register a register which must not be available to the register allocator
 751      * @return the value of {@code register} as a word
 752      */
 753     public static Word registerAsWord(@ConstantNodeParameter Register register) {
 754         return registerAsWord(register, true, false);
 755     }
 756 
 757     @NodeIntrinsic(value = ReadRegisterNode.class)
 758     public static native Word registerAsWord(@ConstantNodeParameter Register register, @ConstantNodeParameter boolean directUse, @ConstantNodeParameter boolean incoming);
 759 
 760     @NodeIntrinsic(value = WriteRegisterNode.class)
 761     public static native void writeRegisterAsWord(@ConstantNodeParameter Register register, Word value);
 762 
 763     @NodeIntrinsic(value = RawLoadNode.class)
 764     private static native Word loadWordFromObjectIntrinsic(Object object, long offset, @ConstantNodeParameter LocationIdentity locationIdentity, @ConstantNodeParameter JavaKind wordKind);
 765 
 766     @NodeIntrinsic(value = RawLoadNode.class)
 767     private static native KlassPointer loadKlassFromObjectIntrinsic(Object object, long offset, @ConstantNodeParameter LocationIdentity locationIdentity, @ConstantNodeParameter JavaKind wordKind);
 768 
 769     @NodeIntrinsic(value = LoadHubNode.class)
 770     public static native KlassPointer loadHubIntrinsic(Object object);
 771 
 772     public static final LocationIdentity CLASS_STATE_LOCATION = NamedLocationIdentity.mutable("ClassState");
 773 
 774     @Fold
 775     public static int instanceKlassInitStateOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 776         return config.instanceKlassInitStateOffset;
 777     }
 778 
 779     @Fold
 780     public static int instanceKlassStateFullyInitialized(@InjectedParameter GraalHotSpotVMConfig config) {
 781         return config.instanceKlassStateFullyInitialized;
 782     }
 783 
 784     /**
 785      *
 786      * @param hub the hub of an InstanceKlass
 787      * @return true is the InstanceKlass represented by hub is fully initialized
 788      */
 789     public static boolean isInstanceKlassFullyInitialized(KlassPointer hub) {
 790         return readInstanceKlassState(hub) == instanceKlassStateFullyInitialized(INJECTED_VMCONFIG);
 791     }
 792 
 793     private static byte readInstanceKlassState(KlassPointer hub) {
 794         return hub.readByte(instanceKlassInitStateOffset(INJECTED_VMCONFIG), CLASS_STATE_LOCATION);
 795     }
 796 
 797     public static final LocationIdentity KLASS_MODIFIER_FLAGS_LOCATION = NamedLocationIdentity.immutable("Klass::_modifier_flags");
 798 
 799     @Fold
 800     public static int klassModifierFlagsOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 801         return config.klassModifierFlagsOffset;
 802     }
 803 
 804     public static final LocationIdentity CLASS_KLASS_LOCATION = new HotSpotOptimizingLocationIdentity("Class._klass") {
 805         @Override
 806         public ValueNode canonicalizeRead(ValueNode read, AddressNode location, ValueNode object, CanonicalizerTool tool) {
 807             return foldIndirection(read, object, CLASS_MIRROR_LOCATION);
 808         }
 809     };
 810 
 811     public static final LocationIdentity CLASS_ARRAY_KLASS_LOCATION = new HotSpotOptimizingLocationIdentity("Class._array_klass") {
 812         @Override
 813         public ValueNode canonicalizeRead(ValueNode read, AddressNode location, ValueNode object, CanonicalizerTool tool) {
 814             return foldIndirection(read, object, ARRAY_KLASS_COMPONENT_MIRROR);
 815         }
 816     };
 817 
 818     @Fold
 819     public static int arrayKlassOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 820         return config.arrayKlassOffset;
 821     }
 822 
 823     public static final LocationIdentity CLASS_MIRROR_LOCATION = NamedLocationIdentity.immutable("Klass::_java_mirror");
 824 
 825     public static final LocationIdentity CLASS_MIRROR_HANDLE_LOCATION = NamedLocationIdentity.immutable("Klass::_java_mirror handle");
 826 
 827     @Fold
 828     public static int layoutHelperHeaderSizeShift(@InjectedParameter GraalHotSpotVMConfig config) {
 829         return config.layoutHelperHeaderSizeShift;
 830     }
 831 
 832     @Fold
 833     public static int layoutHelperHeaderSizeMask(@InjectedParameter GraalHotSpotVMConfig config) {
 834         return config.layoutHelperHeaderSizeMask;
 835     }
 836 
 837     @Fold
 838     public static int layoutHelperLog2ElementSizeShift(@InjectedParameter GraalHotSpotVMConfig config) {
 839         return config.layoutHelperLog2ElementSizeShift;
 840     }
 841 
 842     @Fold
 843     public static int layoutHelperLog2ElementSizeMask(@InjectedParameter GraalHotSpotVMConfig config) {
 844         return config.layoutHelperLog2ElementSizeMask;
 845     }
 846 
 847     @NodeIntrinsic(ForeignCallNode.class)
 848     public static native int identityHashCode(@ConstantNodeParameter ForeignCallDescriptor descriptor, Object object);
 849 
 850     @Fold
 851     public static long gcTotalCollectionsAddress(@InjectedParameter GraalHotSpotVMConfig config) {
 852         return config.gcTotalCollectionsAddress();
 853     }
 854 
 855     @Fold
 856     public static long referentOffset(@InjectedParameter MetaAccessProvider metaAccessProvider) {
 857         return getFieldOffset(metaAccessProvider.lookupJavaType(Reference.class), "referent");
 858     }
 859 
 860     public static final LocationIdentity OBJ_ARRAY_KLASS_ELEMENT_KLASS_LOCATION = new HotSpotOptimizingLocationIdentity("ObjArrayKlass::_element_klass") {
 861         @Override
 862         public ValueNode canonicalizeRead(ValueNode read, AddressNode location, ValueNode object, CanonicalizerTool tool) {
 863             ValueNode javaObject = findReadHub(object);
 864             if (javaObject != null) {
 865                 ResolvedJavaType type = StampTool.typeOrNull(javaObject);
 866                 if (type != null && type.isArray()) {
 867                     ResolvedJavaType element = type.getComponentType();
 868                     if (element != null && !element.isPrimitive() && !element.getElementalType().isInterface()) {
 869                         Assumptions assumptions = object.graph().getAssumptions();
 870                         AssumptionResult<ResolvedJavaType> leafType = element.findLeafConcreteSubtype();
 871                         if (leafType != null && leafType.canRecordTo(assumptions)) {
 872                             leafType.recordTo(assumptions);
 873                             return ConstantNode.forConstant(read.stamp(NodeView.DEFAULT), tool.getConstantReflection().asObjectHub(leafType.getResult()), tool.getMetaAccess());
 874                         }
 875                     }
 876                 }
 877             }
 878             return read;
 879         }
 880     };
 881 
 882     @Fold
 883     public static int arrayClassElementOffset(@InjectedParameter GraalHotSpotVMConfig config) {
 884         return config.arrayClassElementOffset;
 885     }
 886 
 887     public static final LocationIdentity PRIMARY_SUPERS_LOCATION = NamedLocationIdentity.immutable("PrimarySupers");
 888 
 889     public static final LocationIdentity METASPACE_ARRAY_LENGTH_LOCATION = NamedLocationIdentity.immutable("MetaspaceArrayLength");
 890 
 891     public static final LocationIdentity SECONDARY_SUPERS_ELEMENT_LOCATION = NamedLocationIdentity.immutable("SecondarySupersElement");
 892 }