src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/stubs/NewInstanceStub.java

*** 21,31 ****
   * questions.
   */
  package org.graalvm.compiler.hotspot.stubs;
  
  import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFIG;
- import static org.graalvm.compiler.hotspot.nodes.DirectCompareAndSwapNode.compareAndSwap;
  import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.HEAP_END_LOCATION;
  import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.HEAP_TOP_LOCATION;
  import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.PROTOTYPE_MARK_WORD_LOCATION;
  import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_FAST_REFILL_WASTE_LOCATION;
  import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_NOF_REFILLS_LOCATION;
--- 21,30 ----
*** 35,45 ****
  import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_THREAD_ALLOCATED_BYTES_LOCATION;
  import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayBaseOffset;
  import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.getAndClearObjectResult;
  import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.initializeTlab;
  import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.isInstanceKlassFullyInitialized;
! import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.loadKlassLayoutHelperIntrinsic;
  import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.log2WordSize;
  import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.prototypeMarkWordOffset;
  import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabEnd;
  import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabStart;
  import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabTop;
--- 34,44 ----
  import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_THREAD_ALLOCATED_BYTES_LOCATION;
  import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayBaseOffset;
  import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.getAndClearObjectResult;
  import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.initializeTlab;
  import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.isInstanceKlassFullyInitialized;
! import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readLayoutHelper;
  import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.log2WordSize;
  import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.prototypeMarkWordOffset;
  import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabEnd;
  import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabStart;
  import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabTop;
*** 62,72 ****
  import static org.graalvm.compiler.hotspot.stubs.StubUtil.newDescriptor;
  import static org.graalvm.compiler.hotspot.stubs.StubUtil.printf;
  import static org.graalvm.compiler.hotspot.stubs.StubUtil.verifyObject;
  import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FAST_PATH_PROBABILITY;
  import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;
- 
  import org.graalvm.compiler.api.replacements.Fold;
  import org.graalvm.compiler.api.replacements.Snippet;
  import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter;
  import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor;
  import org.graalvm.compiler.graph.Node.ConstantNodeParameter;
--- 61,70 ----
*** 77,87 ****
  import org.graalvm.compiler.hotspot.nodes.StubForeignCallNode;
  import org.graalvm.compiler.hotspot.nodes.type.KlassPointerStamp;
  import org.graalvm.compiler.hotspot.replacements.NewObjectSnippets;
  import org.graalvm.compiler.hotspot.word.KlassPointer;
  import org.graalvm.compiler.nodes.ConstantNode;
! import org.graalvm.compiler.nodes.memory.address.RawAddressNode;
  import org.graalvm.compiler.word.Word;
  
  import jdk.vm.ci.code.Register;
  import jdk.vm.ci.hotspot.HotSpotResolvedObjectType;
  import jdk.vm.ci.meta.JavaKind;
--- 75,85 ----
  import org.graalvm.compiler.hotspot.nodes.StubForeignCallNode;
  import org.graalvm.compiler.hotspot.nodes.type.KlassPointerStamp;
  import org.graalvm.compiler.hotspot.replacements.NewObjectSnippets;
  import org.graalvm.compiler.hotspot.word.KlassPointer;
  import org.graalvm.compiler.nodes.ConstantNode;
! import org.graalvm.compiler.options.OptionValues;
  import org.graalvm.compiler.word.Word;
  
  import jdk.vm.ci.code.Register;
  import jdk.vm.ci.hotspot.HotSpotResolvedObjectType;
  import jdk.vm.ci.meta.JavaKind;
*** 92,114 ****
   * If this stub fails to refill the TLAB or allocate the object, it calls out to the HotSpot C++
   * runtime for to complete the allocation.
   */
  public class NewInstanceStub extends SnippetStub {
  
!     public NewInstanceStub(HotSpotProviders providers, HotSpotForeignCallLinkage linkage) {
!         super("newInstance", providers, linkage);
      }
  
      @Override
      protected Object[] makeConstArgs() {
          HotSpotResolvedObjectType intArrayType = (HotSpotResolvedObjectType) providers.getMetaAccess().lookupJavaType(int[].class);
          int count = method.getSignature().getParameterCount(false);
          Object[] args = new Object[count];
          assert checkConstArg(1, "intArrayHub");
          assert checkConstArg(2, "threadRegister");
          args[1] = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), intArrayType.klass(), null);
          args[2] = providers.getRegisters().getThreadRegister();
          return args;
      }
  
      private static Word allocate(Word thread, int size) {
          Word top = readTlabTop(thread);
--- 90,114 ----
   * If this stub fails to refill the TLAB or allocate the object, it calls out to the HotSpot C++
   * runtime for to complete the allocation.
   */
  public class NewInstanceStub extends SnippetStub {
  
!     public NewInstanceStub(OptionValues options, HotSpotProviders providers, HotSpotForeignCallLinkage linkage) {
!         super("newInstance", options, providers, linkage);
      }
  
      @Override
      protected Object[] makeConstArgs() {
          HotSpotResolvedObjectType intArrayType = (HotSpotResolvedObjectType) providers.getMetaAccess().lookupJavaType(int[].class);
          int count = method.getSignature().getParameterCount(false);
          Object[] args = new Object[count];
          assert checkConstArg(1, "intArrayHub");
          assert checkConstArg(2, "threadRegister");
+         assert checkConstArg(3, "options");
          args[1] = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), intArrayType.klass(), null);
          args[2] = providers.getRegisters().getThreadRegister();
+         args[3] = options;
          return args;
      }
  
      private static Word allocate(Word thread, int size) {
          Word top = readTlabTop(thread);
*** 124,165 ****
          }
          return Word.zero();
      }
  
      @Fold
!     static boolean logging() {
!         return StubOptions.TraceNewInstanceStub.getValue();
      }
  
      /**
       * Re-attempts allocation after an initial TLAB allocation failed or was skipped (e.g., due to
       * -XX:-UseTLAB).
       *
       * @param hub the hub of the object to be allocated
       * @param intArrayHub the hub for {@code int[].class}
       */
      @Snippet
!     private static Object newInstance(KlassPointer hub, @ConstantParameter KlassPointer intArrayHub, @ConstantParameter Register threadRegister) {
          /*
           * The type is known to be an instance so Klass::_layout_helper is the instance size as a
           * raw number
           */
-         int sizeInBytes = loadKlassLayoutHelperIntrinsic(hub);
          Word thread = registerAsWord(threadRegister);
          boolean inlineContiguousAllocationSupported = GraalHotSpotVMConfigNode.inlineContiguousAllocationSupported();
!         if (!forceSlowPath() && inlineContiguousAllocationSupported) {
              if (isInstanceKlassFullyInitialized(hub)) {
!                 Word memory = refillAllocate(thread, intArrayHub, sizeInBytes, logging());
                  if (memory.notEqual(0)) {
                      Word prototypeMarkWord = hub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION);
                      NewObjectSnippets.formatObjectForStub(hub, sizeInBytes, memory, prototypeMarkWord);
                      return verifyObject(memory.toObject());
                  }
              }
          }
  
!         if (logging()) {
              printf("newInstance: calling new_instance_c\n");
          }
  
          newInstanceC(NEW_INSTANCE_C, thread, hub);
          handlePendingException(thread, true);
--- 124,165 ----
          }
          return Word.zero();
      }
  
      @Fold
!     static boolean logging(OptionValues options) {
!         return StubOptions.TraceNewInstanceStub.getValue(options);
      }
  
      /**
       * Re-attempts allocation after an initial TLAB allocation failed or was skipped (e.g., due to
       * -XX:-UseTLAB).
       *
       * @param hub the hub of the object to be allocated
       * @param intArrayHub the hub for {@code int[].class}
       */
      @Snippet
!     private static Object newInstance(KlassPointer hub, @ConstantParameter KlassPointer intArrayHub, @ConstantParameter Register threadRegister, @ConstantParameter OptionValues options) {
          /*
           * The type is known to be an instance so Klass::_layout_helper is the instance size as a
           * raw number
           */
          Word thread = registerAsWord(threadRegister);
          boolean inlineContiguousAllocationSupported = GraalHotSpotVMConfigNode.inlineContiguousAllocationSupported();
!         if (!forceSlowPath(options) && inlineContiguousAllocationSupported) {
              if (isInstanceKlassFullyInitialized(hub)) {
!                 int sizeInBytes = readLayoutHelper(hub);
!                 Word memory = refillAllocate(thread, intArrayHub, sizeInBytes, logging(options));
                  if (memory.notEqual(0)) {
                      Word prototypeMarkWord = hub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION);
                      NewObjectSnippets.formatObjectForStub(hub, sizeInBytes, memory, prototypeMarkWord);
                      return verifyObject(memory.toObject());
                  }
              }
          }
  
!         if (logging(options)) {
              printf("newInstance: calling new_instance_c\n");
          }
  
          newInstanceC(NEW_INSTANCE_C, thread, hub);
          handlePendingException(thread, true);
*** 227,237 ****
                  int headerSize = arrayBaseOffset(JavaKind.Int);
                  // just like the HotSpot assembler stubs, assumes that tlabFreeSpaceInInts fits in
                  // an int
                  int tlabFreeSpaceInInts = (int) tlabFreeSpaceInBytes >>> 2;
                  int length = ((alignmentReserveInBytes - headerSize) >>> 2) + tlabFreeSpaceInInts;
!                 NewObjectSnippets.formatArray(intArrayHub, 0, length, headerSize, top, intArrayMarkWord, false, false, false);
  
                  long allocated = thread.readLong(threadAllocatedBytesOffset(INJECTED_VMCONFIG), TLAB_THREAD_ALLOCATED_BYTES_LOCATION);
                  allocated = allocated + top.subtract(readTlabStart(thread)).rawValue();
                  thread.writeLong(threadAllocatedBytesOffset(INJECTED_VMCONFIG), allocated, TLAB_THREAD_ALLOCATED_BYTES_LOCATION);
              }
--- 227,237 ----
                  int headerSize = arrayBaseOffset(JavaKind.Int);
                  // just like the HotSpot assembler stubs, assumes that tlabFreeSpaceInInts fits in
                  // an int
                  int tlabFreeSpaceInInts = (int) tlabFreeSpaceInBytes >>> 2;
                  int length = ((alignmentReserveInBytes - headerSize) >>> 2) + tlabFreeSpaceInInts;
!                 NewObjectSnippets.formatArray(intArrayHub, 0, length, headerSize, top, intArrayMarkWord, false, false, null);
  
                  long allocated = thread.readLong(threadAllocatedBytesOffset(INJECTED_VMCONFIG), TLAB_THREAD_ALLOCATED_BYTES_LOCATION);
                  allocated = allocated + top.subtract(readTlabStart(thread)).rawValue();
                  thread.writeLong(threadAllocatedBytesOffset(INJECTED_VMCONFIG), allocated, TLAB_THREAD_ALLOCATED_BYTES_LOCATION);
              }
*** 289,308 ****
  
              Word heapEnd = heapEndAddress.readWord(0, HEAP_END_LOCATION);
              if (newHeapTop.aboveThan(heapEnd)) {
                  return Word.zero();
              }
! 
!             if (compareAndSwap(RawAddressNode.address(heapTopAddress), heapTop, newHeapTop, HEAP_TOP_LOCATION).equal(heapTop)) {
                  return heapTop;
              }
          }
      }
  
      @Fold
!     static boolean forceSlowPath() {
!         return StubOptions.ForceUseOfNewInstanceStub.getValue();
      }
  
      public static final ForeignCallDescriptor NEW_INSTANCE_C = newDescriptor(NewInstanceStub.class, "newInstanceC", void.class, Word.class, KlassPointer.class);
  
      @NodeIntrinsic(StubForeignCallNode.class)
--- 289,307 ----
  
              Word heapEnd = heapEndAddress.readWord(0, HEAP_END_LOCATION);
              if (newHeapTop.aboveThan(heapEnd)) {
                  return Word.zero();
              }
!             if (heapTopAddress.logicCompareAndSwapWord(0, heapTop, newHeapTop, HEAP_TOP_LOCATION)) {
                  return heapTop;
              }
          }
      }
  
      @Fold
!     static boolean forceSlowPath(OptionValues options) {
!         return StubOptions.ForceUseOfNewInstanceStub.getValue(options);
      }
  
      public static final ForeignCallDescriptor NEW_INSTANCE_C = newDescriptor(NewInstanceStub.class, "newInstanceC", void.class, Word.class, KlassPointer.class);
  
      @NodeIntrinsic(StubForeignCallNode.class)