1 /*
   2  * Copyright (c) 2012, 2015, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  */
  23 package org.graalvm.compiler.hotspot.stubs;
  24 
  25 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFIG;
  26 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.HEAP_END_LOCATION;
  27 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.HEAP_TOP_LOCATION;
  28 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.PROTOTYPE_MARK_WORD_LOCATION;
  29 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_FAST_REFILL_WASTE_LOCATION;
  30 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_NOF_REFILLS_LOCATION;
  31 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_REFILL_WASTE_LIMIT_LOCATION;
  32 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_SIZE_LOCATION;
  33 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_SLOW_ALLOCATIONS_LOCATION;
  34 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_THREAD_ALLOCATED_BYTES_LOCATION;
  35 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayBaseOffset;
  36 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.getAndClearObjectResult;
  37 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.initializeTlab;
  38 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.isInstanceKlassFullyInitialized;
  39 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readLayoutHelper;
  40 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.log2WordSize;
  41 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.prototypeMarkWordOffset;
  42 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabEnd;
  43 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabStart;
  44 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabTop;
  45 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.registerAsWord;
  46 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.threadAllocatedBytesOffset;
  47 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.threadTlabSizeOffset;
  48 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabAlignmentReserveInHeapWords;
  49 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabFastRefillWasteOffset;
  50 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabIntArrayMarkWord;
  51 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabNumberOfRefillsOffset;
  52 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabRefillWasteIncrement;
  53 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabRefillWasteLimitOffset;
  54 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabSlowAllocationsOffset;
  55 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabStats;

  56 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useG1GC;
  57 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useTLAB;
  58 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.wordSize;
  59 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.writeTlabTop;
  60 import static org.graalvm.compiler.hotspot.stubs.StubUtil.handlePendingException;
  61 import static org.graalvm.compiler.hotspot.stubs.StubUtil.newDescriptor;
  62 import static org.graalvm.compiler.hotspot.stubs.StubUtil.printf;
  63 import static org.graalvm.compiler.hotspot.stubs.StubUtil.verifyObject;
  64 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FAST_PATH_PROBABILITY;
  65 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;
  66 
  67 import org.graalvm.compiler.api.replacements.Fold;
  68 import org.graalvm.compiler.api.replacements.Snippet;
  69 import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter;
  70 import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor;
  71 import org.graalvm.compiler.graph.Node.ConstantNodeParameter;
  72 import org.graalvm.compiler.graph.Node.NodeIntrinsic;
  73 import org.graalvm.compiler.hotspot.HotSpotForeignCallLinkage;
  74 import org.graalvm.compiler.hotspot.meta.HotSpotProviders;
  75 import org.graalvm.compiler.hotspot.nodes.GraalHotSpotVMConfigNode;
  76 import org.graalvm.compiler.hotspot.nodes.StubForeignCallNode;
  77 import org.graalvm.compiler.hotspot.nodes.type.KlassPointerStamp;
  78 import org.graalvm.compiler.hotspot.replacements.NewObjectSnippets;
  79 import org.graalvm.compiler.hotspot.word.KlassPointer;
  80 import org.graalvm.compiler.nodes.ConstantNode;
  81 import org.graalvm.compiler.options.OptionValues;
  82 import org.graalvm.compiler.word.Word;
  83 import org.graalvm.word.WordFactory;
  84 
  85 import jdk.vm.ci.code.Register;
  86 import jdk.vm.ci.hotspot.HotSpotResolvedObjectType;
  87 import jdk.vm.ci.meta.JavaKind;
  88 
  89 /**
  90  * Stub implementing the fast path for TLAB refill during instance class allocation. This stub is
  91  * called from the {@linkplain NewObjectSnippets inline} allocation code when TLAB allocation fails.
  92  * If this stub fails to refill the TLAB or allocate the object, it calls out to the HotSpot C++
  93  * runtime for to complete the allocation.
  94  */
  95 public class NewInstanceStub extends SnippetStub {
  96 
  97     public NewInstanceStub(OptionValues options, HotSpotProviders providers, HotSpotForeignCallLinkage linkage) {
  98         super("newInstance", options, providers, linkage);
  99     }
 100 
 101     @Override
 102     protected Object[] makeConstArgs() {
 103         HotSpotResolvedObjectType intArrayType = (HotSpotResolvedObjectType) providers.getMetaAccess().lookupJavaType(int[].class);
 104         int count = method.getSignature().getParameterCount(false);
 105         Object[] args = new Object[count];
 106         assert checkConstArg(1, "intArrayHub");
 107         assert checkConstArg(2, "threadRegister");
 108         assert checkConstArg(3, "options");
 109         args[1] = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), intArrayType.klass(), null);
 110         args[2] = providers.getRegisters().getThreadRegister();
 111         args[3] = options;
 112         return args;
 113     }
 114 
 115     private static Word allocate(Word thread, int size) {
 116         Word top = readTlabTop(thread);
 117         Word end = readTlabEnd(thread);
 118         Word newTop = top.add(size);
 119         /*
 120          * this check might lead to problems if the TLAB is within 16GB of the address space end
 121          * (checked in c++ code)
 122          */
 123         if (probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
 124             writeTlabTop(thread, newTop);
 125             return top;
 126         }
 127         return WordFactory.zero();
 128     }
 129 
 130     @Fold
 131     static boolean logging(OptionValues options) {
 132         return StubOptions.TraceNewInstanceStub.getValue(options);
 133     }
 134 
 135     /**
 136      * Re-attempts allocation after an initial TLAB allocation failed or was skipped (e.g., due to
 137      * -XX:-UseTLAB).
 138      *
 139      * @param hub the hub of the object to be allocated
 140      * @param intArrayHub the hub for {@code int[].class}
 141      */
 142     @Snippet
 143     private static Object newInstance(KlassPointer hub, @ConstantParameter KlassPointer intArrayHub, @ConstantParameter Register threadRegister, @ConstantParameter OptionValues options) {
 144         /*
 145          * The type is known to be an instance so Klass::_layout_helper is the instance size as a
 146          * raw number
 147          */
 148         Word thread = registerAsWord(threadRegister);
 149         boolean inlineContiguousAllocationSupported = GraalHotSpotVMConfigNode.inlineContiguousAllocationSupported();
 150         if (!forceSlowPath(options) && inlineContiguousAllocationSupported) {
 151             if (isInstanceKlassFullyInitialized(hub)) {
 152                 int sizeInBytes = readLayoutHelper(hub);
 153                 Word memory = refillAllocate(thread, intArrayHub, sizeInBytes, logging(options));
 154                 if (memory.notEqual(0)) {
 155                     Word prototypeMarkWord = hub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION);
 156                     NewObjectSnippets.formatObjectForStub(hub, sizeInBytes, memory, prototypeMarkWord);
 157                     return verifyObject(memory.toObject());
 158                 }
 159             }
 160         }
 161 
 162         if (logging(options)) {
 163             printf("newInstance: calling new_instance_c\n");
 164         }
 165 
 166         newInstanceC(NEW_INSTANCE_C, thread, hub);
 167         handlePendingException(thread, true);
 168         return verifyObject(getAndClearObjectResult(thread));
 169     }
 170 
 171     /**
 172      * Attempts to refill the current thread's TLAB and retries the allocation.
 173      *
 174      * @param intArrayHub the hub for {@code int[].class}
 175      * @param sizeInBytes the size of the allocation
 176      * @param log specifies if logging is enabled
 177      *
 178      * @return the newly allocated, uninitialized chunk of memory, or {@link WordFactory#zero()} if
 179      *         the operation was unsuccessful
 180      */
 181     static Word refillAllocate(Word thread, KlassPointer intArrayHub, int sizeInBytes, boolean log) {
 182         // If G1 is enabled, the "eden" allocation space is not the same always
 183         // and therefore we have to go to slowpath to allocate a new TLAB.
 184         if (useG1GC(INJECTED_VMCONFIG)) {
 185             return WordFactory.zero();
 186         }
 187         if (!useTLAB(INJECTED_VMCONFIG)) {
 188             return edenAllocate(WordFactory.unsigned(sizeInBytes), log);
 189         }
 190         Word intArrayMarkWord = WordFactory.unsigned(tlabIntArrayMarkWord(INJECTED_VMCONFIG));
 191         int alignmentReserveInBytes = tlabAlignmentReserveInHeapWords(INJECTED_VMCONFIG) * wordSize();
 192 
 193         Word top = readTlabTop(thread);
 194         Word end = readTlabEnd(thread);
 195 
 196         // calculate amount of free space
 197         long tlabFreeSpaceInBytes = end.subtract(top).rawValue();
 198 
 199         if (log) {
 200             printf("refillTLAB: thread=%p\n", thread.rawValue());
 201             printf("refillTLAB: top=%p\n", top.rawValue());
 202             printf("refillTLAB: end=%p\n", end.rawValue());
 203             printf("refillTLAB: tlabFreeSpaceInBytes=%ld\n", tlabFreeSpaceInBytes);
 204         }
 205 
 206         long tlabFreeSpaceInWords = tlabFreeSpaceInBytes >>> log2WordSize();
 207 
 208         // Retain TLAB and allocate object in shared space if
 209         // the amount free in the TLAB is too large to discard.
 210         Word refillWasteLimit = thread.readWord(tlabRefillWasteLimitOffset(INJECTED_VMCONFIG), TLAB_REFILL_WASTE_LIMIT_LOCATION);
 211         if (tlabFreeSpaceInWords <= refillWasteLimit.rawValue()) {
 212             if (tlabStats(INJECTED_VMCONFIG)) {
 213                 // increment number of refills
 214                 thread.writeInt(tlabNumberOfRefillsOffset(INJECTED_VMCONFIG), thread.readInt(tlabNumberOfRefillsOffset(INJECTED_VMCONFIG), TLAB_NOF_REFILLS_LOCATION) + 1, TLAB_NOF_REFILLS_LOCATION);
 215                 if (log) {
 216                     printf("thread: %p -- number_of_refills %d\n", thread.rawValue(), thread.readInt(tlabNumberOfRefillsOffset(INJECTED_VMCONFIG), TLAB_NOF_REFILLS_LOCATION));
 217                 }
 218                 // accumulate wastage
 219                 int wastage = thread.readInt(tlabFastRefillWasteOffset(INJECTED_VMCONFIG), TLAB_FAST_REFILL_WASTE_LOCATION) + (int) tlabFreeSpaceInWords;
 220                 if (log) {
 221                     printf("thread: %p -- accumulated wastage %d\n", thread.rawValue(), wastage);
 222                 }
 223                 thread.writeInt(tlabFastRefillWasteOffset(INJECTED_VMCONFIG), wastage, TLAB_FAST_REFILL_WASTE_LOCATION);
 224             }
 225 
 226             // if TLAB is currently allocated (top or end != null) then
 227             // fill [top, end + alignment_reserve) with array object
 228             if (top.notEqual(0)) {
 229                 int headerSize = arrayBaseOffset(JavaKind.Int);
 230                 // just like the HotSpot assembler stubs, assumes that tlabFreeSpaceInInts fits in
 231                 // an int
 232                 int tlabFreeSpaceInInts = (int) tlabFreeSpaceInBytes >>> 2;
 233                 int length = ((alignmentReserveInBytes - headerSize) >>> 2) + tlabFreeSpaceInInts;
 234                 NewObjectSnippets.formatArray(intArrayHub, 0, length, headerSize, top, intArrayMarkWord, false, false, null);
 235 
 236                 long allocated = thread.readLong(threadAllocatedBytesOffset(INJECTED_VMCONFIG), TLAB_THREAD_ALLOCATED_BYTES_LOCATION);
 237                 allocated = allocated + top.subtract(readTlabStart(thread)).rawValue();
 238                 thread.writeLong(threadAllocatedBytesOffset(INJECTED_VMCONFIG), allocated, TLAB_THREAD_ALLOCATED_BYTES_LOCATION);
 239             }
 240 
 241             // refill the TLAB with an eden allocation
 242             Word tlabRefillSizeInWords = thread.readWord(threadTlabSizeOffset(INJECTED_VMCONFIG), TLAB_SIZE_LOCATION);
 243             Word tlabRefillSizeInBytes = tlabRefillSizeInWords.multiply(wordSize());
 244             // allocate new TLAB, address returned in top
 245             top = edenAllocate(tlabRefillSizeInBytes, log);
 246             if (top.notEqual(0)) {
 247                 end = top.add(tlabRefillSizeInBytes.subtract(alignmentReserveInBytes));
 248                 initializeTlab(thread, top, end);
 249 
 250                 return NewInstanceStub.allocate(thread, sizeInBytes);
 251             } else {
 252                 return WordFactory.zero();
 253             }
 254         } else {
 255             // Retain TLAB
 256             Word newRefillWasteLimit = refillWasteLimit.add(tlabRefillWasteIncrement(INJECTED_VMCONFIG));
 257             thread.writeWord(tlabRefillWasteLimitOffset(INJECTED_VMCONFIG), newRefillWasteLimit, TLAB_REFILL_WASTE_LIMIT_LOCATION);
 258             if (log) {
 259                 printf("refillTLAB: retaining TLAB - newRefillWasteLimit=%p\n", newRefillWasteLimit.rawValue());
 260             }
 261 
 262             if (tlabStats(INJECTED_VMCONFIG)) {
 263                 thread.writeInt(tlabSlowAllocationsOffset(INJECTED_VMCONFIG), thread.readInt(tlabSlowAllocationsOffset(INJECTED_VMCONFIG), TLAB_SLOW_ALLOCATIONS_LOCATION) + 1,
 264                                 TLAB_SLOW_ALLOCATIONS_LOCATION);
 265             }
 266 
 267             return edenAllocate(WordFactory.unsigned(sizeInBytes), log);
 268         }
 269     }
 270 
 271     /**
 272      * Attempts to allocate a chunk of memory from Eden space.
 273      *
 274      * @param sizeInBytes the size of the chunk to allocate
 275      * @param log specifies if logging is enabled
 276      * @return the allocated chunk or {@link WordFactory#zero()} if allocation fails
 277      */
 278     public static Word edenAllocate(Word sizeInBytes, boolean log) {
 279         final long heapTopRawAddress = GraalHotSpotVMConfigNode.heapTopAddress();
 280         final long heapEndRawAddress = GraalHotSpotVMConfigNode.heapEndAddress();
 281 
 282         Word heapTopAddress = WordFactory.unsigned(heapTopRawAddress);
 283         Word heapEndAddress = WordFactory.unsigned(heapEndRawAddress);
 284 
 285         while (true) {
 286             Word heapTop = heapTopAddress.readWord(0, HEAP_TOP_LOCATION);
 287             Word newHeapTop = heapTop.add(sizeInBytes);
 288             if (newHeapTop.belowOrEqual(heapTop)) {
 289                 return WordFactory.zero();
 290             }
 291 
 292             Word heapEnd = heapEndAddress.readWord(0, HEAP_END_LOCATION);
 293             if (newHeapTop.aboveThan(heapEnd)) {
 294                 return WordFactory.zero();
 295             }
 296             if (heapTopAddress.logicCompareAndSwapWord(0, heapTop, newHeapTop, HEAP_TOP_LOCATION)) {
 297                 return heapTop;
 298             }
 299         }
 300     }
 301 
 302     @Fold
 303     static boolean forceSlowPath(OptionValues options) {
 304         return StubOptions.ForceUseOfNewInstanceStub.getValue(options);
 305     }
 306 
 307     public static final ForeignCallDescriptor NEW_INSTANCE_C = newDescriptor(NewInstanceStub.class, "newInstanceC", void.class, Word.class, KlassPointer.class);
 308 
 309     @NodeIntrinsic(StubForeignCallNode.class)
 310     public static native void newInstanceC(@ConstantNodeParameter ForeignCallDescriptor newInstanceC, Word thread, KlassPointer hub);
 311 }
--- EOF ---