1 /*
   2  * Copyright (c) 2012, 2015, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  */
  23 package org.graalvm.compiler.hotspot.stubs;
  24 
  25 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFIG;
  26 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.HEAP_END_LOCATION;
  27 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.HEAP_TOP_LOCATION;
  28 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.PROTOTYPE_MARK_WORD_LOCATION;
  29 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_FAST_REFILL_WASTE_LOCATION;
  30 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_NOF_REFILLS_LOCATION;
  31 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_REFILL_WASTE_LIMIT_LOCATION;
  32 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_SIZE_LOCATION;
  33 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_SLOW_ALLOCATIONS_LOCATION;
  34 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_THREAD_ALLOCATED_BYTES_LOCATION;
  35 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayBaseOffset;
  36 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.getAndClearObjectResult;
  37 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.initializeTlab;
  38 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.isInstanceKlassFullyInitialized;
  39 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readLayoutHelper;
  40 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.log2WordSize;
  41 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.prototypeMarkWordOffset;
  42 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabEnd;
  43 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabStart;
  44 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabTop;
  45 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.registerAsWord;
  46 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.threadAllocatedBytesOffset;
  47 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.threadTlabSizeOffset;
  48 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabAlignmentReserveInHeapWords;
  49 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabFastRefillWasteOffset;
  50 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabIntArrayMarkWord;
  51 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabNumberOfRefillsOffset;
  52 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabRefillWasteIncrement;
  53 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabRefillWasteLimitOffset;
  54 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabSlowAllocationsOffset;
  55 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabStats;
  56 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useCMSIncrementalMode;
  57 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useG1GC;
  58 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useTLAB;
  59 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.wordSize;
  60 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.writeTlabTop;
  61 import static org.graalvm.compiler.hotspot.stubs.StubUtil.handlePendingException;
  62 import static org.graalvm.compiler.hotspot.stubs.StubUtil.newDescriptor;
  63 import static org.graalvm.compiler.hotspot.stubs.StubUtil.printf;
  64 import static org.graalvm.compiler.hotspot.stubs.StubUtil.verifyObject;
  65 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FAST_PATH_PROBABILITY;
  66 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;
  67 
  68 import org.graalvm.compiler.api.replacements.Fold;
  69 import org.graalvm.compiler.api.replacements.Snippet;
  70 import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter;
  71 import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor;
  72 import org.graalvm.compiler.graph.Node.ConstantNodeParameter;
  73 import org.graalvm.compiler.graph.Node.NodeIntrinsic;
  74 import org.graalvm.compiler.hotspot.HotSpotForeignCallLinkage;
  75 import org.graalvm.compiler.hotspot.meta.HotSpotProviders;
  76 import org.graalvm.compiler.hotspot.nodes.GraalHotSpotVMConfigNode;
  77 import org.graalvm.compiler.hotspot.nodes.StubForeignCallNode;
  78 import org.graalvm.compiler.hotspot.nodes.type.KlassPointerStamp;
  79 import org.graalvm.compiler.hotspot.replacements.NewObjectSnippets;
  80 import org.graalvm.compiler.hotspot.word.KlassPointer;
  81 import org.graalvm.compiler.nodes.ConstantNode;
  82 import org.graalvm.compiler.options.OptionValues;
  83 import org.graalvm.compiler.word.Word;
  84 import org.graalvm.word.WordFactory;
  85 
  86 import jdk.vm.ci.code.Register;
  87 import jdk.vm.ci.hotspot.HotSpotResolvedObjectType;
  88 import jdk.vm.ci.meta.JavaKind;
  89 
  90 /**
  91  * Stub implementing the fast path for TLAB refill during instance class allocation. This stub is
  92  * called from the {@linkplain NewObjectSnippets inline} allocation code when TLAB allocation fails.
  93  * If this stub fails to refill the TLAB or allocate the object, it calls out to the HotSpot C++
  94  * runtime for to complete the allocation.
  95  */
  96 public class NewInstanceStub extends SnippetStub {
  97 
  98     public NewInstanceStub(OptionValues options, HotSpotProviders providers, HotSpotForeignCallLinkage linkage) {
  99         super("newInstance", options, providers, linkage);
 100     }
 101 
 102     @Override
 103     protected Object[] makeConstArgs() {
 104         HotSpotResolvedObjectType intArrayType = (HotSpotResolvedObjectType) providers.getMetaAccess().lookupJavaType(int[].class);
 105         int count = method.getSignature().getParameterCount(false);
 106         Object[] args = new Object[count];
 107         assert checkConstArg(1, "intArrayHub");
 108         assert checkConstArg(2, "threadRegister");
 109         assert checkConstArg(3, "options");
 110         args[1] = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), intArrayType.klass(), null);
 111         args[2] = providers.getRegisters().getThreadRegister();
 112         args[3] = options;
 113         return args;
 114     }
 115 
 116     private static Word allocate(Word thread, int size) {
 117         Word top = readTlabTop(thread);
 118         Word end = readTlabEnd(thread);
 119         Word newTop = top.add(size);
 120         /*
 121          * this check might lead to problems if the TLAB is within 16GB of the address space end
 122          * (checked in c++ code)
 123          */
 124         if (probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
 125             writeTlabTop(thread, newTop);
 126             return top;
 127         }
 128         return WordFactory.zero();
 129     }
 130 
 131     @Fold
 132     static boolean logging(OptionValues options) {
 133         return StubOptions.TraceNewInstanceStub.getValue(options);
 134     }
 135 
 136     /**
 137      * Re-attempts allocation after an initial TLAB allocation failed or was skipped (e.g., due to
 138      * -XX:-UseTLAB).
 139      *
 140      * @param hub the hub of the object to be allocated
 141      * @param intArrayHub the hub for {@code int[].class}
 142      */
 143     @Snippet
 144     private static Object newInstance(KlassPointer hub, @ConstantParameter KlassPointer intArrayHub, @ConstantParameter Register threadRegister, @ConstantParameter OptionValues options) {
 145         /*
 146          * The type is known to be an instance so Klass::_layout_helper is the instance size as a
 147          * raw number
 148          */
 149         Word thread = registerAsWord(threadRegister);
 150         boolean inlineContiguousAllocationSupported = GraalHotSpotVMConfigNode.inlineContiguousAllocationSupported();
 151         if (!forceSlowPath(options) && inlineContiguousAllocationSupported && !useCMSIncrementalMode(INJECTED_VMCONFIG)) {
 152             if (isInstanceKlassFullyInitialized(hub)) {
 153                 int sizeInBytes = readLayoutHelper(hub);
 154                 Word memory = refillAllocate(thread, intArrayHub, sizeInBytes, logging(options));
 155                 if (memory.notEqual(0)) {
 156                     Word prototypeMarkWord = hub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION);
 157                     NewObjectSnippets.formatObjectForStub(hub, sizeInBytes, memory, prototypeMarkWord);
 158                     return verifyObject(memory.toObject());
 159                 }
 160             }
 161         }
 162 
 163         if (logging(options)) {
 164             printf("newInstance: calling new_instance_c\n");
 165         }
 166 
 167         newInstanceC(NEW_INSTANCE_C, thread, hub);
 168         handlePendingException(thread, true);
 169         return verifyObject(getAndClearObjectResult(thread));
 170     }
 171 
 172     /**
 173      * Attempts to refill the current thread's TLAB and retries the allocation.
 174      *
 175      * @param intArrayHub the hub for {@code int[].class}
 176      * @param sizeInBytes the size of the allocation
 177      * @param log specifies if logging is enabled
 178      *
 179      * @return the newly allocated, uninitialized chunk of memory, or {@link WordFactory#zero()} if
 180      *         the operation was unsuccessful
 181      */
 182     static Word refillAllocate(Word thread, KlassPointer intArrayHub, int sizeInBytes, boolean log) {
 183         // If G1 is enabled, the "eden" allocation space is not the same always
 184         // and therefore we have to go to slowpath to allocate a new TLAB.
 185         if (useG1GC(INJECTED_VMCONFIG)) {
 186             return WordFactory.zero();
 187         }
 188         if (!useTLAB(INJECTED_VMCONFIG)) {
 189             return edenAllocate(WordFactory.unsigned(sizeInBytes), log);
 190         }
 191         Word intArrayMarkWord = WordFactory.unsigned(tlabIntArrayMarkWord(INJECTED_VMCONFIG));
 192         int alignmentReserveInBytes = tlabAlignmentReserveInHeapWords(INJECTED_VMCONFIG) * wordSize();
 193 
 194         Word top = readTlabTop(thread);
 195         Word end = readTlabEnd(thread);
 196 
 197         // calculate amount of free space
 198         long tlabFreeSpaceInBytes = end.subtract(top).rawValue();
 199 
 200         if (log) {
 201             printf("refillTLAB: thread=%p\n", thread.rawValue());
 202             printf("refillTLAB: top=%p\n", top.rawValue());
 203             printf("refillTLAB: end=%p\n", end.rawValue());
 204             printf("refillTLAB: tlabFreeSpaceInBytes=%ld\n", tlabFreeSpaceInBytes);
 205         }
 206 
 207         long tlabFreeSpaceInWords = tlabFreeSpaceInBytes >>> log2WordSize();
 208 
 209         // Retain TLAB and allocate object in shared space if
 210         // the amount free in the TLAB is too large to discard.
 211         Word refillWasteLimit = thread.readWord(tlabRefillWasteLimitOffset(INJECTED_VMCONFIG), TLAB_REFILL_WASTE_LIMIT_LOCATION);
 212         if (tlabFreeSpaceInWords <= refillWasteLimit.rawValue()) {
 213             if (tlabStats(INJECTED_VMCONFIG)) {
 214                 // increment number of refills
 215                 thread.writeInt(tlabNumberOfRefillsOffset(INJECTED_VMCONFIG), thread.readInt(tlabNumberOfRefillsOffset(INJECTED_VMCONFIG), TLAB_NOF_REFILLS_LOCATION) + 1, TLAB_NOF_REFILLS_LOCATION);
 216                 if (log) {
 217                     printf("thread: %p -- number_of_refills %d\n", thread.rawValue(), thread.readInt(tlabNumberOfRefillsOffset(INJECTED_VMCONFIG), TLAB_NOF_REFILLS_LOCATION));
 218                 }
 219                 // accumulate wastage
 220                 int wastage = thread.readInt(tlabFastRefillWasteOffset(INJECTED_VMCONFIG), TLAB_FAST_REFILL_WASTE_LOCATION) + (int) tlabFreeSpaceInWords;
 221                 if (log) {
 222                     printf("thread: %p -- accumulated wastage %d\n", thread.rawValue(), wastage);
 223                 }
 224                 thread.writeInt(tlabFastRefillWasteOffset(INJECTED_VMCONFIG), wastage, TLAB_FAST_REFILL_WASTE_LOCATION);
 225             }
 226 
 227             // if TLAB is currently allocated (top or end != null) then
 228             // fill [top, end + alignment_reserve) with array object
 229             if (top.notEqual(0)) {
 230                 int headerSize = arrayBaseOffset(JavaKind.Int);
 231                 // just like the HotSpot assembler stubs, assumes that tlabFreeSpaceInInts fits in
 232                 // an int
 233                 int tlabFreeSpaceInInts = (int) tlabFreeSpaceInBytes >>> 2;
 234                 int length = ((alignmentReserveInBytes - headerSize) >>> 2) + tlabFreeSpaceInInts;
 235                 NewObjectSnippets.formatArray(intArrayHub, 0, length, headerSize, top, intArrayMarkWord, false, false, null);
 236 
 237                 long allocated = thread.readLong(threadAllocatedBytesOffset(INJECTED_VMCONFIG), TLAB_THREAD_ALLOCATED_BYTES_LOCATION);
 238                 allocated = allocated + top.subtract(readTlabStart(thread)).rawValue();
 239                 thread.writeLong(threadAllocatedBytesOffset(INJECTED_VMCONFIG), allocated, TLAB_THREAD_ALLOCATED_BYTES_LOCATION);
 240             }
 241 
 242             // refill the TLAB with an eden allocation
 243             Word tlabRefillSizeInWords = thread.readWord(threadTlabSizeOffset(INJECTED_VMCONFIG), TLAB_SIZE_LOCATION);
 244             Word tlabRefillSizeInBytes = tlabRefillSizeInWords.multiply(wordSize());
 245             // allocate new TLAB, address returned in top
 246             top = edenAllocate(tlabRefillSizeInBytes, log);
 247             if (top.notEqual(0)) {
 248                 end = top.add(tlabRefillSizeInBytes.subtract(alignmentReserveInBytes));
 249                 initializeTlab(thread, top, end);
 250 
 251                 return NewInstanceStub.allocate(thread, sizeInBytes);
 252             } else {
 253                 return WordFactory.zero();
 254             }
 255         } else {
 256             // Retain TLAB
 257             Word newRefillWasteLimit = refillWasteLimit.add(tlabRefillWasteIncrement(INJECTED_VMCONFIG));
 258             thread.writeWord(tlabRefillWasteLimitOffset(INJECTED_VMCONFIG), newRefillWasteLimit, TLAB_REFILL_WASTE_LIMIT_LOCATION);
 259             if (log) {
 260                 printf("refillTLAB: retaining TLAB - newRefillWasteLimit=%p\n", newRefillWasteLimit.rawValue());
 261             }
 262 
 263             if (tlabStats(INJECTED_VMCONFIG)) {
 264                 thread.writeInt(tlabSlowAllocationsOffset(INJECTED_VMCONFIG), thread.readInt(tlabSlowAllocationsOffset(INJECTED_VMCONFIG), TLAB_SLOW_ALLOCATIONS_LOCATION) + 1,
 265                                 TLAB_SLOW_ALLOCATIONS_LOCATION);
 266             }
 267 
 268             return edenAllocate(WordFactory.unsigned(sizeInBytes), log);
 269         }
 270     }
 271 
 272     /**
 273      * Attempts to allocate a chunk of memory from Eden space.
 274      *
 275      * @param sizeInBytes the size of the chunk to allocate
 276      * @param log specifies if logging is enabled
 277      * @return the allocated chunk or {@link WordFactory#zero()} if allocation fails
 278      */
 279     public static Word edenAllocate(Word sizeInBytes, boolean log) {
 280         final long heapTopRawAddress = GraalHotSpotVMConfigNode.heapTopAddress();
 281         final long heapEndRawAddress = GraalHotSpotVMConfigNode.heapEndAddress();
 282 
 283         Word heapTopAddress = WordFactory.unsigned(heapTopRawAddress);
 284         Word heapEndAddress = WordFactory.unsigned(heapEndRawAddress);
 285 
 286         while (true) {
 287             Word heapTop = heapTopAddress.readWord(0, HEAP_TOP_LOCATION);
 288             Word newHeapTop = heapTop.add(sizeInBytes);
 289             if (newHeapTop.belowOrEqual(heapTop)) {
 290                 return WordFactory.zero();
 291             }
 292 
 293             Word heapEnd = heapEndAddress.readWord(0, HEAP_END_LOCATION);
 294             if (newHeapTop.aboveThan(heapEnd)) {
 295                 return WordFactory.zero();
 296             }
 297             if (heapTopAddress.logicCompareAndSwapWord(0, heapTop, newHeapTop, HEAP_TOP_LOCATION)) {
 298                 return heapTop;
 299             }
 300         }
 301     }
 302 
 303     @Fold
 304     static boolean forceSlowPath(OptionValues options) {
 305         return StubOptions.ForceUseOfNewInstanceStub.getValue(options);
 306     }
 307 
 308     public static final ForeignCallDescriptor NEW_INSTANCE_C = newDescriptor(NewInstanceStub.class, "newInstanceC", void.class, Word.class, KlassPointer.class);
 309 
 310     @NodeIntrinsic(StubForeignCallNode.class)
 311     public static native void newInstanceC(@ConstantNodeParameter ForeignCallDescriptor newInstanceC, Word thread, KlassPointer hub);
 312 }