1 /*
   2  * Copyright (c) 2012, 2018, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  */
  23 
  24 
  25 package org.graalvm.compiler.hotspot.replacements;
  26 
  27 import static jdk.vm.ci.code.MemoryBarriers.STORE_LOAD;
  28 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFIG;
  29 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfigBase.INJECTED_METAACCESS;
  30 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.cardTableShift;
  31 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.dirtyCardValue;
  32 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.g1CardQueueBufferOffset;
  33 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.g1CardQueueIndexOffset;
  34 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.g1SATBQueueBufferOffset;
  35 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.g1SATBQueueIndexOffset;
  36 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.g1SATBQueueMarkingOffset;
  37 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.g1YoungCardValue;
  38 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.registerAsWord;
  39 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.verifyOop;
  40 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.verifyOops;
  41 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.wordSize;
  42 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FREQUENT_PROBABILITY;
  43 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.LIKELY_PROBABILITY;
  44 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.NOT_FREQUENT_PROBABILITY;
  45 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;
  46 import static org.graalvm.compiler.replacements.SnippetTemplate.DEFAULT_REPLACER;
  47 
  48 import org.graalvm.compiler.api.replacements.Snippet;
  49 import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter;
  50 import org.graalvm.compiler.core.common.CompressEncoding;
  51 import org.graalvm.compiler.core.common.GraalOptions;
  52 import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor;
  53 import org.graalvm.compiler.debug.DebugHandlersFactory;
  54 import org.graalvm.compiler.graph.Node.ConstantNodeParameter;
  55 import org.graalvm.compiler.graph.Node.NodeIntrinsic;
  56 import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig;
  57 import org.graalvm.compiler.hotspot.gc.g1.G1ArrayRangePostWriteBarrier;
  58 import org.graalvm.compiler.hotspot.gc.g1.G1ArrayRangePreWriteBarrier;
  59 import org.graalvm.compiler.hotspot.gc.g1.G1PostWriteBarrier;
  60 import org.graalvm.compiler.hotspot.gc.g1.G1PreWriteBarrier;
  61 import org.graalvm.compiler.hotspot.gc.g1.G1ReferentFieldReadBarrier;
  62 import org.graalvm.compiler.hotspot.gc.shared.SerialArrayRangeWriteBarrier;
  63 import org.graalvm.compiler.hotspot.gc.shared.SerialWriteBarrier;
  64 import org.graalvm.compiler.hotspot.meta.HotSpotProviders;
  65 import org.graalvm.compiler.hotspot.meta.HotSpotRegistersProvider;





  66 import org.graalvm.compiler.hotspot.nodes.GraalHotSpotVMConfigNode;
  67 import org.graalvm.compiler.hotspot.nodes.HotSpotCompressionNode;


  68 import org.graalvm.compiler.hotspot.nodes.VMErrorNode;
  69 import org.graalvm.compiler.nodes.NamedLocationIdentity;
  70 import org.graalvm.compiler.nodes.NodeView;
  71 import org.graalvm.compiler.nodes.PiNode;
  72 import org.graalvm.compiler.nodes.SnippetAnchorNode;
  73 import org.graalvm.compiler.nodes.StructuredGraph;
  74 import org.graalvm.compiler.nodes.ValueNode;
  75 import org.graalvm.compiler.nodes.extended.FixedValueAnchorNode;
  76 import org.graalvm.compiler.nodes.extended.ForeignCallNode;
  77 import org.graalvm.compiler.nodes.extended.MembarNode;
  78 import org.graalvm.compiler.nodes.extended.NullCheckNode;
  79 import org.graalvm.compiler.nodes.memory.HeapAccess.BarrierType;
  80 import org.graalvm.compiler.nodes.memory.address.AddressNode;
  81 import org.graalvm.compiler.nodes.memory.address.AddressNode.Address;
  82 import org.graalvm.compiler.nodes.memory.address.OffsetAddressNode;
  83 import org.graalvm.compiler.nodes.spi.LoweringTool;
  84 import org.graalvm.compiler.nodes.type.NarrowOopStamp;
  85 import org.graalvm.compiler.options.OptionValues;
  86 import org.graalvm.compiler.replacements.Log;
  87 import org.graalvm.compiler.replacements.ReplacementsUtil;
  88 import org.graalvm.compiler.replacements.SnippetCounter;
  89 import org.graalvm.compiler.replacements.SnippetCounter.Group;
  90 import org.graalvm.compiler.replacements.SnippetTemplate.AbstractTemplates;
  91 import org.graalvm.compiler.replacements.SnippetTemplate.Arguments;
  92 import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo;
  93 import org.graalvm.compiler.replacements.Snippets;
  94 import org.graalvm.compiler.replacements.nodes.AssertionNode;
  95 import org.graalvm.compiler.replacements.nodes.DirectStoreNode;
  96 import org.graalvm.compiler.word.Word;
  97 import jdk.internal.vm.compiler.word.LocationIdentity;
  98 import jdk.internal.vm.compiler.word.Pointer;
  99 import jdk.internal.vm.compiler.word.UnsignedWord;
 100 import jdk.internal.vm.compiler.word.WordFactory;
 101 
 102 import jdk.vm.ci.code.Register;
 103 import jdk.vm.ci.code.TargetDescription;
 104 import jdk.vm.ci.meta.JavaKind;
 105 
 106 public class WriteBarrierSnippets implements Snippets {
 107 
 108     static class Counters {
 109         Counters(SnippetCounter.Group.Factory factory) {
 110             Group countersWriteBarriers = factory.createSnippetCounterGroup("WriteBarriers");
 111             serialWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "serialWriteBarrier", "Number of Serial Write Barriers");
 112             g1AttemptedPreWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "g1AttemptedPreWriteBarrier", "Number of attempted G1 Pre Write Barriers");
 113             g1EffectivePreWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "g1EffectivePreWriteBarrier", "Number of effective G1 Pre Write Barriers");
 114             g1ExecutedPreWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "g1ExecutedPreWriteBarrier", "Number of executed G1 Pre Write Barriers");
 115             g1AttemptedPostWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "g1AttemptedPostWriteBarrier", "Number of attempted G1 Post Write Barriers");
 116             g1EffectiveAfterXORPostWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "g1EffectiveAfterXORPostWriteBarrier",
 117                             "Number of effective G1 Post Write Barriers (after passing the XOR test)");
 118             g1EffectiveAfterNullPostWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "g1EffectiveAfterNullPostWriteBarrier",
 119                             "Number of effective G1 Post Write Barriers (after passing the NULL test)");
 120             g1ExecutedPostWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "g1ExecutedPostWriteBarrier", "Number of executed G1 Post Write Barriers");
 121 
 122         }
 123 
 124         final SnippetCounter serialWriteBarrierCounter;
 125         final SnippetCounter g1AttemptedPreWriteBarrierCounter;
 126         final SnippetCounter g1EffectivePreWriteBarrierCounter;
 127         final SnippetCounter g1ExecutedPreWriteBarrierCounter;
 128         final SnippetCounter g1AttemptedPostWriteBarrierCounter;
 129         final SnippetCounter g1EffectiveAfterXORPostWriteBarrierCounter;
 130         final SnippetCounter g1EffectiveAfterNullPostWriteBarrierCounter;
 131         final SnippetCounter g1ExecutedPostWriteBarrierCounter;
 132     }
 133 
 134     public static final LocationIdentity GC_CARD_LOCATION = NamedLocationIdentity.mutable("GC-Card");
 135     public static final LocationIdentity GC_LOG_LOCATION = NamedLocationIdentity.mutable("GC-Log");
 136     public static final LocationIdentity GC_INDEX_LOCATION = NamedLocationIdentity.mutable("GC-Index");
 137 
 138     private static void serialWriteBarrier(Pointer ptr, Counters counters) {
 139         counters.serialWriteBarrierCounter.inc();
 140         final long startAddress = GraalHotSpotVMConfigNode.cardTableAddress();
 141         Word base = (Word) ptr.unsignedShiftRight(cardTableShift(INJECTED_VMCONFIG));
 142         if (((int) startAddress) == startAddress && GraalHotSpotVMConfigNode.isCardTableAddressConstant()) {
 143             base.writeByte((int) startAddress, (byte) 0, GC_CARD_LOCATION);
 144         } else {
 145             base.writeByte(WordFactory.unsigned(startAddress), (byte) 0, GC_CARD_LOCATION);
 146         }
 147     }
 148 
 149     @Snippet
 150     public static void serialImpreciseWriteBarrier(Object object, @ConstantParameter boolean verifyBarrier, @ConstantParameter Counters counters) {
 151         if (verifyBarrier) {
 152             verifyNotArray(object);
 153         }
 154         serialWriteBarrier(Word.objectToTrackedPointer(object), counters);
 155     }
 156 
 157     @Snippet
 158     public static void serialPreciseWriteBarrier(Address address, @ConstantParameter Counters counters) {
 159         serialWriteBarrier(Word.fromAddress(address), counters);
 160     }
 161 
 162     @Snippet
 163     public static void serialArrayRangeWriteBarrier(Address address, int length, @ConstantParameter int elementStride) {
 164         if (length == 0) {
 165             return;
 166         }
 167         int cardShift = cardTableShift(INJECTED_VMCONFIG);
 168         final long cardStart = GraalHotSpotVMConfigNode.cardTableAddress();
 169         long start = getPointerToFirstArrayElement(address, length, elementStride) >>> cardShift;
 170         long end = getPointerToLastArrayElement(address, length, elementStride) >>> cardShift;
 171         long count = end - start + 1;
 172         while (count-- > 0) {
 173             DirectStoreNode.storeBoolean((start + cardStart) + count, false, JavaKind.Boolean);
 174         }
 175     }
 176 
 177     @Snippet
 178     public static void g1PreWriteBarrier(Address address, Object object, Object expectedObject, @ConstantParameter boolean doLoad, @ConstantParameter boolean nullCheck,
 179                     @ConstantParameter Register threadRegister, @ConstantParameter boolean trace, @ConstantParameter Counters counters) {
 180         if (nullCheck) {
 181             NullCheckNode.nullCheck(address);
 182         }
 183         Word thread = registerAsWord(threadRegister);
 184         verifyOop(object);
 185         Object fixedExpectedObject = FixedValueAnchorNode.getObject(expectedObject);
 186         Word field = Word.fromAddress(address);
 187         Pointer previousOop = Word.objectToTrackedPointer(fixedExpectedObject);
 188         byte markingValue = thread.readByte(g1SATBQueueMarkingOffset(INJECTED_VMCONFIG));
 189         int gcCycle = 0;
 190         if (trace) {
 191             Pointer gcTotalCollectionsAddress = WordFactory.pointer(HotSpotReplacementsUtil.gcTotalCollectionsAddress(INJECTED_VMCONFIG));
 192             gcCycle = (int) gcTotalCollectionsAddress.readLong(0);
 193             log(trace, "[%d] G1-Pre Thread %p Object %p\n", gcCycle, thread.rawValue(), Word.objectToTrackedPointer(object).rawValue());
 194             log(trace, "[%d] G1-Pre Thread %p Expected Object %p\n", gcCycle, thread.rawValue(), Word.objectToTrackedPointer(fixedExpectedObject).rawValue());
 195             log(trace, "[%d] G1-Pre Thread %p Field %p\n", gcCycle, thread.rawValue(), field.rawValue());
 196             log(trace, "[%d] G1-Pre Thread %p Marking %d\n", gcCycle, thread.rawValue(), markingValue);
 197             log(trace, "[%d] G1-Pre Thread %p DoLoad %d\n", gcCycle, thread.rawValue(), doLoad ? 1L : 0L);
 198         }
 199         counters.g1AttemptedPreWriteBarrierCounter.inc();
 200         // If the concurrent marker is enabled, the barrier is issued.
 201         if (probability(NOT_FREQUENT_PROBABILITY, markingValue != (byte) 0)) {
 202             // If the previous value has to be loaded (before the write), the load is issued.
 203             // The load is always issued except the cases of CAS and referent field.
 204             if (probability(LIKELY_PROBABILITY, doLoad)) {
 205                 previousOop = Word.objectToTrackedPointer(field.readObject(0, BarrierType.NONE));
 206                 if (trace) {
 207                     log(trace, "[%d] G1-Pre Thread %p Previous Object %p\n ", gcCycle, thread.rawValue(), previousOop.rawValue());
 208                     verifyOop(previousOop.toObject());
 209                 }
 210             }
 211             counters.g1EffectivePreWriteBarrierCounter.inc();
 212             // If the previous value is null the barrier should not be issued.
 213             if (probability(FREQUENT_PROBABILITY, previousOop.notEqual(0))) {
 214                 counters.g1ExecutedPreWriteBarrierCounter.inc();
 215                 // If the thread-local SATB buffer is full issue a native call which will
 216                 // initialize a new one and add the entry.
 217                 Word indexAddress = thread.add(g1SATBQueueIndexOffset(INJECTED_VMCONFIG));
 218                 Word indexValue = indexAddress.readWord(0);
 219                 if (probability(FREQUENT_PROBABILITY, indexValue.notEqual(0))) {
 220                     Word bufferAddress = thread.readWord(g1SATBQueueBufferOffset(INJECTED_VMCONFIG));
 221                     Word nextIndex = indexValue.subtract(wordSize());
 222                     Word logAddress = bufferAddress.add(nextIndex);
 223                     // Log the object to be marked as well as update the SATB's buffer next index.
 224                     logAddress.writeWord(0, previousOop, GC_LOG_LOCATION);
 225                     indexAddress.writeWord(0, nextIndex, GC_INDEX_LOCATION);
 226                 } else {
 227                     g1PreBarrierStub(G1WBPRECALL, previousOop.toObject());
 228                 }
 229             }
 230         }
 231     }
 232 
 233     @Snippet
 234     public static void g1PostWriteBarrier(Address address, Object object, Object value, @ConstantParameter boolean usePrecise, @ConstantParameter boolean verifyBarrier,
 235                     @ConstantParameter Register threadRegister, @ConstantParameter boolean trace, @ConstantParameter Counters counters) {
 236         Word thread = registerAsWord(threadRegister);
 237         Object fixedValue = FixedValueAnchorNode.getObject(value);
 238         verifyOop(object);
 239         verifyOop(fixedValue);
 240         validateObject(object, fixedValue);
 241         Pointer oop;
 242         if (usePrecise) {
 243             oop = Word.fromAddress(address);
 244         } else {
 245             if (verifyBarrier) {
 246                 verifyNotArray(object);
 247             }
 248             oop = Word.objectToTrackedPointer(object);
 249         }
 250         int gcCycle = 0;
 251         if (trace) {
 252             Pointer gcTotalCollectionsAddress = WordFactory.pointer(HotSpotReplacementsUtil.gcTotalCollectionsAddress(INJECTED_VMCONFIG));
 253             gcCycle = (int) gcTotalCollectionsAddress.readLong(0);
 254             log(trace, "[%d] G1-Post Thread: %p Object: %p\n", gcCycle, thread.rawValue(), Word.objectToTrackedPointer(object).rawValue());
 255             log(trace, "[%d] G1-Post Thread: %p Field: %p\n", gcCycle, thread.rawValue(), oop.rawValue());
 256         }
 257         Pointer writtenValue = Word.objectToTrackedPointer(fixedValue);
 258         // The result of the xor reveals whether the installed pointer crosses heap regions.
 259         // In case it does the write barrier has to be issued.
 260         final int logOfHeapRegionGrainBytes = GraalHotSpotVMConfigNode.logOfHeapRegionGrainBytes();
 261         UnsignedWord xorResult = (oop.xor(writtenValue)).unsignedShiftRight(logOfHeapRegionGrainBytes);
 262 
 263         // Calculate the address of the card to be enqueued to the
 264         // thread local card queue.
 265         UnsignedWord cardBase = oop.unsignedShiftRight(cardTableShift(INJECTED_VMCONFIG));
 266         final long startAddress = GraalHotSpotVMConfigNode.cardTableAddress();
 267         int displacement = 0;
 268         if (((int) startAddress) == startAddress && GraalHotSpotVMConfigNode.isCardTableAddressConstant()) {
 269             displacement = (int) startAddress;
 270         } else {
 271             cardBase = cardBase.add(WordFactory.unsigned(startAddress));
 272         }
 273         Word cardAddress = (Word) cardBase.add(displacement);
 274 
 275         counters.g1AttemptedPostWriteBarrierCounter.inc();
 276         if (probability(FREQUENT_PROBABILITY, xorResult.notEqual(0))) {
 277             counters.g1EffectiveAfterXORPostWriteBarrierCounter.inc();
 278 
 279             // If the written value is not null continue with the barrier addition.
 280             if (probability(FREQUENT_PROBABILITY, writtenValue.notEqual(0))) {
 281                 byte cardByte = cardAddress.readByte(0, GC_CARD_LOCATION);
 282                 counters.g1EffectiveAfterNullPostWriteBarrierCounter.inc();
 283 
 284                 // If the card is already dirty, (hence already enqueued) skip the insertion.
 285                 if (probability(NOT_FREQUENT_PROBABILITY, cardByte != g1YoungCardValue(INJECTED_VMCONFIG))) {
 286                     MembarNode.memoryBarrier(STORE_LOAD, GC_CARD_LOCATION);
 287                     byte cardByteReload = cardAddress.readByte(0, GC_CARD_LOCATION);
 288                     if (probability(NOT_FREQUENT_PROBABILITY, cardByteReload != dirtyCardValue(INJECTED_VMCONFIG))) {
 289                         log(trace, "[%d] G1-Post Thread: %p Card: %p \n", gcCycle, thread.rawValue(), WordFactory.unsigned((int) cardByte).rawValue());
 290                         cardAddress.writeByte(0, (byte) 0, GC_CARD_LOCATION);
 291                         counters.g1ExecutedPostWriteBarrierCounter.inc();
 292 
 293                         // If the thread local card queue is full, issue a native call which will
 294                         // initialize a new one and add the card entry.
 295                         Word indexAddress = thread.add(g1CardQueueIndexOffset(INJECTED_VMCONFIG));
 296                         Word indexValue = thread.readWord(g1CardQueueIndexOffset(INJECTED_VMCONFIG));
 297                         if (probability(FREQUENT_PROBABILITY, indexValue.notEqual(0))) {
 298                             Word bufferAddress = thread.readWord(g1CardQueueBufferOffset(INJECTED_VMCONFIG));
 299                             Word nextIndex = indexValue.subtract(wordSize());
 300                             Word logAddress = bufferAddress.add(nextIndex);
 301                             // Log the object to be scanned as well as update
 302                             // the card queue's next index.
 303                             logAddress.writeWord(0, cardAddress, GC_LOG_LOCATION);
 304                             indexAddress.writeWord(0, nextIndex, GC_INDEX_LOCATION);
 305                         } else {
 306                             g1PostBarrierStub(G1WBPOSTCALL, cardAddress);
 307                         }
 308                     }
 309                 }
 310             }
 311         }
 312     }
 313 
 314     private static void verifyNotArray(Object object) {
 315         if (object != null) {
 316             // Manually build the null check and cast because we're in snippet that's lowered late.
 317             AssertionNode.assertion(false, !PiNode.piCastNonNull(object, SnippetAnchorNode.anchor()).getClass().isArray(), "imprecise card mark used with array");
 318         }
 319     }
 320 
 321     @Snippet
 322     public static void g1ArrayRangePreWriteBarrier(Address address, int length, @ConstantParameter int elementStride, @ConstantParameter Register threadRegister) {
 323         Word thread = registerAsWord(threadRegister);
 324         byte markingValue = thread.readByte(g1SATBQueueMarkingOffset(INJECTED_VMCONFIG));
 325         // If the concurrent marker is not enabled or the vector length is zero, return.
 326         if (markingValue == (byte) 0 || length == 0) {
 327             return;
 328         }
 329         Word bufferAddress = thread.readWord(g1SATBQueueBufferOffset(INJECTED_VMCONFIG));
 330         Word indexAddress = thread.add(g1SATBQueueIndexOffset(INJECTED_VMCONFIG));
 331         long indexValue = indexAddress.readWord(0).rawValue();
 332         final int scale = ReplacementsUtil.arrayIndexScale(INJECTED_METAACCESS, JavaKind.Object);
 333         long start = getPointerToFirstArrayElement(address, length, elementStride);
 334 
 335         for (int i = 0; i < length; i++) {
 336             Word arrElemPtr = WordFactory.pointer(start + i * scale);
 337             Pointer oop = Word.objectToTrackedPointer(arrElemPtr.readObject(0, BarrierType.NONE));
 338             verifyOop(oop.toObject());
 339             if (oop.notEqual(0)) {
 340                 if (indexValue != 0) {
 341                     indexValue = indexValue - wordSize();
 342                     Word logAddress = bufferAddress.add(WordFactory.unsigned(indexValue));
 343                     // Log the object to be marked as well as update the SATB's buffer next index.
 344                     logAddress.writeWord(0, oop, GC_LOG_LOCATION);
 345                     indexAddress.writeWord(0, WordFactory.unsigned(indexValue), GC_INDEX_LOCATION);
 346                 } else {
 347                     g1PreBarrierStub(G1WBPRECALL, oop.toObject());
 348                 }
 349             }
 350         }
 351     }
 352 
 353     @Snippet
 354     public static void g1ArrayRangePostWriteBarrier(Address address, int length, @ConstantParameter int elementStride, @ConstantParameter Register threadRegister) {
 355         if (length == 0) {
 356             return;
 357         }
 358         Word thread = registerAsWord(threadRegister);
 359         Word bufferAddress = thread.readWord(g1CardQueueBufferOffset(INJECTED_VMCONFIG));
 360         Word indexAddress = thread.add(g1CardQueueIndexOffset(INJECTED_VMCONFIG));
 361         long indexValue = thread.readWord(g1CardQueueIndexOffset(INJECTED_VMCONFIG)).rawValue();
 362 
 363         int cardShift = cardTableShift(INJECTED_VMCONFIG);
 364         final long cardStart = GraalHotSpotVMConfigNode.cardTableAddress();
 365         long start = getPointerToFirstArrayElement(address, length, elementStride) >>> cardShift;
 366         long end = getPointerToLastArrayElement(address, length, elementStride) >>> cardShift;
 367         long count = end - start + 1;
 368 
 369         while (count-- > 0) {
 370             Word cardAddress = WordFactory.unsigned((start + cardStart) + count);
 371             byte cardByte = cardAddress.readByte(0, GC_CARD_LOCATION);
 372             // If the card is already dirty, (hence already enqueued) skip the insertion.
 373             if (probability(NOT_FREQUENT_PROBABILITY, cardByte != g1YoungCardValue(INJECTED_VMCONFIG))) {
 374                 MembarNode.memoryBarrier(STORE_LOAD, GC_CARD_LOCATION);
 375                 byte cardByteReload = cardAddress.readByte(0, GC_CARD_LOCATION);
 376                 if (probability(NOT_FREQUENT_PROBABILITY, cardByteReload != dirtyCardValue(INJECTED_VMCONFIG))) {
 377                     cardAddress.writeByte(0, (byte) 0, GC_CARD_LOCATION);
 378                     // If the thread local card queue is full, issue a native call which will
 379                     // initialize a new one and add the card entry.
 380                     if (indexValue != 0) {
 381                         indexValue = indexValue - wordSize();
 382                         Word logAddress = bufferAddress.add(WordFactory.unsigned(indexValue));
 383                         // Log the object to be scanned as well as update
 384                         // the card queue's next index.
 385                         logAddress.writeWord(0, cardAddress, GC_LOG_LOCATION);
 386                         indexAddress.writeWord(0, WordFactory.unsigned(indexValue), GC_INDEX_LOCATION);
 387                     } else {
 388                         g1PostBarrierStub(G1WBPOSTCALL, cardAddress);
 389                     }
 390                 }
 391             }
 392         }
 393     }
 394 
 395     private static long getPointerToFirstArrayElement(Address address, int length, int elementStride) {
 396         long result = Word.fromAddress(address).rawValue();
 397         if (elementStride < 0) {
 398             // the address points to the place after the last array element
 399             result = result + elementStride * length;
 400         }
 401         return result;
 402     }
 403 
 404     private static long getPointerToLastArrayElement(Address address, int length, int elementStride) {
 405         long result = Word.fromAddress(address).rawValue();
 406         if (elementStride < 0) {
 407             // the address points to the place after the last array element
 408             result = result + elementStride;
 409         } else {
 410             result = result + (length - 1) * elementStride;
 411         }
 412         return result;
 413     }
 414 
 415     public static final ForeignCallDescriptor G1WBPRECALL = new ForeignCallDescriptor("write_barrier_pre", void.class, Object.class);
 416 
 417     @NodeIntrinsic(ForeignCallNode.class)
 418     private static native void g1PreBarrierStub(@ConstantNodeParameter ForeignCallDescriptor descriptor, Object object);
 419 
 420     public static final ForeignCallDescriptor G1WBPOSTCALL = new ForeignCallDescriptor("write_barrier_post", void.class, Word.class);
 421 
 422     @NodeIntrinsic(ForeignCallNode.class)
 423     public static native void g1PostBarrierStub(@ConstantNodeParameter ForeignCallDescriptor descriptor, Word card);
 424 
 425     public static class Templates extends AbstractTemplates {
 426 
 427         private final SnippetInfo serialImpreciseWriteBarrier = snippet(WriteBarrierSnippets.class, "serialImpreciseWriteBarrier", GC_CARD_LOCATION);
 428         private final SnippetInfo serialPreciseWriteBarrier = snippet(WriteBarrierSnippets.class, "serialPreciseWriteBarrier", GC_CARD_LOCATION);
 429         private final SnippetInfo serialArrayRangeWriteBarrier = snippet(WriteBarrierSnippets.class, "serialArrayRangeWriteBarrier");
 430         private final SnippetInfo g1PreWriteBarrier = snippet(WriteBarrierSnippets.class, "g1PreWriteBarrier", GC_INDEX_LOCATION, GC_LOG_LOCATION);
 431         private final SnippetInfo g1ReferentReadBarrier = g1PreWriteBarrier;
 432         private final SnippetInfo g1PostWriteBarrier = snippet(WriteBarrierSnippets.class, "g1PostWriteBarrier", GC_CARD_LOCATION, GC_INDEX_LOCATION, GC_LOG_LOCATION);
 433         private final SnippetInfo g1ArrayRangePreWriteBarrier = snippet(WriteBarrierSnippets.class, "g1ArrayRangePreWriteBarrier", GC_INDEX_LOCATION, GC_LOG_LOCATION);
 434         private final SnippetInfo g1ArrayRangePostWriteBarrier = snippet(WriteBarrierSnippets.class, "g1ArrayRangePostWriteBarrier", GC_CARD_LOCATION, GC_INDEX_LOCATION, GC_LOG_LOCATION);
 435 
 436         private final CompressEncoding oopEncoding;
 437         private final Counters counters;
 438         private final boolean verifyBarrier;
 439         private final long gcTotalCollectionsAddress;
 440 
 441         public Templates(OptionValues options, Iterable<DebugHandlersFactory> factories, Group.Factory factory, HotSpotProviders providers, TargetDescription target,
 442                         GraalHotSpotVMConfig config) {
 443             super(options, factories, providers, providers.getSnippetReflection(), target);
 444             this.oopEncoding = config.useCompressedOops ? config.getOopEncoding() : null;
 445             this.verifyBarrier = ReplacementsUtil.REPLACEMENTS_ASSERTIONS_ENABLED || config.verifyBeforeGC || config.verifyAfterGC;
 446             this.gcTotalCollectionsAddress = config.gcTotalCollectionsAddress();
 447             this.counters = new Counters(factory);
 448         }
 449 
 450         public boolean traceBarrier(StructuredGraph graph) {
 451             long startCycle = GraalOptions.GCDebugStartCycle.getValue(graph.getOptions());
 452             return startCycle > 0 && ((Pointer) WordFactory.pointer(gcTotalCollectionsAddress)).readLong(0) > startCycle;
 453         }
 454 
 455         public void lower(SerialWriteBarrier writeBarrier, LoweringTool tool) {
 456             Arguments args;
 457             if (writeBarrier.usePrecise()) {
 458                 args = new Arguments(serialPreciseWriteBarrier, writeBarrier.graph().getGuardsStage(), tool.getLoweringStage());
 459                 args.add("address", writeBarrier.getAddress());
 460             } else {
 461                 args = new Arguments(serialImpreciseWriteBarrier, writeBarrier.graph().getGuardsStage(), tool.getLoweringStage());
 462                 OffsetAddressNode address = (OffsetAddressNode) writeBarrier.getAddress();
 463                 args.add("object", address.getBase());
 464                 args.addConst("verifyBarrier", verifyBarrier);
 465             }
 466             args.addConst("counters", counters);
 467             template(writeBarrier, args).instantiate(providers.getMetaAccess(), writeBarrier, DEFAULT_REPLACER, args);
 468         }
 469 
 470         public void lower(SerialArrayRangeWriteBarrier arrayRangeWriteBarrier, LoweringTool tool) {
 471             Arguments args = new Arguments(serialArrayRangeWriteBarrier, arrayRangeWriteBarrier.graph().getGuardsStage(), tool.getLoweringStage());
 472             args.add("address", arrayRangeWriteBarrier.getAddress());
 473             args.add("length", arrayRangeWriteBarrier.getLength());
 474             args.addConst("elementStride", arrayRangeWriteBarrier.getElementStride());
 475             template(arrayRangeWriteBarrier, args).instantiate(providers.getMetaAccess(), arrayRangeWriteBarrier, DEFAULT_REPLACER, args);
 476         }
 477 
 478         public void lower(G1PreWriteBarrier writeBarrierPre, HotSpotRegistersProvider registers, LoweringTool tool) {
 479             Arguments args = new Arguments(g1PreWriteBarrier, writeBarrierPre.graph().getGuardsStage(), tool.getLoweringStage());
 480             AddressNode address = writeBarrierPre.getAddress();
 481             args.add("address", address);
 482             if (address instanceof OffsetAddressNode) {
 483                 args.add("object", ((OffsetAddressNode) address).getBase());
 484             } else {
 485                 args.add("object", null);
 486             }
 487 
 488             ValueNode expected = writeBarrierPre.getExpectedObject();
 489             if (expected != null && expected.stamp(NodeView.DEFAULT) instanceof NarrowOopStamp) {
 490                 assert oopEncoding != null;
 491                 expected = HotSpotCompressionNode.uncompress(expected, oopEncoding);
 492             }
 493             args.add("expectedObject", expected);
 494 
 495             args.addConst("doLoad", writeBarrierPre.doLoad());
 496             args.addConst("nullCheck", writeBarrierPre.getNullCheck());
 497             args.addConst("threadRegister", registers.getThreadRegister());
 498             args.addConst("trace", traceBarrier(writeBarrierPre.graph()));
 499             args.addConst("counters", counters);
 500             template(writeBarrierPre, args).instantiate(providers.getMetaAccess(), writeBarrierPre, DEFAULT_REPLACER, args);
 501         }
 502 
 503         public void lower(G1ReferentFieldReadBarrier readBarrier, HotSpotRegistersProvider registers, LoweringTool tool) {
 504             Arguments args = new Arguments(g1ReferentReadBarrier, readBarrier.graph().getGuardsStage(), tool.getLoweringStage());
 505             AddressNode address = readBarrier.getAddress();
 506             args.add("address", address);
 507             if (address instanceof OffsetAddressNode) {
 508                 args.add("object", ((OffsetAddressNode) address).getBase());
 509             } else {
 510                 args.add("object", null);
 511             }
 512 
 513             ValueNode expected = readBarrier.getExpectedObject();
 514             if (expected != null && expected.stamp(NodeView.DEFAULT) instanceof NarrowOopStamp) {
 515                 assert oopEncoding != null;
 516                 expected = HotSpotCompressionNode.uncompress(expected, oopEncoding);
 517             }
 518 
 519             args.add("expectedObject", expected);
 520             args.addConst("doLoad", readBarrier.doLoad());
 521             args.addConst("nullCheck", false);
 522             args.addConst("threadRegister", registers.getThreadRegister());
 523             args.addConst("trace", traceBarrier(readBarrier.graph()));
 524             args.addConst("counters", counters);
 525             template(readBarrier, args).instantiate(providers.getMetaAccess(), readBarrier, DEFAULT_REPLACER, args);
 526         }
 527 
 528         public void lower(G1PostWriteBarrier writeBarrierPost, HotSpotRegistersProvider registers, LoweringTool tool) {
 529             StructuredGraph graph = writeBarrierPost.graph();
 530             if (writeBarrierPost.alwaysNull()) {
 531                 graph.removeFixed(writeBarrierPost);
 532                 return;
 533             }
 534             Arguments args = new Arguments(g1PostWriteBarrier, graph.getGuardsStage(), tool.getLoweringStage());
 535             AddressNode address = writeBarrierPost.getAddress();
 536             args.add("address", address);
 537             if (address instanceof OffsetAddressNode) {
 538                 args.add("object", ((OffsetAddressNode) address).getBase());
 539             } else {
 540                 assert writeBarrierPost.usePrecise() : "found imprecise barrier that's not an object access " + writeBarrierPost;
 541                 args.add("object", null);
 542             }
 543 
 544             ValueNode value = writeBarrierPost.getValue();
 545             if (value.stamp(NodeView.DEFAULT) instanceof NarrowOopStamp) {
 546                 assert oopEncoding != null;
 547                 value = HotSpotCompressionNode.uncompress(value, oopEncoding);
 548             }
 549             args.add("value", value);
 550 
 551             args.addConst("usePrecise", writeBarrierPost.usePrecise());
 552             args.addConst("verifyBarrier", verifyBarrier);
 553             args.addConst("threadRegister", registers.getThreadRegister());
 554             args.addConst("trace", traceBarrier(writeBarrierPost.graph()));
 555             args.addConst("counters", counters);
 556             template(writeBarrierPost, args).instantiate(providers.getMetaAccess(), writeBarrierPost, DEFAULT_REPLACER, args);
 557         }
 558 
 559         public void lower(G1ArrayRangePreWriteBarrier arrayRangeWriteBarrier, HotSpotRegistersProvider registers, LoweringTool tool) {
 560             Arguments args = new Arguments(g1ArrayRangePreWriteBarrier, arrayRangeWriteBarrier.graph().getGuardsStage(), tool.getLoweringStage());
 561             args.add("address", arrayRangeWriteBarrier.getAddress());
 562             args.add("length", arrayRangeWriteBarrier.getLength());
 563             args.addConst("elementStride", arrayRangeWriteBarrier.getElementStride());
 564             args.addConst("threadRegister", registers.getThreadRegister());
 565             template(arrayRangeWriteBarrier, args).instantiate(providers.getMetaAccess(), arrayRangeWriteBarrier, DEFAULT_REPLACER, args);
 566         }
 567 
 568         public void lower(G1ArrayRangePostWriteBarrier arrayRangeWriteBarrier, HotSpotRegistersProvider registers, LoweringTool tool) {
 569             Arguments args = new Arguments(g1ArrayRangePostWriteBarrier, arrayRangeWriteBarrier.graph().getGuardsStage(), tool.getLoweringStage());
 570             args.add("address", arrayRangeWriteBarrier.getAddress());
 571             args.add("length", arrayRangeWriteBarrier.getLength());
 572             args.addConst("elementStride", arrayRangeWriteBarrier.getElementStride());
 573             args.addConst("threadRegister", registers.getThreadRegister());
 574             template(arrayRangeWriteBarrier, args).instantiate(providers.getMetaAccess(), arrayRangeWriteBarrier, DEFAULT_REPLACER, args);
 575         }
 576     }
 577 
 578     /**
 579      * Log method of debugging purposes.
 580      */
 581     public static void log(boolean enabled, String format, long value) {
 582         if (enabled) {
 583             Log.printf(format, value);
 584         }
 585     }
 586 
 587     public static void log(boolean enabled, String format, long value1, long value2) {
 588         if (enabled) {
 589             Log.printf(format, value1, value2);
 590         }
 591     }
 592 
 593     public static void log(boolean enabled, String format, long value1, long value2, long value3) {
 594         if (enabled) {
 595             Log.printf(format, value1, value2, value3);
 596         }
 597     }
 598 
 599     /**
 600      * Validation helper method which performs sanity checks on write operations. The addresses of
 601      * both the object and the value being written are checked in order to determine if they reside
 602      * in a valid heap region. If an object is stale, an invalid access is performed in order to
 603      * prematurely crash the VM and debug the stack trace of the faulty method.
 604      */
 605     public static void validateObject(Object parent, Object child) {
 606         if (verifyOops(INJECTED_VMCONFIG) && child != null) {
 607             Word parentWord = Word.objectToTrackedPointer(parent);
 608             Word childWord = Word.objectToTrackedPointer(child);
 609             if (!validateOop(VALIDATE_OBJECT, parentWord, childWord)) {
 610                 log(true, "Verification ERROR, Parent: %p Child: %p\n", parentWord.rawValue(), childWord.rawValue());
 611                 VMErrorNode.vmError("Verification ERROR, Parent: %p\n", parentWord.rawValue());
 612             }
 613         }
 614     }
 615 
 616     public static final ForeignCallDescriptor VALIDATE_OBJECT = new ForeignCallDescriptor("validate_object", boolean.class, Word.class, Word.class);
 617 
 618     @NodeIntrinsic(ForeignCallNode.class)
 619     private static native boolean validateOop(@ConstantNodeParameter ForeignCallDescriptor descriptor, Word parent, Word object);
 620 
 621 }
--- EOF ---