52 import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor;
53 import org.graalvm.compiler.debug.DebugHandlersFactory;
54 import org.graalvm.compiler.graph.Node.ConstantNodeParameter;
55 import org.graalvm.compiler.graph.Node.NodeIntrinsic;
56 import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig;
57 import org.graalvm.compiler.hotspot.meta.HotSpotProviders;
58 import org.graalvm.compiler.hotspot.meta.HotSpotRegistersProvider;
59 import org.graalvm.compiler.hotspot.nodes.G1ArrayRangePostWriteBarrier;
60 import org.graalvm.compiler.hotspot.nodes.G1ArrayRangePreWriteBarrier;
61 import org.graalvm.compiler.hotspot.nodes.G1PostWriteBarrier;
62 import org.graalvm.compiler.hotspot.nodes.G1PreWriteBarrier;
63 import org.graalvm.compiler.hotspot.nodes.G1ReferentFieldReadBarrier;
64 import org.graalvm.compiler.hotspot.nodes.GraalHotSpotVMConfigNode;
65 import org.graalvm.compiler.hotspot.nodes.HotSpotCompressionNode;
66 import org.graalvm.compiler.hotspot.nodes.SerialArrayRangeWriteBarrier;
67 import org.graalvm.compiler.hotspot.nodes.SerialWriteBarrier;
68 import org.graalvm.compiler.hotspot.nodes.VMErrorNode;
69 import org.graalvm.compiler.nodes.NamedLocationIdentity;
70 import org.graalvm.compiler.nodes.NodeView;
71 import org.graalvm.compiler.nodes.PiNode;
72 import org.graalvm.compiler.nodes.StructuredGraph;
73 import org.graalvm.compiler.nodes.ValueNode;
74 import org.graalvm.compiler.nodes.extended.FixedValueAnchorNode;
75 import org.graalvm.compiler.nodes.extended.ForeignCallNode;
76 import org.graalvm.compiler.nodes.extended.MembarNode;
77 import org.graalvm.compiler.nodes.extended.NullCheckNode;
78 import org.graalvm.compiler.nodes.memory.HeapAccess.BarrierType;
79 import org.graalvm.compiler.nodes.memory.address.AddressNode;
80 import org.graalvm.compiler.nodes.memory.address.AddressNode.Address;
81 import org.graalvm.compiler.nodes.memory.address.OffsetAddressNode;
82 import org.graalvm.compiler.nodes.spi.LoweringTool;
83 import org.graalvm.compiler.nodes.type.NarrowOopStamp;
84 import org.graalvm.compiler.options.OptionValues;
85 import org.graalvm.compiler.replacements.Log;
86 import org.graalvm.compiler.replacements.ReplacementsUtil;
87 import org.graalvm.compiler.replacements.SnippetCounter;
88 import org.graalvm.compiler.replacements.SnippetCounter.Group;
89 import org.graalvm.compiler.replacements.SnippetTemplate.AbstractTemplates;
90 import org.graalvm.compiler.replacements.SnippetTemplate.Arguments;
91 import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo;
296 if (probability(FREQUENT_PROBABILITY, indexValue.notEqual(0))) {
297 Word bufferAddress = thread.readWord(g1CardQueueBufferOffset(INJECTED_VMCONFIG));
298 Word nextIndex = indexValue.subtract(wordSize());
299 Word logAddress = bufferAddress.add(nextIndex);
300 // Log the object to be scanned as well as update
301 // the card queue's next index.
302 logAddress.writeWord(0, cardAddress, GC_LOG_LOCATION);
303 indexAddress.writeWord(0, nextIndex, GC_INDEX_LOCATION);
304 } else {
305 g1PostBarrierStub(G1WBPOSTCALL, cardAddress);
306 }
307 }
308 }
309 }
310 }
311 }
312
313 private static void verifyNotArray(Object object) {
314 if (object != null) {
315 // Manually build the null check and cast because we're in snippet that's lowered late.
316 AssertionNode.assertion(false, !PiNode.piCastNonNull(object, Object.class).getClass().isArray(), "imprecise card mark used with array");
317 }
318 }
319
320 @Snippet
321 public static void g1ArrayRangePreWriteBarrier(Address address, int length, @ConstantParameter int elementStride, @ConstantParameter Register threadRegister) {
322 Word thread = registerAsWord(threadRegister);
323 byte markingValue = thread.readByte(g1SATBQueueMarkingOffset(INJECTED_VMCONFIG));
324 // If the concurrent marker is not enabled or the vector length is zero, return.
325 if (markingValue == (byte) 0 || length == 0) {
326 return;
327 }
328 Word bufferAddress = thread.readWord(g1SATBQueueBufferOffset(INJECTED_VMCONFIG));
329 Word indexAddress = thread.add(g1SATBQueueIndexOffset(INJECTED_VMCONFIG));
330 long indexValue = indexAddress.readWord(0).rawValue();
331 final int scale = ReplacementsUtil.arrayIndexScale(INJECTED_METAACCESS, JavaKind.Object);
332 long start = getPointerToFirstArrayElement(address, length, elementStride);
333
334 for (int i = 0; i < length; i++) {
335 Word arrElemPtr = WordFactory.pointer(start + i * scale);
336 Pointer oop = Word.objectToTrackedPointer(arrElemPtr.readObject(0, BarrierType.NONE));
410 }
411 return result;
412 }
413
414 public static final ForeignCallDescriptor G1WBPRECALL = new ForeignCallDescriptor("write_barrier_pre", void.class, Object.class);
415
416 @NodeIntrinsic(ForeignCallNode.class)
417 private static native void g1PreBarrierStub(@ConstantNodeParameter ForeignCallDescriptor descriptor, Object object);
418
419 public static final ForeignCallDescriptor G1WBPOSTCALL = new ForeignCallDescriptor("write_barrier_post", void.class, Word.class);
420
421 @NodeIntrinsic(ForeignCallNode.class)
422 public static native void g1PostBarrierStub(@ConstantNodeParameter ForeignCallDescriptor descriptor, Word card);
423
424 public static class Templates extends AbstractTemplates {
425
426 private final SnippetInfo serialImpreciseWriteBarrier = snippet(WriteBarrierSnippets.class, "serialImpreciseWriteBarrier", GC_CARD_LOCATION);
427 private final SnippetInfo serialPreciseWriteBarrier = snippet(WriteBarrierSnippets.class, "serialPreciseWriteBarrier", GC_CARD_LOCATION);
428 private final SnippetInfo serialArrayRangeWriteBarrier = snippet(WriteBarrierSnippets.class, "serialArrayRangeWriteBarrier");
429 private final SnippetInfo g1PreWriteBarrier = snippet(WriteBarrierSnippets.class, "g1PreWriteBarrier", GC_INDEX_LOCATION, GC_LOG_LOCATION);
430 private final SnippetInfo g1ReferentReadBarrier = snippet(WriteBarrierSnippets.class, "g1PreWriteBarrier", GC_INDEX_LOCATION, GC_LOG_LOCATION);
431 private final SnippetInfo g1PostWriteBarrier = snippet(WriteBarrierSnippets.class, "g1PostWriteBarrier", GC_CARD_LOCATION, GC_INDEX_LOCATION, GC_LOG_LOCATION);
432 private final SnippetInfo g1ArrayRangePreWriteBarrier = snippet(WriteBarrierSnippets.class, "g1ArrayRangePreWriteBarrier", GC_INDEX_LOCATION, GC_LOG_LOCATION);
433 private final SnippetInfo g1ArrayRangePostWriteBarrier = snippet(WriteBarrierSnippets.class, "g1ArrayRangePostWriteBarrier", GC_CARD_LOCATION, GC_INDEX_LOCATION, GC_LOG_LOCATION);
434
435 private final CompressEncoding oopEncoding;
436 private final Counters counters;
437 private final boolean verifyBarrier;
438 private final long gcTotalCollectionsAddress;
439
440 public Templates(OptionValues options, Iterable<DebugHandlersFactory> factories, Group.Factory factory, HotSpotProviders providers, TargetDescription target,
441 GraalHotSpotVMConfig config) {
442 super(options, factories, providers, providers.getSnippetReflection(), target);
443 this.oopEncoding = config.useCompressedOops ? config.getOopEncoding() : null;
444 this.verifyBarrier = ReplacementsUtil.REPLACEMENTS_ASSERTIONS_ENABLED || config.verifyBeforeGC || config.verifyAfterGC;
445 this.gcTotalCollectionsAddress = config.gcTotalCollectionsAddress();
446 this.counters = new Counters(factory);
447 }
448
449 public boolean traceBarrier(StructuredGraph graph) {
450 long startCycle = GraalOptions.GCDebugStartCycle.getValue(graph.getOptions());
|
52 import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor;
53 import org.graalvm.compiler.debug.DebugHandlersFactory;
54 import org.graalvm.compiler.graph.Node.ConstantNodeParameter;
55 import org.graalvm.compiler.graph.Node.NodeIntrinsic;
56 import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig;
57 import org.graalvm.compiler.hotspot.meta.HotSpotProviders;
58 import org.graalvm.compiler.hotspot.meta.HotSpotRegistersProvider;
59 import org.graalvm.compiler.hotspot.nodes.G1ArrayRangePostWriteBarrier;
60 import org.graalvm.compiler.hotspot.nodes.G1ArrayRangePreWriteBarrier;
61 import org.graalvm.compiler.hotspot.nodes.G1PostWriteBarrier;
62 import org.graalvm.compiler.hotspot.nodes.G1PreWriteBarrier;
63 import org.graalvm.compiler.hotspot.nodes.G1ReferentFieldReadBarrier;
64 import org.graalvm.compiler.hotspot.nodes.GraalHotSpotVMConfigNode;
65 import org.graalvm.compiler.hotspot.nodes.HotSpotCompressionNode;
66 import org.graalvm.compiler.hotspot.nodes.SerialArrayRangeWriteBarrier;
67 import org.graalvm.compiler.hotspot.nodes.SerialWriteBarrier;
68 import org.graalvm.compiler.hotspot.nodes.VMErrorNode;
69 import org.graalvm.compiler.nodes.NamedLocationIdentity;
70 import org.graalvm.compiler.nodes.NodeView;
71 import org.graalvm.compiler.nodes.PiNode;
72 import org.graalvm.compiler.nodes.SnippetAnchorNode;
73 import org.graalvm.compiler.nodes.StructuredGraph;
74 import org.graalvm.compiler.nodes.ValueNode;
75 import org.graalvm.compiler.nodes.extended.FixedValueAnchorNode;
76 import org.graalvm.compiler.nodes.extended.ForeignCallNode;
77 import org.graalvm.compiler.nodes.extended.MembarNode;
78 import org.graalvm.compiler.nodes.extended.NullCheckNode;
79 import org.graalvm.compiler.nodes.memory.HeapAccess.BarrierType;
80 import org.graalvm.compiler.nodes.memory.address.AddressNode;
81 import org.graalvm.compiler.nodes.memory.address.AddressNode.Address;
82 import org.graalvm.compiler.nodes.memory.address.OffsetAddressNode;
83 import org.graalvm.compiler.nodes.spi.LoweringTool;
84 import org.graalvm.compiler.nodes.type.NarrowOopStamp;
85 import org.graalvm.compiler.options.OptionValues;
86 import org.graalvm.compiler.replacements.Log;
87 import org.graalvm.compiler.replacements.ReplacementsUtil;
88 import org.graalvm.compiler.replacements.SnippetCounter;
89 import org.graalvm.compiler.replacements.SnippetCounter.Group;
90 import org.graalvm.compiler.replacements.SnippetTemplate.AbstractTemplates;
91 import org.graalvm.compiler.replacements.SnippetTemplate.Arguments;
92 import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo;
297 if (probability(FREQUENT_PROBABILITY, indexValue.notEqual(0))) {
298 Word bufferAddress = thread.readWord(g1CardQueueBufferOffset(INJECTED_VMCONFIG));
299 Word nextIndex = indexValue.subtract(wordSize());
300 Word logAddress = bufferAddress.add(nextIndex);
301 // Log the object to be scanned as well as update
302 // the card queue's next index.
303 logAddress.writeWord(0, cardAddress, GC_LOG_LOCATION);
304 indexAddress.writeWord(0, nextIndex, GC_INDEX_LOCATION);
305 } else {
306 g1PostBarrierStub(G1WBPOSTCALL, cardAddress);
307 }
308 }
309 }
310 }
311 }
312 }
313
314 private static void verifyNotArray(Object object) {
315 if (object != null) {
316 // Manually build the null check and cast because we're in snippet that's lowered late.
317 AssertionNode.assertion(false, !PiNode.piCastNonNull(object, SnippetAnchorNode.anchor()).getClass().isArray(), "imprecise card mark used with array");
318 }
319 }
320
321 @Snippet
322 public static void g1ArrayRangePreWriteBarrier(Address address, int length, @ConstantParameter int elementStride, @ConstantParameter Register threadRegister) {
323 Word thread = registerAsWord(threadRegister);
324 byte markingValue = thread.readByte(g1SATBQueueMarkingOffset(INJECTED_VMCONFIG));
325 // If the concurrent marker is not enabled or the vector length is zero, return.
326 if (markingValue == (byte) 0 || length == 0) {
327 return;
328 }
329 Word bufferAddress = thread.readWord(g1SATBQueueBufferOffset(INJECTED_VMCONFIG));
330 Word indexAddress = thread.add(g1SATBQueueIndexOffset(INJECTED_VMCONFIG));
331 long indexValue = indexAddress.readWord(0).rawValue();
332 final int scale = ReplacementsUtil.arrayIndexScale(INJECTED_METAACCESS, JavaKind.Object);
333 long start = getPointerToFirstArrayElement(address, length, elementStride);
334
335 for (int i = 0; i < length; i++) {
336 Word arrElemPtr = WordFactory.pointer(start + i * scale);
337 Pointer oop = Word.objectToTrackedPointer(arrElemPtr.readObject(0, BarrierType.NONE));
411 }
412 return result;
413 }
414
415 public static final ForeignCallDescriptor G1WBPRECALL = new ForeignCallDescriptor("write_barrier_pre", void.class, Object.class);
416
417 @NodeIntrinsic(ForeignCallNode.class)
418 private static native void g1PreBarrierStub(@ConstantNodeParameter ForeignCallDescriptor descriptor, Object object);
419
420 public static final ForeignCallDescriptor G1WBPOSTCALL = new ForeignCallDescriptor("write_barrier_post", void.class, Word.class);
421
422 @NodeIntrinsic(ForeignCallNode.class)
423 public static native void g1PostBarrierStub(@ConstantNodeParameter ForeignCallDescriptor descriptor, Word card);
424
425 public static class Templates extends AbstractTemplates {
426
427 private final SnippetInfo serialImpreciseWriteBarrier = snippet(WriteBarrierSnippets.class, "serialImpreciseWriteBarrier", GC_CARD_LOCATION);
428 private final SnippetInfo serialPreciseWriteBarrier = snippet(WriteBarrierSnippets.class, "serialPreciseWriteBarrier", GC_CARD_LOCATION);
429 private final SnippetInfo serialArrayRangeWriteBarrier = snippet(WriteBarrierSnippets.class, "serialArrayRangeWriteBarrier");
430 private final SnippetInfo g1PreWriteBarrier = snippet(WriteBarrierSnippets.class, "g1PreWriteBarrier", GC_INDEX_LOCATION, GC_LOG_LOCATION);
431 private final SnippetInfo g1ReferentReadBarrier = g1PreWriteBarrier;
432 private final SnippetInfo g1PostWriteBarrier = snippet(WriteBarrierSnippets.class, "g1PostWriteBarrier", GC_CARD_LOCATION, GC_INDEX_LOCATION, GC_LOG_LOCATION);
433 private final SnippetInfo g1ArrayRangePreWriteBarrier = snippet(WriteBarrierSnippets.class, "g1ArrayRangePreWriteBarrier", GC_INDEX_LOCATION, GC_LOG_LOCATION);
434 private final SnippetInfo g1ArrayRangePostWriteBarrier = snippet(WriteBarrierSnippets.class, "g1ArrayRangePostWriteBarrier", GC_CARD_LOCATION, GC_INDEX_LOCATION, GC_LOG_LOCATION);
435
436 private final CompressEncoding oopEncoding;
437 private final Counters counters;
438 private final boolean verifyBarrier;
439 private final long gcTotalCollectionsAddress;
440
441 public Templates(OptionValues options, Iterable<DebugHandlersFactory> factories, Group.Factory factory, HotSpotProviders providers, TargetDescription target,
442 GraalHotSpotVMConfig config) {
443 super(options, factories, providers, providers.getSnippetReflection(), target);
444 this.oopEncoding = config.useCompressedOops ? config.getOopEncoding() : null;
445 this.verifyBarrier = ReplacementsUtil.REPLACEMENTS_ASSERTIONS_ENABLED || config.verifyBeforeGC || config.verifyAfterGC;
446 this.gcTotalCollectionsAddress = config.gcTotalCollectionsAddress();
447 this.counters = new Counters(factory);
448 }
449
450 public boolean traceBarrier(StructuredGraph graph) {
451 long startCycle = GraalOptions.GCDebugStartCycle.getValue(graph.getOptions());
|