src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/WriteBarrierSnippets.java
Index
Unified diffs
Context diffs
Sdiffs
Patch
New
Old
Previous File
Next File
*** old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/WriteBarrierSnippets.java Mon Mar 20 17:39:02 2017
--- new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/WriteBarrierSnippets.java Mon Mar 20 17:39:02 2017
*** 20,30 ****
--- 20,30 ----
* or visit www.oracle.com if you need additional information or have any
* questions.
*/
package org.graalvm.compiler.hotspot.replacements;
! import static org.graalvm.compiler.core.common.GraalOptions.SnippetCounters;
! import static jdk.vm.ci.code.MemoryBarriers.STORE_LOAD;
import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFIG;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayBaseOffset;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayIndexScale;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.cardTableShift;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.dirtyCardValue;
*** 41,60 ****
--- 41,59 ----
import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FREQUENT_PROBABILITY;
import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.LIKELY_PROBABILITY;
import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.NOT_FREQUENT_PROBABILITY;
import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;
import static org.graalvm.compiler.replacements.SnippetTemplate.DEFAULT_REPLACER;
import static jdk.vm.ci.code.MemoryBarriers.STORE_LOAD;
import org.graalvm.compiler.api.replacements.Snippet;
import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter;
+ import org.graalvm.compiler.core.common.CompressEncoding;
import org.graalvm.compiler.core.common.GraalOptions;
import org.graalvm.compiler.core.common.LocationIdentity;
import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor;
import org.graalvm.compiler.graph.Node.ConstantNodeParameter;
import org.graalvm.compiler.graph.Node.NodeIntrinsic;
import org.graalvm.compiler.hotspot.CompressEncoding;
import org.graalvm.compiler.hotspot.meta.HotSpotProviders;
import org.graalvm.compiler.hotspot.meta.HotSpotRegistersProvider;
import org.graalvm.compiler.hotspot.nodes.CompressionNode;
import org.graalvm.compiler.hotspot.nodes.G1ArrayRangePostWriteBarrier;
import org.graalvm.compiler.hotspot.nodes.G1ArrayRangePreWriteBarrier;
*** 63,72 ****
--- 62,72 ----
import org.graalvm.compiler.hotspot.nodes.G1ReferentFieldReadBarrier;
import org.graalvm.compiler.hotspot.nodes.GetObjectAddressNode;
import org.graalvm.compiler.hotspot.nodes.GraalHotSpotVMConfigNode;
import org.graalvm.compiler.hotspot.nodes.SerialArrayRangeWriteBarrier;
import org.graalvm.compiler.hotspot.nodes.SerialWriteBarrier;
+ import org.graalvm.compiler.hotspot.nodes.VMErrorNode;
import org.graalvm.compiler.hotspot.nodes.type.NarrowOopStamp;
import org.graalvm.compiler.nodes.NamedLocationIdentity;
import org.graalvm.compiler.nodes.StructuredGraph;
import org.graalvm.compiler.nodes.ValueNode;
import org.graalvm.compiler.nodes.extended.FixedValueAnchorNode;
*** 76,92 ****
--- 76,93 ----
import org.graalvm.compiler.nodes.memory.HeapAccess.BarrierType;
import org.graalvm.compiler.nodes.memory.address.AddressNode;
import org.graalvm.compiler.nodes.memory.address.AddressNode.Address;
import org.graalvm.compiler.nodes.memory.address.OffsetAddressNode;
import org.graalvm.compiler.nodes.spi.LoweringTool;
+ import org.graalvm.compiler.options.OptionValues;
import org.graalvm.compiler.replacements.Log;
import org.graalvm.compiler.replacements.SnippetCounter;
+ import org.graalvm.compiler.replacements.SnippetCounter.Group;
import org.graalvm.compiler.replacements.SnippetTemplate.AbstractTemplates;
import org.graalvm.compiler.replacements.SnippetTemplate.Arguments;
import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo;
import org.graalvm.compiler.replacements.Snippets;
import org.graalvm.compiler.replacements.nodes.DirectObjectStoreNode;
import org.graalvm.compiler.replacements.nodes.DirectStoreNode;
import org.graalvm.compiler.word.Pointer;
import org.graalvm.compiler.word.Unsigned;
import org.graalvm.compiler.word.Word;
*** 94,138 ****
--- 95,153 ----
import jdk.vm.ci.code.TargetDescription;
import jdk.vm.ci.meta.JavaKind;
public class WriteBarrierSnippets implements Snippets {
private static final SnippetCounter.Group countersWriteBarriers = SnippetCounters.getValue() ? new SnippetCounter.Group("WriteBarriers") : null;
private static final SnippetCounter serialWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "serialWriteBarrier", "Number of Serial Write Barriers");
! private static final SnippetCounter g1AttemptedPreWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "g1AttemptedPreWriteBarrier", "Number of attempted G1 Pre Write Barriers");
! private static final SnippetCounter g1EffectivePreWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "g1EffectivePreWriteBarrier", "Number of effective G1 Pre Write Barriers");
! private static final SnippetCounter g1ExecutedPreWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "g1ExecutedPreWriteBarrier", "Number of executed G1 Pre Write Barriers");
! private static final SnippetCounter g1AttemptedPostWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "g1AttemptedPostWriteBarrier", "Number of attempted G1 Post Write Barriers");
private static final SnippetCounter g1EffectiveAfterXORPostWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "g1EffectiveAfterXORPostWriteBarrier",
+ static class Counters {
+ Counters(SnippetCounter.Group.Factory factory) {
! Group countersWriteBarriers = factory.createSnippetCounterGroup("WriteBarriers");
! serialWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "serialWriteBarrier", "Number of Serial Write Barriers");
! g1AttemptedPreWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "g1AttemptedPreWriteBarrier", "Number of attempted G1 Pre Write Barriers");
! g1EffectivePreWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "g1EffectivePreWriteBarrier", "Number of effective G1 Pre Write Barriers");
+ g1ExecutedPreWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "g1ExecutedPreWriteBarrier", "Number of executed G1 Pre Write Barriers");
+ g1AttemptedPostWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "g1AttemptedPostWriteBarrier", "Number of attempted G1 Post Write Barriers");
+ g1EffectiveAfterXORPostWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "g1EffectiveAfterXORPostWriteBarrier",
"Number of effective G1 Post Write Barriers (after passing the XOR test)");
! private static final SnippetCounter g1EffectiveAfterNullPostWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "g1EffectiveAfterNullPostWriteBarrier",
! g1EffectiveAfterNullPostWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "g1EffectiveAfterNullPostWriteBarrier",
"Number of effective G1 Post Write Barriers (after passing the NULL test)");
! private static final SnippetCounter g1ExecutedPostWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "g1ExecutedPostWriteBarrier", "Number of executed G1 Post Write Barriers");
! g1ExecutedPostWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "g1ExecutedPostWriteBarrier", "Number of executed G1 Post Write Barriers");
+
+ }
+
+ final SnippetCounter serialWriteBarrierCounter;
+ final SnippetCounter g1AttemptedPreWriteBarrierCounter;
+ final SnippetCounter g1EffectivePreWriteBarrierCounter;
+ final SnippetCounter g1ExecutedPreWriteBarrierCounter;
+ final SnippetCounter g1AttemptedPostWriteBarrierCounter;
+ final SnippetCounter g1EffectiveAfterXORPostWriteBarrierCounter;
+ final SnippetCounter g1EffectiveAfterNullPostWriteBarrierCounter;
+ final SnippetCounter g1ExecutedPostWriteBarrierCounter;
+ }
public static final LocationIdentity GC_CARD_LOCATION = NamedLocationIdentity.mutable("GC-Card");
public static final LocationIdentity GC_LOG_LOCATION = NamedLocationIdentity.mutable("GC-Log");
public static final LocationIdentity GC_INDEX_LOCATION = NamedLocationIdentity.mutable("GC-Index");
! private static void serialWriteBarrier(Pointer ptr, Counters counters) {
! counters.serialWriteBarrierCounter.inc();
final long startAddress = GraalHotSpotVMConfigNode.cardTableAddress();
Word base = (Word) ptr.unsignedShiftRight(cardTableShift(INJECTED_VMCONFIG));
if (((int) startAddress) == startAddress && GraalHotSpotVMConfigNode.isCardTableAddressConstant()) {
base.writeByte((int) startAddress, (byte) 0, GC_CARD_LOCATION);
} else {
base.writeByte(Word.unsigned(startAddress), (byte) 0, GC_CARD_LOCATION);
}
}
@Snippet
! public static void serialImpreciseWriteBarrier(Object object, @ConstantParameter Counters counters) {
! serialWriteBarrier(Word.objectToTrackedPointer(object), counters);
}
@Snippet
! public static void serialPreciseWriteBarrier(Address address, @ConstantParameter Counters counters) {
! serialWriteBarrier(Word.fromAddress(address), counters);
}
@Snippet
public static void serialArrayRangeWriteBarrier(Object object, int startIndex, int length) {
if (length == 0) {
*** 152,162 ****
--- 167,177 ----
}
}
@Snippet
public static void g1PreWriteBarrier(Address address, Object object, Object expectedObject, @ConstantParameter boolean doLoad, @ConstantParameter boolean nullCheck,
! @ConstantParameter Register threadRegister, @ConstantParameter boolean trace, @ConstantParameter Counters counters) {
if (nullCheck) {
NullCheckNode.nullCheck(address);
}
Word thread = registerAsWord(threadRegister);
verifyOop(object);
*** 171,181 ****
--- 186,196 ----
log(trace, "[%d] G1-Pre Thread %p Expected Object %p\n", gcCycle, thread.rawValue(), Word.objectToTrackedPointer(fixedExpectedObject).rawValue());
log(trace, "[%d] G1-Pre Thread %p Field %p\n", gcCycle, thread.rawValue(), field.rawValue());
log(trace, "[%d] G1-Pre Thread %p Marking %d\n", gcCycle, thread.rawValue(), markingValue);
log(trace, "[%d] G1-Pre Thread %p DoLoad %d\n", gcCycle, thread.rawValue(), doLoad ? 1L : 0L);
}
! counters.g1AttemptedPreWriteBarrierCounter.inc();
// If the concurrent marker is enabled, the barrier is issued.
if (probability(NOT_FREQUENT_PROBABILITY, markingValue != (byte) 0)) {
// If the previous value has to be loaded (before the write), the load is issued.
// The load is always issued except the cases of CAS and referent field.
if (probability(LIKELY_PROBABILITY, doLoad)) {
*** 183,196 ****
--- 198,211 ----
if (trace) {
log(trace, "[%d] G1-Pre Thread %p Previous Object %p\n ", gcCycle, thread.rawValue(), previousOop.rawValue());
verifyOop(previousOop.toObject());
}
}
! counters.g1EffectivePreWriteBarrierCounter.inc();
// If the previous value is null the barrier should not be issued.
if (probability(FREQUENT_PROBABILITY, previousOop.notEqual(0))) {
! counters.g1ExecutedPreWriteBarrierCounter.inc();
// If the thread-local SATB buffer is full issue a native call which will
// initialize a new one and add the entry.
Word indexAddress = thread.add(g1SATBQueueIndexOffset(INJECTED_VMCONFIG));
Word indexValue = indexAddress.readWord(0);
if (probability(FREQUENT_PROBABILITY, indexValue.notEqual(0))) {
*** 207,217 ****
--- 222,232 ----
}
}
@Snippet
public static void g1PostWriteBarrier(Address address, Object object, Object value, @ConstantParameter boolean usePrecise, @ConstantParameter Register threadRegister,
! @ConstantParameter boolean trace, @ConstantParameter Counters counters) {
Word thread = registerAsWord(threadRegister);
Object fixedValue = FixedValueAnchorNode.getObject(value);
verifyOop(object);
verifyOop(fixedValue);
validateObject(object, fixedValue);
*** 243,269 ****
--- 258,284 ----
} else {
cardBase = cardBase.add(Word.unsigned(startAddress));
}
Word cardAddress = (Word) cardBase.add(displacement);
! counters.g1AttemptedPostWriteBarrierCounter.inc();
if (probability(FREQUENT_PROBABILITY, xorResult.notEqual(0))) {
! counters.g1EffectiveAfterXORPostWriteBarrierCounter.inc();
// If the written value is not null continue with the barrier addition.
if (probability(FREQUENT_PROBABILITY, writtenValue.notEqual(0))) {
byte cardByte = cardAddress.readByte(0, GC_CARD_LOCATION);
! counters.g1EffectiveAfterNullPostWriteBarrierCounter.inc();
// If the card is already dirty, (hence already enqueued) skip the insertion.
if (probability(NOT_FREQUENT_PROBABILITY, cardByte != g1YoungCardValue(INJECTED_VMCONFIG))) {
MembarNode.memoryBarrier(STORE_LOAD, GC_CARD_LOCATION);
byte cardByteReload = cardAddress.readByte(0, GC_CARD_LOCATION);
if (probability(NOT_FREQUENT_PROBABILITY, cardByteReload != dirtyCardValue(INJECTED_VMCONFIG))) {
log(trace, "[%d] G1-Post Thread: %p Card: %p \n", gcCycle, thread.rawValue(), Word.unsigned(cardByte).rawValue());
cardAddress.writeByte(0, (byte) 0, GC_CARD_LOCATION);
! counters.g1ExecutedPostWriteBarrierCounter.inc();
// If the thread local card queue is full, issue a native call which will
// initialize a new one and add the card entry.
Word indexAddress = thread.add(g1CardQueueIndexOffset(INJECTED_VMCONFIG));
Word indexValue = thread.readWord(g1CardQueueIndexOffset(INJECTED_VMCONFIG));
*** 384,397 ****
--- 399,414 ----
private final SnippetInfo g1PostWriteBarrier = snippet(WriteBarrierSnippets.class, "g1PostWriteBarrier", GC_CARD_LOCATION, GC_INDEX_LOCATION, GC_LOG_LOCATION);
private final SnippetInfo g1ArrayRangePreWriteBarrier = snippet(WriteBarrierSnippets.class, "g1ArrayRangePreWriteBarrier", GC_INDEX_LOCATION, GC_LOG_LOCATION);
private final SnippetInfo g1ArrayRangePostWriteBarrier = snippet(WriteBarrierSnippets.class, "g1ArrayRangePostWriteBarrier", GC_CARD_LOCATION, GC_INDEX_LOCATION, GC_LOG_LOCATION);
private final CompressEncoding oopEncoding;
+ private final Counters counters;
! public Templates(OptionValues options, SnippetCounter.Group.Factory factory, HotSpotProviders providers, TargetDescription target, CompressEncoding oopEncoding) {
! super(options, providers, providers.getSnippetReflection(), target);
this.oopEncoding = oopEncoding;
+ this.counters = new Counters(factory);
}
public void lower(SerialWriteBarrier writeBarrier, LoweringTool tool) {
Arguments args;
if (writeBarrier.usePrecise()) {
*** 400,409 ****
--- 417,427 ----
} else {
args = new Arguments(serialImpreciseWriteBarrier, writeBarrier.graph().getGuardsStage(), tool.getLoweringStage());
OffsetAddressNode address = (OffsetAddressNode) writeBarrier.getAddress();
args.add("object", address.getBase());
}
+ args.addConst("counters", counters);
template(args).instantiate(providers.getMetaAccess(), writeBarrier, DEFAULT_REPLACER, args);
}
public void lower(SerialArrayRangeWriteBarrier arrayRangeWriteBarrier, LoweringTool tool) {
Arguments args = new Arguments(serialArrayRangeWriteBarrier, arrayRangeWriteBarrier.graph().getGuardsStage(), tool.getLoweringStage());
*** 431,441 ****
--- 449,460 ----
args.add("expectedObject", expected);
args.addConst("doLoad", writeBarrierPre.doLoad());
args.addConst("nullCheck", writeBarrierPre.getNullCheck());
args.addConst("threadRegister", registers.getThreadRegister());
! args.addConst("trace", traceBarrier(writeBarrierPre.graph()));
+ args.addConst("counters", counters);
template(args).instantiate(providers.getMetaAccess(), writeBarrierPre, DEFAULT_REPLACER, args);
}
public void lower(G1ReferentFieldReadBarrier readBarrier, HotSpotRegistersProvider registers, LoweringTool tool) {
Arguments args = new Arguments(g1ReferentReadBarrier, readBarrier.graph().getGuardsStage(), tool.getLoweringStage());
*** 455,465 ****
--- 474,485 ----
args.add("expectedObject", expected);
args.addConst("doLoad", readBarrier.doLoad());
args.addConst("nullCheck", false);
args.addConst("threadRegister", registers.getThreadRegister());
! args.addConst("trace", traceBarrier(readBarrier.graph()));
+ args.addConst("counters", counters);
template(args).instantiate(providers.getMetaAccess(), readBarrier, DEFAULT_REPLACER, args);
}
public void lower(G1PostWriteBarrier writeBarrierPost, HotSpotRegistersProvider registers, LoweringTool tool) {
StructuredGraph graph = writeBarrierPost.graph();
*** 484,494 ****
--- 504,515 ----
}
args.add("value", value);
args.addConst("usePrecise", writeBarrierPost.usePrecise());
args.addConst("threadRegister", registers.getThreadRegister());
! args.addConst("trace", traceBarrier(writeBarrierPost.graph()));
+ args.addConst("counters", counters);
template(args).instantiate(providers.getMetaAccess(), writeBarrierPost, DEFAULT_REPLACER, args);
}
public void lower(G1ArrayRangePreWriteBarrier arrayRangeWriteBarrier, HotSpotRegistersProvider registers, LoweringTool tool) {
Arguments args = new Arguments(g1ArrayRangePreWriteBarrier, arrayRangeWriteBarrier.graph().getGuardsStage(), tool.getLoweringStage());
*** 528,540 ****
--- 549,561 ----
if (enabled) {
Log.printf(format, value1, value2, value3);
}
}
! public static boolean traceBarrier(StructuredGraph graph) {
! return GraalOptions.GCDebugStartCycle.getValue(graph.getOptions()) > 0 &&
! ((int) Word.unsigned(HotSpotReplacementsUtil.gcTotalCollectionsAddress(INJECTED_VMCONFIG)).readLong(0) > GraalOptions.GCDebugStartCycle.getValue(graph.getOptions()));
}
/**
* Validation helper method which performs sanity checks on write operations. The addresses of
* both the object and the value being written are checked in order to determine if they reside
*** 542,552 ****
--- 563,573 ----
* prematurely crash the VM and debug the stack trace of the faulty method.
*/
public static void validateObject(Object parent, Object child) {
if (verifyOops(INJECTED_VMCONFIG) && child != null && !validateOop(VALIDATE_OBJECT, parent, child)) {
log(true, "Verification ERROR, Parent: %p Child: %p\n", Word.objectToTrackedPointer(parent).rawValue(), Word.objectToTrackedPointer(child).rawValue());
! DirectObjectStoreNode.storeObject(null, 0, 0, null, LocationIdentity.any(), JavaKind.Object);
! VMErrorNode.vmError("Verification ERROR, Parent: %p\n", Word.objectToTrackedPointer(parent).rawValue());
}
}
public static final ForeignCallDescriptor VALIDATE_OBJECT = new ForeignCallDescriptor("validate_object", boolean.class, Word.class, Word.class);
src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/WriteBarrierSnippets.java
Index
Unified diffs
Context diffs
Sdiffs
Patch
New
Old
Previous File
Next File