< prev index next >
src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/MonitorSnippets.java
Print this page
*** 24,34 ****
--- 24,36 ----
package org.graalvm.compiler.hotspot.replacements;
import static jdk.vm.ci.code.MemoryBarriers.LOAD_STORE;
import static jdk.vm.ci.code.MemoryBarriers.STORE_STORE;
+ import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_OPTIONVALUES;
import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFIG;
+ import static org.graalvm.compiler.hotspot.nodes.AcquiredCASLockNode.mark;
import static org.graalvm.compiler.hotspot.nodes.BeginLockScopeNode.beginLockScope;
import static org.graalvm.compiler.hotspot.nodes.EndLockScopeNode.endLockScope;
import static org.graalvm.compiler.hotspot.nodes.VMErrorNode.vmError;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.DISPLACED_MARK_WORD_LOCATION;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.MARK_WORD_LOCATION;
*** 38,48 ****
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.OBJECT_MONITOR_RECURSION_LOCATION;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.PROTOTYPE_MARK_WORD_LOCATION;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.ageMaskInPlace;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.biasedLockMaskInPlace;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.biasedLockPattern;
- import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.config;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.epochMaskInPlace;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.loadWordFromObject;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.lockDisplacedMarkOffset;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.markOffset;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.monitorMask;
--- 40,49 ----
*** 51,60 ****
--- 52,62 ----
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.objectMonitorOwnerOffset;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.objectMonitorRecursionsOffset;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.pageSize;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.prototypeMarkWordOffset;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.registerAsWord;
+ import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.stackBias;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.unlockedMask;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useBiasedLocking;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.verifyOop;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.wordSize;
import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.ProfileMonitors;
*** 67,77 ****
--- 69,82 ----
import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.NOT_FREQUENT_PROBABILITY;
import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.NOT_LIKELY_PROBABILITY;
import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY;
import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.VERY_FAST_PATH_PROBABILITY;
import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;
+ import static org.graalvm.compiler.nodes.extended.MembarNode.memoryBarrier;
import static org.graalvm.compiler.replacements.SnippetTemplate.DEFAULT_REPLACER;
+ import static jdk.internal.vm.compiler.word.WordFactory.unsigned;
+ import static jdk.internal.vm.compiler.word.WordFactory.zero;
import java.util.List;
import org.graalvm.compiler.api.replacements.Fold;
import org.graalvm.compiler.api.replacements.Snippet;
*** 87,97 ****
import org.graalvm.compiler.graph.Node.NodeIntrinsic;
import org.graalvm.compiler.graph.iterators.NodeIterable;
import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig;
import org.graalvm.compiler.hotspot.meta.HotSpotProviders;
import org.graalvm.compiler.hotspot.meta.HotSpotRegistersProvider;
- import org.graalvm.compiler.hotspot.nodes.AcquiredCASLockNode;
import org.graalvm.compiler.hotspot.nodes.CurrentLockNode;
import org.graalvm.compiler.hotspot.nodes.FastAcquireBiasedLockNode;
import org.graalvm.compiler.hotspot.nodes.MonitorCounterNode;
import org.graalvm.compiler.hotspot.word.KlassPointer;
import org.graalvm.compiler.nodes.BreakpointNode;
--- 92,101 ----
*** 217,233 ****
public class MonitorSnippets implements Snippets {
private static final boolean PROFILE_CONTEXT = false;
@Fold
! static boolean doProfile(OptionValues options) {
return ProfileMonitors.getValue(options);
}
@Snippet
public static void monitorenter(Object object, KlassPointer hub, @ConstantParameter int lockDepth, @ConstantParameter Register threadRegister, @ConstantParameter Register stackPointerRegister,
! @ConstantParameter boolean trace, @ConstantParameter OptionValues options, @ConstantParameter Counters counters) {
verifyOop(object);
// Load the mark word - this includes a null-check on object
final Word mark = loadWordFromObject(object, markOffset(INJECTED_VMCONFIG));
--- 221,237 ----
public class MonitorSnippets implements Snippets {
private static final boolean PROFILE_CONTEXT = false;
@Fold
! static boolean doProfile(@Fold.InjectedParameter OptionValues options) {
return ProfileMonitors.getValue(options);
}
@Snippet
public static void monitorenter(Object object, KlassPointer hub, @ConstantParameter int lockDepth, @ConstantParameter Register threadRegister, @ConstantParameter Register stackPointerRegister,
! @ConstantParameter boolean trace, @ConstantParameter Counters counters) {
verifyOop(object);
// Load the mark word - this includes a null-check on object
final Word mark = loadWordFromObject(object, markOffset(INJECTED_VMCONFIG));
*** 236,256 ****
Pointer objectPointer = Word.objectToTrackedPointer(object);
trace(trace, " object: 0x%016lx\n", objectPointer);
trace(trace, " lock: 0x%016lx\n", lock);
trace(trace, " mark: 0x%016lx\n", mark);
! incCounter(options);
if (useBiasedLocking(INJECTED_VMCONFIG)) {
! if (tryEnterBiased(object, hub, lock, mark, threadRegister, trace, options, counters)) {
return;
}
// not biased, fall-through
}
! if (inlineFastLockSupported(options) && probability(SLOW_PATH_PROBABILITY, mark.and(monitorMask(INJECTED_VMCONFIG)).notEqual(0))) {
// Inflated case
! if (tryEnterInflated(object, lock, mark, threadRegister, trace, options, counters)) {
return;
}
} else {
// Create the unlocked mark word pattern
Word unlockedMark = mark.or(unlockedMask(INJECTED_VMCONFIG));
--- 240,260 ----
Pointer objectPointer = Word.objectToTrackedPointer(object);
trace(trace, " object: 0x%016lx\n", objectPointer);
trace(trace, " lock: 0x%016lx\n", lock);
trace(trace, " mark: 0x%016lx\n", mark);
! incCounter();
if (useBiasedLocking(INJECTED_VMCONFIG)) {
! if (tryEnterBiased(object, hub, lock, mark, threadRegister, trace, counters)) {
return;
}
// not biased, fall-through
}
! if (inlineFastLockSupported() && probability(SLOW_PATH_PROBABILITY, mark.and(monitorMask(INJECTED_VMCONFIG)).notEqual(0))) {
// Inflated case
! if (tryEnterInflated(object, lock, mark, threadRegister, trace, counters)) {
return;
}
} else {
// Create the unlocked mark word pattern
Word unlockedMark = mark.or(unlockedMask(INJECTED_VMCONFIG));
*** 264,276 ****
// Test if the object's mark word is unlocked, and if so, store the
// (address of) the lock slot into the object's mark word.
Word currentMark = objectPointer.compareAndSwapWord(markOffset(INJECTED_VMCONFIG), unlockedMark, lock, MARK_WORD_LOCATION);
if (probability(FAST_PATH_PROBABILITY, currentMark.equal(unlockedMark))) {
! traceObject(trace, "+lock{cas}", object, true, options);
counters.lockCas.inc();
! AcquiredCASLockNode.mark(object);
return;
} else {
trace(trace, " currentMark: 0x%016lx\n", currentMark);
// The mark word in the object header was not the same.
// Either the object is locked by another thread or is already locked
--- 268,280 ----
// Test if the object's mark word is unlocked, and if so, store the
// (address of) the lock slot into the object's mark word.
Word currentMark = objectPointer.compareAndSwapWord(markOffset(INJECTED_VMCONFIG), unlockedMark, lock, MARK_WORD_LOCATION);
if (probability(FAST_PATH_PROBABILITY, currentMark.equal(unlockedMark))) {
! traceObject(trace, "+lock{cas}", object, true);
counters.lockCas.inc();
! mark(object);
return;
} else {
trace(trace, " currentMark: 0x%016lx\n", currentMark);
// The mark word in the object header was not the same.
// Either the object is locked by another thread or is already locked
*** 285,312 ****
//
// (currentMark - rsp) & (aligned_mask - page_size)
//
// assuming both the stack pointer and page_size have their least
// significant 2 bits cleared and page_size is a power of 2
! final Word alignedMask = WordFactory.unsigned(wordSize() - 1);
! final Word stackPointer = registerAsWord(stackPointerRegister).add(config(INJECTED_VMCONFIG).stackBias);
! if (probability(FAST_PATH_PROBABILITY, currentMark.subtract(stackPointer).and(alignedMask.subtract(pageSize())).equal(0))) {
// Recursively locked => write 0 to the lock slot
! lock.writeWord(lockDisplacedMarkOffset(INJECTED_VMCONFIG), WordFactory.zero(), DISPLACED_MARK_WORD_LOCATION);
! traceObject(trace, "+lock{cas:recursive}", object, true, options);
counters.lockCasRecursive.inc();
return;
}
! traceObject(trace, "+lock{stub:failed-cas/stack}", object, true, options);
counters.lockStubFailedCas.inc();
}
}
// slow-path runtime-call
monitorenterStubC(MONITORENTER, object, lock);
}
! private static boolean tryEnterBiased(Object object, KlassPointer hub, Word lock, Word mark, Register threadRegister, boolean trace, OptionValues options, Counters counters) {
// See whether the lock is currently biased toward our thread and
// whether the epoch is still valid.
// Note that the runtime guarantees sufficient alignment of JavaThread
// pointers to allow age to be placed into low bits.
final Word biasableLockBits = mark.and(biasedLockMaskInPlace(INJECTED_VMCONFIG));
--- 289,316 ----
//
// (currentMark - rsp) & (aligned_mask - page_size)
//
// assuming both the stack pointer and page_size have their least
// significant 2 bits cleared and page_size is a power of 2
! final Word alignedMask = unsigned(wordSize() - 1);
! final Word stackPointer = registerAsWord(stackPointerRegister).add(stackBias(INJECTED_VMCONFIG));
! if (probability(FAST_PATH_PROBABILITY, currentMark.subtract(stackPointer).and(alignedMask.subtract(pageSize(INJECTED_VMCONFIG))).equal(0))) {
// Recursively locked => write 0 to the lock slot
! lock.writeWord(lockDisplacedMarkOffset(INJECTED_VMCONFIG), zero(), DISPLACED_MARK_WORD_LOCATION);
! traceObject(trace, "+lock{cas:recursive}", object, true);
counters.lockCasRecursive.inc();
return;
}
! traceObject(trace, "+lock{stub:failed-cas/stack}", object, true);
counters.lockStubFailedCas.inc();
}
}
// slow-path runtime-call
monitorenterStubC(MONITORENTER, object, lock);
}
! private static boolean tryEnterBiased(Object object, KlassPointer hub, Word lock, Word mark, Register threadRegister, boolean trace, Counters counters) {
// See whether the lock is currently biased toward our thread and
// whether the epoch is still valid.
// Note that the runtime guarantees sufficient alignment of JavaThread
// pointers to allow age to be placed into low bits.
final Word biasableLockBits = mark.and(biasedLockMaskInPlace(INJECTED_VMCONFIG));
*** 319,329 ****
trace(trace, "prototypeMarkWord: 0x%016lx\n", prototypeMarkWord);
trace(trace, " thread: 0x%016lx\n", thread);
trace(trace, " tmp: 0x%016lx\n", tmp);
if (probability(FAST_PATH_PROBABILITY, tmp.equal(0))) {
// Object is already biased to current thread -> done
! traceObject(trace, "+lock{bias:existing}", object, true, options);
counters.lockBiasExisting.inc();
FastAcquireBiasedLockNode.mark(object);
return true;
}
--- 323,333 ----
trace(trace, "prototypeMarkWord: 0x%016lx\n", prototypeMarkWord);
trace(trace, " thread: 0x%016lx\n", thread);
trace(trace, " tmp: 0x%016lx\n", tmp);
if (probability(FAST_PATH_PROBABILITY, tmp.equal(0))) {
// Object is already biased to current thread -> done
! traceObject(trace, "+lock{bias:existing}", object, true);
counters.lockBiasExisting.inc();
FastAcquireBiasedLockNode.mark(object);
return true;
}
*** 360,377 ****
Word biasedMark = unbiasedMark.or(thread);
trace(trace, " unbiasedMark: 0x%016lx\n", unbiasedMark);
trace(trace, " biasedMark: 0x%016lx\n", biasedMark);
if (probability(VERY_FAST_PATH_PROBABILITY, objectPointer.logicCompareAndSwapWord(markOffset(INJECTED_VMCONFIG), unbiasedMark, biasedMark, MARK_WORD_LOCATION))) {
// Object is now biased to current thread -> done
! traceObject(trace, "+lock{bias:acquired}", object, true, options);
counters.lockBiasAcquired.inc();
return true;
}
// If the biasing toward our thread failed, this means that another thread
// owns the bias and we need to revoke that bias. The revocation will occur
// in the interpreter runtime.
! traceObject(trace, "+lock{stub:revoke}", object, true, options);
counters.lockStubRevoke.inc();
} else {
// At this point we know the epoch has expired, meaning that the
// current bias owner, if any, is actually invalid. Under these
// circumstances _only_, are we allowed to use the current mark word
--- 364,381 ----
Word biasedMark = unbiasedMark.or(thread);
trace(trace, " unbiasedMark: 0x%016lx\n", unbiasedMark);
trace(trace, " biasedMark: 0x%016lx\n", biasedMark);
if (probability(VERY_FAST_PATH_PROBABILITY, objectPointer.logicCompareAndSwapWord(markOffset(INJECTED_VMCONFIG), unbiasedMark, biasedMark, MARK_WORD_LOCATION))) {
// Object is now biased to current thread -> done
! traceObject(trace, "+lock{bias:acquired}", object, true);
counters.lockBiasAcquired.inc();
return true;
}
// If the biasing toward our thread failed, this means that another thread
// owns the bias and we need to revoke that bias. The revocation will occur
// in the interpreter runtime.
! traceObject(trace, "+lock{stub:revoke}", object, true);
counters.lockStubRevoke.inc();
} else {
// At this point we know the epoch has expired, meaning that the
// current bias owner, if any, is actually invalid. Under these
// circumstances _only_, are we allowed to use the current mark word
*** 380,397 ****
// the bias from one thread to another directly in this situation.
Word biasedMark = prototypeMarkWord.or(thread);
trace(trace, " biasedMark: 0x%016lx\n", biasedMark);
if (probability(VERY_FAST_PATH_PROBABILITY, objectPointer.logicCompareAndSwapWord(markOffset(INJECTED_VMCONFIG), mark, biasedMark, MARK_WORD_LOCATION))) {
// Object is now biased to current thread -> done
! traceObject(trace, "+lock{bias:transfer}", object, true, options);
counters.lockBiasTransfer.inc();
return true;
}
// If the biasing toward our thread failed, then another thread
// succeeded in biasing it toward itself and we need to revoke that
// bias. The revocation will occur in the runtime in the slow case.
! traceObject(trace, "+lock{stub:epoch-expired}", object, true, options);
counters.lockStubEpochExpired.inc();
}
// slow-path runtime-call
monitorenterStubC(MONITORENTER, object, lock);
return true;
--- 384,401 ----
// the bias from one thread to another directly in this situation.
Word biasedMark = prototypeMarkWord.or(thread);
trace(trace, " biasedMark: 0x%016lx\n", biasedMark);
if (probability(VERY_FAST_PATH_PROBABILITY, objectPointer.logicCompareAndSwapWord(markOffset(INJECTED_VMCONFIG), mark, biasedMark, MARK_WORD_LOCATION))) {
// Object is now biased to current thread -> done
! traceObject(trace, "+lock{bias:transfer}", object, true);
counters.lockBiasTransfer.inc();
return true;
}
// If the biasing toward our thread failed, then another thread
// succeeded in biasing it toward itself and we need to revoke that
// bias. The revocation will occur in the runtime in the slow case.
! traceObject(trace, "+lock{stub:epoch-expired}", object, true);
counters.lockStubEpochExpired.inc();
}
// slow-path runtime-call
monitorenterStubC(MONITORENTER, object, lock);
return true;
*** 422,488 ****
return false;
}
}
@Fold
! public static boolean useFastInflatedLocking(OptionValues options) {
return SimpleFastInflatedLocking.getValue(options);
}
! private static boolean inlineFastLockSupported(OptionValues options) {
! return inlineFastLockSupported(INJECTED_VMCONFIG, options);
}
private static boolean inlineFastLockSupported(GraalHotSpotVMConfig config, OptionValues options) {
return useFastInflatedLocking(options) && monitorMask(config) >= 0 && objectMonitorOwnerOffset(config) >= 0;
}
! private static boolean tryEnterInflated(Object object, Word lock, Word mark, Register threadRegister, boolean trace, OptionValues options, Counters counters) {
// write non-zero value to lock slot
lock.writeWord(lockDisplacedMarkOffset(INJECTED_VMCONFIG), lock, DISPLACED_MARK_WORD_LOCATION);
// mark is a pointer to the ObjectMonitor + monitorMask
Word monitor = mark.subtract(monitorMask(INJECTED_VMCONFIG));
int ownerOffset = objectMonitorOwnerOffset(INJECTED_VMCONFIG);
Word owner = monitor.readWord(ownerOffset, OBJECT_MONITOR_OWNER_LOCATION);
if (probability(FREQUENT_PROBABILITY, owner.equal(0))) {
// it appears unlocked (owner == 0)
if (probability(FREQUENT_PROBABILITY, monitor.logicCompareAndSwapWord(ownerOffset, owner, registerAsWord(threadRegister), OBJECT_MONITOR_OWNER_LOCATION))) {
// success
! traceObject(trace, "+lock{inflated:cas}", object, true, options);
counters.inflatedCas.inc();
return true;
} else {
! traceObject(trace, "+lock{stub:inflated:failed-cas}", object, true, options);
counters.inflatedFailedCas.inc();
}
} else {
! traceObject(trace, "+lock{stub:inflated:owned}", object, true, options);
counters.inflatedOwned.inc();
}
return false;
}
/**
* Calls straight out to the monitorenter stub.
*/
@Snippet
! public static void monitorenterStub(Object object, @ConstantParameter int lockDepth, @ConstantParameter boolean trace, @ConstantParameter OptionValues options) {
verifyOop(object);
! incCounter(options);
if (object == null) {
DeoptimizeNode.deopt(DeoptimizationAction.InvalidateReprofile, DeoptimizationReason.NullCheckException);
}
// BeginLockScope nodes do not read from object so a use of object
// cannot float about the null check above
final Word lock = beginLockScope(lockDepth);
! traceObject(trace, "+lock{stub}", object, true, options);
monitorenterStubC(MONITORENTER, object, lock);
}
@Snippet
public static void monitorexit(Object object, @ConstantParameter int lockDepth, @ConstantParameter Register threadRegister, @ConstantParameter boolean trace,
! @ConstantParameter OptionValues options, @ConstantParameter Counters counters) {
trace(trace, " object: 0x%016lx\n", Word.objectToTrackedPointer(object));
final Word mark = loadWordFromObject(object, markOffset(INJECTED_VMCONFIG));
if (useBiasedLocking(INJECTED_VMCONFIG)) {
// Check for biased locking unlock case, which is a no-op
// Note: we do not have to check the thread ID for two reasons.
--- 426,492 ----
return false;
}
}
@Fold
! public static boolean useFastInflatedLocking(@Fold.InjectedParameter OptionValues options) {
return SimpleFastInflatedLocking.getValue(options);
}
! private static boolean inlineFastLockSupported() {
! return inlineFastLockSupported(INJECTED_VMCONFIG, INJECTED_OPTIONVALUES);
}
private static boolean inlineFastLockSupported(GraalHotSpotVMConfig config, OptionValues options) {
return useFastInflatedLocking(options) && monitorMask(config) >= 0 && objectMonitorOwnerOffset(config) >= 0;
}
! private static boolean tryEnterInflated(Object object, Word lock, Word mark, Register threadRegister, boolean trace, Counters counters) {
// write non-zero value to lock slot
lock.writeWord(lockDisplacedMarkOffset(INJECTED_VMCONFIG), lock, DISPLACED_MARK_WORD_LOCATION);
// mark is a pointer to the ObjectMonitor + monitorMask
Word monitor = mark.subtract(monitorMask(INJECTED_VMCONFIG));
int ownerOffset = objectMonitorOwnerOffset(INJECTED_VMCONFIG);
Word owner = monitor.readWord(ownerOffset, OBJECT_MONITOR_OWNER_LOCATION);
if (probability(FREQUENT_PROBABILITY, owner.equal(0))) {
// it appears unlocked (owner == 0)
if (probability(FREQUENT_PROBABILITY, monitor.logicCompareAndSwapWord(ownerOffset, owner, registerAsWord(threadRegister), OBJECT_MONITOR_OWNER_LOCATION))) {
// success
! traceObject(trace, "+lock{inflated:cas}", object, true);
counters.inflatedCas.inc();
return true;
} else {
! traceObject(trace, "+lock{stub:inflated:failed-cas}", object, true);
counters.inflatedFailedCas.inc();
}
} else {
! traceObject(trace, "+lock{stub:inflated:owned}", object, true);
counters.inflatedOwned.inc();
}
return false;
}
/**
* Calls straight out to the monitorenter stub.
*/
@Snippet
! public static void monitorenterStub(Object object, @ConstantParameter int lockDepth, @ConstantParameter boolean trace) {
verifyOop(object);
! incCounter();
if (object == null) {
DeoptimizeNode.deopt(DeoptimizationAction.InvalidateReprofile, DeoptimizationReason.NullCheckException);
}
// BeginLockScope nodes do not read from object so a use of object
// cannot float about the null check above
final Word lock = beginLockScope(lockDepth);
! traceObject(trace, "+lock{stub}", object, true);
monitorenterStubC(MONITORENTER, object, lock);
}
@Snippet
public static void monitorexit(Object object, @ConstantParameter int lockDepth, @ConstantParameter Register threadRegister, @ConstantParameter boolean trace,
! @ConstantParameter Counters counters) {
trace(trace, " object: 0x%016lx\n", Word.objectToTrackedPointer(object));
final Word mark = loadWordFromObject(object, markOffset(INJECTED_VMCONFIG));
if (useBiasedLocking(INJECTED_VMCONFIG)) {
// Check for biased locking unlock case, which is a no-op
// Note: we do not have to check the thread ID for two reasons.
*** 491,502 ****
// lock, the object could not be rebiased toward another thread, so
// the bias bit would be clear.
trace(trace, " mark: 0x%016lx\n", mark);
if (probability(FREQUENT_PROBABILITY, mark.and(biasedLockMaskInPlace(INJECTED_VMCONFIG)).equal(WordFactory.unsigned(biasedLockPattern(INJECTED_VMCONFIG))))) {
endLockScope();
! decCounter(options);
! traceObject(trace, "-lock{bias}", object, false, options);
counters.unlockBias.inc();
return;
}
}
--- 495,506 ----
// lock, the object could not be rebiased toward another thread, so
// the bias bit would be clear.
trace(trace, " mark: 0x%016lx\n", mark);
if (probability(FREQUENT_PROBABILITY, mark.and(biasedLockMaskInPlace(INJECTED_VMCONFIG)).equal(WordFactory.unsigned(biasedLockPattern(INJECTED_VMCONFIG))))) {
endLockScope();
! decCounter();
! traceObject(trace, "-lock{bias}", object, false);
counters.unlockBias.inc();
return;
}
}
*** 506,538 ****
final Word displacedMark = lock.readWord(lockDisplacedMarkOffset(INJECTED_VMCONFIG), DISPLACED_MARK_WORD_LOCATION);
trace(trace, " displacedMark: 0x%016lx\n", displacedMark);
if (probability(NOT_LIKELY_PROBABILITY, displacedMark.equal(0))) {
// Recursive locking => done
! traceObject(trace, "-lock{recursive}", object, false, options);
counters.unlockCasRecursive.inc();
} else {
! if (!tryExitInflated(object, mark, lock, threadRegister, trace, options, counters)) {
verifyOop(object);
// Test if object's mark word is pointing to the displaced mark word, and if so,
// restore
// the displaced mark in the object - if the object's mark word is not pointing to
// the displaced mark word, do unlocking via runtime call.
Pointer objectPointer = Word.objectToTrackedPointer(object);
if (probability(VERY_FAST_PATH_PROBABILITY, objectPointer.logicCompareAndSwapWord(markOffset(INJECTED_VMCONFIG), lock, displacedMark, MARK_WORD_LOCATION))) {
! traceObject(trace, "-lock{cas}", object, false, options);
counters.unlockCas.inc();
} else {
// The object's mark word was not pointing to the displaced header
! traceObject(trace, "-lock{stub}", object, false, options);
counters.unlockStub.inc();
monitorexitStubC(MONITOREXIT, object, lock);
}
}
}
endLockScope();
! decCounter(options);
}
private static boolean inlineFastUnlockSupported(OptionValues options) {
return inlineFastUnlockSupported(INJECTED_VMCONFIG, options);
}
--- 510,542 ----
final Word displacedMark = lock.readWord(lockDisplacedMarkOffset(INJECTED_VMCONFIG), DISPLACED_MARK_WORD_LOCATION);
trace(trace, " displacedMark: 0x%016lx\n", displacedMark);
if (probability(NOT_LIKELY_PROBABILITY, displacedMark.equal(0))) {
// Recursive locking => done
! traceObject(trace, "-lock{recursive}", object, false);
counters.unlockCasRecursive.inc();
} else {
! if (!tryExitInflated(object, mark, lock, threadRegister, trace, counters)) {
verifyOop(object);
// Test if object's mark word is pointing to the displaced mark word, and if so,
// restore
// the displaced mark in the object - if the object's mark word is not pointing to
// the displaced mark word, do unlocking via runtime call.
Pointer objectPointer = Word.objectToTrackedPointer(object);
if (probability(VERY_FAST_PATH_PROBABILITY, objectPointer.logicCompareAndSwapWord(markOffset(INJECTED_VMCONFIG), lock, displacedMark, MARK_WORD_LOCATION))) {
! traceObject(trace, "-lock{cas}", object, false);
counters.unlockCas.inc();
} else {
// The object's mark word was not pointing to the displaced header
! traceObject(trace, "-lock{stub}", object, false);
counters.unlockStub.inc();
monitorexitStubC(MONITOREXIT, object, lock);
}
}
}
endLockScope();
! decCounter();
}
private static boolean inlineFastUnlockSupported(OptionValues options) {
return inlineFastUnlockSupported(INJECTED_VMCONFIG, options);
}
*** 540,551 ****
private static boolean inlineFastUnlockSupported(GraalHotSpotVMConfig config, OptionValues options) {
return useFastInflatedLocking(options) && objectMonitorEntryListOffset(config) >= 0 && objectMonitorCxqOffset(config) >= 0 && monitorMask(config) >= 0 &&
objectMonitorOwnerOffset(config) >= 0 && objectMonitorRecursionsOffset(config) >= 0;
}
! private static boolean tryExitInflated(Object object, Word mark, Word lock, Register threadRegister, boolean trace, OptionValues options, Counters counters) {
! if (!inlineFastUnlockSupported(options)) {
return false;
}
if (probability(SLOW_PATH_PROBABILITY, mark.and(monitorMask(INJECTED_VMCONFIG)).notEqual(0))) {
// Inflated case
// mark is a pointer to the ObjectMonitor + monitorMask
--- 544,555 ----
private static boolean inlineFastUnlockSupported(GraalHotSpotVMConfig config, OptionValues options) {
return useFastInflatedLocking(options) && objectMonitorEntryListOffset(config) >= 0 && objectMonitorCxqOffset(config) >= 0 && monitorMask(config) >= 0 &&
objectMonitorOwnerOffset(config) >= 0 && objectMonitorRecursionsOffset(config) >= 0;
}
! private static boolean tryExitInflated(Object object, Word mark, Word lock, Register threadRegister, boolean trace, Counters counters) {
! if (!inlineFastUnlockSupported(INJECTED_OPTIONVALUES)) {
return false;
}
if (probability(SLOW_PATH_PROBABILITY, mark.and(monitorMask(INJECTED_VMCONFIG)).notEqual(0))) {
// Inflated case
// mark is a pointer to the ObjectMonitor + monitorMask
*** 563,602 ****
Word entryList = monitor.readWord(entryListOffset, OBJECT_MONITOR_ENTRY_LIST_LOCATION);
if (probability(FREQUENT_PROBABILITY, cxq.or(entryList).equal(0))) {
// cxq == 0 && entryList == 0
// Nobody is waiting, success
// release_store
! MembarNode.memoryBarrier(LOAD_STORE | STORE_STORE);
! monitor.writeWord(ownerOffset, WordFactory.zero());
! traceObject(trace, "-lock{inflated:simple}", object, false, options);
counters.unlockInflatedSimple.inc();
return true;
}
}
counters.unlockStubInflated.inc();
! traceObject(trace, "-lock{stub:inflated}", object, false, options);
monitorexitStubC(MONITOREXIT, object, lock);
return true;
}
return false;
}
/**
* Calls straight out to the monitorexit stub.
*/
@Snippet
! public static void monitorexitStub(Object object, @ConstantParameter int lockDepth, @ConstantParameter boolean trace, @ConstantParameter OptionValues options) {
verifyOop(object);
! traceObject(trace, "-lock{stub}", object, false, options);
final Word lock = CurrentLockNode.currentLock(lockDepth);
monitorexitStubC(MONITOREXIT, object, lock);
endLockScope();
! decCounter(options);
}
! public static void traceObject(boolean enabled, String action, Object object, boolean enter, OptionValues options) {
! if (doProfile(options)) {
DynamicCounterNode.counter(enter ? "number of monitor enters" : "number of monitor exits", action, 1, PROFILE_CONTEXT);
}
if (enabled) {
Log.print(action);
Log.print(' ');
--- 567,606 ----
Word entryList = monitor.readWord(entryListOffset, OBJECT_MONITOR_ENTRY_LIST_LOCATION);
if (probability(FREQUENT_PROBABILITY, cxq.or(entryList).equal(0))) {
// cxq == 0 && entryList == 0
// Nobody is waiting, success
// release_store
! memoryBarrier(LOAD_STORE | STORE_STORE);
! monitor.writeWord(ownerOffset, zero());
! traceObject(trace, "-lock{inflated:simple}", object, false);
counters.unlockInflatedSimple.inc();
return true;
}
}
counters.unlockStubInflated.inc();
! traceObject(trace, "-lock{stub:inflated}", object, false);
monitorexitStubC(MONITOREXIT, object, lock);
return true;
}
return false;
}
/**
* Calls straight out to the monitorexit stub.
*/
@Snippet
! public static void monitorexitStub(Object object, @ConstantParameter int lockDepth, @ConstantParameter boolean trace) {
verifyOop(object);
! traceObject(trace, "-lock{stub}", object, false);
final Word lock = CurrentLockNode.currentLock(lockDepth);
monitorexitStubC(MONITOREXIT, object, lock);
endLockScope();
! decCounter();
}
! public static void traceObject(boolean enabled, String action, Object object, boolean enter) {
! if (doProfile(INJECTED_OPTIONVALUES)) {
DynamicCounterNode.counter(enter ? "number of monitor enters" : "number of monitor exits", action, 1, PROFILE_CONTEXT);
}
if (enabled) {
Log.print(action);
Log.print(' ');
*** 620,643 ****
@NodeIntrinsic(BreakpointNode.class)
static native void bkpt(Object object, Word mark, Word tmp, Word value);
@Fold
! static boolean verifyBalancedMonitors(OptionValues options) {
return VerifyBalancedMonitors.getValue(options);
}
! public static void incCounter(OptionValues options) {
! if (verifyBalancedMonitors(options)) {
final Word counter = MonitorCounterNode.counter();
final int count = counter.readInt(0, MONITOR_COUNTER_LOCATION);
counter.writeInt(0, count + 1, MONITOR_COUNTER_LOCATION);
}
}
! public static void decCounter(OptionValues options) {
! if (verifyBalancedMonitors(options)) {
final Word counter = MonitorCounterNode.counter();
final int count = counter.readInt(0, MONITOR_COUNTER_LOCATION);
counter.writeInt(0, count - 1, MONITOR_COUNTER_LOCATION);
}
}
--- 624,647 ----
@NodeIntrinsic(BreakpointNode.class)
static native void bkpt(Object object, Word mark, Word tmp, Word value);
@Fold
! static boolean verifyBalancedMonitors(@Fold.InjectedParameter OptionValues options) {
return VerifyBalancedMonitors.getValue(options);
}
! static void incCounter() {
! if (verifyBalancedMonitors(INJECTED_OPTIONVALUES)) {
final Word counter = MonitorCounterNode.counter();
final int count = counter.readInt(0, MONITOR_COUNTER_LOCATION);
counter.writeInt(0, count + 1, MONITOR_COUNTER_LOCATION);
}
}
! public static void decCounter() {
! if (verifyBalancedMonitors(INJECTED_OPTIONVALUES)) {
final Word counter = MonitorCounterNode.counter();
final int count = counter.readInt(0, MONITOR_COUNTER_LOCATION);
counter.writeInt(0, count - 1, MONITOR_COUNTER_LOCATION);
}
}
*** 748,765 ****
args.add("hub", monitorenterNode.getHub());
args.addConst("lockDepth", monitorenterNode.getMonitorId().getLockDepth());
args.addConst("threadRegister", registers.getThreadRegister());
args.addConst("stackPointerRegister", registers.getStackPointerRegister());
args.addConst("trace", isTracingEnabledForType(monitorenterNode.object()) || isTracingEnabledForMethod(graph));
- args.addConst("options", graph.getOptions());
args.addConst("counters", counters);
} else {
args = new Arguments(monitorenterStub, graph.getGuardsStage(), tool.getLoweringStage());
args.add("object", monitorenterNode.object());
args.addConst("lockDepth", monitorenterNode.getMonitorId().getLockDepth());
args.addConst("trace", isTracingEnabledForType(monitorenterNode.object()) || isTracingEnabledForMethod(graph));
- args.addConst("options", graph.getOptions());
args.addConst("counters", counters);
}
template(monitorenterNode, args).instantiate(providers.getMetaAccess(), monitorenterNode, DEFAULT_REPLACER, args);
}
--- 752,767 ----
*** 775,785 ****
}
args.add("object", monitorexitNode.object());
args.addConst("lockDepth", monitorexitNode.getMonitorId().getLockDepth());
args.addConst("threadRegister", registers.getThreadRegister());
args.addConst("trace", isTracingEnabledForType(monitorexitNode.object()) || isTracingEnabledForMethod(graph));
- args.addConst("options", graph.getOptions());
args.addConst("counters", counters);
template(monitorexitNode, args).instantiate(providers.getMetaAccess(), monitorexitNode, DEFAULT_REPLACER, args);
}
--- 777,786 ----
< prev index next >