--- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/MonitorSnippets.java 2017-03-20 17:38:58.000000000 -0700 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/MonitorSnippets.java 2017-03-20 17:38:58.000000000 -0700 @@ -22,14 +22,18 @@ */ package org.graalvm.compiler.hotspot.replacements; -import static org.graalvm.compiler.core.common.GraalOptions.SnippetCounters; +import static jdk.vm.ci.code.MemoryBarriers.LOAD_STORE; +import static jdk.vm.ci.code.MemoryBarriers.STORE_STORE; import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFIG; import static org.graalvm.compiler.hotspot.nodes.BeginLockScopeNode.beginLockScope; -import static org.graalvm.compiler.hotspot.nodes.DirectCompareAndSwapNode.compareAndSwap; import static org.graalvm.compiler.hotspot.nodes.EndLockScopeNode.endLockScope; import static org.graalvm.compiler.hotspot.nodes.VMErrorNode.vmError; import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.DISPLACED_MARK_WORD_LOCATION; import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.MARK_WORD_LOCATION; +import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.OBJECT_MONITOR_CXQ_LOCATION; +import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.OBJECT_MONITOR_ENTRY_LIST_LOCATION; +import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.OBJECT_MONITOR_OWNER_LOCATION; +import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.OBJECT_MONITOR_RECURSION_LOCATION; import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.PROTOTYPE_MARK_WORD_LOCATION; import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.ageMaskInPlace; import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.biasedLockMaskInPlace; @@ -39,6 +43,11 @@ import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.loadWordFromObject; import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.lockDisplacedMarkOffset; import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.markOffset; +import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.monitorMask; +import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.objectMonitorCxqOffset; +import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.objectMonitorEntryListOffset; +import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.objectMonitorOwnerOffset; +import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.objectMonitorRecursionsOffset; import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.pageSize; import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.prototypeMarkWordOffset; import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.registerAsWord; @@ -47,12 +56,16 @@ import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.verifyOop; import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.wordSize; import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.ProfileMonitors; +import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.SimpleFastInflatedLocking; import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.TraceMonitorsMethodFilter; import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.TraceMonitorsTypeFilter; import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.VerifyBalancedMonitors; +import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FAST_PATH_PROBABILITY; import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FREQUENT_PROBABILITY; +import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.NOT_FREQUENT_PROBABILITY; +import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.NOT_LIKELY_PROBABILITY; +import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY; import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.VERY_FAST_PATH_PROBABILITY; -import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.VERY_SLOW_PATH_PROBABILITY; import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability; import static org.graalvm.compiler.replacements.SnippetTemplate.DEFAULT_REPLACER; @@ -71,11 +84,11 @@ import org.graalvm.compiler.graph.Node.ConstantNodeParameter; import org.graalvm.compiler.graph.Node.NodeIntrinsic; import org.graalvm.compiler.graph.iterators.NodeIterable; +import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig; import org.graalvm.compiler.hotspot.meta.HotSpotProviders; import org.graalvm.compiler.hotspot.meta.HotSpotRegistersProvider; import org.graalvm.compiler.hotspot.nodes.AcquiredCASLockNode; import org.graalvm.compiler.hotspot.nodes.CurrentLockNode; -import org.graalvm.compiler.hotspot.nodes.DirectCompareAndSwapNode; import org.graalvm.compiler.hotspot.nodes.FastAcquireBiasedLockNode; import org.graalvm.compiler.hotspot.nodes.MonitorCounterNode; import org.graalvm.compiler.hotspot.word.KlassPointer; @@ -90,14 +103,14 @@ import org.graalvm.compiler.nodes.StructuredGraph; import org.graalvm.compiler.nodes.ValueNode; import org.graalvm.compiler.nodes.debug.DynamicCounterNode; -import org.graalvm.compiler.nodes.extended.BranchProbabilityNode; import org.graalvm.compiler.nodes.extended.ForeignCallNode; +import org.graalvm.compiler.nodes.extended.MembarNode; import org.graalvm.compiler.nodes.java.MethodCallTargetNode; import org.graalvm.compiler.nodes.java.MonitorExitNode; import org.graalvm.compiler.nodes.java.RawMonitorEnterNode; -import org.graalvm.compiler.nodes.memory.address.OffsetAddressNode; import org.graalvm.compiler.nodes.spi.LoweringTool; import org.graalvm.compiler.nodes.type.StampTool; +import org.graalvm.compiler.options.OptionValues; import org.graalvm.compiler.phases.common.inlining.InliningUtil; import org.graalvm.compiler.replacements.Log; import org.graalvm.compiler.replacements.SnippetCounter; @@ -105,6 +118,7 @@ import org.graalvm.compiler.replacements.SnippetTemplate.Arguments; import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo; import org.graalvm.compiler.replacements.Snippets; +import org.graalvm.compiler.word.Pointer; import org.graalvm.compiler.word.Word; import org.graalvm.compiler.word.WordBase; @@ -114,7 +128,6 @@ import jdk.vm.ci.meta.DeoptimizationAction; import jdk.vm.ci.meta.DeoptimizationReason; import jdk.vm.ci.meta.JavaType; -import jdk.vm.ci.meta.ResolvedJavaMethod; import jdk.vm.ci.meta.ResolvedJavaType; /** @@ -201,13 +214,13 @@ private static final boolean PROFILE_CONTEXT = false; @Fold - static boolean doProfile() { - return ProfileMonitors.getValue(); + static boolean doProfile(OptionValues options) { + return ProfileMonitors.getValue(options); } @Snippet public static void monitorenter(Object object, KlassPointer hub, @ConstantParameter int lockDepth, @ConstantParameter Register threadRegister, @ConstantParameter Register stackPointerRegister, - @ConstantParameter boolean trace) { + @ConstantParameter boolean trace, @ConstantParameter OptionValues options, @ConstantParameter Counters counters) { verifyOop(object); // Load the mark word - this includes a null-check on object @@ -215,197 +228,256 @@ final Word lock = beginLockScope(lockDepth); - trace(trace, " object: 0x%016lx\n", Word.objectToTrackedPointer(object)); + Pointer objectPointer = Word.objectToTrackedPointer(object); + trace(trace, " object: 0x%016lx\n", objectPointer); trace(trace, " lock: 0x%016lx\n", lock); trace(trace, " mark: 0x%016lx\n", mark); - incCounter(); + incCounter(options); if (useBiasedLocking(INJECTED_VMCONFIG)) { - // See whether the lock is currently biased toward our thread and - // whether the epoch is still valid. - // Note that the runtime guarantees sufficient alignment of JavaThread - // pointers to allow age to be placed into low bits. - final Word biasableLockBits = mark.and(biasedLockMaskInPlace(INJECTED_VMCONFIG)); - - // Check whether the bias pattern is present in the object's mark word - // and the bias owner and the epoch are both still current. - final Word prototypeMarkWord = hub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION); - final Word thread = registerAsWord(threadRegister); - final Word tmp = prototypeMarkWord.or(thread).xor(mark).and(~ageMaskInPlace(INJECTED_VMCONFIG)); - trace(trace, "prototypeMarkWord: 0x%016lx\n", prototypeMarkWord); - trace(trace, " thread: 0x%016lx\n", thread); - trace(trace, " tmp: 0x%016lx\n", tmp); - if (probability(BranchProbabilityNode.NOT_LIKELY_PROBABILITY, tmp.equal(0))) { - // Object is already biased to current thread -> done - traceObject(trace, "+lock{bias:existing}", object, true); - lockBiasExisting.inc(); - FastAcquireBiasedLockNode.mark(object); + if (tryEnterBiased(object, hub, lock, mark, threadRegister, trace, options, counters)) { return; } - - // Now check to see whether biasing is enabled for this object - if (probability(BranchProbabilityNode.FAST_PATH_PROBABILITY, biasableLockBits.notEqual(Word.unsigned(biasedLockPattern(INJECTED_VMCONFIG))))) { - // Biasing not enabled -> fall through to lightweight locking - unbiasable.inc(); + // not biased, fall-through + } + if (inlineFastLockSupported(options) && probability(SLOW_PATH_PROBABILITY, mark.and(monitorMask(INJECTED_VMCONFIG)).notEqual(0))) { + // Inflated case + if (tryEnterInflated(object, lock, mark, threadRegister, trace, options, counters)) { + return; + } + } else { + // Create the unlocked mark word pattern + Word unlockedMark = mark.or(unlockedMask(INJECTED_VMCONFIG)); + trace(trace, " unlockedMark: 0x%016lx\n", unlockedMark); + + // Copy this unlocked mark word into the lock slot on the stack + lock.writeWord(lockDisplacedMarkOffset(INJECTED_VMCONFIG), unlockedMark, DISPLACED_MARK_WORD_LOCATION); + + // make sure previous store does not float below compareAndSwap + MembarNode.memoryBarrier(STORE_STORE); + + // Test if the object's mark word is unlocked, and if so, store the + // (address of) the lock slot into the object's mark word. + Word currentMark = objectPointer.compareAndSwapWord(markOffset(INJECTED_VMCONFIG), unlockedMark, lock, MARK_WORD_LOCATION); + if (probability(FAST_PATH_PROBABILITY, currentMark.equal(unlockedMark))) { + traceObject(trace, "+lock{cas}", object, true, options); + counters.lockCas.inc(); + AcquiredCASLockNode.mark(object); + return; } else { - // At this point we know that the mark word has the bias pattern and - // that we are not the bias owner in the current epoch. We need to - // figure out more details about the state of the mark word in order to - // know what operations can be legally performed on the object's - // mark word. - - // If the low three bits in the xor result aren't clear, that means - // the prototype header is no longer biasable and we have to revoke - // the bias on this object. - if (probability(FREQUENT_PROBABILITY, tmp.and(biasedLockMaskInPlace(INJECTED_VMCONFIG)).equal(0))) { - // Biasing is still enabled for object's type. See whether the - // epoch of the current bias is still valid, meaning that the epoch - // bits of the mark word are equal to the epoch bits of the - // prototype mark word. (Note that the prototype mark word's epoch bits - // only change at a safepoint.) If not, attempt to rebias the object - // toward the current thread. Note that we must be absolutely sure - // that the current epoch is invalid in order to do this because - // otherwise the manipulations it performs on the mark word are - // illegal. - if (probability(FREQUENT_PROBABILITY, tmp.and(epochMaskInPlace(INJECTED_VMCONFIG)).equal(0))) { - // The epoch of the current bias is still valid but we know nothing - // about the owner; it might be set or it might be clear. Try to - // acquire the bias of the object using an atomic operation. If this - // fails we will go in to the runtime to revoke the object's bias. - // Note that we first construct the presumed unbiased header so we - // don't accidentally blow away another thread's valid bias. - Word unbiasedMark = mark.and(biasedLockMaskInPlace(INJECTED_VMCONFIG) | ageMaskInPlace(INJECTED_VMCONFIG) | epochMaskInPlace(INJECTED_VMCONFIG)); - Word biasedMark = unbiasedMark.or(thread); - trace(trace, " unbiasedMark: 0x%016lx\n", unbiasedMark); - trace(trace, " biasedMark: 0x%016lx\n", biasedMark); - if (probability(VERY_FAST_PATH_PROBABILITY, - compareAndSwap(OffsetAddressNode.address(object, markOffset(INJECTED_VMCONFIG)), unbiasedMark, biasedMark, MARK_WORD_LOCATION).equal(unbiasedMark))) { - // Object is now biased to current thread -> done - traceObject(trace, "+lock{bias:acquired}", object, true); - lockBiasAcquired.inc(); - return; - } - // If the biasing toward our thread failed, this means that another thread - // owns the bias and we need to revoke that bias. The revocation will occur - // in the interpreter runtime. - traceObject(trace, "+lock{stub:revoke}", object, true); - lockStubRevoke.inc(); - } else { - // At this point we know the epoch has expired, meaning that the - // current bias owner, if any, is actually invalid. Under these - // circumstances _only_, are we allowed to use the current mark word - // value as the comparison value when doing the CAS to acquire the - // bias in the current epoch. In other words, we allow transfer of - // the bias from one thread to another directly in this situation. - Word biasedMark = prototypeMarkWord.or(thread); - trace(trace, " biasedMark: 0x%016lx\n", biasedMark); - if (probability(VERY_FAST_PATH_PROBABILITY, - compareAndSwap(OffsetAddressNode.address(object, markOffset(INJECTED_VMCONFIG)), mark, biasedMark, MARK_WORD_LOCATION).equal(mark))) { - // Object is now biased to current thread -> done - traceObject(trace, "+lock{bias:transfer}", object, true); - lockBiasTransfer.inc(); - return; - } - // If the biasing toward our thread failed, then another thread - // succeeded in biasing it toward itself and we need to revoke that - // bias. The revocation will occur in the runtime in the slow case. - traceObject(trace, "+lock{stub:epoch-expired}", object, true); - lockStubEpochExpired.inc(); - } - monitorenterStubC(MONITORENTER, object, lock); + trace(trace, " currentMark: 0x%016lx\n", currentMark); + // The mark word in the object header was not the same. + // Either the object is locked by another thread or is already locked + // by the current thread. The latter is true if the mark word + // is a stack pointer into the current thread's stack, i.e.: + // + // 1) (currentMark & aligned_mask) == 0 + // 2) rsp <= currentMark + // 3) currentMark <= rsp + page_size + // + // These 3 tests can be done by evaluating the following expression: + // + // (currentMark - rsp) & (aligned_mask - page_size) + // + // assuming both the stack pointer and page_size have their least + // significant 2 bits cleared and page_size is a power of 2 + final Word alignedMask = Word.unsigned(wordSize() - 1); + final Word stackPointer = registerAsWord(stackPointerRegister).add(config(INJECTED_VMCONFIG).stackBias); + if (probability(FAST_PATH_PROBABILITY, currentMark.subtract(stackPointer).and(alignedMask.subtract(pageSize())).equal(0))) { + // Recursively locked => write 0 to the lock slot + lock.writeWord(lockDisplacedMarkOffset(INJECTED_VMCONFIG), Word.zero(), DISPLACED_MARK_WORD_LOCATION); + traceObject(trace, "+lock{cas:recursive}", object, true, options); + counters.lockCasRecursive.inc(); return; - } else { - // The prototype mark word doesn't have the bias bit set any - // more, indicating that objects of this data type are not supposed - // to be biased any more. We are going to try to reset the mark of - // this object to the prototype value and fall through to the - // CAS-based locking scheme. Note that if our CAS fails, it means - // that another thread raced us for the privilege of revoking the - // bias of this particular object, so it's okay to continue in the - // normal locking code. - Word result = compareAndSwap(OffsetAddressNode.address(object, markOffset(INJECTED_VMCONFIG)), mark, prototypeMarkWord, MARK_WORD_LOCATION); - - // Fall through to the normal CAS-based lock, because no matter what - // the result of the above CAS, some thread must have succeeded in - // removing the bias bit from the object's header. + } + traceObject(trace, "+lock{stub:failed-cas/stack}", object, true, options); + counters.lockStubFailedCas.inc(); + } + } + // slow-path runtime-call + monitorenterStubC(MONITORENTER, object, lock); + } - if (ENABLE_BREAKPOINT) { - bkpt(object, mark, tmp, result); + private static boolean tryEnterBiased(Object object, KlassPointer hub, Word lock, Word mark, Register threadRegister, boolean trace, OptionValues options, Counters counters) { + // See whether the lock is currently biased toward our thread and + // whether the epoch is still valid. + // Note that the runtime guarantees sufficient alignment of JavaThread + // pointers to allow age to be placed into low bits. + final Word biasableLockBits = mark.and(biasedLockMaskInPlace(INJECTED_VMCONFIG)); + + // Check whether the bias pattern is present in the object's mark word + // and the bias owner and the epoch are both still current. + final Word prototypeMarkWord = hub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION); + final Word thread = registerAsWord(threadRegister); + final Word tmp = prototypeMarkWord.or(thread).xor(mark).and(~ageMaskInPlace(INJECTED_VMCONFIG)); + trace(trace, "prototypeMarkWord: 0x%016lx\n", prototypeMarkWord); + trace(trace, " thread: 0x%016lx\n", thread); + trace(trace, " tmp: 0x%016lx\n", tmp); + if (probability(FAST_PATH_PROBABILITY, tmp.equal(0))) { + // Object is already biased to current thread -> done + traceObject(trace, "+lock{bias:existing}", object, true, options); + counters.lockBiasExisting.inc(); + FastAcquireBiasedLockNode.mark(object); + return true; + } + + // Now check to see whether biasing is enabled for this object + if (probability(NOT_FREQUENT_PROBABILITY, biasableLockBits.equal(Word.unsigned(biasedLockPattern(INJECTED_VMCONFIG))))) { + Pointer objectPointer = Word.objectToTrackedPointer(object); + // At this point we know that the mark word has the bias pattern and + // that we are not the bias owner in the current epoch. We need to + // figure out more details about the state of the mark word in order to + // know what operations can be legally performed on the object's + // mark word. + + // If the low three bits in the xor result aren't clear, that means + // the prototype header is no longer biasable and we have to revoke + // the bias on this object. + if (probability(FREQUENT_PROBABILITY, tmp.and(biasedLockMaskInPlace(INJECTED_VMCONFIG)).equal(0))) { + // Biasing is still enabled for object's type. See whether the + // epoch of the current bias is still valid, meaning that the epoch + // bits of the mark word are equal to the epoch bits of the + // prototype mark word. (Note that the prototype mark word's epoch bits + // only change at a safepoint.) If not, attempt to rebias the object + // toward the current thread. Note that we must be absolutely sure + // that the current epoch is invalid in order to do this because + // otherwise the manipulations it performs on the mark word are + // illegal. + if (probability(FREQUENT_PROBABILITY, tmp.and(epochMaskInPlace(INJECTED_VMCONFIG)).equal(0))) { + // The epoch of the current bias is still valid but we know nothing + // about the owner; it might be set or it might be clear. Try to + // acquire the bias of the object using an atomic operation. If this + // fails we will go in to the runtime to revoke the object's bias. + // Note that we first construct the presumed unbiased header so we + // don't accidentally blow away another thread's valid bias. + Word unbiasedMark = mark.and(biasedLockMaskInPlace(INJECTED_VMCONFIG) | ageMaskInPlace(INJECTED_VMCONFIG) | epochMaskInPlace(INJECTED_VMCONFIG)); + Word biasedMark = unbiasedMark.or(thread); + trace(trace, " unbiasedMark: 0x%016lx\n", unbiasedMark); + trace(trace, " biasedMark: 0x%016lx\n", biasedMark); + if (probability(VERY_FAST_PATH_PROBABILITY, objectPointer.logicCompareAndSwapWord(markOffset(INJECTED_VMCONFIG), unbiasedMark, biasedMark, MARK_WORD_LOCATION))) { + // Object is now biased to current thread -> done + traceObject(trace, "+lock{bias:acquired}", object, true, options); + counters.lockBiasAcquired.inc(); + return true; + } + // If the biasing toward our thread failed, this means that another thread + // owns the bias and we need to revoke that bias. The revocation will occur + // in the interpreter runtime. + traceObject(trace, "+lock{stub:revoke}", object, true, options); + counters.lockStubRevoke.inc(); + } else { + // At this point we know the epoch has expired, meaning that the + // current bias owner, if any, is actually invalid. Under these + // circumstances _only_, are we allowed to use the current mark word + // value as the comparison value when doing the CAS to acquire the + // bias in the current epoch. In other words, we allow transfer of + // the bias from one thread to another directly in this situation. + Word biasedMark = prototypeMarkWord.or(thread); + trace(trace, " biasedMark: 0x%016lx\n", biasedMark); + if (probability(VERY_FAST_PATH_PROBABILITY, objectPointer.logicCompareAndSwapWord(markOffset(INJECTED_VMCONFIG), mark, biasedMark, MARK_WORD_LOCATION))) { + // Object is now biased to current thread -> done + traceObject(trace, "+lock{bias:transfer}", object, true, options); + counters.lockBiasTransfer.inc(); + return true; } - revokeBias.inc(); + // If the biasing toward our thread failed, then another thread + // succeeded in biasing it toward itself and we need to revoke that + // bias. The revocation will occur in the runtime in the slow case. + traceObject(trace, "+lock{stub:epoch-expired}", object, true, options); + counters.lockStubEpochExpired.inc(); + } + // slow-path runtime-call + monitorenterStubC(MONITORENTER, object, lock); + return true; + } else { + // The prototype mark word doesn't have the bias bit set any + // more, indicating that objects of this data type are not supposed + // to be biased any more. We are going to try to reset the mark of + // this object to the prototype value and fall through to the + // CAS-based locking scheme. Note that if our CAS fails, it means + // that another thread raced us for the privilege of revoking the + // bias of this particular object, so it's okay to continue in the + // normal locking code. + Word result = objectPointer.compareAndSwapWord(markOffset(INJECTED_VMCONFIG), mark, prototypeMarkWord, MARK_WORD_LOCATION); + + // Fall through to the normal CAS-based lock, because no matter what + // the result of the above CAS, some thread must have succeeded in + // removing the bias bit from the object's header. + + if (ENABLE_BREAKPOINT) { + bkpt(object, mark, tmp, result); } + counters.revokeBias.inc(); + return false; } + } else { + // Biasing not enabled -> fall through to lightweight locking + counters.unbiasable.inc(); + return false; } + } - // Create the unlocked mark word pattern - Word unlockedMark = mark.or(unlockedMask(INJECTED_VMCONFIG)); - trace(trace, " unlockedMark: 0x%016lx\n", unlockedMark); - - // Copy this unlocked mark word into the lock slot on the stack - lock.writeWord(lockDisplacedMarkOffset(INJECTED_VMCONFIG), unlockedMark, DISPLACED_MARK_WORD_LOCATION); - - // Test if the object's mark word is unlocked, and if so, store the - // (address of) the lock slot into the object's mark word. - Word currentMark = compareAndSwap(OffsetAddressNode.address(object, markOffset(INJECTED_VMCONFIG)), unlockedMark, lock, MARK_WORD_LOCATION); - if (probability(BranchProbabilityNode.SLOW_PATH_PROBABILITY, currentMark.notEqual(unlockedMark))) { - trace(trace, " currentMark: 0x%016lx\n", currentMark); - // The mark word in the object header was not the same. - // Either the object is locked by another thread or is already locked - // by the current thread. The latter is true if the mark word - // is a stack pointer into the current thread's stack, i.e.: - // - // 1) (currentMark & aligned_mask) == 0 - // 2) rsp <= currentMark - // 3) currentMark <= rsp + page_size - // - // These 3 tests can be done by evaluating the following expression: - // - // (currentMark - rsp) & (aligned_mask - page_size) - // - // assuming both the stack pointer and page_size have their least - // significant 2 bits cleared and page_size is a power of 2 - final Word alignedMask = Word.unsigned(wordSize() - 1); - final Word stackPointer = registerAsWord(stackPointerRegister).add(config(INJECTED_VMCONFIG).stackBias); - if (probability(VERY_SLOW_PATH_PROBABILITY, currentMark.subtract(stackPointer).and(alignedMask.subtract(pageSize())).notEqual(0))) { - // Most likely not a recursive lock, go into a slow runtime call - traceObject(trace, "+lock{stub:failed-cas}", object, true); - lockStubFailedCas.inc(); - monitorenterStubC(MONITORENTER, object, lock); - return; + @Fold + public static boolean useFastInflatedLocking(OptionValues options) { + return SimpleFastInflatedLocking.getValue(options); + } + + private static boolean inlineFastLockSupported(OptionValues options) { + return inlineFastLockSupported(INJECTED_VMCONFIG, options); + } + + private static boolean inlineFastLockSupported(GraalHotSpotVMConfig config, OptionValues options) { + return useFastInflatedLocking(options) && monitorMask(config) >= 0 && objectMonitorOwnerOffset(config) >= 0; + } + + private static boolean tryEnterInflated(Object object, Word lock, Word mark, Register threadRegister, boolean trace, OptionValues options, Counters counters) { + // write non-zero value to lock slot + lock.writeWord(lockDisplacedMarkOffset(INJECTED_VMCONFIG), lock, DISPLACED_MARK_WORD_LOCATION); + // mark is a pointer to the ObjectMonitor + monitorMask + Word monitor = mark.subtract(monitorMask(INJECTED_VMCONFIG)); + int ownerOffset = objectMonitorOwnerOffset(INJECTED_VMCONFIG); + Word owner = monitor.readWord(ownerOffset, OBJECT_MONITOR_OWNER_LOCATION); + if (probability(FREQUENT_PROBABILITY, owner.equal(0))) { + // it appears unlocked (owner == 0) + if (probability(FREQUENT_PROBABILITY, monitor.logicCompareAndSwapWord(ownerOffset, owner, registerAsWord(threadRegister), OBJECT_MONITOR_OWNER_LOCATION))) { + // success + traceObject(trace, "+lock{inflated:cas}", object, true, options); + counters.inflatedCas.inc(); + return true; } else { - // Recursively locked => write 0 to the lock slot - lock.writeWord(lockDisplacedMarkOffset(INJECTED_VMCONFIG), Word.zero(), DISPLACED_MARK_WORD_LOCATION); - traceObject(trace, "+lock{cas:recursive}", object, true); - lockCasRecursive.inc(); + traceObject(trace, "+lock{stub:inflated:failed-cas}", object, true, options); + counters.inflatedFailedCas.inc(); } } else { - traceObject(trace, "+lock{cas}", object, true); - lockCas.inc(); - AcquiredCASLockNode.mark(object); + traceObject(trace, "+lock{stub:inflated:owned}", object, true, options); + counters.inflatedOwned.inc(); } + return false; } /** * Calls straight out to the monitorenter stub. */ @Snippet - public static void monitorenterStub(Object object, @ConstantParameter int lockDepth, @ConstantParameter boolean trace) { + public static void monitorenterStub(Object object, @ConstantParameter int lockDepth, @ConstantParameter boolean trace, @ConstantParameter OptionValues options) { verifyOop(object); - incCounter(); + incCounter(options); if (object == null) { DeoptimizeNode.deopt(DeoptimizationAction.InvalidateReprofile, DeoptimizationReason.NullCheckException); } // BeginLockScope nodes do not read from object so a use of object // cannot float about the null check above final Word lock = beginLockScope(lockDepth); - traceObject(trace, "+lock{stub}", object, true); + traceObject(trace, "+lock{stub}", object, true, options); monitorenterStubC(MONITORENTER, object, lock); } @Snippet - public static void monitorexit(Object object, @ConstantParameter int lockDepth, @ConstantParameter boolean trace) { + public static void monitorexit(Object object, @ConstantParameter int lockDepth, @ConstantParameter Register threadRegister, @ConstantParameter boolean trace, + @ConstantParameter OptionValues options, @ConstantParameter Counters counters) { trace(trace, " object: 0x%016lx\n", Word.objectToTrackedPointer(object)); + final Word mark = loadWordFromObject(object, markOffset(INJECTED_VMCONFIG)); if (useBiasedLocking(INJECTED_VMCONFIG)) { // Check for biased locking unlock case, which is a no-op // Note: we do not have to check the thread ID for two reasons. @@ -413,13 +485,12 @@ // a higher level. Second, if the bias was revoked while we held the // lock, the object could not be rebiased toward another thread, so // the bias bit would be clear. - final Word mark = loadWordFromObject(object, markOffset(INJECTED_VMCONFIG)); trace(trace, " mark: 0x%016lx\n", mark); - if (probability(BranchProbabilityNode.NOT_LIKELY_PROBABILITY, mark.and(biasedLockMaskInPlace(INJECTED_VMCONFIG)).equal(Word.unsigned(biasedLockPattern(INJECTED_VMCONFIG))))) { + if (probability(FREQUENT_PROBABILITY, mark.and(biasedLockMaskInPlace(INJECTED_VMCONFIG)).equal(Word.unsigned(biasedLockPattern(INJECTED_VMCONFIG))))) { endLockScope(); - decCounter(); - traceObject(trace, "-lock{bias}", object, false); - unlockBias.inc(); + decCounter(options); + traceObject(trace, "-lock{bias}", object, false, options); + counters.unlockBias.inc(); return; } } @@ -430,46 +501,95 @@ final Word displacedMark = lock.readWord(lockDisplacedMarkOffset(INJECTED_VMCONFIG), DISPLACED_MARK_WORD_LOCATION); trace(trace, " displacedMark: 0x%016lx\n", displacedMark); - if (probability(BranchProbabilityNode.NOT_LIKELY_PROBABILITY, displacedMark.equal(0))) { + if (probability(NOT_LIKELY_PROBABILITY, displacedMark.equal(0))) { // Recursive locking => done - traceObject(trace, "-lock{recursive}", object, false); - unlockCasRecursive.inc(); + traceObject(trace, "-lock{recursive}", object, false, options); + counters.unlockCasRecursive.inc(); } else { - verifyOop(object); - // Test if object's mark word is pointing to the displaced mark word, and if so, restore - // the displaced mark in the object - if the object's mark word is not pointing to - // the displaced mark word, do unlocking via runtime call. - if (probability(VERY_SLOW_PATH_PROBABILITY, - DirectCompareAndSwapNode.compareAndSwap(OffsetAddressNode.address(object, markOffset(INJECTED_VMCONFIG)), lock, displacedMark, MARK_WORD_LOCATION).notEqual(lock))) { - // The object's mark word was not pointing to the displaced header, - // we do unlocking via runtime call. - traceObject(trace, "-lock{stub}", object, false); - unlockStub.inc(); - monitorexitStubC(MONITOREXIT, object, lock); - } else { - traceObject(trace, "-lock{cas}", object, false); - unlockCas.inc(); + if (!tryExitInflated(object, mark, lock, threadRegister, trace, options, counters)) { + verifyOop(object); + // Test if object's mark word is pointing to the displaced mark word, and if so, + // restore + // the displaced mark in the object - if the object's mark word is not pointing to + // the displaced mark word, do unlocking via runtime call. + Pointer objectPointer = Word.objectToTrackedPointer(object); + if (probability(VERY_FAST_PATH_PROBABILITY, objectPointer.logicCompareAndSwapWord(markOffset(INJECTED_VMCONFIG), lock, displacedMark, MARK_WORD_LOCATION))) { + traceObject(trace, "-lock{cas}", object, false, options); + counters.unlockCas.inc(); + } else { + // The object's mark word was not pointing to the displaced header + traceObject(trace, "-lock{stub}", object, false, options); + counters.unlockStub.inc(); + monitorexitStubC(MONITOREXIT, object, lock); + } } } endLockScope(); - decCounter(); + decCounter(options); + } + + private static boolean inlineFastUnlockSupported(OptionValues options) { + return inlineFastUnlockSupported(INJECTED_VMCONFIG, options); + } + + private static boolean inlineFastUnlockSupported(GraalHotSpotVMConfig config, OptionValues options) { + return useFastInflatedLocking(options) && objectMonitorEntryListOffset(config) >= 0 && objectMonitorCxqOffset(config) >= 0 && monitorMask(config) >= 0 && + objectMonitorOwnerOffset(config) >= 0 && objectMonitorRecursionsOffset(config) >= 0; + } + + private static boolean tryExitInflated(Object object, Word mark, Word lock, Register threadRegister, boolean trace, OptionValues options, Counters counters) { + if (!inlineFastUnlockSupported(options)) { + return false; + } + if (probability(SLOW_PATH_PROBABILITY, mark.and(monitorMask(INJECTED_VMCONFIG)).notEqual(0))) { + // Inflated case + // mark is a pointer to the ObjectMonitor + monitorMask + Word monitor = mark.subtract(monitorMask(INJECTED_VMCONFIG)); + int ownerOffset = objectMonitorOwnerOffset(INJECTED_VMCONFIG); + Word owner = monitor.readWord(ownerOffset, OBJECT_MONITOR_OWNER_LOCATION); + int recursionsOffset = objectMonitorRecursionsOffset(INJECTED_VMCONFIG); + Word recursions = monitor.readWord(recursionsOffset, OBJECT_MONITOR_RECURSION_LOCATION); + Word thread = registerAsWord(threadRegister); + if (probability(FAST_PATH_PROBABILITY, owner.xor(thread).or(recursions).equal(0))) { + // owner == thread && recursions == 0 + int cxqOffset = objectMonitorCxqOffset(INJECTED_VMCONFIG); + Word cxq = monitor.readWord(cxqOffset, OBJECT_MONITOR_CXQ_LOCATION); + int entryListOffset = objectMonitorEntryListOffset(INJECTED_VMCONFIG); + Word entryList = monitor.readWord(entryListOffset, OBJECT_MONITOR_ENTRY_LIST_LOCATION); + if (probability(FREQUENT_PROBABILITY, cxq.or(entryList).equal(0))) { + // cxq == 0 && entryList == 0 + // Nobody is waiting, success + // release_store + MembarNode.memoryBarrier(LOAD_STORE | STORE_STORE); + monitor.writeWord(ownerOffset, Word.zero()); + traceObject(trace, "-lock{inflated:simple}", object, false, options); + counters.unlockInflatedSimple.inc(); + return true; + } + } + counters.unlockStubInflated.inc(); + traceObject(trace, "-lock{stub:inflated}", object, false, options); + monitorexitStubC(MONITOREXIT, object, lock); + return true; + } + return false; } /** * Calls straight out to the monitorexit stub. */ @Snippet - public static void monitorexitStub(Object object, @ConstantParameter int lockDepth, @ConstantParameter boolean trace) { + public static void monitorexitStub(Object object, @ConstantParameter int lockDepth, @ConstantParameter boolean trace, @ConstantParameter OptionValues options) { verifyOop(object); - traceObject(trace, "-lock{stub}", object, false); + traceObject(trace, "-lock{stub}", object, false, options); final Word lock = CurrentLockNode.currentLock(lockDepth); monitorexitStubC(MONITOREXIT, object, lock); endLockScope(); - decCounter(); + decCounter(options); } - public static void traceObject(boolean enabled, String action, Object object, boolean enter) { - if (doProfile()) { + public static void traceObject(boolean enabled, String action, Object object, boolean enter, OptionValues options) { + if (doProfile(options)) { DynamicCounterNode.counter(action, enter ? "number of monitor enters" : "number of monitor exits", 1, PROFILE_CONTEXT); } if (enabled) { @@ -496,18 +616,21 @@ @NodeIntrinsic(BreakpointNode.class) static native void bkpt(Object object, Word mark, Word tmp, Word value); - private static final boolean VERIFY_BALANCED_MONITORS = VerifyBalancedMonitors.getValue(); + @Fold + static boolean verifyBalancedMonitors(OptionValues options) { + return VerifyBalancedMonitors.getValue(options); + } - public static void incCounter() { - if (VERIFY_BALANCED_MONITORS) { + public static void incCounter(OptionValues options) { + if (verifyBalancedMonitors(options)) { final Word counter = MonitorCounterNode.counter(); final int count = counter.readInt(0, MONITOR_COUNTER_LOCATION); counter.writeInt(0, count + 1, MONITOR_COUNTER_LOCATION); } } - public static void decCounter() { - if (VERIFY_BALANCED_MONITORS) { + public static void decCounter(OptionValues options) { + if (verifyBalancedMonitors(options)) { final Word counter = MonitorCounterNode.counter(); final int count = counter.readInt(0, MONITOR_COUNTER_LOCATION); counter.writeInt(0, count - 1, MONITOR_COUNTER_LOCATION); @@ -529,6 +652,64 @@ } } + public static class Counters { + /** + * Counters for the various paths for acquiring a lock. The counters whose names start with + * {@code "lock"} are mutually exclusive. The other counters are for paths that may be + * shared. + */ + public final SnippetCounter lockBiasExisting; + public final SnippetCounter lockBiasAcquired; + public final SnippetCounter lockBiasTransfer; + public final SnippetCounter lockCas; + public final SnippetCounter lockCasRecursive; + public final SnippetCounter lockStubEpochExpired; + public final SnippetCounter lockStubRevoke; + public final SnippetCounter lockStubFailedCas; + public final SnippetCounter inflatedCas; + public final SnippetCounter inflatedFailedCas; + public final SnippetCounter inflatedOwned; + public final SnippetCounter unbiasable; + public final SnippetCounter revokeBias; + + /** + * Counters for the various paths for releasing a lock. The counters whose names start with + * {@code "unlock"} are mutually exclusive. The other counters are for paths that may be + * shared. + */ + public final SnippetCounter unlockBias; + public final SnippetCounter unlockCas; + public final SnippetCounter unlockCasRecursive; + public final SnippetCounter unlockStub; + public final SnippetCounter unlockStubInflated; + public final SnippetCounter unlockInflatedSimple; + + public Counters(SnippetCounter.Group.Factory factory) { + SnippetCounter.Group enter = factory.createSnippetCounterGroup("MonitorEnters"); + SnippetCounter.Group exit = factory.createSnippetCounterGroup("MonitorExits"); + lockBiasExisting = new SnippetCounter(enter, "lock{bias:existing}", "bias-locked previously biased object"); + lockBiasAcquired = new SnippetCounter(enter, "lock{bias:acquired}", "bias-locked newly biased object"); + lockBiasTransfer = new SnippetCounter(enter, "lock{bias:transfer}", "bias-locked, biased transferred"); + lockCas = new SnippetCounter(enter, "lock{cas}", "cas-locked an object"); + lockCasRecursive = new SnippetCounter(enter, "lock{cas:recursive}", "cas-locked, recursive"); + lockStubEpochExpired = new SnippetCounter(enter, "lock{stub:epoch-expired}", "stub-locked, epoch expired"); + lockStubRevoke = new SnippetCounter(enter, "lock{stub:revoke}", "stub-locked, biased revoked"); + lockStubFailedCas = new SnippetCounter(enter, "lock{stub:failed-cas/stack}", "stub-locked, failed cas and stack locking"); + inflatedCas = new SnippetCounter(enter, "lock{inflated:cas}", "heavyweight-locked, cas-locked"); + inflatedFailedCas = new SnippetCounter(enter, "lock{inflated:failed-cas}", "heavyweight-locked, failed cas"); + inflatedOwned = new SnippetCounter(enter, "lock{inflated:owned}", "heavyweight-locked, already owned"); + unbiasable = new SnippetCounter(enter, "unbiasable", "object with unbiasable type"); + revokeBias = new SnippetCounter(enter, "revokeBias", "object had bias revoked"); + + unlockBias = new SnippetCounter(exit, "unlock{bias}", "bias-unlocked an object"); + unlockCas = new SnippetCounter(exit, "unlock{cas}", "cas-unlocked an object"); + unlockCasRecursive = new SnippetCounter(exit, "unlock{cas:recursive}", "cas-unlocked an object, recursive"); + unlockStub = new SnippetCounter(exit, "unlock{stub}", "stub-unlocked an object"); + unlockStubInflated = new SnippetCounter(exit, "unlock{stub:inflated}", "stub-unlocked an object with inflated monitor"); + unlockInflatedSimple = new SnippetCounter(exit, "unlock{inflated}", "unlocked an object monitor"); + } + } + public static class Templates extends AbstractTemplates { private final SnippetInfo monitorenter = snippet(MonitorSnippets.class, "monitorenter"); @@ -539,10 +720,13 @@ private final SnippetInfo checkCounter = snippet(MonitorSnippets.class, "checkCounter"); private final boolean useFastLocking; + public final Counters counters; - public Templates(HotSpotProviders providers, TargetDescription target, boolean useFastLocking) { - super(providers, providers.getSnippetReflection(), target); + public Templates(OptionValues options, SnippetCounter.Group.Factory factory, HotSpotProviders providers, TargetDescription target, boolean useFastLocking) { + super(options, providers, providers.getSnippetReflection(), target); this.useFastLocking = useFastLocking; + + this.counters = new Counters(factory); } public void lower(RawMonitorEnterNode monitorenterNode, HotSpotRegistersProvider registers, LoweringTool tool) { @@ -559,18 +743,22 @@ args.addConst("lockDepth", monitorenterNode.getMonitorId().getLockDepth()); args.addConst("threadRegister", registers.getThreadRegister()); args.addConst("stackPointerRegister", registers.getStackPointerRegister()); - args.addConst("trace", isTracingEnabledForType(monitorenterNode.object()) || isTracingEnabledForMethod(graph.method())); + args.addConst("trace", isTracingEnabledForType(monitorenterNode.object()) || isTracingEnabledForMethod(graph)); + args.addConst("options", graph.getOptions()); + args.addConst("counters", counters); } else { args = new Arguments(monitorenterStub, graph.getGuardsStage(), tool.getLoweringStage()); args.add("object", monitorenterNode.object()); args.addConst("lockDepth", monitorenterNode.getMonitorId().getLockDepth()); - args.addConst("trace", isTracingEnabledForType(monitorenterNode.object()) || isTracingEnabledForMethod(graph.method())); + args.addConst("trace", isTracingEnabledForType(monitorenterNode.object()) || isTracingEnabledForMethod(graph)); + args.addConst("options", graph.getOptions()); + args.addConst("counters", counters); } template(args).instantiate(providers.getMetaAccess(), monitorenterNode, DEFAULT_REPLACER, args); } - public void lower(MonitorExitNode monitorexitNode, LoweringTool tool) { + public void lower(MonitorExitNode monitorexitNode, HotSpotRegistersProvider registers, LoweringTool tool) { StructuredGraph graph = monitorexitNode.graph(); Arguments args; @@ -581,14 +769,17 @@ } args.add("object", monitorexitNode.object()); args.addConst("lockDepth", monitorexitNode.getMonitorId().getLockDepth()); - args.addConst("trace", isTracingEnabledForType(monitorexitNode.object()) || isTracingEnabledForMethod(graph.method())); + args.addConst("threadRegister", registers.getThreadRegister()); + args.addConst("trace", isTracingEnabledForType(monitorexitNode.object()) || isTracingEnabledForMethod(graph)); + args.addConst("options", graph.getOptions()); + args.addConst("counters", counters); template(args).instantiate(providers.getMetaAccess(), monitorexitNode, DEFAULT_REPLACER, args); } public static boolean isTracingEnabledForType(ValueNode object) { ResolvedJavaType type = StampTool.typeOrNull(object.stamp()); - String filter = TraceMonitorsTypeFilter.getValue(); + String filter = TraceMonitorsTypeFilter.getValue(object.getOptions()); if (filter == null) { return false; } else { @@ -602,18 +793,18 @@ } } - public static boolean isTracingEnabledForMethod(ResolvedJavaMethod method) { - String filter = TraceMonitorsMethodFilter.getValue(); + public static boolean isTracingEnabledForMethod(StructuredGraph graph) { + String filter = TraceMonitorsMethodFilter.getValue(graph.getOptions()); if (filter == null) { return false; } else { if (filter.length() == 0) { return true; } - if (method == null) { + if (graph.method() == null) { return false; } - return (method.format("%H.%n").contains(filter)); + return (graph.method().format("%H.%n").contains(filter)); } } @@ -622,7 +813,7 @@ * return points of the graph to initialize and check the monitor counter respectively. */ private void checkBalancedMonitors(StructuredGraph graph, LoweringTool tool) { - if (VERIFY_BALANCED_MONITORS) { + if (VerifyBalancedMonitors.getValue(options)) { NodeIterable nodes = graph.getNodes().filter(MonitorCounterNode.class); if (nodes.isEmpty()) { // Only insert the nodes if this is the first monitorenter being lowered. @@ -634,7 +825,7 @@ graph.addAfterFixed(graph.start(), invoke); StructuredGraph inlineeGraph = providers.getReplacements().getSnippet(initCounter.getMethod(), null); - InliningUtil.inline(invoke, inlineeGraph, false, null, null); + InliningUtil.inline(invoke, inlineeGraph, false, null); List rets = graph.getNodes(ReturnNode.TYPE).snapshot(); for (ReturnNode ret : rets) { @@ -652,7 +843,7 @@ Arguments args = new Arguments(checkCounter, graph.getGuardsStage(), tool.getLoweringStage()); args.addConst("errMsg", msg); inlineeGraph = template(args).copySpecializedGraph(); - InliningUtil.inline(invoke, inlineeGraph, false, null, null); + InliningUtil.inline(invoke, inlineeGraph, false, null); } } } @@ -667,31 +858,4 @@ @NodeIntrinsic(ForeignCallNode.class) public static native void monitorexitStubC(@ConstantNodeParameter ForeignCallDescriptor descriptor, Object object, Word lock); - - /** - * Counters for the various paths for acquiring a lock. The counters whose names start with - * {@code "lock"} are mutually exclusive. The other counters are for paths that may be shared. - */ - public static final SnippetCounter.Group lockCounters = SnippetCounters.getValue() ? new SnippetCounter.Group("MonitorEnters") : null; - public static final SnippetCounter lockBiasExisting = new SnippetCounter(lockCounters, "lock{bias:existing}", "bias-locked previously biased object"); - public static final SnippetCounter lockBiasAcquired = new SnippetCounter(lockCounters, "lock{bias:acquired}", "bias-locked newly biased object"); - public static final SnippetCounter lockBiasTransfer = new SnippetCounter(lockCounters, "lock{bias:transfer}", "bias-locked, biased transferred"); - public static final SnippetCounter lockCas = new SnippetCounter(lockCounters, "lock{cas}", "cas-locked an object"); - public static final SnippetCounter lockCasRecursive = new SnippetCounter(lockCounters, "lock{cas:recursive}", "cas-locked, recursive"); - public static final SnippetCounter lockStubEpochExpired = new SnippetCounter(lockCounters, "lock{stub:epoch-expired}", "stub-locked, epoch expired"); - public static final SnippetCounter lockStubRevoke = new SnippetCounter(lockCounters, "lock{stub:revoke}", "stub-locked, biased revoked"); - public static final SnippetCounter lockStubFailedCas = new SnippetCounter(lockCounters, "lock{stub:failed-cas}", "stub-locked, failed cas"); - - public static final SnippetCounter unbiasable = new SnippetCounter(lockCounters, "unbiasable", "object with unbiasable type"); - public static final SnippetCounter revokeBias = new SnippetCounter(lockCounters, "revokeBias", "object had bias revoked"); - - /** - * Counters for the various paths for releasing a lock. The counters whose names start with - * {@code "unlock"} are mutually exclusive. The other counters are for paths that may be shared. - */ - public static final SnippetCounter.Group unlockCounters = SnippetCounters.getValue() ? new SnippetCounter.Group("MonitorExits") : null; - public static final SnippetCounter unlockBias = new SnippetCounter(unlockCounters, "unlock{bias}", "bias-unlocked an object"); - public static final SnippetCounter unlockCas = new SnippetCounter(unlockCounters, "unlock{cas}", "cas-unlocked an object"); - public static final SnippetCounter unlockCasRecursive = new SnippetCounter(unlockCounters, "unlock{cas:recursive}", "cas-unlocked an object, recursive"); - public static final SnippetCounter unlockStub = new SnippetCounter(unlockCounters, "unlock{stub}", "stub-unlocked an object"); }