9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24
25 package org.graalvm.compiler.hotspot.replacements;
26
27 import static jdk.vm.ci.code.MemoryBarriers.LOAD_STORE;
28 import static jdk.vm.ci.code.MemoryBarriers.STORE_STORE;
29 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFIG;
30 import static org.graalvm.compiler.hotspot.nodes.BeginLockScopeNode.beginLockScope;
31 import static org.graalvm.compiler.hotspot.nodes.EndLockScopeNode.endLockScope;
32 import static org.graalvm.compiler.hotspot.nodes.VMErrorNode.vmError;
33 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.DISPLACED_MARK_WORD_LOCATION;
34 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.MARK_WORD_LOCATION;
35 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.OBJECT_MONITOR_CXQ_LOCATION;
36 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.OBJECT_MONITOR_ENTRY_LIST_LOCATION;
37 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.OBJECT_MONITOR_OWNER_LOCATION;
38 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.OBJECT_MONITOR_RECURSION_LOCATION;
39 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.PROTOTYPE_MARK_WORD_LOCATION;
40 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.ageMaskInPlace;
41 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.biasedLockMaskInPlace;
42 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.biasedLockPattern;
43 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.config;
44 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.epochMaskInPlace;
45 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.loadWordFromObject;
46 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.lockDisplacedMarkOffset;
47 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.markOffset;
48 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.monitorMask;
49 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.objectMonitorCxqOffset;
50 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.objectMonitorEntryListOffset;
51 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.objectMonitorOwnerOffset;
52 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.objectMonitorRecursionsOffset;
53 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.pageSize;
54 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.prototypeMarkWordOffset;
55 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.registerAsWord;
56 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.unlockedMask;
57 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useBiasedLocking;
58 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.verifyOop;
59 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.wordSize;
60 import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.ProfileMonitors;
61 import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.SimpleFastInflatedLocking;
62 import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.TraceMonitorsMethodFilter;
63 import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.TraceMonitorsTypeFilter;
64 import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.VerifyBalancedMonitors;
65 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FAST_PATH_PROBABILITY;
66 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FREQUENT_PROBABILITY;
67 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.NOT_FREQUENT_PROBABILITY;
68 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.NOT_LIKELY_PROBABILITY;
69 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY;
70 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.VERY_FAST_PATH_PROBABILITY;
71 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;
72 import static org.graalvm.compiler.replacements.SnippetTemplate.DEFAULT_REPLACER;
73
74 import java.util.List;
75
76 import org.graalvm.compiler.api.replacements.Fold;
77 import org.graalvm.compiler.api.replacements.Snippet;
78 import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter;
79 import org.graalvm.compiler.bytecode.Bytecode;
80 import org.graalvm.compiler.bytecode.ResolvedJavaMethodBytecode;
81 import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor;
82 import org.graalvm.compiler.core.common.type.ObjectStamp;
83 import org.graalvm.compiler.core.common.type.StampFactory;
84 import org.graalvm.compiler.core.common.type.StampPair;
85 import org.graalvm.compiler.debug.DebugHandlersFactory;
86 import org.graalvm.compiler.graph.Node.ConstantNodeParameter;
87 import org.graalvm.compiler.graph.Node.NodeIntrinsic;
88 import org.graalvm.compiler.graph.iterators.NodeIterable;
89 import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig;
90 import org.graalvm.compiler.hotspot.meta.HotSpotProviders;
91 import org.graalvm.compiler.hotspot.meta.HotSpotRegistersProvider;
92 import org.graalvm.compiler.hotspot.nodes.AcquiredCASLockNode;
93 import org.graalvm.compiler.hotspot.nodes.CurrentLockNode;
94 import org.graalvm.compiler.hotspot.nodes.FastAcquireBiasedLockNode;
95 import org.graalvm.compiler.hotspot.nodes.MonitorCounterNode;
96 import org.graalvm.compiler.hotspot.word.KlassPointer;
97 import org.graalvm.compiler.nodes.BreakpointNode;
98 import org.graalvm.compiler.nodes.CallTargetNode.InvokeKind;
99 import org.graalvm.compiler.nodes.ConstantNode;
100 import org.graalvm.compiler.nodes.DeoptimizeNode;
101 import org.graalvm.compiler.nodes.FrameState;
102 import org.graalvm.compiler.nodes.InvokeNode;
103 import org.graalvm.compiler.nodes.NamedLocationIdentity;
104 import org.graalvm.compiler.nodes.NodeView;
105 import org.graalvm.compiler.nodes.ReturnNode;
106 import org.graalvm.compiler.nodes.StructuredGraph;
107 import org.graalvm.compiler.nodes.ValueNode;
108 import org.graalvm.compiler.nodes.debug.DynamicCounterNode;
109 import org.graalvm.compiler.nodes.extended.ForeignCallNode;
110 import org.graalvm.compiler.nodes.extended.MembarNode;
111 import org.graalvm.compiler.nodes.java.MethodCallTargetNode;
112 import org.graalvm.compiler.nodes.java.MonitorExitNode;
202 *
203 * - the two lock bits are used to describe three states: locked/unlocked and monitor.
204 *
205 * [ptr | 00] locked ptr points to real header on stack
206 * [header | 0 | 01] unlocked regular object header
207 * [ptr | 10] monitor inflated lock (header is wapped out)
208 * [ptr | 11] marked used by markSweep to mark an object
209 * not valid at any other time
210 *
211 * We assume that stack/thread pointers have the lowest two bits cleared.
212 * </pre>
213 *
214 * Note that {@code Thread::allocate} enforces {@code JavaThread} objects to be aligned
215 * appropriately to comply with the layouts above.
216 */
217 public class MonitorSnippets implements Snippets {
218
219 private static final boolean PROFILE_CONTEXT = false;
220
221 @Fold
222 static boolean doProfile(OptionValues options) {
223 return ProfileMonitors.getValue(options);
224 }
225
226 @Snippet
227 public static void monitorenter(Object object, KlassPointer hub, @ConstantParameter int lockDepth, @ConstantParameter Register threadRegister, @ConstantParameter Register stackPointerRegister,
228 @ConstantParameter boolean trace, @ConstantParameter OptionValues options, @ConstantParameter Counters counters) {
229 verifyOop(object);
230
231 // Load the mark word - this includes a null-check on object
232 final Word mark = loadWordFromObject(object, markOffset(INJECTED_VMCONFIG));
233
234 final Word lock = beginLockScope(lockDepth);
235
236 Pointer objectPointer = Word.objectToTrackedPointer(object);
237 trace(trace, " object: 0x%016lx\n", objectPointer);
238 trace(trace, " lock: 0x%016lx\n", lock);
239 trace(trace, " mark: 0x%016lx\n", mark);
240
241 incCounter(options);
242
243 if (useBiasedLocking(INJECTED_VMCONFIG)) {
244 if (tryEnterBiased(object, hub, lock, mark, threadRegister, trace, options, counters)) {
245 return;
246 }
247 // not biased, fall-through
248 }
249 if (inlineFastLockSupported(options) && probability(SLOW_PATH_PROBABILITY, mark.and(monitorMask(INJECTED_VMCONFIG)).notEqual(0))) {
250 // Inflated case
251 if (tryEnterInflated(object, lock, mark, threadRegister, trace, options, counters)) {
252 return;
253 }
254 } else {
255 // Create the unlocked mark word pattern
256 Word unlockedMark = mark.or(unlockedMask(INJECTED_VMCONFIG));
257 trace(trace, " unlockedMark: 0x%016lx\n", unlockedMark);
258
259 // Copy this unlocked mark word into the lock slot on the stack
260 lock.writeWord(lockDisplacedMarkOffset(INJECTED_VMCONFIG), unlockedMark, DISPLACED_MARK_WORD_LOCATION);
261
262 // make sure previous store does not float below compareAndSwap
263 MembarNode.memoryBarrier(STORE_STORE);
264
265 // Test if the object's mark word is unlocked, and if so, store the
266 // (address of) the lock slot into the object's mark word.
267 Word currentMark = objectPointer.compareAndSwapWord(markOffset(INJECTED_VMCONFIG), unlockedMark, lock, MARK_WORD_LOCATION);
268 if (probability(FAST_PATH_PROBABILITY, currentMark.equal(unlockedMark))) {
269 traceObject(trace, "+lock{cas}", object, true, options);
270 counters.lockCas.inc();
271 AcquiredCASLockNode.mark(object);
272 return;
273 } else {
274 trace(trace, " currentMark: 0x%016lx\n", currentMark);
275 // The mark word in the object header was not the same.
276 // Either the object is locked by another thread or is already locked
277 // by the current thread. The latter is true if the mark word
278 // is a stack pointer into the current thread's stack, i.e.:
279 //
280 // 1) (currentMark & aligned_mask) == 0
281 // 2) rsp <= currentMark
282 // 3) currentMark <= rsp + page_size
283 //
284 // These 3 tests can be done by evaluating the following expression:
285 //
286 // (currentMark - rsp) & (aligned_mask - page_size)
287 //
288 // assuming both the stack pointer and page_size have their least
289 // significant 2 bits cleared and page_size is a power of 2
290 final Word alignedMask = WordFactory.unsigned(wordSize() - 1);
291 final Word stackPointer = registerAsWord(stackPointerRegister).add(config(INJECTED_VMCONFIG).stackBias);
292 if (probability(FAST_PATH_PROBABILITY, currentMark.subtract(stackPointer).and(alignedMask.subtract(pageSize())).equal(0))) {
293 // Recursively locked => write 0 to the lock slot
294 lock.writeWord(lockDisplacedMarkOffset(INJECTED_VMCONFIG), WordFactory.zero(), DISPLACED_MARK_WORD_LOCATION);
295 traceObject(trace, "+lock{cas:recursive}", object, true, options);
296 counters.lockCasRecursive.inc();
297 return;
298 }
299 traceObject(trace, "+lock{stub:failed-cas/stack}", object, true, options);
300 counters.lockStubFailedCas.inc();
301 }
302 }
303 // slow-path runtime-call
304 monitorenterStubC(MONITORENTER, object, lock);
305 }
306
307 private static boolean tryEnterBiased(Object object, KlassPointer hub, Word lock, Word mark, Register threadRegister, boolean trace, OptionValues options, Counters counters) {
308 // See whether the lock is currently biased toward our thread and
309 // whether the epoch is still valid.
310 // Note that the runtime guarantees sufficient alignment of JavaThread
311 // pointers to allow age to be placed into low bits.
312 final Word biasableLockBits = mark.and(biasedLockMaskInPlace(INJECTED_VMCONFIG));
313
314 // Check whether the bias pattern is present in the object's mark word
315 // and the bias owner and the epoch are both still current.
316 final Word prototypeMarkWord = hub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION);
317 final Word thread = registerAsWord(threadRegister);
318 final Word tmp = prototypeMarkWord.or(thread).xor(mark).and(~ageMaskInPlace(INJECTED_VMCONFIG));
319 trace(trace, "prototypeMarkWord: 0x%016lx\n", prototypeMarkWord);
320 trace(trace, " thread: 0x%016lx\n", thread);
321 trace(trace, " tmp: 0x%016lx\n", tmp);
322 if (probability(FAST_PATH_PROBABILITY, tmp.equal(0))) {
323 // Object is already biased to current thread -> done
324 traceObject(trace, "+lock{bias:existing}", object, true, options);
325 counters.lockBiasExisting.inc();
326 FastAcquireBiasedLockNode.mark(object);
327 return true;
328 }
329
330 // Now check to see whether biasing is enabled for this object
331 if (probability(NOT_FREQUENT_PROBABILITY, biasableLockBits.equal(WordFactory.unsigned(biasedLockPattern(INJECTED_VMCONFIG))))) {
332 Pointer objectPointer = Word.objectToTrackedPointer(object);
333 // At this point we know that the mark word has the bias pattern and
334 // that we are not the bias owner in the current epoch. We need to
335 // figure out more details about the state of the mark word in order to
336 // know what operations can be legally performed on the object's
337 // mark word.
338
339 // If the low three bits in the xor result aren't clear, that means
340 // the prototype header is no longer biasable and we have to revoke
341 // the bias on this object.
342 if (probability(FREQUENT_PROBABILITY, tmp.and(biasedLockMaskInPlace(INJECTED_VMCONFIG)).equal(0))) {
343 // Biasing is still enabled for object's type. See whether the
344 // epoch of the current bias is still valid, meaning that the epoch
345 // bits of the mark word are equal to the epoch bits of the
346 // prototype mark word. (Note that the prototype mark word's epoch bits
347 // only change at a safepoint.) If not, attempt to rebias the object
348 // toward the current thread. Note that we must be absolutely sure
349 // that the current epoch is invalid in order to do this because
350 // otherwise the manipulations it performs on the mark word are
351 // illegal.
352 if (probability(FREQUENT_PROBABILITY, tmp.and(epochMaskInPlace(INJECTED_VMCONFIG)).equal(0))) {
353 // The epoch of the current bias is still valid but we know nothing
354 // about the owner; it might be set or it might be clear. Try to
355 // acquire the bias of the object using an atomic operation. If this
356 // fails we will go in to the runtime to revoke the object's bias.
357 // Note that we first construct the presumed unbiased header so we
358 // don't accidentally blow away another thread's valid bias.
359 Word unbiasedMark = mark.and(biasedLockMaskInPlace(INJECTED_VMCONFIG) | ageMaskInPlace(INJECTED_VMCONFIG) | epochMaskInPlace(INJECTED_VMCONFIG));
360 Word biasedMark = unbiasedMark.or(thread);
361 trace(trace, " unbiasedMark: 0x%016lx\n", unbiasedMark);
362 trace(trace, " biasedMark: 0x%016lx\n", biasedMark);
363 if (probability(VERY_FAST_PATH_PROBABILITY, objectPointer.logicCompareAndSwapWord(markOffset(INJECTED_VMCONFIG), unbiasedMark, biasedMark, MARK_WORD_LOCATION))) {
364 // Object is now biased to current thread -> done
365 traceObject(trace, "+lock{bias:acquired}", object, true, options);
366 counters.lockBiasAcquired.inc();
367 return true;
368 }
369 // If the biasing toward our thread failed, this means that another thread
370 // owns the bias and we need to revoke that bias. The revocation will occur
371 // in the interpreter runtime.
372 traceObject(trace, "+lock{stub:revoke}", object, true, options);
373 counters.lockStubRevoke.inc();
374 } else {
375 // At this point we know the epoch has expired, meaning that the
376 // current bias owner, if any, is actually invalid. Under these
377 // circumstances _only_, are we allowed to use the current mark word
378 // value as the comparison value when doing the CAS to acquire the
379 // bias in the current epoch. In other words, we allow transfer of
380 // the bias from one thread to another directly in this situation.
381 Word biasedMark = prototypeMarkWord.or(thread);
382 trace(trace, " biasedMark: 0x%016lx\n", biasedMark);
383 if (probability(VERY_FAST_PATH_PROBABILITY, objectPointer.logicCompareAndSwapWord(markOffset(INJECTED_VMCONFIG), mark, biasedMark, MARK_WORD_LOCATION))) {
384 // Object is now biased to current thread -> done
385 traceObject(trace, "+lock{bias:transfer}", object, true, options);
386 counters.lockBiasTransfer.inc();
387 return true;
388 }
389 // If the biasing toward our thread failed, then another thread
390 // succeeded in biasing it toward itself and we need to revoke that
391 // bias. The revocation will occur in the runtime in the slow case.
392 traceObject(trace, "+lock{stub:epoch-expired}", object, true, options);
393 counters.lockStubEpochExpired.inc();
394 }
395 // slow-path runtime-call
396 monitorenterStubC(MONITORENTER, object, lock);
397 return true;
398 } else {
399 // The prototype mark word doesn't have the bias bit set any
400 // more, indicating that objects of this data type are not supposed
401 // to be biased any more. We are going to try to reset the mark of
402 // this object to the prototype value and fall through to the
403 // CAS-based locking scheme. Note that if our CAS fails, it means
404 // that another thread raced us for the privilege of revoking the
405 // bias of this particular object, so it's okay to continue in the
406 // normal locking code.
407 Word result = objectPointer.compareAndSwapWord(markOffset(INJECTED_VMCONFIG), mark, prototypeMarkWord, MARK_WORD_LOCATION);
408
409 // Fall through to the normal CAS-based lock, because no matter what
410 // the result of the above CAS, some thread must have succeeded in
411 // removing the bias bit from the object's header.
412
413 if (ENABLE_BREAKPOINT) {
414 bkpt(object, mark, tmp, result);
415 }
416 counters.revokeBias.inc();
417 return false;
418 }
419 } else {
420 // Biasing not enabled -> fall through to lightweight locking
421 counters.unbiasable.inc();
422 return false;
423 }
424 }
425
426 @Fold
427 public static boolean useFastInflatedLocking(OptionValues options) {
428 return SimpleFastInflatedLocking.getValue(options);
429 }
430
431 private static boolean inlineFastLockSupported(OptionValues options) {
432 return inlineFastLockSupported(INJECTED_VMCONFIG, options);
433 }
434
435 private static boolean inlineFastLockSupported(GraalHotSpotVMConfig config, OptionValues options) {
436 return useFastInflatedLocking(options) && monitorMask(config) >= 0 && objectMonitorOwnerOffset(config) >= 0;
437 }
438
439 private static boolean tryEnterInflated(Object object, Word lock, Word mark, Register threadRegister, boolean trace, OptionValues options, Counters counters) {
440 // write non-zero value to lock slot
441 lock.writeWord(lockDisplacedMarkOffset(INJECTED_VMCONFIG), lock, DISPLACED_MARK_WORD_LOCATION);
442 // mark is a pointer to the ObjectMonitor + monitorMask
443 Word monitor = mark.subtract(monitorMask(INJECTED_VMCONFIG));
444 int ownerOffset = objectMonitorOwnerOffset(INJECTED_VMCONFIG);
445 Word owner = monitor.readWord(ownerOffset, OBJECT_MONITOR_OWNER_LOCATION);
446 if (probability(FREQUENT_PROBABILITY, owner.equal(0))) {
447 // it appears unlocked (owner == 0)
448 if (probability(FREQUENT_PROBABILITY, monitor.logicCompareAndSwapWord(ownerOffset, owner, registerAsWord(threadRegister), OBJECT_MONITOR_OWNER_LOCATION))) {
449 // success
450 traceObject(trace, "+lock{inflated:cas}", object, true, options);
451 counters.inflatedCas.inc();
452 return true;
453 } else {
454 traceObject(trace, "+lock{stub:inflated:failed-cas}", object, true, options);
455 counters.inflatedFailedCas.inc();
456 }
457 } else {
458 traceObject(trace, "+lock{stub:inflated:owned}", object, true, options);
459 counters.inflatedOwned.inc();
460 }
461 return false;
462 }
463
464 /**
465 * Calls straight out to the monitorenter stub.
466 */
467 @Snippet
468 public static void monitorenterStub(Object object, @ConstantParameter int lockDepth, @ConstantParameter boolean trace, @ConstantParameter OptionValues options) {
469 verifyOop(object);
470 incCounter(options);
471 if (object == null) {
472 DeoptimizeNode.deopt(DeoptimizationAction.InvalidateReprofile, DeoptimizationReason.NullCheckException);
473 }
474 // BeginLockScope nodes do not read from object so a use of object
475 // cannot float about the null check above
476 final Word lock = beginLockScope(lockDepth);
477 traceObject(trace, "+lock{stub}", object, true, options);
478 monitorenterStubC(MONITORENTER, object, lock);
479 }
480
481 @Snippet
482 public static void monitorexit(Object object, @ConstantParameter int lockDepth, @ConstantParameter Register threadRegister, @ConstantParameter boolean trace,
483 @ConstantParameter OptionValues options, @ConstantParameter Counters counters) {
484 trace(trace, " object: 0x%016lx\n", Word.objectToTrackedPointer(object));
485 final Word mark = loadWordFromObject(object, markOffset(INJECTED_VMCONFIG));
486 if (useBiasedLocking(INJECTED_VMCONFIG)) {
487 // Check for biased locking unlock case, which is a no-op
488 // Note: we do not have to check the thread ID for two reasons.
489 // First, the interpreter checks for IllegalMonitorStateException at
490 // a higher level. Second, if the bias was revoked while we held the
491 // lock, the object could not be rebiased toward another thread, so
492 // the bias bit would be clear.
493 trace(trace, " mark: 0x%016lx\n", mark);
494 if (probability(FREQUENT_PROBABILITY, mark.and(biasedLockMaskInPlace(INJECTED_VMCONFIG)).equal(WordFactory.unsigned(biasedLockPattern(INJECTED_VMCONFIG))))) {
495 endLockScope();
496 decCounter(options);
497 traceObject(trace, "-lock{bias}", object, false, options);
498 counters.unlockBias.inc();
499 return;
500 }
501 }
502
503 final Word lock = CurrentLockNode.currentLock(lockDepth);
504
505 // Load displaced mark
506 final Word displacedMark = lock.readWord(lockDisplacedMarkOffset(INJECTED_VMCONFIG), DISPLACED_MARK_WORD_LOCATION);
507 trace(trace, " displacedMark: 0x%016lx\n", displacedMark);
508
509 if (probability(NOT_LIKELY_PROBABILITY, displacedMark.equal(0))) {
510 // Recursive locking => done
511 traceObject(trace, "-lock{recursive}", object, false, options);
512 counters.unlockCasRecursive.inc();
513 } else {
514 if (!tryExitInflated(object, mark, lock, threadRegister, trace, options, counters)) {
515 verifyOop(object);
516 // Test if object's mark word is pointing to the displaced mark word, and if so,
517 // restore
518 // the displaced mark in the object - if the object's mark word is not pointing to
519 // the displaced mark word, do unlocking via runtime call.
520 Pointer objectPointer = Word.objectToTrackedPointer(object);
521 if (probability(VERY_FAST_PATH_PROBABILITY, objectPointer.logicCompareAndSwapWord(markOffset(INJECTED_VMCONFIG), lock, displacedMark, MARK_WORD_LOCATION))) {
522 traceObject(trace, "-lock{cas}", object, false, options);
523 counters.unlockCas.inc();
524 } else {
525 // The object's mark word was not pointing to the displaced header
526 traceObject(trace, "-lock{stub}", object, false, options);
527 counters.unlockStub.inc();
528 monitorexitStubC(MONITOREXIT, object, lock);
529 }
530 }
531 }
532 endLockScope();
533 decCounter(options);
534 }
535
536 private static boolean inlineFastUnlockSupported(OptionValues options) {
537 return inlineFastUnlockSupported(INJECTED_VMCONFIG, options);
538 }
539
540 private static boolean inlineFastUnlockSupported(GraalHotSpotVMConfig config, OptionValues options) {
541 return useFastInflatedLocking(options) && objectMonitorEntryListOffset(config) >= 0 && objectMonitorCxqOffset(config) >= 0 && monitorMask(config) >= 0 &&
542 objectMonitorOwnerOffset(config) >= 0 && objectMonitorRecursionsOffset(config) >= 0;
543 }
544
545 private static boolean tryExitInflated(Object object, Word mark, Word lock, Register threadRegister, boolean trace, OptionValues options, Counters counters) {
546 if (!inlineFastUnlockSupported(options)) {
547 return false;
548 }
549 if (probability(SLOW_PATH_PROBABILITY, mark.and(monitorMask(INJECTED_VMCONFIG)).notEqual(0))) {
550 // Inflated case
551 // mark is a pointer to the ObjectMonitor + monitorMask
552 Word monitor = mark.subtract(monitorMask(INJECTED_VMCONFIG));
553 int ownerOffset = objectMonitorOwnerOffset(INJECTED_VMCONFIG);
554 Word owner = monitor.readWord(ownerOffset, OBJECT_MONITOR_OWNER_LOCATION);
555 int recursionsOffset = objectMonitorRecursionsOffset(INJECTED_VMCONFIG);
556 Word recursions = monitor.readWord(recursionsOffset, OBJECT_MONITOR_RECURSION_LOCATION);
557 Word thread = registerAsWord(threadRegister);
558 if (probability(FAST_PATH_PROBABILITY, owner.xor(thread).or(recursions).equal(0))) {
559 // owner == thread && recursions == 0
560 int cxqOffset = objectMonitorCxqOffset(INJECTED_VMCONFIG);
561 Word cxq = monitor.readWord(cxqOffset, OBJECT_MONITOR_CXQ_LOCATION);
562 int entryListOffset = objectMonitorEntryListOffset(INJECTED_VMCONFIG);
563 Word entryList = monitor.readWord(entryListOffset, OBJECT_MONITOR_ENTRY_LIST_LOCATION);
564 if (probability(FREQUENT_PROBABILITY, cxq.or(entryList).equal(0))) {
565 // cxq == 0 && entryList == 0
566 // Nobody is waiting, success
567 // release_store
568 MembarNode.memoryBarrier(LOAD_STORE | STORE_STORE);
569 monitor.writeWord(ownerOffset, WordFactory.zero());
570 traceObject(trace, "-lock{inflated:simple}", object, false, options);
571 counters.unlockInflatedSimple.inc();
572 return true;
573 }
574 }
575 counters.unlockStubInflated.inc();
576 traceObject(trace, "-lock{stub:inflated}", object, false, options);
577 monitorexitStubC(MONITOREXIT, object, lock);
578 return true;
579 }
580 return false;
581 }
582
583 /**
584 * Calls straight out to the monitorexit stub.
585 */
586 @Snippet
587 public static void monitorexitStub(Object object, @ConstantParameter int lockDepth, @ConstantParameter boolean trace, @ConstantParameter OptionValues options) {
588 verifyOop(object);
589 traceObject(trace, "-lock{stub}", object, false, options);
590 final Word lock = CurrentLockNode.currentLock(lockDepth);
591 monitorexitStubC(MONITOREXIT, object, lock);
592 endLockScope();
593 decCounter(options);
594 }
595
596 public static void traceObject(boolean enabled, String action, Object object, boolean enter, OptionValues options) {
597 if (doProfile(options)) {
598 DynamicCounterNode.counter(enter ? "number of monitor enters" : "number of monitor exits", action, 1, PROFILE_CONTEXT);
599 }
600 if (enabled) {
601 Log.print(action);
602 Log.print(' ');
603 Log.printlnObject(object);
604 }
605 }
606
607 public static void trace(boolean enabled, String format, WordBase value) {
608 if (enabled) {
609 Log.printf(format, value.rawValue());
610 }
611 }
612
613 /**
614 * Leaving the breakpoint code in to provide an example of how to use the {@link BreakpointNode}
615 * intrinsic.
616 */
617 private static final boolean ENABLE_BREAKPOINT = false;
618
619 private static final LocationIdentity MONITOR_COUNTER_LOCATION = NamedLocationIdentity.mutable("MonitorCounter");
620
621 @NodeIntrinsic(BreakpointNode.class)
622 static native void bkpt(Object object, Word mark, Word tmp, Word value);
623
624 @Fold
625 static boolean verifyBalancedMonitors(OptionValues options) {
626 return VerifyBalancedMonitors.getValue(options);
627 }
628
629 public static void incCounter(OptionValues options) {
630 if (verifyBalancedMonitors(options)) {
631 final Word counter = MonitorCounterNode.counter();
632 final int count = counter.readInt(0, MONITOR_COUNTER_LOCATION);
633 counter.writeInt(0, count + 1, MONITOR_COUNTER_LOCATION);
634 }
635 }
636
637 public static void decCounter(OptionValues options) {
638 if (verifyBalancedMonitors(options)) {
639 final Word counter = MonitorCounterNode.counter();
640 final int count = counter.readInt(0, MONITOR_COUNTER_LOCATION);
641 counter.writeInt(0, count - 1, MONITOR_COUNTER_LOCATION);
642 }
643 }
644
645 @Snippet
646 private static void initCounter() {
647 final Word counter = MonitorCounterNode.counter();
648 counter.writeInt(0, 0, MONITOR_COUNTER_LOCATION);
649 }
650
651 @Snippet
652 private static void checkCounter(@ConstantParameter String errMsg) {
653 final Word counter = MonitorCounterNode.counter();
654 final int count = counter.readInt(0, MONITOR_COUNTER_LOCATION);
655 if (count != 0) {
656 vmError(errMsg, count);
657 }
658 }
733 this.useFastLocking = useFastLocking;
734
735 this.counters = new Counters(factory);
736 }
737
738 public void lower(RawMonitorEnterNode monitorenterNode, HotSpotRegistersProvider registers, LoweringTool tool) {
739 StructuredGraph graph = monitorenterNode.graph();
740 checkBalancedMonitors(graph, tool);
741
742 assert ((ObjectStamp) monitorenterNode.object().stamp(NodeView.DEFAULT)).nonNull();
743
744 Arguments args;
745 if (useFastLocking) {
746 args = new Arguments(monitorenter, graph.getGuardsStage(), tool.getLoweringStage());
747 args.add("object", monitorenterNode.object());
748 args.add("hub", monitorenterNode.getHub());
749 args.addConst("lockDepth", monitorenterNode.getMonitorId().getLockDepth());
750 args.addConst("threadRegister", registers.getThreadRegister());
751 args.addConst("stackPointerRegister", registers.getStackPointerRegister());
752 args.addConst("trace", isTracingEnabledForType(monitorenterNode.object()) || isTracingEnabledForMethod(graph));
753 args.addConst("options", graph.getOptions());
754 args.addConst("counters", counters);
755 } else {
756 args = new Arguments(monitorenterStub, graph.getGuardsStage(), tool.getLoweringStage());
757 args.add("object", monitorenterNode.object());
758 args.addConst("lockDepth", monitorenterNode.getMonitorId().getLockDepth());
759 args.addConst("trace", isTracingEnabledForType(monitorenterNode.object()) || isTracingEnabledForMethod(graph));
760 args.addConst("options", graph.getOptions());
761 args.addConst("counters", counters);
762 }
763
764 template(monitorenterNode, args).instantiate(providers.getMetaAccess(), monitorenterNode, DEFAULT_REPLACER, args);
765 }
766
767 public void lower(MonitorExitNode monitorexitNode, HotSpotRegistersProvider registers, LoweringTool tool) {
768 StructuredGraph graph = monitorexitNode.graph();
769
770 Arguments args;
771 if (useFastLocking) {
772 args = new Arguments(monitorexit, graph.getGuardsStage(), tool.getLoweringStage());
773 } else {
774 args = new Arguments(monitorexitStub, graph.getGuardsStage(), tool.getLoweringStage());
775 }
776 args.add("object", monitorexitNode.object());
777 args.addConst("lockDepth", monitorexitNode.getMonitorId().getLockDepth());
778 args.addConst("threadRegister", registers.getThreadRegister());
779 args.addConst("trace", isTracingEnabledForType(monitorexitNode.object()) || isTracingEnabledForMethod(graph));
780 args.addConst("options", graph.getOptions());
781 args.addConst("counters", counters);
782
783 template(monitorexitNode, args).instantiate(providers.getMetaAccess(), monitorexitNode, DEFAULT_REPLACER, args);
784 }
785
786 public static boolean isTracingEnabledForType(ValueNode object) {
787 ResolvedJavaType type = StampTool.typeOrNull(object.stamp(NodeView.DEFAULT));
788 String filter = TraceMonitorsTypeFilter.getValue(object.getOptions());
789 if (filter == null) {
790 return false;
791 } else {
792 if (filter.length() == 0) {
793 return true;
794 }
795 if (type == null) {
796 return false;
797 }
798 return (type.getName().contains(filter));
799 }
800 }
|
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24
25 package org.graalvm.compiler.hotspot.replacements;
26
27 import static jdk.vm.ci.code.MemoryBarriers.LOAD_STORE;
28 import static jdk.vm.ci.code.MemoryBarriers.STORE_STORE;
29 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_OPTIONVALUES;
30 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFIG;
31 import static org.graalvm.compiler.hotspot.nodes.AcquiredCASLockNode.mark;
32 import static org.graalvm.compiler.hotspot.nodes.BeginLockScopeNode.beginLockScope;
33 import static org.graalvm.compiler.hotspot.nodes.EndLockScopeNode.endLockScope;
34 import static org.graalvm.compiler.hotspot.nodes.VMErrorNode.vmError;
35 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.DISPLACED_MARK_WORD_LOCATION;
36 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.MARK_WORD_LOCATION;
37 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.OBJECT_MONITOR_CXQ_LOCATION;
38 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.OBJECT_MONITOR_ENTRY_LIST_LOCATION;
39 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.OBJECT_MONITOR_OWNER_LOCATION;
40 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.OBJECT_MONITOR_RECURSION_LOCATION;
41 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.PROTOTYPE_MARK_WORD_LOCATION;
42 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.ageMaskInPlace;
43 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.biasedLockMaskInPlace;
44 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.biasedLockPattern;
45 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.epochMaskInPlace;
46 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.loadWordFromObject;
47 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.lockDisplacedMarkOffset;
48 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.markOffset;
49 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.monitorMask;
50 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.objectMonitorCxqOffset;
51 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.objectMonitorEntryListOffset;
52 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.objectMonitorOwnerOffset;
53 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.objectMonitorRecursionsOffset;
54 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.pageSize;
55 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.prototypeMarkWordOffset;
56 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.registerAsWord;
57 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.stackBias;
58 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.unlockedMask;
59 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useBiasedLocking;
60 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.verifyOop;
61 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.wordSize;
62 import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.ProfileMonitors;
63 import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.SimpleFastInflatedLocking;
64 import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.TraceMonitorsMethodFilter;
65 import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.TraceMonitorsTypeFilter;
66 import static org.graalvm.compiler.hotspot.replacements.HotspotSnippetsOptions.VerifyBalancedMonitors;
67 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FAST_PATH_PROBABILITY;
68 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FREQUENT_PROBABILITY;
69 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.NOT_FREQUENT_PROBABILITY;
70 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.NOT_LIKELY_PROBABILITY;
71 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY;
72 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.VERY_FAST_PATH_PROBABILITY;
73 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;
74 import static org.graalvm.compiler.nodes.extended.MembarNode.memoryBarrier;
75 import static org.graalvm.compiler.replacements.SnippetTemplate.DEFAULT_REPLACER;
76 import static jdk.internal.vm.compiler.word.WordFactory.unsigned;
77 import static jdk.internal.vm.compiler.word.WordFactory.zero;
78
79 import java.util.List;
80
81 import org.graalvm.compiler.api.replacements.Fold;
82 import org.graalvm.compiler.api.replacements.Snippet;
83 import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter;
84 import org.graalvm.compiler.bytecode.Bytecode;
85 import org.graalvm.compiler.bytecode.ResolvedJavaMethodBytecode;
86 import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor;
87 import org.graalvm.compiler.core.common.type.ObjectStamp;
88 import org.graalvm.compiler.core.common.type.StampFactory;
89 import org.graalvm.compiler.core.common.type.StampPair;
90 import org.graalvm.compiler.debug.DebugHandlersFactory;
91 import org.graalvm.compiler.graph.Node.ConstantNodeParameter;
92 import org.graalvm.compiler.graph.Node.NodeIntrinsic;
93 import org.graalvm.compiler.graph.iterators.NodeIterable;
94 import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig;
95 import org.graalvm.compiler.hotspot.meta.HotSpotProviders;
96 import org.graalvm.compiler.hotspot.meta.HotSpotRegistersProvider;
97 import org.graalvm.compiler.hotspot.nodes.CurrentLockNode;
98 import org.graalvm.compiler.hotspot.nodes.FastAcquireBiasedLockNode;
99 import org.graalvm.compiler.hotspot.nodes.MonitorCounterNode;
100 import org.graalvm.compiler.hotspot.word.KlassPointer;
101 import org.graalvm.compiler.nodes.BreakpointNode;
102 import org.graalvm.compiler.nodes.CallTargetNode.InvokeKind;
103 import org.graalvm.compiler.nodes.ConstantNode;
104 import org.graalvm.compiler.nodes.DeoptimizeNode;
105 import org.graalvm.compiler.nodes.FrameState;
106 import org.graalvm.compiler.nodes.InvokeNode;
107 import org.graalvm.compiler.nodes.NamedLocationIdentity;
108 import org.graalvm.compiler.nodes.NodeView;
109 import org.graalvm.compiler.nodes.ReturnNode;
110 import org.graalvm.compiler.nodes.StructuredGraph;
111 import org.graalvm.compiler.nodes.ValueNode;
112 import org.graalvm.compiler.nodes.debug.DynamicCounterNode;
113 import org.graalvm.compiler.nodes.extended.ForeignCallNode;
114 import org.graalvm.compiler.nodes.extended.MembarNode;
115 import org.graalvm.compiler.nodes.java.MethodCallTargetNode;
116 import org.graalvm.compiler.nodes.java.MonitorExitNode;
206 *
207 * - the two lock bits are used to describe three states: locked/unlocked and monitor.
208 *
209 * [ptr | 00] locked ptr points to real header on stack
210 * [header | 0 | 01] unlocked regular object header
211 * [ptr | 10] monitor inflated lock (header is wapped out)
212 * [ptr | 11] marked used by markSweep to mark an object
213 * not valid at any other time
214 *
215 * We assume that stack/thread pointers have the lowest two bits cleared.
216 * </pre>
217 *
218 * Note that {@code Thread::allocate} enforces {@code JavaThread} objects to be aligned
219 * appropriately to comply with the layouts above.
220 */
221 public class MonitorSnippets implements Snippets {
222
223 private static final boolean PROFILE_CONTEXT = false;
224
225 @Fold
226 static boolean doProfile(@Fold.InjectedParameter OptionValues options) {
227 return ProfileMonitors.getValue(options);
228 }
229
230 @Snippet
231 public static void monitorenter(Object object, KlassPointer hub, @ConstantParameter int lockDepth, @ConstantParameter Register threadRegister, @ConstantParameter Register stackPointerRegister,
232 @ConstantParameter boolean trace, @ConstantParameter Counters counters) {
233 verifyOop(object);
234
235 // Load the mark word - this includes a null-check on object
236 final Word mark = loadWordFromObject(object, markOffset(INJECTED_VMCONFIG));
237
238 final Word lock = beginLockScope(lockDepth);
239
240 Pointer objectPointer = Word.objectToTrackedPointer(object);
241 trace(trace, " object: 0x%016lx\n", objectPointer);
242 trace(trace, " lock: 0x%016lx\n", lock);
243 trace(trace, " mark: 0x%016lx\n", mark);
244
245 incCounter();
246
247 if (useBiasedLocking(INJECTED_VMCONFIG)) {
248 if (tryEnterBiased(object, hub, lock, mark, threadRegister, trace, counters)) {
249 return;
250 }
251 // not biased, fall-through
252 }
253 if (inlineFastLockSupported() && probability(SLOW_PATH_PROBABILITY, mark.and(monitorMask(INJECTED_VMCONFIG)).notEqual(0))) {
254 // Inflated case
255 if (tryEnterInflated(object, lock, mark, threadRegister, trace, counters)) {
256 return;
257 }
258 } else {
259 // Create the unlocked mark word pattern
260 Word unlockedMark = mark.or(unlockedMask(INJECTED_VMCONFIG));
261 trace(trace, " unlockedMark: 0x%016lx\n", unlockedMark);
262
263 // Copy this unlocked mark word into the lock slot on the stack
264 lock.writeWord(lockDisplacedMarkOffset(INJECTED_VMCONFIG), unlockedMark, DISPLACED_MARK_WORD_LOCATION);
265
266 // make sure previous store does not float below compareAndSwap
267 MembarNode.memoryBarrier(STORE_STORE);
268
269 // Test if the object's mark word is unlocked, and if so, store the
270 // (address of) the lock slot into the object's mark word.
271 Word currentMark = objectPointer.compareAndSwapWord(markOffset(INJECTED_VMCONFIG), unlockedMark, lock, MARK_WORD_LOCATION);
272 if (probability(FAST_PATH_PROBABILITY, currentMark.equal(unlockedMark))) {
273 traceObject(trace, "+lock{cas}", object, true);
274 counters.lockCas.inc();
275 mark(object);
276 return;
277 } else {
278 trace(trace, " currentMark: 0x%016lx\n", currentMark);
279 // The mark word in the object header was not the same.
280 // Either the object is locked by another thread or is already locked
281 // by the current thread. The latter is true if the mark word
282 // is a stack pointer into the current thread's stack, i.e.:
283 //
284 // 1) (currentMark & aligned_mask) == 0
285 // 2) rsp <= currentMark
286 // 3) currentMark <= rsp + page_size
287 //
288 // These 3 tests can be done by evaluating the following expression:
289 //
290 // (currentMark - rsp) & (aligned_mask - page_size)
291 //
292 // assuming both the stack pointer and page_size have their least
293 // significant 2 bits cleared and page_size is a power of 2
294 final Word alignedMask = unsigned(wordSize() - 1);
295 final Word stackPointer = registerAsWord(stackPointerRegister).add(stackBias(INJECTED_VMCONFIG));
296 if (probability(FAST_PATH_PROBABILITY, currentMark.subtract(stackPointer).and(alignedMask.subtract(pageSize(INJECTED_VMCONFIG))).equal(0))) {
297 // Recursively locked => write 0 to the lock slot
298 lock.writeWord(lockDisplacedMarkOffset(INJECTED_VMCONFIG), zero(), DISPLACED_MARK_WORD_LOCATION);
299 traceObject(trace, "+lock{cas:recursive}", object, true);
300 counters.lockCasRecursive.inc();
301 return;
302 }
303 traceObject(trace, "+lock{stub:failed-cas/stack}", object, true);
304 counters.lockStubFailedCas.inc();
305 }
306 }
307 // slow-path runtime-call
308 monitorenterStubC(MONITORENTER, object, lock);
309 }
310
311 private static boolean tryEnterBiased(Object object, KlassPointer hub, Word lock, Word mark, Register threadRegister, boolean trace, Counters counters) {
312 // See whether the lock is currently biased toward our thread and
313 // whether the epoch is still valid.
314 // Note that the runtime guarantees sufficient alignment of JavaThread
315 // pointers to allow age to be placed into low bits.
316 final Word biasableLockBits = mark.and(biasedLockMaskInPlace(INJECTED_VMCONFIG));
317
318 // Check whether the bias pattern is present in the object's mark word
319 // and the bias owner and the epoch are both still current.
320 final Word prototypeMarkWord = hub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION);
321 final Word thread = registerAsWord(threadRegister);
322 final Word tmp = prototypeMarkWord.or(thread).xor(mark).and(~ageMaskInPlace(INJECTED_VMCONFIG));
323 trace(trace, "prototypeMarkWord: 0x%016lx\n", prototypeMarkWord);
324 trace(trace, " thread: 0x%016lx\n", thread);
325 trace(trace, " tmp: 0x%016lx\n", tmp);
326 if (probability(FAST_PATH_PROBABILITY, tmp.equal(0))) {
327 // Object is already biased to current thread -> done
328 traceObject(trace, "+lock{bias:existing}", object, true);
329 counters.lockBiasExisting.inc();
330 FastAcquireBiasedLockNode.mark(object);
331 return true;
332 }
333
334 // Now check to see whether biasing is enabled for this object
335 if (probability(NOT_FREQUENT_PROBABILITY, biasableLockBits.equal(WordFactory.unsigned(biasedLockPattern(INJECTED_VMCONFIG))))) {
336 Pointer objectPointer = Word.objectToTrackedPointer(object);
337 // At this point we know that the mark word has the bias pattern and
338 // that we are not the bias owner in the current epoch. We need to
339 // figure out more details about the state of the mark word in order to
340 // know what operations can be legally performed on the object's
341 // mark word.
342
343 // If the low three bits in the xor result aren't clear, that means
344 // the prototype header is no longer biasable and we have to revoke
345 // the bias on this object.
346 if (probability(FREQUENT_PROBABILITY, tmp.and(biasedLockMaskInPlace(INJECTED_VMCONFIG)).equal(0))) {
347 // Biasing is still enabled for object's type. See whether the
348 // epoch of the current bias is still valid, meaning that the epoch
349 // bits of the mark word are equal to the epoch bits of the
350 // prototype mark word. (Note that the prototype mark word's epoch bits
351 // only change at a safepoint.) If not, attempt to rebias the object
352 // toward the current thread. Note that we must be absolutely sure
353 // that the current epoch is invalid in order to do this because
354 // otherwise the manipulations it performs on the mark word are
355 // illegal.
356 if (probability(FREQUENT_PROBABILITY, tmp.and(epochMaskInPlace(INJECTED_VMCONFIG)).equal(0))) {
357 // The epoch of the current bias is still valid but we know nothing
358 // about the owner; it might be set or it might be clear. Try to
359 // acquire the bias of the object using an atomic operation. If this
360 // fails we will go in to the runtime to revoke the object's bias.
361 // Note that we first construct the presumed unbiased header so we
362 // don't accidentally blow away another thread's valid bias.
363 Word unbiasedMark = mark.and(biasedLockMaskInPlace(INJECTED_VMCONFIG) | ageMaskInPlace(INJECTED_VMCONFIG) | epochMaskInPlace(INJECTED_VMCONFIG));
364 Word biasedMark = unbiasedMark.or(thread);
365 trace(trace, " unbiasedMark: 0x%016lx\n", unbiasedMark);
366 trace(trace, " biasedMark: 0x%016lx\n", biasedMark);
367 if (probability(VERY_FAST_PATH_PROBABILITY, objectPointer.logicCompareAndSwapWord(markOffset(INJECTED_VMCONFIG), unbiasedMark, biasedMark, MARK_WORD_LOCATION))) {
368 // Object is now biased to current thread -> done
369 traceObject(trace, "+lock{bias:acquired}", object, true);
370 counters.lockBiasAcquired.inc();
371 return true;
372 }
373 // If the biasing toward our thread failed, this means that another thread
374 // owns the bias and we need to revoke that bias. The revocation will occur
375 // in the interpreter runtime.
376 traceObject(trace, "+lock{stub:revoke}", object, true);
377 counters.lockStubRevoke.inc();
378 } else {
379 // At this point we know the epoch has expired, meaning that the
380 // current bias owner, if any, is actually invalid. Under these
381 // circumstances _only_, are we allowed to use the current mark word
382 // value as the comparison value when doing the CAS to acquire the
383 // bias in the current epoch. In other words, we allow transfer of
384 // the bias from one thread to another directly in this situation.
385 Word biasedMark = prototypeMarkWord.or(thread);
386 trace(trace, " biasedMark: 0x%016lx\n", biasedMark);
387 if (probability(VERY_FAST_PATH_PROBABILITY, objectPointer.logicCompareAndSwapWord(markOffset(INJECTED_VMCONFIG), mark, biasedMark, MARK_WORD_LOCATION))) {
388 // Object is now biased to current thread -> done
389 traceObject(trace, "+lock{bias:transfer}", object, true);
390 counters.lockBiasTransfer.inc();
391 return true;
392 }
393 // If the biasing toward our thread failed, then another thread
394 // succeeded in biasing it toward itself and we need to revoke that
395 // bias. The revocation will occur in the runtime in the slow case.
396 traceObject(trace, "+lock{stub:epoch-expired}", object, true);
397 counters.lockStubEpochExpired.inc();
398 }
399 // slow-path runtime-call
400 monitorenterStubC(MONITORENTER, object, lock);
401 return true;
402 } else {
403 // The prototype mark word doesn't have the bias bit set any
404 // more, indicating that objects of this data type are not supposed
405 // to be biased any more. We are going to try to reset the mark of
406 // this object to the prototype value and fall through to the
407 // CAS-based locking scheme. Note that if our CAS fails, it means
408 // that another thread raced us for the privilege of revoking the
409 // bias of this particular object, so it's okay to continue in the
410 // normal locking code.
411 Word result = objectPointer.compareAndSwapWord(markOffset(INJECTED_VMCONFIG), mark, prototypeMarkWord, MARK_WORD_LOCATION);
412
413 // Fall through to the normal CAS-based lock, because no matter what
414 // the result of the above CAS, some thread must have succeeded in
415 // removing the bias bit from the object's header.
416
417 if (ENABLE_BREAKPOINT) {
418 bkpt(object, mark, tmp, result);
419 }
420 counters.revokeBias.inc();
421 return false;
422 }
423 } else {
424 // Biasing not enabled -> fall through to lightweight locking
425 counters.unbiasable.inc();
426 return false;
427 }
428 }
429
430 @Fold
431 public static boolean useFastInflatedLocking(@Fold.InjectedParameter OptionValues options) {
432 return SimpleFastInflatedLocking.getValue(options);
433 }
434
435 private static boolean inlineFastLockSupported() {
436 return inlineFastLockSupported(INJECTED_VMCONFIG, INJECTED_OPTIONVALUES);
437 }
438
439 private static boolean inlineFastLockSupported(GraalHotSpotVMConfig config, OptionValues options) {
440 return useFastInflatedLocking(options) && monitorMask(config) >= 0 && objectMonitorOwnerOffset(config) >= 0;
441 }
442
443 private static boolean tryEnterInflated(Object object, Word lock, Word mark, Register threadRegister, boolean trace, Counters counters) {
444 // write non-zero value to lock slot
445 lock.writeWord(lockDisplacedMarkOffset(INJECTED_VMCONFIG), lock, DISPLACED_MARK_WORD_LOCATION);
446 // mark is a pointer to the ObjectMonitor + monitorMask
447 Word monitor = mark.subtract(monitorMask(INJECTED_VMCONFIG));
448 int ownerOffset = objectMonitorOwnerOffset(INJECTED_VMCONFIG);
449 Word owner = monitor.readWord(ownerOffset, OBJECT_MONITOR_OWNER_LOCATION);
450 if (probability(FREQUENT_PROBABILITY, owner.equal(0))) {
451 // it appears unlocked (owner == 0)
452 if (probability(FREQUENT_PROBABILITY, monitor.logicCompareAndSwapWord(ownerOffset, owner, registerAsWord(threadRegister), OBJECT_MONITOR_OWNER_LOCATION))) {
453 // success
454 traceObject(trace, "+lock{inflated:cas}", object, true);
455 counters.inflatedCas.inc();
456 return true;
457 } else {
458 traceObject(trace, "+lock{stub:inflated:failed-cas}", object, true);
459 counters.inflatedFailedCas.inc();
460 }
461 } else {
462 traceObject(trace, "+lock{stub:inflated:owned}", object, true);
463 counters.inflatedOwned.inc();
464 }
465 return false;
466 }
467
468 /**
469 * Calls straight out to the monitorenter stub.
470 */
471 @Snippet
472 public static void monitorenterStub(Object object, @ConstantParameter int lockDepth, @ConstantParameter boolean trace) {
473 verifyOop(object);
474 incCounter();
475 if (object == null) {
476 DeoptimizeNode.deopt(DeoptimizationAction.InvalidateReprofile, DeoptimizationReason.NullCheckException);
477 }
478 // BeginLockScope nodes do not read from object so a use of object
479 // cannot float about the null check above
480 final Word lock = beginLockScope(lockDepth);
481 traceObject(trace, "+lock{stub}", object, true);
482 monitorenterStubC(MONITORENTER, object, lock);
483 }
484
485 @Snippet
486 public static void monitorexit(Object object, @ConstantParameter int lockDepth, @ConstantParameter Register threadRegister, @ConstantParameter boolean trace,
487 @ConstantParameter Counters counters) {
488 trace(trace, " object: 0x%016lx\n", Word.objectToTrackedPointer(object));
489 final Word mark = loadWordFromObject(object, markOffset(INJECTED_VMCONFIG));
490 if (useBiasedLocking(INJECTED_VMCONFIG)) {
491 // Check for biased locking unlock case, which is a no-op
492 // Note: we do not have to check the thread ID for two reasons.
493 // First, the interpreter checks for IllegalMonitorStateException at
494 // a higher level. Second, if the bias was revoked while we held the
495 // lock, the object could not be rebiased toward another thread, so
496 // the bias bit would be clear.
497 trace(trace, " mark: 0x%016lx\n", mark);
498 if (probability(FREQUENT_PROBABILITY, mark.and(biasedLockMaskInPlace(INJECTED_VMCONFIG)).equal(WordFactory.unsigned(biasedLockPattern(INJECTED_VMCONFIG))))) {
499 endLockScope();
500 decCounter();
501 traceObject(trace, "-lock{bias}", object, false);
502 counters.unlockBias.inc();
503 return;
504 }
505 }
506
507 final Word lock = CurrentLockNode.currentLock(lockDepth);
508
509 // Load displaced mark
510 final Word displacedMark = lock.readWord(lockDisplacedMarkOffset(INJECTED_VMCONFIG), DISPLACED_MARK_WORD_LOCATION);
511 trace(trace, " displacedMark: 0x%016lx\n", displacedMark);
512
513 if (probability(NOT_LIKELY_PROBABILITY, displacedMark.equal(0))) {
514 // Recursive locking => done
515 traceObject(trace, "-lock{recursive}", object, false);
516 counters.unlockCasRecursive.inc();
517 } else {
518 if (!tryExitInflated(object, mark, lock, threadRegister, trace, counters)) {
519 verifyOop(object);
520 // Test if object's mark word is pointing to the displaced mark word, and if so,
521 // restore
522 // the displaced mark in the object - if the object's mark word is not pointing to
523 // the displaced mark word, do unlocking via runtime call.
524 Pointer objectPointer = Word.objectToTrackedPointer(object);
525 if (probability(VERY_FAST_PATH_PROBABILITY, objectPointer.logicCompareAndSwapWord(markOffset(INJECTED_VMCONFIG), lock, displacedMark, MARK_WORD_LOCATION))) {
526 traceObject(trace, "-lock{cas}", object, false);
527 counters.unlockCas.inc();
528 } else {
529 // The object's mark word was not pointing to the displaced header
530 traceObject(trace, "-lock{stub}", object, false);
531 counters.unlockStub.inc();
532 monitorexitStubC(MONITOREXIT, object, lock);
533 }
534 }
535 }
536 endLockScope();
537 decCounter();
538 }
539
540 private static boolean inlineFastUnlockSupported(OptionValues options) {
541 return inlineFastUnlockSupported(INJECTED_VMCONFIG, options);
542 }
543
544 private static boolean inlineFastUnlockSupported(GraalHotSpotVMConfig config, OptionValues options) {
545 return useFastInflatedLocking(options) && objectMonitorEntryListOffset(config) >= 0 && objectMonitorCxqOffset(config) >= 0 && monitorMask(config) >= 0 &&
546 objectMonitorOwnerOffset(config) >= 0 && objectMonitorRecursionsOffset(config) >= 0;
547 }
548
549 private static boolean tryExitInflated(Object object, Word mark, Word lock, Register threadRegister, boolean trace, Counters counters) {
550 if (!inlineFastUnlockSupported(INJECTED_OPTIONVALUES)) {
551 return false;
552 }
553 if (probability(SLOW_PATH_PROBABILITY, mark.and(monitorMask(INJECTED_VMCONFIG)).notEqual(0))) {
554 // Inflated case
555 // mark is a pointer to the ObjectMonitor + monitorMask
556 Word monitor = mark.subtract(monitorMask(INJECTED_VMCONFIG));
557 int ownerOffset = objectMonitorOwnerOffset(INJECTED_VMCONFIG);
558 Word owner = monitor.readWord(ownerOffset, OBJECT_MONITOR_OWNER_LOCATION);
559 int recursionsOffset = objectMonitorRecursionsOffset(INJECTED_VMCONFIG);
560 Word recursions = monitor.readWord(recursionsOffset, OBJECT_MONITOR_RECURSION_LOCATION);
561 Word thread = registerAsWord(threadRegister);
562 if (probability(FAST_PATH_PROBABILITY, owner.xor(thread).or(recursions).equal(0))) {
563 // owner == thread && recursions == 0
564 int cxqOffset = objectMonitorCxqOffset(INJECTED_VMCONFIG);
565 Word cxq = monitor.readWord(cxqOffset, OBJECT_MONITOR_CXQ_LOCATION);
566 int entryListOffset = objectMonitorEntryListOffset(INJECTED_VMCONFIG);
567 Word entryList = monitor.readWord(entryListOffset, OBJECT_MONITOR_ENTRY_LIST_LOCATION);
568 if (probability(FREQUENT_PROBABILITY, cxq.or(entryList).equal(0))) {
569 // cxq == 0 && entryList == 0
570 // Nobody is waiting, success
571 // release_store
572 memoryBarrier(LOAD_STORE | STORE_STORE);
573 monitor.writeWord(ownerOffset, zero());
574 traceObject(trace, "-lock{inflated:simple}", object, false);
575 counters.unlockInflatedSimple.inc();
576 return true;
577 }
578 }
579 counters.unlockStubInflated.inc();
580 traceObject(trace, "-lock{stub:inflated}", object, false);
581 monitorexitStubC(MONITOREXIT, object, lock);
582 return true;
583 }
584 return false;
585 }
586
587 /**
588 * Calls straight out to the monitorexit stub.
589 */
590 @Snippet
591 public static void monitorexitStub(Object object, @ConstantParameter int lockDepth, @ConstantParameter boolean trace) {
592 verifyOop(object);
593 traceObject(trace, "-lock{stub}", object, false);
594 final Word lock = CurrentLockNode.currentLock(lockDepth);
595 monitorexitStubC(MONITOREXIT, object, lock);
596 endLockScope();
597 decCounter();
598 }
599
600 public static void traceObject(boolean enabled, String action, Object object, boolean enter) {
601 if (doProfile(INJECTED_OPTIONVALUES)) {
602 DynamicCounterNode.counter(enter ? "number of monitor enters" : "number of monitor exits", action, 1, PROFILE_CONTEXT);
603 }
604 if (enabled) {
605 Log.print(action);
606 Log.print(' ');
607 Log.printlnObject(object);
608 }
609 }
610
611 public static void trace(boolean enabled, String format, WordBase value) {
612 if (enabled) {
613 Log.printf(format, value.rawValue());
614 }
615 }
616
617 /**
618 * Leaving the breakpoint code in to provide an example of how to use the {@link BreakpointNode}
619 * intrinsic.
620 */
621 private static final boolean ENABLE_BREAKPOINT = false;
622
623 private static final LocationIdentity MONITOR_COUNTER_LOCATION = NamedLocationIdentity.mutable("MonitorCounter");
624
625 @NodeIntrinsic(BreakpointNode.class)
626 static native void bkpt(Object object, Word mark, Word tmp, Word value);
627
628 @Fold
629 static boolean verifyBalancedMonitors(@Fold.InjectedParameter OptionValues options) {
630 return VerifyBalancedMonitors.getValue(options);
631 }
632
633 static void incCounter() {
634 if (verifyBalancedMonitors(INJECTED_OPTIONVALUES)) {
635 final Word counter = MonitorCounterNode.counter();
636 final int count = counter.readInt(0, MONITOR_COUNTER_LOCATION);
637 counter.writeInt(0, count + 1, MONITOR_COUNTER_LOCATION);
638 }
639 }
640
641 public static void decCounter() {
642 if (verifyBalancedMonitors(INJECTED_OPTIONVALUES)) {
643 final Word counter = MonitorCounterNode.counter();
644 final int count = counter.readInt(0, MONITOR_COUNTER_LOCATION);
645 counter.writeInt(0, count - 1, MONITOR_COUNTER_LOCATION);
646 }
647 }
648
649 @Snippet
650 private static void initCounter() {
651 final Word counter = MonitorCounterNode.counter();
652 counter.writeInt(0, 0, MONITOR_COUNTER_LOCATION);
653 }
654
655 @Snippet
656 private static void checkCounter(@ConstantParameter String errMsg) {
657 final Word counter = MonitorCounterNode.counter();
658 final int count = counter.readInt(0, MONITOR_COUNTER_LOCATION);
659 if (count != 0) {
660 vmError(errMsg, count);
661 }
662 }
737 this.useFastLocking = useFastLocking;
738
739 this.counters = new Counters(factory);
740 }
741
742 public void lower(RawMonitorEnterNode monitorenterNode, HotSpotRegistersProvider registers, LoweringTool tool) {
743 StructuredGraph graph = monitorenterNode.graph();
744 checkBalancedMonitors(graph, tool);
745
746 assert ((ObjectStamp) monitorenterNode.object().stamp(NodeView.DEFAULT)).nonNull();
747
748 Arguments args;
749 if (useFastLocking) {
750 args = new Arguments(monitorenter, graph.getGuardsStage(), tool.getLoweringStage());
751 args.add("object", monitorenterNode.object());
752 args.add("hub", monitorenterNode.getHub());
753 args.addConst("lockDepth", monitorenterNode.getMonitorId().getLockDepth());
754 args.addConst("threadRegister", registers.getThreadRegister());
755 args.addConst("stackPointerRegister", registers.getStackPointerRegister());
756 args.addConst("trace", isTracingEnabledForType(monitorenterNode.object()) || isTracingEnabledForMethod(graph));
757 args.addConst("counters", counters);
758 } else {
759 args = new Arguments(monitorenterStub, graph.getGuardsStage(), tool.getLoweringStage());
760 args.add("object", monitorenterNode.object());
761 args.addConst("lockDepth", monitorenterNode.getMonitorId().getLockDepth());
762 args.addConst("trace", isTracingEnabledForType(monitorenterNode.object()) || isTracingEnabledForMethod(graph));
763 args.addConst("counters", counters);
764 }
765
766 template(monitorenterNode, args).instantiate(providers.getMetaAccess(), monitorenterNode, DEFAULT_REPLACER, args);
767 }
768
769 public void lower(MonitorExitNode monitorexitNode, HotSpotRegistersProvider registers, LoweringTool tool) {
770 StructuredGraph graph = monitorexitNode.graph();
771
772 Arguments args;
773 if (useFastLocking) {
774 args = new Arguments(monitorexit, graph.getGuardsStage(), tool.getLoweringStage());
775 } else {
776 args = new Arguments(monitorexitStub, graph.getGuardsStage(), tool.getLoweringStage());
777 }
778 args.add("object", monitorexitNode.object());
779 args.addConst("lockDepth", monitorexitNode.getMonitorId().getLockDepth());
780 args.addConst("threadRegister", registers.getThreadRegister());
781 args.addConst("trace", isTracingEnabledForType(monitorexitNode.object()) || isTracingEnabledForMethod(graph));
782 args.addConst("counters", counters);
783
784 template(monitorexitNode, args).instantiate(providers.getMetaAccess(), monitorexitNode, DEFAULT_REPLACER, args);
785 }
786
787 public static boolean isTracingEnabledForType(ValueNode object) {
788 ResolvedJavaType type = StampTool.typeOrNull(object.stamp(NodeView.DEFAULT));
789 String filter = TraceMonitorsTypeFilter.getValue(object.getOptions());
790 if (filter == null) {
791 return false;
792 } else {
793 if (filter.length() == 0) {
794 return true;
795 }
796 if (type == null) {
797 return false;
798 }
799 return (type.getName().contains(filter));
800 }
801 }
|