--- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/DefaultHotSpotLoweringProvider.java 2017-03-20 17:38:38.000000000 -0700 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/DefaultHotSpotLoweringProvider.java 2017-03-20 17:38:38.000000000 -0700 @@ -22,6 +22,7 @@ */ package org.graalvm.compiler.hotspot.meta; +import static jdk.vm.ci.hotspot.HotSpotJVMCIRuntimeProvider.getArrayBaseOffset; import static org.graalvm.compiler.core.common.GraalOptions.AlwaysInlineVTableStubs; import static org.graalvm.compiler.core.common.GraalOptions.GeneratePIC; import static org.graalvm.compiler.core.common.GraalOptions.InlineVTableStubs; @@ -31,13 +32,11 @@ import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.CLASS_KLASS_LOCATION; import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.CLASS_MIRROR_LOCATION; import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.COMPRESSED_HUB_LOCATION; +import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.DISPLACED_MARK_WORD_LOCATION; import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.HUB_LOCATION; import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.HUB_WRITE_LOCATION; import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.KLASS_LAYOUT_HELPER_LOCATION; import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.OBJ_ARRAY_KLASS_ELEMENT_KLASS_LOCATION; -import static org.graalvm.compiler.hotspot.replacements.NewObjectSnippets.INIT_LOCATION; -import static jdk.vm.ci.hotspot.HotSpotJVMCIRuntimeProvider.getArrayBaseOffset; - import java.lang.ref.Reference; import org.graalvm.compiler.api.directives.GraalDirectives; @@ -53,12 +52,9 @@ import org.graalvm.compiler.graph.NodeInputList; import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig; import org.graalvm.compiler.hotspot.HotSpotGraalRuntimeProvider; +import org.graalvm.compiler.hotspot.nodes.BeginLockScopeNode; import org.graalvm.compiler.hotspot.nodes.CompressionNode; import org.graalvm.compiler.hotspot.nodes.CompressionNode.CompressionOp; -import org.graalvm.compiler.hotspot.nodes.aot.InitializeKlassNode; -import org.graalvm.compiler.hotspot.nodes.aot.ResolveConstantNode; -import org.graalvm.compiler.hotspot.nodes.aot.ResolveMethodAndLoadCountersNode; -import org.graalvm.compiler.hotspot.nodes.profiling.ProfileNode; import org.graalvm.compiler.hotspot.nodes.ComputeObjectAddressNode; import org.graalvm.compiler.hotspot.nodes.G1ArrayRangePostWriteBarrier; import org.graalvm.compiler.hotspot.nodes.G1ArrayRangePreWriteBarrier; @@ -70,6 +66,10 @@ import org.graalvm.compiler.hotspot.nodes.HotSpotIndirectCallTargetNode; import org.graalvm.compiler.hotspot.nodes.SerialArrayRangeWriteBarrier; import org.graalvm.compiler.hotspot.nodes.SerialWriteBarrier; +import org.graalvm.compiler.hotspot.nodes.aot.InitializeKlassNode; +import org.graalvm.compiler.hotspot.nodes.aot.ResolveConstantNode; +import org.graalvm.compiler.hotspot.nodes.aot.ResolveMethodAndLoadCountersNode; +import org.graalvm.compiler.hotspot.nodes.profiling.ProfileNode; import org.graalvm.compiler.hotspot.nodes.type.KlassPointerStamp; import org.graalvm.compiler.hotspot.nodes.type.MethodPointerStamp; import org.graalvm.compiler.hotspot.nodes.type.NarrowOopStamp; @@ -102,7 +102,6 @@ import org.graalvm.compiler.nodes.LogicNode; import org.graalvm.compiler.nodes.LoweredCallTargetNode; import org.graalvm.compiler.nodes.ParameterNode; -import org.graalvm.compiler.nodes.PiNode; import org.graalvm.compiler.nodes.SafepointNode; import org.graalvm.compiler.nodes.StartNode; import org.graalvm.compiler.nodes.StructuredGraph; @@ -119,13 +118,14 @@ import org.graalvm.compiler.nodes.extended.ForeignCallNode; import org.graalvm.compiler.nodes.extended.GetClassNode; import org.graalvm.compiler.nodes.extended.GuardedUnsafeLoadNode; -import org.graalvm.compiler.nodes.extended.GuardingNode; import org.graalvm.compiler.nodes.extended.LoadHubNode; import org.graalvm.compiler.nodes.extended.LoadMethodNode; import org.graalvm.compiler.nodes.extended.OSRLocalNode; +import org.graalvm.compiler.nodes.extended.OSRLockNode; +import org.graalvm.compiler.nodes.extended.OSRMonitorEnterNode; import org.graalvm.compiler.nodes.extended.OSRStartNode; import org.graalvm.compiler.nodes.extended.StoreHubNode; -import org.graalvm.compiler.nodes.extended.UnsafeLoadNode; +import org.graalvm.compiler.nodes.extended.RawLoadNode; import org.graalvm.compiler.nodes.java.ClassIsAssignableFromNode; import org.graalvm.compiler.nodes.java.DynamicNewArrayNode; import org.graalvm.compiler.nodes.java.DynamicNewInstanceNode; @@ -134,6 +134,7 @@ import org.graalvm.compiler.nodes.java.LoadExceptionObjectNode; import org.graalvm.compiler.nodes.java.MethodCallTargetNode; import org.graalvm.compiler.nodes.java.MonitorExitNode; +import org.graalvm.compiler.nodes.java.MonitorIdNode; import org.graalvm.compiler.nodes.java.NewArrayNode; import org.graalvm.compiler.nodes.java.NewInstanceNode; import org.graalvm.compiler.nodes.java.NewMultiArrayNode; @@ -148,6 +149,7 @@ import org.graalvm.compiler.nodes.spi.StampProvider; import org.graalvm.compiler.nodes.type.StampTool; import org.graalvm.compiler.nodes.util.GraphUtil; +import org.graalvm.compiler.options.OptionValues; import org.graalvm.compiler.replacements.DefaultJavaLoweringProvider; import org.graalvm.compiler.replacements.nodes.AssertionNode; @@ -194,25 +196,29 @@ } @Override - public void initialize(HotSpotProviders providers, GraalHotSpotVMConfig config) { - super.initialize(providers, providers.getSnippetReflection()); + public void initialize(OptionValues options, HotSpotProviders providers, GraalHotSpotVMConfig config) { + super.initialize(options, runtime, providers, providers.getSnippetReflection()); assert target == providers.getCodeCache().getTarget(); - instanceofSnippets = new InstanceOfSnippets.Templates(providers, target); - newObjectSnippets = new NewObjectSnippets.Templates(providers, target, config); - monitorSnippets = new MonitorSnippets.Templates(providers, target, config.useFastLocking); - writeBarrierSnippets = new WriteBarrierSnippets.Templates(providers, target, config.useCompressedOops ? config.getOopEncoding() : null); - exceptionObjectSnippets = new LoadExceptionObjectSnippets.Templates(providers, target); - unsafeLoadSnippets = new UnsafeLoadSnippets.Templates(providers, target); - assertionSnippets = new AssertionSnippets.Templates(providers, target); - arraycopySnippets = new ArrayCopySnippets.Templates(providers, target); - stringToBytesSnippets = new StringToBytesSnippets.Templates(providers, target); - hashCodeSnippets = new HashCodeSnippets.Templates(providers, target); - if (GeneratePIC.getValue()) { - resolveConstantSnippets = new ResolveConstantSnippets.Templates(providers, target); - profileSnippets = new ProfileSnippets.Templates(providers, target); + instanceofSnippets = new InstanceOfSnippets.Templates(options, runtime, providers, target); + newObjectSnippets = new NewObjectSnippets.Templates(options, runtime, providers, target, config); + monitorSnippets = new MonitorSnippets.Templates(options, runtime, providers, target, config.useFastLocking); + writeBarrierSnippets = new WriteBarrierSnippets.Templates(options, runtime, providers, target, config.useCompressedOops ? config.getOopEncoding() : null); + exceptionObjectSnippets = new LoadExceptionObjectSnippets.Templates(options, providers, target); + unsafeLoadSnippets = new UnsafeLoadSnippets.Templates(options, providers, target); + assertionSnippets = new AssertionSnippets.Templates(options, providers, target); + arraycopySnippets = new ArrayCopySnippets.Templates(options, runtime, providers, target); + stringToBytesSnippets = new StringToBytesSnippets.Templates(options, providers, target); + hashCodeSnippets = new HashCodeSnippets.Templates(options, providers, target); + if (GeneratePIC.getValue(options)) { + resolveConstantSnippets = new ResolveConstantSnippets.Templates(options, providers, target); + profileSnippets = new ProfileSnippets.Templates(options, providers, target); } - providers.getReplacements().registerSnippetTemplateCache(new UnsafeArrayCopySnippets.Templates(providers, target)); + providers.getReplacements().registerSnippetTemplateCache(new UnsafeArrayCopySnippets.Templates(options, providers, target)); + } + + public MonitorSnippets.Templates getMonitorSnippets() { + return monitorSnippets; } @Override @@ -303,7 +309,7 @@ } } else if (n instanceof MonitorExitNode) { if (graph.getGuardsStage().areFrameStatesAtDeopts()) { - monitorSnippets.lower((MonitorExitNode) n, tool); + monitorSnippets.lower((MonitorExitNode) n, registers, tool); } } else if (n instanceof ArrayCopyNode) { arraycopySnippets.lower((ArrayCopyNode) n, tool); @@ -395,7 +401,7 @@ StructuredGraph graph = n.graph(); assert !n.getHub().isConstant(); AddressNode address = createOffsetAddress(graph, n.getHub(), runtime.getVMConfig().klassLayoutHelperOffset); - n.replaceAtUsagesAndDelete(graph.unique(new FloatingReadNode(address, KLASS_LAYOUT_HELPER_LOCATION, null, n.stamp(), n.getGuard(), BarrierType.NONE))); + n.replaceAtUsagesAndDelete(graph.unique(new FloatingReadNode(address, KLASS_LAYOUT_HELPER_LOCATION, null, n.stamp(), null, BarrierType.NONE))); } private void lowerHubGetClassNode(HubGetClassNode n, LoweringTool tool) { @@ -406,7 +412,7 @@ StructuredGraph graph = n.graph(); assert !n.getHub().isConstant(); AddressNode address = createOffsetAddress(graph, n.getHub(), runtime.getVMConfig().classMirrorOffset); - FloatingReadNode read = graph.unique(new FloatingReadNode(address, CLASS_MIRROR_LOCATION, null, n.stamp(), n.getGuard(), BarrierType.NONE)); + FloatingReadNode read = graph.unique(new FloatingReadNode(address, CLASS_MIRROR_LOCATION, null, n.stamp(), null, BarrierType.NONE)); n.replaceAtUsagesAndDelete(read); } @@ -418,7 +424,7 @@ StructuredGraph graph = n.graph(); assert !n.getValue().isConstant(); AddressNode address = createOffsetAddress(graph, n.getValue(), runtime.getVMConfig().klassOffset); - FloatingReadNode read = graph.unique(new FloatingReadNode(address, CLASS_KLASS_LOCATION, null, n.stamp(), n.getGuard(), BarrierType.NONE)); + FloatingReadNode read = graph.unique(new FloatingReadNode(address, CLASS_KLASS_LOCATION, null, n.stamp(), null, BarrierType.NONE)); n.replaceAtUsagesAndDelete(read); } @@ -428,15 +434,15 @@ NodeInputList parameters = callTarget.arguments(); ValueNode receiver = parameters.size() <= 0 ? null : parameters.get(0); if (!callTarget.isStatic() && receiver.stamp() instanceof ObjectStamp && !StampTool.isPointerNonNull(receiver)) { - GuardingNode receiverNullCheck = createNullCheck(receiver, invoke.asNode(), tool); - PiNode nonNullReceiver = graph.unique(new PiNode(receiver, ((ObjectStamp) receiver.stamp()).join(StampFactory.objectNonNull()), (ValueNode) receiverNullCheck)); + ValueNode nonNullReceiver = createNullCheckedValue(receiver, invoke.asNode(), tool); parameters.set(0, nonNullReceiver); receiver = nonNullReceiver; } JavaType[] signature = callTarget.targetMethod().getSignature().toParameterTypes(callTarget.isStatic() ? null : callTarget.targetMethod().getDeclaringClass()); LoweredCallTargetNode loweredCallTarget = null; - if (InlineVTableStubs.getValue() && callTarget.invokeKind().isIndirect() && (AlwaysInlineVTableStubs.getValue() || invoke.isPolymorphic())) { + OptionValues options = graph.getOptions(); + if (InlineVTableStubs.getValue(options) && callTarget.invokeKind().isIndirect() && (AlwaysInlineVTableStubs.getValue(options) || invoke.isPolymorphic())) { HotSpotResolvedJavaMethod hsMethod = (HotSpotResolvedJavaMethod) callTarget.targetMethod(); ResolvedJavaType receiverType = invoke.getReceiverType(); if (hsMethod.isInVirtualMethodTable(receiverType)) { @@ -512,7 +518,7 @@ } @Override - protected void lowerUnsafeLoadNode(UnsafeLoadNode load, LoweringTool tool) { + protected void lowerUnsafeLoadNode(RawLoadNode load, LoweringTool tool) { StructuredGraph graph = load.graph(); if (!(load instanceof GuardedUnsafeLoadNode) && !graph.getGuardsStage().allowsFloatingGuards() && addReadBarrier(load)) { unsafeLoadSnippets.lower(load, tool); @@ -559,23 +565,64 @@ ParameterNode buffer = graph.addWithoutUnique(new ParameterNode(0, StampPair.createSingle(StampFactory.forKind(runtime.getTarget().wordJavaKind)))); ForeignCallNode migrationEnd = graph.add(new ForeignCallNode(foreignCalls, OSR_MIGRATION_END, buffer)); migrationEnd.setStateAfter(osrStart.stateAfter()); - newStart.setNext(migrationEnd); FixedNode next = osrStart.next(); osrStart.setNext(null); migrationEnd.setNext(next); graph.setStart(newStart); - // mirroring the calculations in c1_GraphBuilder.cpp (setup_osr_entry_block) - int localsOffset = (graph.method().getMaxLocals() - 1) * 8; + final int wordSize = target.wordSize; + + // @formatter:off + // taken from c2 locals_addr = osr_buf + (max_locals-1)*wordSize) + // @formatter:on + int localsOffset = (graph.method().getMaxLocals() - 1) * wordSize; for (OSRLocalNode osrLocal : graph.getNodes(OSRLocalNode.TYPE)) { int size = osrLocal.getStackKind().getSlotCount(); - int offset = localsOffset - (osrLocal.index() + size - 1) * 8; + int offset = localsOffset - (osrLocal.index() + size - 1) * wordSize; AddressNode address = createOffsetAddress(graph, buffer, offset); ReadNode load = graph.add(new ReadNode(address, any(), osrLocal.stamp(), BarrierType.NONE)); osrLocal.replaceAndDelete(load); graph.addBeforeFixed(migrationEnd, load); } + + // @formatter:off + // taken from c2 monitors_addr = osr_buf + (max_locals+mcnt*2-1)*wordSize); + // @formatter:on + final int lockCount = osrStart.stateAfter().locksSize(); + final int locksOffset = (graph.method().getMaxLocals() + lockCount * 2 - 1) * wordSize; + + // first initialize the lock slots for all enters with the displaced marks read from the + // buffer + for (OSRMonitorEnterNode osrMonitorEnter : graph.getNodes(OSRMonitorEnterNode.TYPE)) { + MonitorIdNode monitorID = osrMonitorEnter.getMonitorId(); + OSRLockNode lock = (OSRLockNode) osrMonitorEnter.object(); + final int index = lock.index(); + + final int offsetDisplacedHeader = locksOffset - ((index * 2) + 1) * wordSize; + final int offsetLockObject = locksOffset - index * 2 * wordSize; + + // load the displaced mark from the osr buffer + AddressNode addressDisplacedHeader = createOffsetAddress(graph, buffer, offsetDisplacedHeader); + ReadNode loadDisplacedHeader = graph.add(new ReadNode(addressDisplacedHeader, any(), lock.stamp(), BarrierType.NONE)); + graph.addBeforeFixed(migrationEnd, loadDisplacedHeader); + + // we need to initialize the stack slot for the lock + BeginLockScopeNode beginLockScope = graph.add(new BeginLockScopeNode(lock.getStackKind(), monitorID.getLockDepth())); + graph.addBeforeFixed(migrationEnd, beginLockScope); + + // write the displaced mark to the correct stack slot + AddressNode addressDisplacedMark = createOffsetAddress(graph, beginLockScope, runtime.getVMConfig().basicLockDisplacedHeaderOffset); + WriteNode writeStackSlot = graph.add(new WriteNode(addressDisplacedMark, DISPLACED_MARK_WORD_LOCATION, loadDisplacedHeader, BarrierType.NONE)); + graph.addBeforeFixed(migrationEnd, writeStackSlot); + + // load the lock object from the osr buffer + AddressNode addressLockObject = createOffsetAddress(graph, buffer, offsetLockObject); + ReadNode loadObject = graph.add(new ReadNode(addressLockObject, any(), lock.stamp(), BarrierType.NONE)); + lock.replaceAndDelete(loadObject); + graph.addBeforeFixed(migrationEnd, loadObject); + } + osrStart.replaceAtUsagesAndDelete(newStart); } } @@ -616,7 +663,7 @@ } private void lowerBytecodeExceptionNode(BytecodeExceptionNode node) { - if (OmitHotExceptionStacktrace.getValue()) { + if (OmitHotExceptionStacktrace.getValue(node.getOptions())) { if (throwCachedException(node)) { return; } @@ -640,7 +687,7 @@ graph.replaceFixedWithFixed(node, foreignCallNode); } - private boolean addReadBarrier(UnsafeLoadNode load) { + private boolean addReadBarrier(RawLoadNode load) { if (runtime.getVMConfig().useG1GC && load.graph().getGuardsStage() == StructuredGraph.GuardsStage.FIXED_DEOPTS && load.object().getStackKind() == JavaKind.Object && load.accessKind() == JavaKind.Object && !StampTool.isPointerAlwaysNull(load.object())) { ResolvedJavaType type = StampTool.typeOrNull(load.object()); @@ -659,7 +706,7 @@ assert vtableEntryOffset > 0; // We use LocationNode.ANY_LOCATION for the reads that access the vtable // entry as HotSpot does not guarantee that this is a final value. - Stamp methodStamp = MethodPointerStamp.method(); + Stamp methodStamp = MethodPointerStamp.methodNonNull(); AddressNode address = createOffsetAddress(graph, hub, vtableEntryOffset); ReadNode metaspaceMethod = graph.add(new ReadNode(address, any(), methodStamp, BarrierType.NONE)); return metaspaceMethod; @@ -734,9 +781,4 @@ public int arrayLengthOffset() { return runtime.getVMConfig().arrayOopDescLengthOffset(); } - - @Override - public LocationIdentity initLocationIdentity() { - return INIT_LOCATION; - } }