--- /dev/null 2017-01-22 10:16:57.869617664 -0800 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/DefaultJavaLoweringProvider.java 2017-02-15 17:09:04.606022896 -0800 @@ -0,0 +1,986 @@ +/* + * Copyright (c) 2011, 2016, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.graalvm.compiler.replacements; + +import static org.graalvm.compiler.nodes.NamedLocationIdentity.ARRAY_LENGTH_LOCATION; +import static org.graalvm.compiler.nodes.java.ArrayLengthNode.readArrayLength; +import static jdk.vm.ci.code.MemoryBarriers.JMM_POST_VOLATILE_READ; +import static jdk.vm.ci.code.MemoryBarriers.JMM_POST_VOLATILE_WRITE; +import static jdk.vm.ci.code.MemoryBarriers.JMM_PRE_VOLATILE_READ; +import static jdk.vm.ci.code.MemoryBarriers.JMM_PRE_VOLATILE_WRITE; +import static jdk.vm.ci.meta.DeoptimizationAction.InvalidateReprofile; +import static jdk.vm.ci.meta.DeoptimizationReason.BoundsCheckException; +import static jdk.vm.ci.meta.DeoptimizationReason.NullCheckException; + +import java.util.ArrayList; +import java.util.BitSet; +import java.util.List; + +import org.graalvm.compiler.api.directives.GraalDirectives; +import org.graalvm.compiler.api.replacements.Snippet; +import org.graalvm.compiler.api.replacements.SnippetReflectionProvider; +import org.graalvm.compiler.core.common.LocationIdentity; +import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor; +import org.graalvm.compiler.core.common.spi.ForeignCallsProvider; +import org.graalvm.compiler.core.common.type.IntegerStamp; +import org.graalvm.compiler.core.common.type.ObjectStamp; +import org.graalvm.compiler.core.common.type.Stamp; +import org.graalvm.compiler.core.common.type.StampFactory; +import org.graalvm.compiler.core.common.type.TypeReference; +import org.graalvm.compiler.debug.GraalError; +import org.graalvm.compiler.graph.Node; +import org.graalvm.compiler.nodes.ConstantNode; +import org.graalvm.compiler.nodes.FieldLocationIdentity; +import org.graalvm.compiler.nodes.FixedNode; +import org.graalvm.compiler.nodes.LogicNode; +import org.graalvm.compiler.nodes.NamedLocationIdentity; +import org.graalvm.compiler.nodes.PiNode; +import org.graalvm.compiler.nodes.StructuredGraph; +import org.graalvm.compiler.nodes.ValueNode; +import org.graalvm.compiler.nodes.calc.AddNode; +import org.graalvm.compiler.nodes.calc.IntegerBelowNode; +import org.graalvm.compiler.nodes.calc.IntegerConvertNode; +import org.graalvm.compiler.nodes.calc.IsNullNode; +import org.graalvm.compiler.nodes.calc.LeftShiftNode; +import org.graalvm.compiler.nodes.calc.NarrowNode; +import org.graalvm.compiler.nodes.calc.RightShiftNode; +import org.graalvm.compiler.nodes.calc.SignExtendNode; +import org.graalvm.compiler.nodes.calc.SubNode; +import org.graalvm.compiler.nodes.calc.ZeroExtendNode; +import org.graalvm.compiler.nodes.debug.VerifyHeapNode; +import org.graalvm.compiler.nodes.extended.BoxNode; +import org.graalvm.compiler.nodes.extended.FixedValueAnchorNode; +import org.graalvm.compiler.nodes.extended.ForeignCallNode; +import org.graalvm.compiler.nodes.extended.GuardedUnsafeLoadNode; +import org.graalvm.compiler.nodes.extended.GuardingNode; +import org.graalvm.compiler.nodes.extended.JavaReadNode; +import org.graalvm.compiler.nodes.extended.JavaWriteNode; +import org.graalvm.compiler.nodes.extended.LoadHubNode; +import org.graalvm.compiler.nodes.extended.MembarNode; +import org.graalvm.compiler.nodes.extended.UnboxNode; +import org.graalvm.compiler.nodes.extended.UnsafeLoadNode; +import org.graalvm.compiler.nodes.extended.UnsafeMemoryLoadNode; +import org.graalvm.compiler.nodes.extended.UnsafeMemoryStoreNode; +import org.graalvm.compiler.nodes.extended.UnsafeStoreNode; +import org.graalvm.compiler.nodes.java.AbstractNewArrayNode; +import org.graalvm.compiler.nodes.java.AbstractNewObjectNode; +import org.graalvm.compiler.nodes.java.AccessIndexedNode; +import org.graalvm.compiler.nodes.java.ArrayLengthNode; +import org.graalvm.compiler.nodes.java.AtomicReadAndWriteNode; +import org.graalvm.compiler.nodes.java.CompareAndSwapNode; +import org.graalvm.compiler.nodes.java.FinalFieldBarrierNode; +import org.graalvm.compiler.nodes.java.InstanceOfDynamicNode; +import org.graalvm.compiler.nodes.java.InstanceOfNode; +import org.graalvm.compiler.nodes.java.LoadFieldNode; +import org.graalvm.compiler.nodes.java.LoadIndexedNode; +import org.graalvm.compiler.nodes.java.LoweredAtomicReadAndWriteNode; +import org.graalvm.compiler.nodes.java.LoweredCompareAndSwapNode; +import org.graalvm.compiler.nodes.java.MonitorEnterNode; +import org.graalvm.compiler.nodes.java.MonitorIdNode; +import org.graalvm.compiler.nodes.java.NewArrayNode; +import org.graalvm.compiler.nodes.java.NewInstanceNode; +import org.graalvm.compiler.nodes.java.RawMonitorEnterNode; +import org.graalvm.compiler.nodes.java.StoreFieldNode; +import org.graalvm.compiler.nodes.java.StoreIndexedNode; +import org.graalvm.compiler.nodes.memory.HeapAccess.BarrierType; +import org.graalvm.compiler.nodes.memory.ReadNode; +import org.graalvm.compiler.nodes.memory.WriteNode; +import org.graalvm.compiler.nodes.memory.address.AddressNode; +import org.graalvm.compiler.nodes.memory.address.OffsetAddressNode; +import org.graalvm.compiler.nodes.memory.address.RawAddressNode; +import org.graalvm.compiler.nodes.spi.Lowerable; +import org.graalvm.compiler.nodes.spi.LoweringProvider; +import org.graalvm.compiler.nodes.spi.LoweringTool; +import org.graalvm.compiler.nodes.type.StampTool; +import org.graalvm.compiler.nodes.util.GraphUtil; +import org.graalvm.compiler.nodes.virtual.AllocatedObjectNode; +import org.graalvm.compiler.nodes.virtual.CommitAllocationNode; +import org.graalvm.compiler.nodes.virtual.VirtualArrayNode; +import org.graalvm.compiler.nodes.virtual.VirtualInstanceNode; +import org.graalvm.compiler.nodes.virtual.VirtualObjectNode; +import org.graalvm.compiler.phases.util.Providers; +import org.graalvm.compiler.replacements.nodes.BinaryMathIntrinsicNode; +import org.graalvm.compiler.replacements.nodes.BinaryMathIntrinsicNode.BinaryOperation; +import org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode; +import org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.UnaryOperation; + +import jdk.vm.ci.code.CodeUtil; +import jdk.vm.ci.code.MemoryBarriers; +import jdk.vm.ci.code.TargetDescription; +import jdk.vm.ci.meta.DeoptimizationAction; +import jdk.vm.ci.meta.DeoptimizationReason; +import jdk.vm.ci.meta.JavaConstant; +import jdk.vm.ci.meta.JavaKind; +import jdk.vm.ci.meta.MetaAccessProvider; +import jdk.vm.ci.meta.ResolvedJavaField; +import jdk.vm.ci.meta.ResolvedJavaMethod; +import jdk.vm.ci.meta.ResolvedJavaType; + +/** + * VM-independent lowerings for standard Java nodes. VM-specific methods are abstract and must be + * implemented by VM-specific subclasses. + */ +public abstract class DefaultJavaLoweringProvider implements LoweringProvider { + + protected final MetaAccessProvider metaAccess; + protected final ForeignCallsProvider foreignCalls; + protected final TargetDescription target; + + private BoxingSnippets.Templates boxingSnippets; + + public DefaultJavaLoweringProvider(MetaAccessProvider metaAccess, ForeignCallsProvider foreignCalls, TargetDescription target) { + this.metaAccess = metaAccess; + this.foreignCalls = foreignCalls; + this.target = target; + } + + public void initialize(Providers providers, SnippetReflectionProvider snippetReflection) { + boxingSnippets = new BoxingSnippets.Templates(providers, snippetReflection, target); + providers.getReplacements().registerSnippetTemplateCache(new SnippetCounterNode.SnippetCounterSnippets.Templates(providers, snippetReflection, target)); + } + + public final TargetDescription getTarget() { + return target; + } + + @Override + public void lower(Node n, LoweringTool tool) { + assert n instanceof Lowerable; + StructuredGraph graph = (StructuredGraph) n.graph(); + if (n instanceof LoadFieldNode) { + lowerLoadFieldNode((LoadFieldNode) n, tool); + } else if (n instanceof StoreFieldNode) { + lowerStoreFieldNode((StoreFieldNode) n, tool); + } else if (n instanceof LoadIndexedNode) { + lowerLoadIndexedNode((LoadIndexedNode) n, tool); + } else if (n instanceof StoreIndexedNode) { + lowerStoreIndexedNode((StoreIndexedNode) n, tool); + } else if (n instanceof ArrayLengthNode) { + lowerArrayLengthNode((ArrayLengthNode) n, tool); + } else if (n instanceof LoadHubNode) { + lowerLoadHubNode((LoadHubNode) n, tool); + } else if (n instanceof MonitorEnterNode) { + lowerMonitorEnterNode((MonitorEnterNode) n, tool, graph); + } else if (n instanceof CompareAndSwapNode) { + lowerCompareAndSwapNode((CompareAndSwapNode) n); + } else if (n instanceof AtomicReadAndWriteNode) { + lowerAtomicReadAndWriteNode((AtomicReadAndWriteNode) n); + } else if (n instanceof UnsafeLoadNode) { + lowerUnsafeLoadNode((UnsafeLoadNode) n, tool); + } else if (n instanceof UnsafeMemoryLoadNode) { + lowerUnsafeMemoryLoadNode((UnsafeMemoryLoadNode) n); + } else if (n instanceof UnsafeStoreNode) { + lowerUnsafeStoreNode((UnsafeStoreNode) n); + } else if (n instanceof UnsafeMemoryStoreNode) { + lowerUnsafeMemoryStoreNode((UnsafeMemoryStoreNode) n); + } else if (n instanceof JavaReadNode) { + lowerJavaReadNode((JavaReadNode) n); + } else if (n instanceof JavaWriteNode) { + lowerJavaWriteNode((JavaWriteNode) n); + } else if (n instanceof CommitAllocationNode) { + lowerCommitAllocationNode((CommitAllocationNode) n, tool); + } else if (n instanceof BoxNode) { + boxingSnippets.lower((BoxNode) n, tool); + } else if (n instanceof UnboxNode) { + boxingSnippets.lower((UnboxNode) n, tool); + } else if (n instanceof VerifyHeapNode) { + lowerVerifyHeap((VerifyHeapNode) n); + } else if (n instanceof UnaryMathIntrinsicNode) { + lowerUnaryMath((UnaryMathIntrinsicNode) n, tool); + } else if (n instanceof BinaryMathIntrinsicNode) { + lowerBinaryMath((BinaryMathIntrinsicNode) n, tool); + } else { + throw GraalError.shouldNotReachHere("Node implementing Lowerable not handled: " + n); + } + } + + private void lowerBinaryMath(BinaryMathIntrinsicNode math, LoweringTool tool) { + if (tool.getLoweringStage() == LoweringTool.StandardLoweringStage.HIGH_TIER) { + return; + } + ResolvedJavaMethod method = math.graph().method(); + if (method != null) { + if (method.getAnnotation(Snippet.class) != null) { + /* + * In the context of the snippet use the LIR lowering instead of the Node lowering. + */ + return; + } + if (method.getName().equalsIgnoreCase(math.getOperation().name()) && tool.getMetaAccess().lookupJavaType(Math.class).equals(method.getDeclaringClass())) { + /* + * A root compilation of the intrinsic method should emit the full assembly + * implementation. + */ + return; + } + + } + ForeignCallDescriptor foreignCall = toForeignCall(math.getOperation()); + if (foreignCall != null) { + StructuredGraph graph = math.graph(); + ForeignCallNode call = graph.add(new ForeignCallNode(foreignCalls, toForeignCall(math.getOperation()), math.getX(), math.getY())); + graph.addAfterFixed(tool.lastFixedNode(), call); + math.replaceAtUsages(call); + } + } + + private void lowerUnaryMath(UnaryMathIntrinsicNode math, LoweringTool tool) { + if (tool.getLoweringStage() == LoweringTool.StandardLoweringStage.HIGH_TIER) { + return; + } + ResolvedJavaMethod method = math.graph().method(); + if (method != null) { + if (method.getAnnotation(Snippet.class) != null) { + /* + * In the context of the snippet use the LIR lowering instead of the Node lowering. + */ + return; + } + if (method.getName().equalsIgnoreCase(math.getOperation().name()) && tool.getMetaAccess().lookupJavaType(Math.class).equals(method.getDeclaringClass())) { + /* + * A root compilation of the intrinsic method should emit the full assembly + * implementation. + */ + return; + } + + } + ForeignCallDescriptor foreignCall = toForeignCall(math.getOperation()); + if (foreignCall != null) { + StructuredGraph graph = math.graph(); + ForeignCallNode call = math.graph().add(new ForeignCallNode(foreignCalls, foreignCall, math.getValue())); + graph.addAfterFixed(tool.lastFixedNode(), call); + math.replaceAtUsages(call); + } + } + + protected ForeignCallDescriptor toForeignCall(UnaryOperation operation) { + return operation.foreignCallDescriptor; + } + + protected ForeignCallDescriptor toForeignCall(BinaryOperation operation) { + return operation.foreignCallDescriptor; + } + + protected void lowerVerifyHeap(VerifyHeapNode n) { + GraphUtil.removeFixedWithUnusedInputs(n); + } + + protected AddressNode createOffsetAddress(StructuredGraph graph, ValueNode object, long offset) { + ValueNode o = ConstantNode.forIntegerKind(target.wordJavaKind, offset, graph); + return graph.unique(new OffsetAddressNode(object, o)); + } + + protected AddressNode createFieldAddress(StructuredGraph graph, ValueNode object, ResolvedJavaField field) { + int offset = fieldOffset(field); + if (offset >= 0) { + return createOffsetAddress(graph, object, offset); + } else { + return null; + } + } + + protected void lowerLoadFieldNode(LoadFieldNode loadField, LoweringTool tool) { + assert loadField.getStackKind() != JavaKind.Illegal; + StructuredGraph graph = loadField.graph(); + ResolvedJavaField field = loadField.field(); + ValueNode object = loadField.isStatic() ? staticFieldBase(graph, field) : loadField.object(); + Stamp loadStamp = loadStamp(loadField.stamp(), field.getJavaKind()); + + AddressNode address = createFieldAddress(graph, object, field); + assert address != null : "Field that is loaded must not be eliminated: " + field.getDeclaringClass().toJavaName(true) + "." + field.getName(); + + ReadNode memoryRead = graph.add(new ReadNode(address, fieldLocationIdentity(field), loadStamp, fieldLoadBarrierType(field))); + ValueNode readValue = implicitLoadConvert(graph, field.getJavaKind(), memoryRead); + loadField.replaceAtUsages(readValue); + graph.replaceFixed(loadField, memoryRead); + + memoryRead.setGuard(createNullCheck(object, memoryRead, tool)); + + if (loadField.isVolatile()) { + MembarNode preMembar = graph.add(new MembarNode(JMM_PRE_VOLATILE_READ)); + graph.addBeforeFixed(memoryRead, preMembar); + MembarNode postMembar = graph.add(new MembarNode(JMM_POST_VOLATILE_READ)); + graph.addAfterFixed(memoryRead, postMembar); + } + } + + protected void lowerStoreFieldNode(StoreFieldNode storeField, LoweringTool tool) { + StructuredGraph graph = storeField.graph(); + ResolvedJavaField field = storeField.field(); + ValueNode object = storeField.isStatic() ? staticFieldBase(graph, field) : storeField.object(); + ValueNode value = implicitStoreConvert(graph, storeField.field().getJavaKind(), storeField.value()); + AddressNode address = createFieldAddress(graph, object, field); + assert address != null; + + WriteNode memoryWrite = graph.add(new WriteNode(address, fieldLocationIdentity(field), value, fieldStoreBarrierType(storeField.field()))); + memoryWrite.setStateAfter(storeField.stateAfter()); + graph.replaceFixedWithFixed(storeField, memoryWrite); + memoryWrite.setGuard(createNullCheck(object, memoryWrite, tool)); + + if (storeField.isVolatile()) { + MembarNode preMembar = graph.add(new MembarNode(JMM_PRE_VOLATILE_WRITE)); + graph.addBeforeFixed(memoryWrite, preMembar); + MembarNode postMembar = graph.add(new MembarNode(JMM_POST_VOLATILE_WRITE)); + graph.addAfterFixed(memoryWrite, postMembar); + } + } + + public AddressNode createArrayAddress(StructuredGraph graph, ValueNode array, JavaKind elementKind, ValueNode index) { + ValueNode wordIndex; + if (target.wordSize > 4) { + wordIndex = graph.unique(new SignExtendNode(index, target.wordSize * 8)); + } else { + assert target.wordSize == 4 : "unsupported word size"; + wordIndex = index; + } + + int shift = CodeUtil.log2(arrayScalingFactor(elementKind)); + ValueNode scaledIndex = graph.unique(new LeftShiftNode(wordIndex, ConstantNode.forInt(shift, graph))); + + int base = arrayBaseOffset(elementKind); + ValueNode offset = graph.unique(new AddNode(scaledIndex, ConstantNode.forIntegerKind(target.wordJavaKind, base, graph))); + + return graph.unique(new OffsetAddressNode(array, offset)); + } + + protected void lowerLoadIndexedNode(LoadIndexedNode loadIndexed, LoweringTool tool) { + StructuredGraph graph = loadIndexed.graph(); + JavaKind elementKind = loadIndexed.elementKind(); + Stamp loadStamp = loadStamp(loadIndexed.stamp(), elementKind); + + PiNode pi = getBoundsCheckedIndex(loadIndexed, tool, null); + ValueNode checkedIndex = pi; + if (checkedIndex == null) { + checkedIndex = loadIndexed.index(); + } + + AddressNode address = createArrayAddress(graph, loadIndexed.array(), elementKind, checkedIndex); + ReadNode memoryRead = graph.add(new ReadNode(address, NamedLocationIdentity.getArrayLocation(elementKind), loadStamp, BarrierType.NONE)); + ValueNode readValue = implicitLoadConvert(graph, elementKind, memoryRead); + + if (pi != null) { + memoryRead.setGuard(pi.getGuard()); + } + + loadIndexed.replaceAtUsages(readValue); + graph.replaceFixed(loadIndexed, memoryRead); + } + + protected void lowerStoreIndexedNode(StoreIndexedNode storeIndexed, LoweringTool tool) { + StructuredGraph graph = storeIndexed.graph(); + + GuardingNode[] nullCheckReturn = new GuardingNode[1]; + PiNode pi = getBoundsCheckedIndex(storeIndexed, tool, nullCheckReturn); + ValueNode checkedIndex; + GuardingNode boundsCheck; + if (pi == null) { + checkedIndex = storeIndexed.index(); + boundsCheck = null; + } else { + checkedIndex = pi; + boundsCheck = pi.getGuard(); + } + + JavaKind elementKind = storeIndexed.elementKind(); + + ValueNode value = storeIndexed.value(); + ValueNode array = storeIndexed.array(); + LogicNode condition = null; + if (elementKind == JavaKind.Object && !StampTool.isPointerAlwaysNull(value)) { + /* Array store check. */ + TypeReference arrayType = StampTool.typeReferenceOrNull(array); + if (arrayType != null && arrayType.isExact()) { + ResolvedJavaType elementType = arrayType.getType().getComponentType(); + if (!elementType.isJavaLangObject()) { + TypeReference typeReference = TypeReference.createTrusted(storeIndexed.graph().getAssumptions(), elementType); + LogicNode typeTest = graph.addOrUniqueWithInputs(InstanceOfNode.create(typeReference, value)); + condition = LogicNode.or(graph.unique(IsNullNode.create(value)), typeTest, GraalDirectives.UNLIKELY_PROBABILITY); + } + } else { + /* + * The guard on the read hub should be the null check of the array that was + * introduced earlier. + */ + GuardingNode nullCheck = nullCheckReturn[0]; + assert nullCheckReturn[0] != null || createNullCheck(array, storeIndexed, tool) == null; + ValueNode arrayClass = createReadHub(graph, graph.unique(new PiNode(array, (ValueNode) nullCheck)), tool); + ValueNode componentHub = createReadArrayComponentHub(graph, arrayClass, storeIndexed); + LogicNode typeTest = graph.unique(InstanceOfDynamicNode.create(graph.getAssumptions(), tool.getConstantReflection(), componentHub, value, false)); + condition = LogicNode.or(graph.unique(IsNullNode.create(value)), typeTest, GraalDirectives.UNLIKELY_PROBABILITY); + } + } + + AddressNode address = createArrayAddress(graph, array, elementKind, checkedIndex); + WriteNode memoryWrite = graph.add(new WriteNode(address, NamedLocationIdentity.getArrayLocation(elementKind), implicitStoreConvert(graph, elementKind, value), + arrayStoreBarrierType(storeIndexed.elementKind()))); + memoryWrite.setGuard(boundsCheck); + if (condition != null) { + GuardingNode storeCheckGuard = tool.createGuard(storeIndexed, condition, DeoptimizationReason.ArrayStoreException, DeoptimizationAction.InvalidateReprofile); + memoryWrite.setStoreCheckGuard(storeCheckGuard); + } + memoryWrite.setStateAfter(storeIndexed.stateAfter()); + graph.replaceFixedWithFixed(storeIndexed, memoryWrite); + } + + protected void lowerArrayLengthNode(ArrayLengthNode arrayLengthNode, LoweringTool tool) { + StructuredGraph graph = arrayLengthNode.graph(); + ValueNode array = arrayLengthNode.array(); + + AddressNode address = createOffsetAddress(graph, array, arrayLengthOffset()); + ReadNode arrayLengthRead = graph.add(new ReadNode(address, ARRAY_LENGTH_LOCATION, StampFactory.positiveInt(), BarrierType.NONE)); + arrayLengthRead.setGuard(createNullCheck(array, arrayLengthNode, tool)); + graph.replaceFixedWithFixed(arrayLengthNode, arrayLengthRead); + } + + protected void lowerLoadHubNode(LoadHubNode loadHub, LoweringTool tool) { + StructuredGraph graph = loadHub.graph(); + if (tool.getLoweringStage() != LoweringTool.StandardLoweringStage.LOW_TIER) { + return; + } + if (graph.getGuardsStage().allowsFloatingGuards()) { + return; + } + ValueNode hub = createReadHub(graph, loadHub.getValue(), tool); + loadHub.replaceAtUsagesAndDelete(hub); + } + + protected void lowerMonitorEnterNode(MonitorEnterNode monitorEnter, LoweringTool tool, StructuredGraph graph) { + ValueNode object = monitorEnter.object(); + GuardingNode nullCheck = createNullCheck(object, monitorEnter, tool); + if (nullCheck != null) { + object = graph.unique(new PiNode(object, ((ObjectStamp) object.stamp()).improveWith(StampFactory.objectNonNull()), (ValueNode) nullCheck)); + } + ValueNode hub = graph.addOrUnique(LoadHubNode.create(object, tool.getStampProvider(), tool.getMetaAccess(), tool.getConstantReflection())); + RawMonitorEnterNode rawMonitorEnter = graph.add(new RawMonitorEnterNode(object, hub, monitorEnter.getMonitorId())); + rawMonitorEnter.setStateBefore(monitorEnter.stateBefore()); + rawMonitorEnter.setStateAfter(monitorEnter.stateAfter()); + graph.replaceFixedWithFixed(monitorEnter, rawMonitorEnter); + } + + protected void lowerCompareAndSwapNode(CompareAndSwapNode cas) { + StructuredGraph graph = cas.graph(); + JavaKind valueKind = cas.getValueKind(); + + ValueNode expectedValue = implicitStoreConvert(graph, valueKind, cas.expected()); + ValueNode newValue = implicitStoreConvert(graph, valueKind, cas.newValue()); + + AddressNode address = graph.unique(new OffsetAddressNode(cas.object(), cas.offset())); + LoweredCompareAndSwapNode atomicNode = graph.add(new LoweredCompareAndSwapNode(address, cas.getLocationIdentity(), expectedValue, newValue, compareAndSwapBarrierType(cas))); + atomicNode.setStateAfter(cas.stateAfter()); + graph.replaceFixedWithFixed(cas, atomicNode); + } + + protected void lowerAtomicReadAndWriteNode(AtomicReadAndWriteNode n) { + StructuredGraph graph = n.graph(); + JavaKind valueKind = n.getValueKind(); + + ValueNode newValue = implicitStoreConvert(graph, valueKind, n.newValue()); + + AddressNode address = graph.unique(new OffsetAddressNode(n.object(), n.offset())); + LoweredAtomicReadAndWriteNode memoryRead = graph.add(new LoweredAtomicReadAndWriteNode(address, n.getLocationIdentity(), newValue, atomicReadAndWriteBarrierType(n))); + memoryRead.setStateAfter(n.stateAfter()); + + ValueNode readValue = implicitLoadConvert(graph, valueKind, memoryRead); + n.stateAfter().replaceFirstInput(n, memoryRead); + n.replaceAtUsages(readValue); + graph.replaceFixedWithFixed(n, memoryRead); + } + + /** + * @param tool utility for performing the lowering + */ + protected void lowerUnsafeLoadNode(UnsafeLoadNode load, LoweringTool tool) { + StructuredGraph graph = load.graph(); + if (load instanceof GuardedUnsafeLoadNode) { + GuardedUnsafeLoadNode guardedLoad = (GuardedUnsafeLoadNode) load; + GuardingNode guard = guardedLoad.getGuard(); + if (guard == null) { + // can float freely if the guard folded away + ReadNode memoryRead = createUnsafeRead(graph, load, null); + memoryRead.setForceFixed(false); + graph.replaceFixedWithFixed(load, memoryRead); + } else { + // must be guarded, but flows below the guard + ReadNode memoryRead = createUnsafeRead(graph, load, guard); + graph.replaceFixedWithFixed(load, memoryRead); + } + } else { + // never had a guarding condition so it must be fixed, creation of the read will force + // it to be fixed + ReadNode memoryRead = createUnsafeRead(graph, load, null); + graph.replaceFixedWithFixed(load, memoryRead); + } + } + + protected AddressNode createUnsafeAddress(StructuredGraph graph, ValueNode object, ValueNode offset) { + if (object.isConstant() && object.asConstant().isDefaultForKind()) { + return graph.unique(new RawAddressNode(offset)); + } else { + return graph.unique(new OffsetAddressNode(object, offset)); + } + } + + protected ReadNode createUnsafeRead(StructuredGraph graph, UnsafeLoadNode load, GuardingNode guard) { + boolean compressible = load.accessKind() == JavaKind.Object; + JavaKind readKind = load.accessKind(); + Stamp loadStamp = loadStamp(load.stamp(), readKind, compressible); + AddressNode address = createUnsafeAddress(graph, load.object(), load.offset()); + ReadNode memoryRead = graph.add(new ReadNode(address, load.getLocationIdentity(), loadStamp, guard, BarrierType.NONE)); + if (guard == null) { + // An unsafe read must not float otherwise it may float above + // a test guaranteeing the read is safe. + memoryRead.setForceFixed(true); + } + ValueNode readValue = implicitLoadConvert(graph, readKind, memoryRead, compressible); + load.replaceAtUsages(readValue); + return memoryRead; + } + + protected void lowerUnsafeMemoryLoadNode(UnsafeMemoryLoadNode load) { + StructuredGraph graph = load.graph(); + JavaKind readKind = load.getKind(); + assert readKind != JavaKind.Object; + Stamp loadStamp = loadStamp(load.stamp(), readKind, false); + AddressNode address = graph.unique(new RawAddressNode(load.getAddress())); + ReadNode memoryRead = graph.add(new ReadNode(address, load.getLocationIdentity(), loadStamp, BarrierType.NONE)); + // An unsafe read must not float otherwise it may float above + // a test guaranteeing the read is safe. + memoryRead.setForceFixed(true); + ValueNode readValue = implicitLoadConvert(graph, readKind, memoryRead, false); + load.replaceAtUsages(readValue); + graph.replaceFixedWithFixed(load, memoryRead); + } + + protected void lowerUnsafeStoreNode(UnsafeStoreNode store) { + StructuredGraph graph = store.graph(); + boolean compressible = store.value().getStackKind() == JavaKind.Object; + JavaKind valueKind = store.accessKind(); + ValueNode value = implicitStoreConvert(graph, valueKind, store.value(), compressible); + AddressNode address = createUnsafeAddress(graph, store.object(), store.offset()); + WriteNode write = graph.add(new WriteNode(address, store.getLocationIdentity(), value, unsafeStoreBarrierType(store))); + write.setStateAfter(store.stateAfter()); + graph.replaceFixedWithFixed(store, write); + } + + protected void lowerUnsafeMemoryStoreNode(UnsafeMemoryStoreNode store) { + StructuredGraph graph = store.graph(); + assert store.getValue().getStackKind() != JavaKind.Object; + JavaKind valueKind = store.getKind(); + ValueNode value = implicitStoreConvert(graph, valueKind, store.getValue(), false); + AddressNode address = graph.unique(new RawAddressNode(store.getAddress())); + WriteNode write = graph.add(new WriteNode(address, store.getLocationIdentity(), value, BarrierType.NONE)); + write.setStateAfter(store.stateAfter()); + graph.replaceFixedWithFixed(store, write); + } + + protected void lowerJavaReadNode(JavaReadNode read) { + StructuredGraph graph = read.graph(); + JavaKind valueKind = read.getReadKind(); + Stamp loadStamp = loadStamp(read.stamp(), valueKind, read.isCompressible()); + + ReadNode memoryRead = graph.add(new ReadNode(read.getAddress(), read.getLocationIdentity(), loadStamp, read.getBarrierType())); + GuardingNode guard = read.getGuard(); + ValueNode readValue = implicitLoadConvert(graph, valueKind, memoryRead, read.isCompressible()); + if (guard == null) { + // An unsafe read must not float otherwise it may float above + // a test guaranteeing the read is safe. + memoryRead.setForceFixed(true); + } else { + memoryRead.setGuard(guard); + } + read.replaceAtUsages(readValue); + graph.replaceFixed(read, memoryRead); + } + + protected void lowerJavaWriteNode(JavaWriteNode write) { + StructuredGraph graph = write.graph(); + JavaKind valueKind = write.getWriteKind(); + ValueNode value = implicitStoreConvert(graph, valueKind, write.value(), write.isCompressible()); + + WriteNode memoryWrite = graph.add(new WriteNode(write.getAddress(), write.getLocationIdentity(), value, write.getBarrierType(), write.isInitialization())); + memoryWrite.setStateAfter(write.stateAfter()); + graph.replaceFixedWithFixed(write, memoryWrite); + memoryWrite.setGuard(write.getGuard()); + } + + protected void lowerCommitAllocationNode(CommitAllocationNode commit, LoweringTool tool) { + StructuredGraph graph = commit.graph(); + if (graph.getGuardsStage() == StructuredGraph.GuardsStage.FIXED_DEOPTS) { + List recursiveLowerings = new ArrayList<>(); + + ValueNode[] allocations = new ValueNode[commit.getVirtualObjects().size()]; + BitSet omittedValues = new BitSet(); + int valuePos = 0; + for (int objIndex = 0; objIndex < commit.getVirtualObjects().size(); objIndex++) { + VirtualObjectNode virtual = commit.getVirtualObjects().get(objIndex); + int entryCount = virtual.entryCount(); + AbstractNewObjectNode newObject; + if (virtual instanceof VirtualInstanceNode) { + newObject = graph.add(createNewInstanceFromVirtual(virtual)); + } else { + newObject = graph.add(createNewArrayFromVirtual(virtual, ConstantNode.forInt(entryCount, graph))); + } + recursiveLowerings.add(newObject); + graph.addBeforeFixed(commit, newObject); + allocations[objIndex] = newObject; + for (int i = 0; i < entryCount; i++) { + ValueNode value = commit.getValues().get(valuePos); + if (value instanceof VirtualObjectNode) { + value = allocations[commit.getVirtualObjects().indexOf(value)]; + } + if (value == null) { + omittedValues.set(valuePos); + } else if (!(value.isConstant() && value.asConstant().isDefaultForKind())) { + // Constant.illegal is always the defaultForKind, so it is skipped + JavaKind valueKind = value.getStackKind(); + JavaKind entryKind = virtual.entryKind(i); + + // Truffle requires some leniency in terms of what can be put where: + assert valueKind.getStackKind() == entryKind.getStackKind() || + (valueKind == JavaKind.Long || valueKind == JavaKind.Double || (valueKind == JavaKind.Int && virtual instanceof VirtualArrayNode)); + AddressNode address = null; + BarrierType barrierType = null; + if (virtual instanceof VirtualInstanceNode) { + ResolvedJavaField field = ((VirtualInstanceNode) virtual).field(i); + long offset = fieldOffset(field); + if (offset >= 0) { + address = createOffsetAddress(graph, newObject, offset); + barrierType = fieldInitializationBarrier(entryKind); + } + } else { + address = createOffsetAddress(graph, newObject, arrayBaseOffset(entryKind) + i * arrayScalingFactor(entryKind)); + barrierType = arrayInitializationBarrier(entryKind); + } + if (address != null) { + WriteNode write = new WriteNode(address, initLocationIdentity(), implicitStoreConvert(graph, entryKind, value), barrierType); + graph.addAfterFixed(newObject, graph.add(write)); + } + } + valuePos++; + + } + } + valuePos = 0; + + for (int objIndex = 0; objIndex < commit.getVirtualObjects().size(); objIndex++) { + VirtualObjectNode virtual = commit.getVirtualObjects().get(objIndex); + int entryCount = virtual.entryCount(); + ValueNode newObject = allocations[objIndex]; + for (int i = 0; i < entryCount; i++) { + if (omittedValues.get(valuePos)) { + ValueNode value = commit.getValues().get(valuePos); + assert value instanceof VirtualObjectNode; + ValueNode allocValue = allocations[commit.getVirtualObjects().indexOf(value)]; + if (!(allocValue.isConstant() && allocValue.asConstant().isDefaultForKind())) { + assert virtual.entryKind(i) == JavaKind.Object && allocValue.getStackKind() == JavaKind.Object; + AddressNode address; + BarrierType barrierType; + if (virtual instanceof VirtualInstanceNode) { + VirtualInstanceNode virtualInstance = (VirtualInstanceNode) virtual; + address = createFieldAddress(graph, newObject, virtualInstance.field(i)); + barrierType = BarrierType.IMPRECISE; + } else { + address = createArrayAddress(graph, newObject, virtual.entryKind(i), ConstantNode.forInt(i, graph)); + barrierType = BarrierType.PRECISE; + } + if (address != null) { + WriteNode write = new WriteNode(address, initLocationIdentity(), implicitStoreConvert(graph, JavaKind.Object, allocValue), barrierType); + graph.addBeforeFixed(commit, graph.add(write)); + } + } + } + valuePos++; + } + } + + finishAllocatedObjects(tool, commit, allocations); + graph.removeFixed(commit); + + for (AbstractNewObjectNode recursiveLowering : recursiveLowerings) { + recursiveLowering.lower(tool); + } + } + } + + public NewInstanceNode createNewInstanceFromVirtual(VirtualObjectNode virtual) { + return new NewInstanceNode(virtual.type(), true); + } + + protected NewArrayNode createNewArrayFromVirtual(VirtualObjectNode virtual, ValueNode length) { + return new NewArrayNode(((VirtualArrayNode) virtual).componentType(), length, true); + } + + public void finishAllocatedObjects(LoweringTool tool, CommitAllocationNode commit, ValueNode[] allocations) { + StructuredGraph graph = commit.graph(); + for (int objIndex = 0; objIndex < commit.getVirtualObjects().size(); objIndex++) { + FixedValueAnchorNode anchor = graph.add(new FixedValueAnchorNode(allocations[objIndex])); + allocations[objIndex] = anchor; + graph.addBeforeFixed(commit, anchor); + } + for (int objIndex = 0; objIndex < commit.getVirtualObjects().size(); objIndex++) { + for (MonitorIdNode monitorId : commit.getLocks(objIndex)) { + MonitorEnterNode enter = graph.add(new MonitorEnterNode(allocations[objIndex], monitorId)); + graph.addBeforeFixed(commit, enter); + enter.lower(tool); + } + } + for (Node usage : commit.usages().snapshot()) { + AllocatedObjectNode addObject = (AllocatedObjectNode) usage; + int index = commit.getVirtualObjects().indexOf(addObject.getVirtualObject()); + addObject.replaceAtUsagesAndDelete(allocations[index]); + } + insertAllocationBarrier(commit, graph); + } + + /** + * Insert the required {@link MemoryBarriers#STORE_STORE} barrier for an allocation and also + * include the {@link MemoryBarriers#LOAD_STORE} required for final fields if any final fields + * are being written, as if {@link FinalFieldBarrierNode} were emitted. + */ + private void insertAllocationBarrier(CommitAllocationNode commit, StructuredGraph graph) { + int barrier = MemoryBarriers.STORE_STORE; + outer: for (VirtualObjectNode vobj : commit.getVirtualObjects()) { + for (ResolvedJavaField field : vobj.type().getInstanceFields(true)) { + if (field.isFinal()) { + barrier = barrier | MemoryBarriers.LOAD_STORE; + break outer; + } + } + } + graph.addAfterFixed(commit, graph.add(new MembarNode(barrier, initLocationIdentity()))); + } + + /** + * @param field the field whose barrier type should be returned + */ + protected BarrierType fieldLoadBarrierType(ResolvedJavaField field) { + return BarrierType.NONE; + } + + protected BarrierType fieldStoreBarrierType(ResolvedJavaField field) { + if (field.getJavaKind() == JavaKind.Object) { + return BarrierType.IMPRECISE; + } + return BarrierType.NONE; + } + + protected BarrierType arrayStoreBarrierType(JavaKind elementKind) { + if (elementKind == JavaKind.Object) { + return BarrierType.PRECISE; + } + return BarrierType.NONE; + } + + public BarrierType fieldInitializationBarrier(JavaKind entryKind) { + return entryKind == JavaKind.Object ? BarrierType.IMPRECISE : BarrierType.NONE; + } + + public BarrierType arrayInitializationBarrier(JavaKind entryKind) { + return entryKind == JavaKind.Object ? BarrierType.PRECISE : BarrierType.NONE; + } + + protected BarrierType unsafeStoreBarrierType(UnsafeStoreNode store) { + return storeBarrierType(store.object(), store.value()); + } + + protected BarrierType compareAndSwapBarrierType(CompareAndSwapNode cas) { + return storeBarrierType(cas.object(), cas.expected()); + } + + protected BarrierType atomicReadAndWriteBarrierType(AtomicReadAndWriteNode n) { + return storeBarrierType(n.object(), n.newValue()); + } + + protected BarrierType storeBarrierType(ValueNode object, ValueNode value) { + if (value.getStackKind() == JavaKind.Object) { + ResolvedJavaType type = StampTool.typeOrNull(object); + if (type != null && !type.isArray()) { + return BarrierType.IMPRECISE; + } else { + return BarrierType.PRECISE; + } + } + return BarrierType.NONE; + } + + public abstract int fieldOffset(ResolvedJavaField field); + + public FieldLocationIdentity fieldLocationIdentity(ResolvedJavaField field) { + return new FieldLocationIdentity(field); + } + + public abstract ValueNode staticFieldBase(StructuredGraph graph, ResolvedJavaField field); + + public abstract int arrayLengthOffset(); + + public abstract int arrayBaseOffset(JavaKind elementKind); + + public int arrayScalingFactor(JavaKind elementKind) { + return target.arch.getPlatformKind(elementKind).getSizeInBytes(); + } + + public abstract LocationIdentity initLocationIdentity(); + + public Stamp loadStamp(Stamp stamp, JavaKind kind) { + return loadStamp(stamp, kind, true); + } + + /** + * @param compressible whether the stamp should be compressible + */ + protected Stamp loadStamp(Stamp stamp, JavaKind kind, boolean compressible) { + switch (kind) { + case Boolean: + case Byte: + return IntegerStamp.OPS.getNarrow().foldStamp(32, 8, stamp); + case Char: + case Short: + return IntegerStamp.OPS.getNarrow().foldStamp(32, 16, stamp); + } + return stamp; + } + + public final ValueNode implicitLoadConvert(StructuredGraph graph, JavaKind kind, ValueNode value) { + return implicitLoadConvert(graph, kind, value, true); + } + + public ValueNode implicitLoadConvert(JavaKind kind, ValueNode value) { + return implicitLoadConvert(kind, value, true); + } + + protected final ValueNode implicitLoadConvert(StructuredGraph graph, JavaKind kind, ValueNode value, boolean compressible) { + ValueNode ret = implicitLoadConvert(kind, value, compressible); + if (!ret.isAlive()) { + ret = graph.addOrUnique(ret); + } + return ret; + } + + /** + * @param compressible whether the covert should be compressible + */ + protected ValueNode implicitLoadConvert(JavaKind kind, ValueNode value, boolean compressible) { + switch (kind) { + case Byte: + case Short: + return new SignExtendNode(value, 32); + case Boolean: + case Char: + return new ZeroExtendNode(value, 32); + } + return value; + } + + public final ValueNode implicitStoreConvert(StructuredGraph graph, JavaKind kind, ValueNode value) { + return implicitStoreConvert(graph, kind, value, true); + } + + public ValueNode implicitStoreConvert(JavaKind kind, ValueNode value) { + return implicitStoreConvert(kind, value, true); + } + + protected final ValueNode implicitStoreConvert(StructuredGraph graph, JavaKind kind, ValueNode value, boolean compressible) { + ValueNode ret = implicitStoreConvert(kind, value, compressible); + if (!ret.isAlive()) { + ret = graph.addOrUnique(ret); + } + return ret; + } + + /** + * @param compressible whether the covert should be compressible + */ + protected ValueNode implicitStoreConvert(JavaKind kind, ValueNode value, boolean compressible) { + switch (kind) { + case Boolean: + case Byte: + return new NarrowNode(value, 8); + case Char: + case Short: + return new NarrowNode(value, 16); + } + return value; + } + + protected abstract ValueNode createReadHub(StructuredGraph graph, ValueNode object, LoweringTool tool); + + protected abstract ValueNode createReadArrayComponentHub(StructuredGraph graph, ValueNode arrayHub, FixedNode anchor); + + protected PiNode getBoundsCheckedIndex(AccessIndexedNode n, LoweringTool tool, GuardingNode[] nullCheckReturn) { + StructuredGraph graph = n.graph(); + ValueNode array = n.array(); + ValueNode arrayLength = readArrayLength(array, tool.getConstantReflection()); + if (arrayLength == null) { + Stamp stamp = StampFactory.positiveInt(); + AddressNode address = createOffsetAddress(graph, array, arrayLengthOffset()); + ReadNode readArrayLength = graph.add(new ReadNode(address, ARRAY_LENGTH_LOCATION, stamp, BarrierType.NONE)); + graph.addBeforeFixed(n, readArrayLength); + GuardingNode nullCheck = createNullCheck(array, readArrayLength, tool); + if (nullCheckReturn != null) { + nullCheckReturn[0] = nullCheck; + } + readArrayLength.setGuard(nullCheck); + arrayLength = readArrayLength; + } else { + if (array instanceof AbstractNewArrayNode) { + arrayLength = n.graph().addOrUnique(new PiNode(arrayLength, StampFactory.positiveInt())); + } + arrayLength = arrayLength.isAlive() ? arrayLength : graph.addOrUniqueWithInputs(arrayLength); + } + + if (arrayLength.isConstant() && n.index().isConstant()) { + int l = arrayLength.asJavaConstant().asInt(); + int i = n.index().asJavaConstant().asInt(); + if (i >= 0 && i < l) { + // unneeded range check + return null; + } + } + + GuardingNode guard = tool.createGuard(n, graph.unique(new IntegerBelowNode(n.index(), arrayLength)), BoundsCheckException, InvalidateReprofile); + IntegerStamp lengthStamp = (IntegerStamp) arrayLength.stamp(); + IntegerStamp indexStamp = StampFactory.forInteger(32, 0, lengthStamp.upperBound() - 1); + return graph.unique(new PiNode(n.index(), indexStamp, guard.asNode())); + } + + protected GuardingNode createNullCheck(ValueNode object, FixedNode before, LoweringTool tool) { + if (StampTool.isPointerNonNull(object)) { + return null; + } + return tool.createGuard(before, before.graph().unique(IsNullNode.create(object)), NullCheckException, InvalidateReprofile, JavaConstant.NULL_POINTER, true); + } + + @Override + public ValueNode reconstructArrayIndex(JavaKind elementKind, AddressNode address) { + StructuredGraph graph = address.graph(); + ValueNode offset = ((OffsetAddressNode) address).getOffset(); + + int base = arrayBaseOffset(elementKind); + ValueNode scaledIndex = graph.unique(new SubNode(offset, ConstantNode.forIntegerStamp(offset.stamp(), base, graph))); + + int shift = CodeUtil.log2(arrayScalingFactor(elementKind)); + ValueNode ret = graph.unique(new RightShiftNode(scaledIndex, ConstantNode.forInt(shift, graph))); + return IntegerConvertNode.convert(ret, StampFactory.forKind(JavaKind.Int), graph); + } +}