1 /* 2 * Copyright (c) 2011, 2019, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 */ 23 24 25 package org.graalvm.compiler.replacements; 26 27 import static jdk.vm.ci.code.MemoryBarriers.JMM_POST_VOLATILE_READ; 28 import static jdk.vm.ci.code.MemoryBarriers.JMM_POST_VOLATILE_WRITE; 29 import static jdk.vm.ci.code.MemoryBarriers.JMM_PRE_VOLATILE_READ; 30 import static jdk.vm.ci.code.MemoryBarriers.JMM_PRE_VOLATILE_WRITE; 31 import static jdk.vm.ci.meta.DeoptimizationAction.InvalidateReprofile; 32 import static jdk.vm.ci.meta.DeoptimizationReason.BoundsCheckException; 33 import static jdk.vm.ci.meta.DeoptimizationReason.NullCheckException; 34 import static org.graalvm.compiler.core.common.SpeculativeExecutionAttacksMitigations.Options.UseIndexMasking; 35 import static org.graalvm.compiler.nodes.NamedLocationIdentity.ARRAY_LENGTH_LOCATION; 36 import static org.graalvm.compiler.nodes.calc.BinaryArithmeticNode.branchlessMax; 37 import static org.graalvm.compiler.nodes.calc.BinaryArithmeticNode.branchlessMin; 38 import static org.graalvm.compiler.nodes.java.ArrayLengthNode.readArrayLength; 39 import static org.graalvm.compiler.nodes.util.GraphUtil.skipPiWhileNonNull; 40 41 import java.nio.ByteOrder; 42 import java.util.ArrayList; 43 import java.util.BitSet; 44 import java.util.List; 45 46 import jdk.vm.ci.meta.JavaConstant; 47 import org.graalvm.compiler.api.directives.GraalDirectives; 48 import org.graalvm.compiler.api.replacements.Snippet; 49 import org.graalvm.compiler.api.replacements.SnippetReflectionProvider; 50 import org.graalvm.compiler.core.common.LIRKind; 51 import org.graalvm.compiler.core.common.spi.ForeignCallsProvider; 52 import org.graalvm.compiler.core.common.type.AbstractPointerStamp; 53 import org.graalvm.compiler.core.common.type.IntegerStamp; 54 import org.graalvm.compiler.core.common.type.ObjectStamp; 55 import org.graalvm.compiler.core.common.type.Stamp; 56 import org.graalvm.compiler.core.common.type.StampFactory; 57 import org.graalvm.compiler.core.common.type.TypeReference; 58 import org.graalvm.compiler.debug.DebugCloseable; 59 import org.graalvm.compiler.debug.DebugHandlersFactory; 60 import org.graalvm.compiler.debug.GraalError; 61 import org.graalvm.compiler.graph.Node; 62 import org.graalvm.compiler.nodeinfo.InputType; 63 import org.graalvm.compiler.nodes.CompressionNode.CompressionOp; 64 import org.graalvm.compiler.nodes.ConstantNode; 65 import org.graalvm.compiler.nodes.EndNode; 66 import org.graalvm.compiler.nodes.FieldLocationIdentity; 67 import org.graalvm.compiler.nodes.FixedNode; 68 import org.graalvm.compiler.nodes.FixedWithNextNode; 69 import org.graalvm.compiler.nodes.IfNode; 70 import org.graalvm.compiler.nodes.LogicNode; 71 import org.graalvm.compiler.nodes.MergeNode; 72 import org.graalvm.compiler.nodes.NamedLocationIdentity; 73 import org.graalvm.compiler.nodes.NodeView; 74 import org.graalvm.compiler.nodes.PhiNode; 75 import org.graalvm.compiler.nodes.PiNode; 76 import org.graalvm.compiler.nodes.StructuredGraph; 77 import org.graalvm.compiler.nodes.ValueNode; 78 import org.graalvm.compiler.nodes.ValuePhiNode; 79 import org.graalvm.compiler.nodes.calc.AddNode; 80 import org.graalvm.compiler.nodes.calc.ConditionalNode; 81 import org.graalvm.compiler.nodes.calc.IntegerBelowNode; 82 import org.graalvm.compiler.nodes.calc.IntegerConvertNode; 83 import org.graalvm.compiler.nodes.calc.IntegerEqualsNode; 84 import org.graalvm.compiler.nodes.calc.IsNullNode; 85 import org.graalvm.compiler.nodes.calc.LeftShiftNode; 86 import org.graalvm.compiler.nodes.calc.NarrowNode; 87 import org.graalvm.compiler.nodes.calc.RightShiftNode; 88 import org.graalvm.compiler.nodes.calc.SignExtendNode; 89 import org.graalvm.compiler.nodes.calc.SubNode; 90 import org.graalvm.compiler.nodes.calc.UnpackEndianHalfNode; 91 import org.graalvm.compiler.nodes.calc.ZeroExtendNode; 92 import org.graalvm.compiler.nodes.debug.VerifyHeapNode; 93 import org.graalvm.compiler.nodes.extended.BoxNode; 94 import org.graalvm.compiler.nodes.extended.FixedValueAnchorNode; 95 import org.graalvm.compiler.nodes.extended.ForeignCallNode; 96 import org.graalvm.compiler.nodes.extended.GuardedUnsafeLoadNode; 97 import org.graalvm.compiler.nodes.extended.GuardingNode; 98 import org.graalvm.compiler.nodes.extended.JavaReadNode; 99 import org.graalvm.compiler.nodes.extended.JavaWriteNode; 100 import org.graalvm.compiler.nodes.extended.LoadArrayComponentHubNode; 101 import org.graalvm.compiler.nodes.extended.LoadHubNode; 102 import org.graalvm.compiler.nodes.extended.LoadHubOrNullNode; 103 import org.graalvm.compiler.nodes.extended.MembarNode; 104 import org.graalvm.compiler.nodes.extended.RawLoadNode; 105 import org.graalvm.compiler.nodes.extended.RawStoreNode; 106 import org.graalvm.compiler.nodes.extended.UnboxNode; 107 import org.graalvm.compiler.nodes.extended.UnsafeMemoryLoadNode; 108 import org.graalvm.compiler.nodes.extended.UnsafeMemoryStoreNode; 109 import org.graalvm.compiler.nodes.java.AbstractNewObjectNode; 110 import org.graalvm.compiler.nodes.java.AccessIndexedNode; 111 import org.graalvm.compiler.nodes.java.ArrayLengthNode; 112 import org.graalvm.compiler.nodes.java.AtomicReadAndWriteNode; 113 import org.graalvm.compiler.nodes.java.FinalFieldBarrierNode; 114 import org.graalvm.compiler.nodes.java.InstanceOfDynamicNode; 115 import org.graalvm.compiler.nodes.java.InstanceOfNode; 116 import org.graalvm.compiler.nodes.java.LoadFieldNode; 117 import org.graalvm.compiler.nodes.java.LoadIndexedNode; 118 import org.graalvm.compiler.nodes.java.LogicCompareAndSwapNode; 119 import org.graalvm.compiler.nodes.java.LoweredAtomicReadAndWriteNode; 120 import org.graalvm.compiler.nodes.java.MonitorEnterNode; 121 import org.graalvm.compiler.nodes.java.MonitorIdNode; 122 import org.graalvm.compiler.nodes.java.NewArrayNode; 123 import org.graalvm.compiler.nodes.java.NewInstanceNode; 124 import org.graalvm.compiler.nodes.java.RawMonitorEnterNode; 125 import org.graalvm.compiler.nodes.java.StoreFieldNode; 126 import org.graalvm.compiler.nodes.java.StoreIndexedNode; 127 import org.graalvm.compiler.nodes.java.UnsafeCompareAndExchangeNode; 128 import org.graalvm.compiler.nodes.java.UnsafeCompareAndSwapNode; 129 import org.graalvm.compiler.nodes.java.ValueCompareAndSwapNode; 130 import org.graalvm.compiler.nodes.memory.HeapAccess.BarrierType; 131 import org.graalvm.compiler.nodes.memory.ReadNode; 132 import org.graalvm.compiler.nodes.memory.WriteNode; 133 import org.graalvm.compiler.nodes.memory.address.AddressNode; 134 import org.graalvm.compiler.nodes.memory.address.IndexAddressNode; 135 import org.graalvm.compiler.nodes.memory.address.OffsetAddressNode; 136 import org.graalvm.compiler.nodes.spi.Lowerable; 137 import org.graalvm.compiler.nodes.spi.LoweringProvider; 138 import org.graalvm.compiler.nodes.spi.LoweringTool; 139 import org.graalvm.compiler.nodes.type.StampTool; 140 import org.graalvm.compiler.nodes.util.GraphUtil; 141 import org.graalvm.compiler.nodes.virtual.AllocatedObjectNode; 142 import org.graalvm.compiler.nodes.virtual.CommitAllocationNode; 143 import org.graalvm.compiler.nodes.virtual.VirtualArrayNode; 144 import org.graalvm.compiler.nodes.virtual.VirtualInstanceNode; 145 import org.graalvm.compiler.nodes.virtual.VirtualObjectNode; 146 import org.graalvm.compiler.options.OptionValues; 147 import org.graalvm.compiler.phases.util.Providers; 148 import org.graalvm.compiler.replacements.SnippetLowerableMemoryNode.SnippetLowering; 149 import org.graalvm.compiler.replacements.nodes.BinaryMathIntrinsicNode; 150 import org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode; 151 import jdk.internal.vm.compiler.word.LocationIdentity; 152 153 import jdk.vm.ci.code.CodeUtil; 154 import jdk.vm.ci.code.MemoryBarriers; 155 import jdk.vm.ci.code.TargetDescription; 156 import jdk.vm.ci.meta.DeoptimizationAction; 157 import jdk.vm.ci.meta.DeoptimizationReason; 158 import jdk.vm.ci.meta.JavaKind; 159 import jdk.vm.ci.meta.MetaAccessProvider; 160 import jdk.vm.ci.meta.ResolvedJavaField; 161 import jdk.vm.ci.meta.ResolvedJavaMethod; 162 import jdk.vm.ci.meta.ResolvedJavaType; 163 import jdk.vm.ci.meta.SpeculationLog; 164 165 /** 166 * VM-independent lowerings for standard Java nodes. VM-specific methods are abstract and must be 167 * implemented by VM-specific subclasses. 168 */ 169 public abstract class DefaultJavaLoweringProvider implements LoweringProvider { 170 171 protected final MetaAccessProvider metaAccess; 172 protected final ForeignCallsProvider foreignCalls; 173 protected final TargetDescription target; 174 private final boolean useCompressedOops; 175 private final ResolvedJavaType objectArrayType; 176 177 private BoxingSnippets.Templates boxingSnippets; 178 private ConstantStringIndexOfSnippets.Templates indexOfSnippets; 179 180 public DefaultJavaLoweringProvider(MetaAccessProvider metaAccess, ForeignCallsProvider foreignCalls, TargetDescription target, boolean useCompressedOops) { 181 this.metaAccess = metaAccess; 182 this.foreignCalls = foreignCalls; 183 this.target = target; 184 this.useCompressedOops = useCompressedOops; 185 this.objectArrayType = metaAccess.lookupJavaType(Object[].class); 186 } 187 188 public void initialize(OptionValues options, Iterable<DebugHandlersFactory> factories, SnippetCounter.Group.Factory factory, Providers providers, SnippetReflectionProvider snippetReflection) { 189 boxingSnippets = new BoxingSnippets.Templates(options, factories, factory, providers, snippetReflection, target); 190 indexOfSnippets = new ConstantStringIndexOfSnippets.Templates(options, factories, providers, snippetReflection, target); 191 providers.getReplacements().registerSnippetTemplateCache(new SnippetCounterNode.SnippetCounterSnippets.Templates(options, factories, providers, snippetReflection, target)); 192 } 193 194 public final TargetDescription getTarget() { 195 return target; 196 } 197 198 public MetaAccessProvider getMetaAccess() { 199 return metaAccess; 200 } 201 202 @Override 203 @SuppressWarnings("try") 204 public void lower(Node n, LoweringTool tool) { 205 assert n instanceof Lowerable; 206 StructuredGraph graph = (StructuredGraph) n.graph(); 207 try (DebugCloseable context = n.withNodeSourcePosition()) { 208 if (n instanceof LoadFieldNode) { 209 lowerLoadFieldNode((LoadFieldNode) n, tool); 210 } else if (n instanceof StoreFieldNode) { 211 lowerStoreFieldNode((StoreFieldNode) n, tool); 212 } else if (n instanceof LoadIndexedNode) { 213 lowerLoadIndexedNode((LoadIndexedNode) n, tool); 214 } else if (n instanceof StoreIndexedNode) { 215 lowerStoreIndexedNode((StoreIndexedNode) n, tool); 216 } else if (n instanceof IndexAddressNode) { 217 lowerIndexAddressNode((IndexAddressNode) n); 218 } else if (n instanceof ArrayLengthNode) { 219 lowerArrayLengthNode((ArrayLengthNode) n, tool); 220 } else if (n instanceof LoadHubNode) { 221 lowerLoadHubNode((LoadHubNode) n, tool); 222 } else if (n instanceof LoadHubOrNullNode) { 223 lowerLoadHubOrNullNode((LoadHubOrNullNode) n, tool); 224 } else if (n instanceof LoadArrayComponentHubNode) { 225 lowerLoadArrayComponentHubNode((LoadArrayComponentHubNode) n); 226 } else if (n instanceof MonitorEnterNode) { 227 lowerMonitorEnterNode((MonitorEnterNode) n, tool, graph); 228 } else if (n instanceof UnsafeCompareAndSwapNode) { 229 lowerCompareAndSwapNode((UnsafeCompareAndSwapNode) n); 230 } else if (n instanceof UnsafeCompareAndExchangeNode) { 231 lowerCompareAndExchangeNode((UnsafeCompareAndExchangeNode) n); 232 } else if (n instanceof AtomicReadAndWriteNode) { 233 lowerAtomicReadAndWriteNode((AtomicReadAndWriteNode) n); 234 } else if (n instanceof RawLoadNode) { 235 lowerUnsafeLoadNode((RawLoadNode) n, tool); 236 } else if (n instanceof UnsafeMemoryLoadNode) { 237 lowerUnsafeMemoryLoadNode((UnsafeMemoryLoadNode) n); 238 } else if (n instanceof RawStoreNode) { 239 lowerUnsafeStoreNode((RawStoreNode) n); 240 } else if (n instanceof UnsafeMemoryStoreNode) { 241 lowerUnsafeMemoryStoreNode((UnsafeMemoryStoreNode) n); 242 } else if (n instanceof JavaReadNode) { 243 lowerJavaReadNode((JavaReadNode) n); 244 } else if (n instanceof JavaWriteNode) { 245 lowerJavaWriteNode((JavaWriteNode) n); 246 } else if (n instanceof CommitAllocationNode) { 247 lowerCommitAllocationNode((CommitAllocationNode) n, tool); 248 } else if (n instanceof BoxNode) { 249 boxingSnippets.lower((BoxNode) n, tool); 250 } else if (n instanceof UnboxNode) { 251 boxingSnippets.lower((UnboxNode) n, tool); 252 } else if (n instanceof VerifyHeapNode) { 253 lowerVerifyHeap((VerifyHeapNode) n); 254 } else if (n instanceof UnaryMathIntrinsicNode) { 255 lowerUnaryMath((UnaryMathIntrinsicNode) n, tool); 256 } else if (n instanceof BinaryMathIntrinsicNode) { 257 lowerBinaryMath((BinaryMathIntrinsicNode) n, tool); 258 } else if (n instanceof StringIndexOfNode) { 259 lowerIndexOf((StringIndexOfNode) n); 260 } else if (n instanceof StringLatin1IndexOfNode) { 261 lowerLatin1IndexOf((StringLatin1IndexOfNode) n); 262 } else if (n instanceof StringUTF16IndexOfNode) { 263 lowerUTF16IndexOf((StringUTF16IndexOfNode) n); 264 } else if (n instanceof UnpackEndianHalfNode) { 265 lowerSecondHalf((UnpackEndianHalfNode) n); 266 } else { 267 throw GraalError.shouldNotReachHere("Node implementing Lowerable not handled: " + n); 268 } 269 } 270 } 271 272 private void lowerSecondHalf(UnpackEndianHalfNode n) { 273 ByteOrder byteOrder = target.arch.getByteOrder(); 274 n.lower(byteOrder); 275 } 276 277 private void lowerIndexOf(StringIndexOfNode n) { 278 if (n.getArgument(3).isConstant()) { 279 SnippetLowering lowering = new SnippetLowering() { 280 @Override 281 public void lower(SnippetLowerableMemoryNode node, LoweringTool tool) { 282 if (tool.getLoweringStage() != LoweringTool.StandardLoweringStage.LOW_TIER) { 283 return; 284 } 285 indexOfSnippets.lower(node, tool); 286 } 287 }; 288 SnippetLowerableMemoryNode snippetLower = new SnippetLowerableMemoryNode(lowering, NamedLocationIdentity.getArrayLocation(JavaKind.Char), n.stamp(NodeView.DEFAULT), n.toArgumentArray()); 289 n.graph().add(snippetLower); 290 n.graph().replaceFixedWithFixed(n, snippetLower); 291 } 292 } 293 294 private void lowerLatin1IndexOf(StringLatin1IndexOfNode n) { 295 if (n.getArgument(2).isConstant()) { 296 SnippetLowering lowering = new SnippetLowering() { 297 @Override 298 public void lower(SnippetLowerableMemoryNode node, LoweringTool tool) { 299 if (tool.getLoweringStage() != LoweringTool.StandardLoweringStage.LOW_TIER) { 300 return; 301 } 302 indexOfSnippets.lowerLatin1(node, tool); 303 } 304 }; 305 SnippetLowerableMemoryNode snippetLower = new SnippetLowerableMemoryNode(lowering, NamedLocationIdentity.getArrayLocation(JavaKind.Byte), n.stamp(NodeView.DEFAULT), n.toArgumentArray()); 306 n.graph().add(snippetLower); 307 n.graph().replaceFixedWithFixed(n, snippetLower); 308 } 309 } 310 311 private void lowerUTF16IndexOf(StringUTF16IndexOfNode n) { 312 if (n.getArgument(2).isConstant()) { 313 SnippetLowering lowering = new SnippetLowering() { 314 @Override 315 public void lower(SnippetLowerableMemoryNode node, LoweringTool tool) { 316 if (tool.getLoweringStage() != LoweringTool.StandardLoweringStage.LOW_TIER) { 317 return; 318 } 319 indexOfSnippets.lowerUTF16(node, tool); 320 } 321 }; 322 SnippetLowerableMemoryNode snippetLower = new SnippetLowerableMemoryNode(lowering, NamedLocationIdentity.getArrayLocation(JavaKind.Byte), n.stamp(NodeView.DEFAULT), n.toArgumentArray()); 323 n.graph().add(snippetLower); 324 n.graph().replaceFixedWithFixed(n, snippetLower); 325 } 326 } 327 328 private void lowerBinaryMath(BinaryMathIntrinsicNode math, LoweringTool tool) { 329 if (tool.getLoweringStage() == LoweringTool.StandardLoweringStage.HIGH_TIER) { 330 return; 331 } 332 ResolvedJavaMethod method = math.graph().method(); 333 if (method != null) { 334 if (method.getAnnotation(Snippet.class) != null) { 335 // In the context of SnippetStub, i.e., Graal-generated stubs, use the LIR 336 // lowering to emit the stub assembly code instead of the Node lowering. 337 return; 338 } 339 if (method.getName().equalsIgnoreCase(math.getOperation().name()) && tool.getMetaAccess().lookupJavaType(Math.class).equals(method.getDeclaringClass())) { 340 // A root compilation of the intrinsic method should emit the full assembly 341 // implementation. 342 return; 343 } 344 } 345 StructuredGraph graph = math.graph(); 346 ForeignCallNode call = graph.add(new ForeignCallNode(foreignCalls, math.getOperation().foreignCallDescriptor, math.getX(), math.getY())); 347 graph.addAfterFixed(tool.lastFixedNode(), call); 348 math.replaceAtUsages(call); 349 } 350 351 private void lowerUnaryMath(UnaryMathIntrinsicNode math, LoweringTool tool) { 352 if (tool.getLoweringStage() == LoweringTool.StandardLoweringStage.HIGH_TIER) { 353 return; 354 } 355 ResolvedJavaMethod method = math.graph().method(); 356 if (method != null) { 357 if (method.getName().equalsIgnoreCase(math.getOperation().name()) && tool.getMetaAccess().lookupJavaType(Math.class).equals(method.getDeclaringClass())) { 358 // A root compilation of the intrinsic method should emit the full assembly 359 // implementation. 360 return; 361 } 362 } 363 StructuredGraph graph = math.graph(); 364 ForeignCallNode call = math.graph().add(new ForeignCallNode(foreignCalls, math.getOperation().foreignCallDescriptor, math.getValue())); 365 graph.addAfterFixed(tool.lastFixedNode(), call); 366 math.replaceAtUsages(call); 367 } 368 369 protected void lowerVerifyHeap(VerifyHeapNode n) { 370 GraphUtil.removeFixedWithUnusedInputs(n); 371 } 372 373 protected AddressNode createOffsetAddress(StructuredGraph graph, ValueNode object, long offset) { 374 ValueNode o = ConstantNode.forIntegerKind(target.wordJavaKind, offset, graph); 375 return graph.unique(new OffsetAddressNode(object, o)); 376 } 377 378 protected AddressNode createFieldAddress(StructuredGraph graph, ValueNode object, ResolvedJavaField field) { 379 int offset = fieldOffset(field); 380 if (offset >= 0) { 381 return createOffsetAddress(graph, object, offset); 382 } else { 383 return null; 384 } 385 } 386 387 protected abstract JavaKind getStorageKind(ResolvedJavaField field); 388 389 protected void lowerLoadFieldNode(LoadFieldNode loadField, LoweringTool tool) { 390 assert loadField.getStackKind() != JavaKind.Illegal; 391 StructuredGraph graph = loadField.graph(); 392 ResolvedJavaField field = loadField.field(); 393 ValueNode object = loadField.isStatic() ? staticFieldBase(graph, field) : loadField.object(); 394 object = createNullCheckedValue(object, loadField, tool); 395 Stamp loadStamp = loadStamp(loadField.stamp(NodeView.DEFAULT), getStorageKind(field)); 396 397 AddressNode address = createFieldAddress(graph, object, field); 398 assert address != null : "Field that is loaded must not be eliminated: " + field.getDeclaringClass().toJavaName(true) + "." + field.getName(); 399 400 ReadNode memoryRead = graph.add(new ReadNode(address, fieldLocationIdentity(field), loadStamp, fieldLoadBarrierType(field))); 401 ValueNode readValue = implicitLoadConvert(graph, getStorageKind(field), memoryRead); 402 loadField.replaceAtUsages(readValue); 403 graph.replaceFixed(loadField, memoryRead); 404 405 if (loadField.isVolatile()) { 406 MembarNode preMembar = graph.add(new MembarNode(JMM_PRE_VOLATILE_READ)); 407 graph.addBeforeFixed(memoryRead, preMembar); 408 MembarNode postMembar = graph.add(new MembarNode(JMM_POST_VOLATILE_READ)); 409 graph.addAfterFixed(memoryRead, postMembar); 410 } 411 } 412 413 protected void lowerStoreFieldNode(StoreFieldNode storeField, LoweringTool tool) { 414 StructuredGraph graph = storeField.graph(); 415 ResolvedJavaField field = storeField.field(); 416 ValueNode object = storeField.isStatic() ? staticFieldBase(graph, field) : storeField.object(); 417 object = createNullCheckedValue(object, storeField, tool); 418 ValueNode value = implicitStoreConvert(graph, getStorageKind(storeField.field()), storeField.value()); 419 AddressNode address = createFieldAddress(graph, object, field); 420 assert address != null; 421 422 WriteNode memoryWrite = graph.add(new WriteNode(address, fieldLocationIdentity(field), value, fieldStoreBarrierType(storeField.field()))); 423 memoryWrite.setStateAfter(storeField.stateAfter()); 424 graph.replaceFixedWithFixed(storeField, memoryWrite); 425 426 if (storeField.isVolatile()) { 427 MembarNode preMembar = graph.add(new MembarNode(JMM_PRE_VOLATILE_WRITE)); 428 graph.addBeforeFixed(memoryWrite, preMembar); 429 MembarNode postMembar = graph.add(new MembarNode(JMM_POST_VOLATILE_WRITE)); 430 graph.addAfterFixed(memoryWrite, postMembar); 431 } 432 } 433 434 public static final IntegerStamp POSITIVE_ARRAY_INDEX_STAMP = StampFactory.forInteger(32, 0, Integer.MAX_VALUE - 1); 435 436 /** 437 * Create a PiNode on the index proving that the index is positive. On some platforms this is 438 * important to allow the index to be used as an int in the address mode. 439 */ 440 public AddressNode createArrayIndexAddress(StructuredGraph graph, ValueNode array, JavaKind elementKind, ValueNode index, GuardingNode boundsCheck) { 441 ValueNode positiveIndex = graph.maybeAddOrUnique(PiNode.create(index, POSITIVE_ARRAY_INDEX_STAMP, boundsCheck != null ? boundsCheck.asNode() : null)); 442 return createArrayAddress(graph, array, elementKind, positiveIndex); 443 } 444 445 public AddressNode createArrayAddress(StructuredGraph graph, ValueNode array, JavaKind elementKind, ValueNode index) { 446 return createArrayAddress(graph, array, elementKind, elementKind, index); 447 } 448 449 public AddressNode createArrayAddress(StructuredGraph graph, ValueNode array, JavaKind arrayKind, JavaKind elementKind, ValueNode index) { 450 ValueNode wordIndex; 451 if (target.wordSize > 4) { 452 wordIndex = graph.unique(new SignExtendNode(index, target.wordSize * 8)); 453 } else { 454 assert target.wordSize == 4 : "unsupported word size"; 455 wordIndex = index; 456 } 457 458 int shift = CodeUtil.log2(metaAccess.getArrayIndexScale(elementKind)); 459 ValueNode scaledIndex = graph.unique(new LeftShiftNode(wordIndex, ConstantNode.forInt(shift, graph))); 460 461 int base = metaAccess.getArrayBaseOffset(arrayKind); 462 ValueNode offset = graph.unique(new AddNode(scaledIndex, ConstantNode.forIntegerKind(target.wordJavaKind, base, graph))); 463 464 return graph.unique(new OffsetAddressNode(array, offset)); 465 } 466 467 protected void lowerIndexAddressNode(IndexAddressNode indexAddress) { 468 AddressNode lowered = createArrayAddress(indexAddress.graph(), indexAddress.getArray(), indexAddress.getArrayKind(), indexAddress.getElementKind(), indexAddress.getIndex()); 469 indexAddress.replaceAndDelete(lowered); 470 } 471 472 protected void lowerLoadIndexedNode(LoadIndexedNode loadIndexed, LoweringTool tool) { 473 StructuredGraph graph = loadIndexed.graph(); 474 ValueNode array = loadIndexed.array(); 475 array = createNullCheckedValue(array, loadIndexed, tool); 476 JavaKind elementKind = loadIndexed.elementKind(); 477 Stamp loadStamp = loadStamp(loadIndexed.stamp(NodeView.DEFAULT), elementKind); 478 479 GuardingNode boundsCheck = getBoundsCheck(loadIndexed, array, tool); 480 ValueNode index = loadIndexed.index(); 481 if (UseIndexMasking.getValue(graph.getOptions())) { 482 index = proxyIndex(loadIndexed, index, array, tool); 483 } 484 AddressNode address = createArrayIndexAddress(graph, array, elementKind, index, boundsCheck); 485 486 ReadNode memoryRead = graph.add(new ReadNode(address, NamedLocationIdentity.getArrayLocation(elementKind), loadStamp, BarrierType.NONE)); 487 memoryRead.setGuard(boundsCheck); 488 ValueNode readValue = implicitLoadConvert(graph, elementKind, memoryRead); 489 490 loadIndexed.replaceAtUsages(readValue); 491 graph.replaceFixed(loadIndexed, memoryRead); 492 } 493 494 protected void lowerStoreIndexedNode(StoreIndexedNode storeIndexed, LoweringTool tool) { 495 StructuredGraph graph = storeIndexed.graph(); 496 497 ValueNode value = storeIndexed.value(); 498 ValueNode array = storeIndexed.array(); 499 500 array = this.createNullCheckedValue(array, storeIndexed, tool); 501 502 GuardingNode boundsCheck = getBoundsCheck(storeIndexed, array, tool); 503 504 JavaKind elementKind = storeIndexed.elementKind(); 505 506 LogicNode condition = null; 507 if (storeIndexed.getStoreCheck() == null && elementKind == JavaKind.Object && !StampTool.isPointerAlwaysNull(value)) { 508 /* Array store check. */ 509 TypeReference arrayType = StampTool.typeReferenceOrNull(array); 510 if (arrayType != null && arrayType.isExact()) { 511 ResolvedJavaType elementType = arrayType.getType().getComponentType(); 512 if (!elementType.isJavaLangObject()) { 513 TypeReference typeReference = TypeReference.createTrusted(storeIndexed.graph().getAssumptions(), elementType); 514 LogicNode typeTest = graph.addOrUniqueWithInputs(InstanceOfNode.create(typeReference, value)); 515 condition = LogicNode.or(graph.unique(IsNullNode.create(value)), typeTest, GraalDirectives.UNLIKELY_PROBABILITY); 516 } 517 } else { 518 /* 519 * The guard on the read hub should be the null check of the array that was 520 * introduced earlier. 521 */ 522 ValueNode arrayClass = createReadHub(graph, array, tool); 523 ValueNode componentHub = createReadArrayComponentHub(graph, arrayClass, storeIndexed); 524 LogicNode typeTest = graph.unique(InstanceOfDynamicNode.create(graph.getAssumptions(), tool.getConstantReflection(), componentHub, value, false)); 525 condition = LogicNode.or(graph.unique(IsNullNode.create(value)), typeTest, GraalDirectives.UNLIKELY_PROBABILITY); 526 } 527 } 528 529 AddressNode address = createArrayIndexAddress(graph, array, elementKind, storeIndexed.index(), boundsCheck); 530 WriteNode memoryWrite = graph.add(new WriteNode(address, NamedLocationIdentity.getArrayLocation(elementKind), implicitStoreConvert(graph, elementKind, value), 531 arrayStoreBarrierType(storeIndexed.elementKind()))); 532 memoryWrite.setGuard(boundsCheck); 533 if (condition != null) { 534 tool.createGuard(storeIndexed, condition, DeoptimizationReason.ArrayStoreException, DeoptimizationAction.InvalidateReprofile); 535 } 536 memoryWrite.setStateAfter(storeIndexed.stateAfter()); 537 graph.replaceFixedWithFixed(storeIndexed, memoryWrite); 538 } 539 540 protected void lowerArrayLengthNode(ArrayLengthNode arrayLengthNode, LoweringTool tool) { 541 arrayLengthNode.replaceAtUsages(createReadArrayLength(arrayLengthNode.array(), arrayLengthNode, tool)); 542 StructuredGraph graph = arrayLengthNode.graph(); 543 graph.removeFixed(arrayLengthNode); 544 } 545 546 /** 547 * Creates a read node that read the array length and is guarded by a null-check. 548 * 549 * The created node is placed before {@code before} in the CFG. 550 */ 551 protected ReadNode createReadArrayLength(ValueNode array, FixedNode before, LoweringTool tool) { 552 StructuredGraph graph = array.graph(); 553 ValueNode canonicalArray = this.createNullCheckedValue(skipPiWhileNonNull(array), before, tool); 554 AddressNode address = createOffsetAddress(graph, canonicalArray, arrayLengthOffset()); 555 ReadNode readArrayLength = graph.add(new ReadNode(address, ARRAY_LENGTH_LOCATION, StampFactory.positiveInt(), BarrierType.NONE)); 556 graph.addBeforeFixed(before, readArrayLength); 557 return readArrayLength; 558 } 559 560 protected void lowerLoadHubNode(LoadHubNode loadHub, LoweringTool tool) { 561 StructuredGraph graph = loadHub.graph(); 562 if (tool.getLoweringStage() != LoweringTool.StandardLoweringStage.LOW_TIER) { 563 return; 564 } 565 if (graph.getGuardsStage().allowsFloatingGuards()) { 566 return; 567 } 568 ValueNode hub = createReadHub(graph, loadHub.getValue(), tool); 569 loadHub.replaceAtUsagesAndDelete(hub); 570 } 571 572 protected void lowerLoadHubOrNullNode(LoadHubOrNullNode loadHubOrNullNode, LoweringTool tool) { 573 StructuredGraph graph = loadHubOrNullNode.graph(); 574 if (tool.getLoweringStage() != LoweringTool.StandardLoweringStage.LOW_TIER) { 575 return; 576 } 577 if (graph.getGuardsStage().allowsFloatingGuards()) { 578 return; 579 } 580 final FixedWithNextNode predecessor = tool.lastFixedNode(); 581 final ValueNode value = loadHubOrNullNode.getValue(); 582 AbstractPointerStamp stamp = (AbstractPointerStamp) value.stamp(NodeView.DEFAULT); 583 final LogicNode isNull = graph.addOrUniqueWithInputs(IsNullNode.create(value)); 584 final EndNode trueEnd = graph.add(new EndNode()); 585 final EndNode falseEnd = graph.add(new EndNode()); 586 final IfNode ifNode = graph.add(new IfNode(isNull, trueEnd, falseEnd, 0.5)); 587 final MergeNode merge = graph.add(new MergeNode()); 588 merge.addForwardEnd(trueEnd); 589 merge.addForwardEnd(falseEnd); 590 final AbstractPointerStamp hubStamp = (AbstractPointerStamp) loadHubOrNullNode.stamp(NodeView.DEFAULT); 591 ValueNode nullHub = ConstantNode.forConstant(hubStamp.asAlwaysNull(), JavaConstant.NULL_POINTER, tool.getMetaAccess(), graph); 592 final ValueNode nonNullValue = graph.addOrUniqueWithInputs(PiNode.create(value, stamp.asNonNull(), ifNode.falseSuccessor())); 593 ValueNode hub = createReadHub(graph, nonNullValue, tool); 594 ValueNode[] values = new ValueNode[]{nullHub, hub}; 595 final PhiNode hubPhi = graph.unique(new ValuePhiNode(hubStamp, merge, values)); 596 final FixedNode oldNext = predecessor.next(); 597 predecessor.setNext(ifNode); 598 merge.setNext(oldNext); 599 loadHubOrNullNode.replaceAtUsagesAndDelete(hubPhi); 600 } 601 602 protected void lowerLoadArrayComponentHubNode(LoadArrayComponentHubNode loadHub) { 603 StructuredGraph graph = loadHub.graph(); 604 ValueNode hub = createReadArrayComponentHub(graph, loadHub.getValue(), loadHub); 605 graph.replaceFixed(loadHub, hub); 606 } 607 608 protected void lowerMonitorEnterNode(MonitorEnterNode monitorEnter, LoweringTool tool, StructuredGraph graph) { 609 ValueNode object = createNullCheckedValue(monitorEnter.object(), monitorEnter, tool); 610 ValueNode hub = graph.addOrUnique(LoadHubNode.create(object, tool.getStampProvider(), tool.getMetaAccess(), tool.getConstantReflection())); 611 RawMonitorEnterNode rawMonitorEnter = graph.add(new RawMonitorEnterNode(object, hub, monitorEnter.getMonitorId())); 612 rawMonitorEnter.setStateBefore(monitorEnter.stateBefore()); 613 rawMonitorEnter.setStateAfter(monitorEnter.stateAfter()); 614 graph.replaceFixedWithFixed(monitorEnter, rawMonitorEnter); 615 } 616 617 protected void lowerCompareAndSwapNode(UnsafeCompareAndSwapNode cas) { 618 StructuredGraph graph = cas.graph(); 619 JavaKind valueKind = cas.getValueKind(); 620 621 ValueNode expectedValue = implicitStoreConvert(graph, valueKind, cas.expected()); 622 ValueNode newValue = implicitStoreConvert(graph, valueKind, cas.newValue()); 623 624 AddressNode address = graph.unique(new OffsetAddressNode(cas.object(), cas.offset())); 625 BarrierType barrierType = guessStoreBarrierType(cas.object(), expectedValue); 626 LogicCompareAndSwapNode atomicNode = graph.add(new LogicCompareAndSwapNode(address, cas.getLocationIdentity(), expectedValue, newValue, barrierType)); 627 atomicNode.setStateAfter(cas.stateAfter()); 628 graph.replaceFixedWithFixed(cas, atomicNode); 629 } 630 631 protected void lowerCompareAndExchangeNode(UnsafeCompareAndExchangeNode cas) { 632 StructuredGraph graph = cas.graph(); 633 JavaKind valueKind = cas.getValueKind(); 634 635 ValueNode expectedValue = implicitStoreConvert(graph, valueKind, cas.expected()); 636 ValueNode newValue = implicitStoreConvert(graph, valueKind, cas.newValue()); 637 638 AddressNode address = graph.unique(new OffsetAddressNode(cas.object(), cas.offset())); 639 BarrierType barrierType = guessStoreBarrierType(cas.object(), expectedValue); 640 ValueCompareAndSwapNode atomicNode = graph.add(new ValueCompareAndSwapNode(address, expectedValue, newValue, cas.getLocationIdentity(), barrierType)); 641 ValueNode coercedNode = implicitLoadConvert(graph, valueKind, atomicNode, true); 642 atomicNode.setStateAfter(cas.stateAfter()); 643 cas.replaceAtUsages(coercedNode); 644 graph.replaceFixedWithFixed(cas, atomicNode); 645 } 646 647 protected void lowerAtomicReadAndWriteNode(AtomicReadAndWriteNode n) { 648 StructuredGraph graph = n.graph(); 649 JavaKind valueKind = n.getValueKind(); 650 651 ValueNode newValue = implicitStoreConvert(graph, valueKind, n.newValue()); 652 653 AddressNode address = graph.unique(new OffsetAddressNode(n.object(), n.offset())); 654 BarrierType barrierType = guessStoreBarrierType(n.object(), n.newValue()); 655 LIRKind lirAccessKind = LIRKind.fromJavaKind(target.arch, valueKind); 656 LoweredAtomicReadAndWriteNode memoryRead = graph.add(new LoweredAtomicReadAndWriteNode(address, n.getLocationIdentity(), newValue, lirAccessKind, barrierType)); 657 memoryRead.setStateAfter(n.stateAfter()); 658 659 ValueNode readValue = implicitLoadConvert(graph, valueKind, memoryRead); 660 n.stateAfter().replaceFirstInput(n, memoryRead); 661 n.replaceAtUsages(readValue); 662 graph.replaceFixedWithFixed(n, memoryRead); 663 } 664 665 /** 666 * @param tool utility for performing the lowering 667 */ 668 protected void lowerUnsafeLoadNode(RawLoadNode load, LoweringTool tool) { 669 StructuredGraph graph = load.graph(); 670 if (load instanceof GuardedUnsafeLoadNode) { 671 GuardedUnsafeLoadNode guardedLoad = (GuardedUnsafeLoadNode) load; 672 GuardingNode guard = guardedLoad.getGuard(); 673 if (guard == null) { 674 // can float freely if the guard folded away 675 ReadNode memoryRead = createUnsafeRead(graph, load, null); 676 memoryRead.setForceFixed(false); 677 graph.replaceFixedWithFixed(load, memoryRead); 678 } else { 679 // must be guarded, but flows below the guard 680 ReadNode memoryRead = createUnsafeRead(graph, load, guard); 681 graph.replaceFixedWithFixed(load, memoryRead); 682 } 683 } else { 684 // never had a guarding condition so it must be fixed, creation of the read will force 685 // it to be fixed 686 ReadNode memoryRead = createUnsafeRead(graph, load, null); 687 graph.replaceFixedWithFixed(load, memoryRead); 688 } 689 } 690 691 protected AddressNode createUnsafeAddress(StructuredGraph graph, ValueNode object, ValueNode offset) { 692 if (object.isConstant() && object.asConstant().isDefaultForKind()) { 693 return graph.addOrUniqueWithInputs(OffsetAddressNode.create(offset)); 694 } else { 695 return graph.unique(new OffsetAddressNode(object, offset)); 696 } 697 } 698 699 protected ReadNode createUnsafeRead(StructuredGraph graph, RawLoadNode load, GuardingNode guard) { 700 boolean compressible = load.accessKind() == JavaKind.Object; 701 JavaKind readKind = load.accessKind(); 702 Stamp loadStamp = loadStamp(load.stamp(NodeView.DEFAULT), readKind, compressible); 703 AddressNode address = createUnsafeAddress(graph, load.object(), load.offset()); 704 ReadNode memoryRead = graph.add(new ReadNode(address, load.getLocationIdentity(), loadStamp, BarrierType.NONE)); 705 if (guard == null) { 706 // An unsafe read must not float otherwise it may float above 707 // a test guaranteeing the read is safe. 708 memoryRead.setForceFixed(true); 709 } else { 710 memoryRead.setGuard(guard); 711 } 712 ValueNode readValue = performBooleanCoercionIfNecessary(implicitLoadConvert(graph, readKind, memoryRead, compressible), readKind); 713 load.replaceAtUsages(readValue); 714 return memoryRead; 715 } 716 717 protected void lowerUnsafeMemoryLoadNode(UnsafeMemoryLoadNode load) { 718 StructuredGraph graph = load.graph(); 719 JavaKind readKind = load.getKind(); 720 assert readKind != JavaKind.Object; 721 Stamp loadStamp = loadStamp(load.stamp(NodeView.DEFAULT), readKind, false); 722 AddressNode address = graph.addOrUniqueWithInputs(OffsetAddressNode.create(load.getAddress())); 723 ReadNode memoryRead = graph.add(new ReadNode(address, load.getLocationIdentity(), loadStamp, BarrierType.NONE)); 724 // An unsafe read must not float otherwise it may float above 725 // a test guaranteeing the read is safe. 726 memoryRead.setForceFixed(true); 727 ValueNode readValue = performBooleanCoercionIfNecessary(implicitLoadConvert(graph, readKind, memoryRead, false), readKind); 728 load.replaceAtUsages(readValue); 729 graph.replaceFixedWithFixed(load, memoryRead); 730 } 731 732 private static ValueNode performBooleanCoercionIfNecessary(ValueNode readValue, JavaKind readKind) { 733 if (readKind == JavaKind.Boolean) { 734 StructuredGraph graph = readValue.graph(); 735 IntegerEqualsNode eq = graph.addOrUnique(new IntegerEqualsNode(readValue, ConstantNode.forInt(0, graph))); 736 return graph.addOrUnique(new ConditionalNode(eq, ConstantNode.forBoolean(false, graph), ConstantNode.forBoolean(true, graph))); 737 } 738 return readValue; 739 } 740 741 protected void lowerUnsafeStoreNode(RawStoreNode store) { 742 StructuredGraph graph = store.graph(); 743 boolean compressible = store.value().getStackKind() == JavaKind.Object; 744 JavaKind valueKind = store.accessKind(); 745 ValueNode value = implicitStoreConvert(graph, valueKind, store.value(), compressible); 746 AddressNode address = createUnsafeAddress(graph, store.object(), store.offset()); 747 WriteNode write = graph.add(new WriteNode(address, store.getLocationIdentity(), value, unsafeStoreBarrierType(store))); 748 write.setStateAfter(store.stateAfter()); 749 graph.replaceFixedWithFixed(store, write); 750 } 751 752 protected void lowerUnsafeMemoryStoreNode(UnsafeMemoryStoreNode store) { 753 StructuredGraph graph = store.graph(); 754 assert store.getValue().getStackKind() != JavaKind.Object; 755 JavaKind valueKind = store.getKind(); 756 ValueNode value = implicitStoreConvert(graph, valueKind, store.getValue(), false); 757 AddressNode address = graph.addOrUniqueWithInputs(OffsetAddressNode.create(store.getAddress())); 758 WriteNode write = graph.add(new WriteNode(address, store.getLocationIdentity(), value, BarrierType.NONE)); 759 write.setStateAfter(store.stateAfter()); 760 graph.replaceFixedWithFixed(store, write); 761 } 762 763 protected void lowerJavaReadNode(JavaReadNode read) { 764 StructuredGraph graph = read.graph(); 765 JavaKind valueKind = read.getReadKind(); 766 Stamp loadStamp = loadStamp(read.stamp(NodeView.DEFAULT), valueKind, read.isCompressible()); 767 768 ReadNode memoryRead = graph.add(new ReadNode(read.getAddress(), read.getLocationIdentity(), loadStamp, read.getBarrierType())); 769 GuardingNode guard = read.getGuard(); 770 ValueNode readValue = implicitLoadConvert(graph, valueKind, memoryRead, read.isCompressible()); 771 if (guard == null) { 772 // An unsafe read must not float otherwise it may float above 773 // a test guaranteeing the read is safe. 774 memoryRead.setForceFixed(true); 775 } else { 776 memoryRead.setGuard(guard); 777 } 778 read.replaceAtUsages(readValue); 779 graph.replaceFixed(read, memoryRead); 780 } 781 782 protected void lowerJavaWriteNode(JavaWriteNode write) { 783 StructuredGraph graph = write.graph(); 784 ValueNode value = implicitStoreConvert(graph, write.getWriteKind(), write.value(), write.isCompressible()); 785 WriteNode memoryWrite = graph.add(new WriteNode(write.getAddress(), write.getLocationIdentity(), value, write.getBarrierType())); 786 memoryWrite.setStateAfter(write.stateAfter()); 787 graph.replaceFixedWithFixed(write, memoryWrite); 788 memoryWrite.setGuard(write.getGuard()); 789 } 790 791 @SuppressWarnings("try") 792 protected void lowerCommitAllocationNode(CommitAllocationNode commit, LoweringTool tool) { 793 StructuredGraph graph = commit.graph(); 794 if (graph.getGuardsStage() == StructuredGraph.GuardsStage.FIXED_DEOPTS) { 795 List<AbstractNewObjectNode> recursiveLowerings = new ArrayList<>(); 796 797 ValueNode[] allocations = new ValueNode[commit.getVirtualObjects().size()]; 798 BitSet omittedValues = new BitSet(); 799 int valuePos = 0; 800 for (int objIndex = 0; objIndex < commit.getVirtualObjects().size(); objIndex++) { 801 VirtualObjectNode virtual = commit.getVirtualObjects().get(objIndex); 802 try (DebugCloseable nsp = graph.withNodeSourcePosition(virtual)) { 803 int entryCount = virtual.entryCount(); 804 AbstractNewObjectNode newObject; 805 if (virtual instanceof VirtualInstanceNode) { 806 newObject = graph.add(createNewInstanceFromVirtual(virtual)); 807 } else { 808 newObject = graph.add(createNewArrayFromVirtual(virtual, ConstantNode.forInt(entryCount, graph))); 809 } 810 // The final STORE_STORE barrier will be emitted by finishAllocatedObjects 811 newObject.clearEmitMemoryBarrier(); 812 813 recursiveLowerings.add(newObject); 814 graph.addBeforeFixed(commit, newObject); 815 allocations[objIndex] = newObject; 816 for (int i = 0; i < entryCount; i++) { 817 ValueNode value = commit.getValues().get(valuePos); 818 if (value instanceof VirtualObjectNode) { 819 value = allocations[commit.getVirtualObjects().indexOf(value)]; 820 } 821 if (value == null) { 822 omittedValues.set(valuePos); 823 } else if (!(value.isConstant() && value.asConstant().isDefaultForKind())) { 824 // Constant.illegal is always the defaultForKind, so it is skipped 825 JavaKind valueKind = value.getStackKind(); 826 JavaKind entryKind = virtual.entryKind(i); 827 828 // Truffle requires some leniency in terms of what can be put where: 829 assert valueKind.getStackKind() == entryKind.getStackKind() || 830 (valueKind == JavaKind.Long || valueKind == JavaKind.Double || (valueKind == JavaKind.Int && virtual instanceof VirtualArrayNode)); 831 AddressNode address = null; 832 BarrierType barrierType = null; 833 if (virtual instanceof VirtualInstanceNode) { 834 ResolvedJavaField field = ((VirtualInstanceNode) virtual).field(i); 835 long offset = fieldOffset(field); 836 if (offset >= 0) { 837 address = createOffsetAddress(graph, newObject, offset); 838 barrierType = fieldInitializationBarrier(entryKind); 839 } 840 } else { 841 address = createOffsetAddress(graph, newObject, metaAccess.getArrayBaseOffset(entryKind) + i * metaAccess.getArrayIndexScale(entryKind)); 842 barrierType = arrayInitializationBarrier(entryKind); 843 } 844 if (address != null) { 845 WriteNode write = new WriteNode(address, LocationIdentity.init(), implicitStoreConvert(graph, entryKind, value), barrierType); 846 graph.addAfterFixed(newObject, graph.add(write)); 847 } 848 } 849 valuePos++; 850 } 851 } 852 } 853 valuePos = 0; 854 855 for (int objIndex = 0; objIndex < commit.getVirtualObjects().size(); objIndex++) { 856 VirtualObjectNode virtual = commit.getVirtualObjects().get(objIndex); 857 try (DebugCloseable nsp = graph.withNodeSourcePosition(virtual)) { 858 int entryCount = virtual.entryCount(); 859 ValueNode newObject = allocations[objIndex]; 860 for (int i = 0; i < entryCount; i++) { 861 if (omittedValues.get(valuePos)) { 862 ValueNode value = commit.getValues().get(valuePos); 863 assert value instanceof VirtualObjectNode; 864 ValueNode allocValue = allocations[commit.getVirtualObjects().indexOf(value)]; 865 if (!(allocValue.isConstant() && allocValue.asConstant().isDefaultForKind())) { 866 assert virtual.entryKind(i) == JavaKind.Object && allocValue.getStackKind() == JavaKind.Object; 867 AddressNode address; 868 BarrierType barrierType; 869 if (virtual instanceof VirtualInstanceNode) { 870 VirtualInstanceNode virtualInstance = (VirtualInstanceNode) virtual; 871 address = createFieldAddress(graph, newObject, virtualInstance.field(i)); 872 barrierType = fieldStoreBarrierType(virtualInstance.field(i)); 873 } else { 874 address = createArrayAddress(graph, newObject, virtual.entryKind(i), ConstantNode.forInt(i, graph)); 875 barrierType = arrayStoreBarrierType(virtual.entryKind(i)); 876 } 877 if (address != null) { 878 WriteNode write = new WriteNode(address, LocationIdentity.init(), implicitStoreConvert(graph, JavaKind.Object, allocValue), barrierType); 879 graph.addBeforeFixed(commit, graph.add(write)); 880 } 881 } 882 } 883 valuePos++; 884 } 885 } 886 } 887 888 finishAllocatedObjects(tool, commit, allocations); 889 graph.removeFixed(commit); 890 891 for (AbstractNewObjectNode recursiveLowering : recursiveLowerings) { 892 recursiveLowering.lower(tool); 893 } 894 } 895 896 } 897 898 public NewInstanceNode createNewInstanceFromVirtual(VirtualObjectNode virtual) { 899 return new NewInstanceNode(virtual.type(), true); 900 } 901 902 protected NewArrayNode createNewArrayFromVirtual(VirtualObjectNode virtual, ValueNode length) { 903 return new NewArrayNode(((VirtualArrayNode) virtual).componentType(), length, true); 904 } 905 906 public void finishAllocatedObjects(LoweringTool tool, CommitAllocationNode commit, ValueNode[] allocations) { 907 StructuredGraph graph = commit.graph(); 908 for (int objIndex = 0; objIndex < commit.getVirtualObjects().size(); objIndex++) { 909 FixedValueAnchorNode anchor = graph.add(new FixedValueAnchorNode(allocations[objIndex])); 910 allocations[objIndex] = anchor; 911 graph.addBeforeFixed(commit, anchor); 912 } 913 /* 914 * Note that the FrameState that is assigned to these MonitorEnterNodes isn't the correct 915 * state. It will be the state from before the allocation occurred instead of a valid state 916 * after the locking is performed. In practice this should be fine since these are newly 917 * allocated objects. The bytecodes themselves permit allocating an object, doing a 918 * monitorenter and then dropping all references to the object which would produce the same 919 * state, though that would normally produce an IllegalMonitorStateException. In HotSpot 920 * some form of fast path locking should always occur so the FrameState should never 921 * actually be used. 922 */ 923 ArrayList<MonitorEnterNode> enters = null; 924 for (int objIndex = 0; objIndex < commit.getVirtualObjects().size(); objIndex++) { 925 List<MonitorIdNode> locks = commit.getLocks(objIndex); 926 if (locks.size() > 1) { 927 // Ensure that the lock operations are performed in lock depth order 928 ArrayList<MonitorIdNode> newList = new ArrayList<>(locks); 929 newList.sort((a, b) -> Integer.compare(a.getLockDepth(), b.getLockDepth())); 930 locks = newList; 931 } 932 int lastDepth = -1; 933 for (MonitorIdNode monitorId : locks) { 934 assert lastDepth < monitorId.getLockDepth(); 935 lastDepth = monitorId.getLockDepth(); 936 MonitorEnterNode enter = graph.add(new MonitorEnterNode(allocations[objIndex], monitorId)); 937 graph.addBeforeFixed(commit, enter); 938 if (enters == null) { 939 enters = new ArrayList<>(); 940 } 941 enters.add(enter); 942 } 943 } 944 for (Node usage : commit.usages().snapshot()) { 945 if (usage instanceof AllocatedObjectNode) { 946 AllocatedObjectNode addObject = (AllocatedObjectNode) usage; 947 int index = commit.getVirtualObjects().indexOf(addObject.getVirtualObject()); 948 addObject.replaceAtUsagesAndDelete(allocations[index]); 949 } else { 950 assert enters != null; 951 commit.replaceAtUsages(InputType.Memory, enters.get(enters.size() - 1)); 952 } 953 } 954 if (enters != null) { 955 for (MonitorEnterNode enter : enters) { 956 enter.lower(tool); 957 } 958 } 959 assert commit.hasNoUsages(); 960 insertAllocationBarrier(commit, graph); 961 } 962 963 /** 964 * Insert the required {@link MemoryBarriers#STORE_STORE} barrier for an allocation and also 965 * include the {@link MemoryBarriers#LOAD_STORE} required for final fields if any final fields 966 * are being written, as if {@link FinalFieldBarrierNode} were emitted. 967 */ 968 private static void insertAllocationBarrier(CommitAllocationNode commit, StructuredGraph graph) { 969 int barrier = MemoryBarriers.STORE_STORE; 970 outer: for (VirtualObjectNode vobj : commit.getVirtualObjects()) { 971 for (ResolvedJavaField field : vobj.type().getInstanceFields(true)) { 972 if (field.isFinal()) { 973 barrier = barrier | MemoryBarriers.LOAD_STORE; 974 break outer; 975 } 976 } 977 } 978 graph.addAfterFixed(commit, graph.add(new MembarNode(barrier, LocationIdentity.init()))); 979 } 980 981 /** 982 * @param field the field whose barrier type should be returned 983 */ 984 protected BarrierType fieldLoadBarrierType(ResolvedJavaField field) { 985 return BarrierType.NONE; 986 } 987 988 protected BarrierType fieldStoreBarrierType(ResolvedJavaField field) { 989 if (getStorageKind(field) == JavaKind.Object) { 990 return BarrierType.FIELD; 991 } 992 return BarrierType.NONE; 993 } 994 995 protected BarrierType arrayStoreBarrierType(JavaKind elementKind) { 996 if (elementKind == JavaKind.Object) { 997 return BarrierType.ARRAY; 998 } 999 return BarrierType.NONE; 1000 } 1001 1002 public BarrierType fieldInitializationBarrier(JavaKind entryKind) { 1003 return entryKind == JavaKind.Object ? BarrierType.FIELD : BarrierType.NONE; 1004 } 1005 1006 public BarrierType arrayInitializationBarrier(JavaKind entryKind) { 1007 return entryKind == JavaKind.Object ? BarrierType.ARRAY : BarrierType.NONE; 1008 } 1009 1010 private BarrierType unsafeStoreBarrierType(RawStoreNode store) { 1011 if (!store.needsBarrier()) { 1012 return BarrierType.NONE; 1013 } 1014 return guessStoreBarrierType(store.object(), store.value()); 1015 } 1016 1017 private BarrierType guessStoreBarrierType(ValueNode object, ValueNode value) { 1018 if (value.getStackKind() == JavaKind.Object && object.getStackKind() == JavaKind.Object) { 1019 ResolvedJavaType type = StampTool.typeOrNull(object); 1020 // Array types must use a precise barrier, so if the type is unknown or is a supertype 1021 // of Object[] then treat it as an array. 1022 if (type != null && type.isArray()) { 1023 return BarrierType.ARRAY; 1024 } else if (type == null || type.isAssignableFrom(objectArrayType)) { 1025 return BarrierType.UNKNOWN; 1026 } else { 1027 return BarrierType.FIELD; 1028 } 1029 } 1030 return BarrierType.NONE; 1031 } 1032 1033 public abstract int fieldOffset(ResolvedJavaField field); 1034 1035 public FieldLocationIdentity fieldLocationIdentity(ResolvedJavaField field) { 1036 return new FieldLocationIdentity(field); 1037 } 1038 1039 public abstract ValueNode staticFieldBase(StructuredGraph graph, ResolvedJavaField field); 1040 1041 public abstract int arrayLengthOffset(); 1042 1043 public Stamp loadStamp(Stamp stamp, JavaKind kind) { 1044 return loadStamp(stamp, kind, true); 1045 } 1046 1047 private boolean useCompressedOops(JavaKind kind, boolean compressible) { 1048 return kind == JavaKind.Object && compressible && useCompressedOops; 1049 } 1050 1051 protected abstract Stamp loadCompressedStamp(ObjectStamp stamp); 1052 1053 /** 1054 * @param compressible whether the stamp should be compressible 1055 */ 1056 protected Stamp loadStamp(Stamp stamp, JavaKind kind, boolean compressible) { 1057 if (useCompressedOops(kind, compressible)) { 1058 return loadCompressedStamp((ObjectStamp) stamp); 1059 } 1060 1061 switch (kind) { 1062 case Boolean: 1063 case Byte: 1064 return IntegerStamp.OPS.getNarrow().foldStamp(32, 8, stamp); 1065 case Char: 1066 case Short: 1067 return IntegerStamp.OPS.getNarrow().foldStamp(32, 16, stamp); 1068 } 1069 return stamp; 1070 } 1071 1072 public final ValueNode implicitLoadConvert(StructuredGraph graph, JavaKind kind, ValueNode value) { 1073 return implicitLoadConvert(graph, kind, value, true); 1074 } 1075 1076 public ValueNode implicitLoadConvert(JavaKind kind, ValueNode value) { 1077 return implicitLoadConvert(kind, value, true); 1078 } 1079 1080 protected final ValueNode implicitLoadConvert(StructuredGraph graph, JavaKind kind, ValueNode value, boolean compressible) { 1081 ValueNode ret = implicitLoadConvert(kind, value, compressible); 1082 if (!ret.isAlive()) { 1083 ret = graph.addOrUnique(ret); 1084 } 1085 return ret; 1086 } 1087 1088 protected abstract ValueNode newCompressionNode(CompressionOp op, ValueNode value); 1089 1090 /** 1091 * @param compressible whether the convert should be compressible 1092 */ 1093 protected ValueNode implicitLoadConvert(JavaKind kind, ValueNode value, boolean compressible) { 1094 if (useCompressedOops(kind, compressible)) { 1095 return newCompressionNode(CompressionOp.Uncompress, value); 1096 } 1097 1098 switch (kind) { 1099 case Byte: 1100 case Short: 1101 return new SignExtendNode(value, 32); 1102 case Boolean: 1103 case Char: 1104 return new ZeroExtendNode(value, 32); 1105 } 1106 return value; 1107 } 1108 1109 public final ValueNode implicitStoreConvert(StructuredGraph graph, JavaKind kind, ValueNode value) { 1110 return implicitStoreConvert(graph, kind, value, true); 1111 } 1112 1113 public ValueNode implicitStoreConvert(JavaKind kind, ValueNode value) { 1114 return implicitStoreConvert(kind, value, true); 1115 } 1116 1117 protected final ValueNode implicitStoreConvert(StructuredGraph graph, JavaKind kind, ValueNode value, boolean compressible) { 1118 ValueNode ret = implicitStoreConvert(kind, value, compressible); 1119 if (!ret.isAlive()) { 1120 ret = graph.addOrUnique(ret); 1121 } 1122 return ret; 1123 } 1124 1125 /** 1126 * @param compressible whether the covert should be compressible 1127 */ 1128 protected ValueNode implicitStoreConvert(JavaKind kind, ValueNode value, boolean compressible) { 1129 if (useCompressedOops(kind, compressible)) { 1130 return newCompressionNode(CompressionOp.Compress, value); 1131 } 1132 1133 switch (kind) { 1134 case Boolean: 1135 case Byte: 1136 return new NarrowNode(value, 8); 1137 case Char: 1138 case Short: 1139 return new NarrowNode(value, 16); 1140 } 1141 return value; 1142 } 1143 1144 protected abstract ValueNode createReadHub(StructuredGraph graph, ValueNode object, LoweringTool tool); 1145 1146 protected abstract ValueNode createReadArrayComponentHub(StructuredGraph graph, ValueNode arrayHub, FixedNode anchor); 1147 1148 protected ValueNode proxyIndex(AccessIndexedNode n, ValueNode index, ValueNode array, LoweringTool tool) { 1149 StructuredGraph graph = index.graph(); 1150 ValueNode arrayLength = readOrCreateArrayLength(n, array, tool, graph); 1151 ValueNode lengthMinusOne = SubNode.create(arrayLength, ConstantNode.forInt(1), NodeView.DEFAULT); 1152 return branchlessMax(branchlessMin(index, lengthMinusOne, NodeView.DEFAULT), ConstantNode.forInt(0), NodeView.DEFAULT); 1153 } 1154 1155 protected GuardingNode getBoundsCheck(AccessIndexedNode n, ValueNode array, LoweringTool tool) { 1156 if (n.getBoundsCheck() != null) { 1157 return n.getBoundsCheck(); 1158 } 1159 1160 StructuredGraph graph = n.graph(); 1161 ValueNode arrayLength = readOrCreateArrayLength(n, array, tool, graph); 1162 1163 LogicNode boundsCheck = IntegerBelowNode.create(n.index(), arrayLength, NodeView.DEFAULT); 1164 if (boundsCheck.isTautology()) { 1165 return null; 1166 } 1167 return tool.createGuard(n, graph.addOrUniqueWithInputs(boundsCheck), BoundsCheckException, InvalidateReprofile); 1168 } 1169 1170 private ValueNode readOrCreateArrayLength(AccessIndexedNode n, ValueNode array, LoweringTool tool, StructuredGraph graph) { 1171 ValueNode arrayLength = readArrayLength(array, tool.getConstantReflection()); 1172 if (arrayLength == null) { 1173 arrayLength = createReadArrayLength(array, n, tool); 1174 } else { 1175 arrayLength = arrayLength.isAlive() ? arrayLength : graph.addOrUniqueWithInputs(arrayLength); 1176 } 1177 return arrayLength; 1178 } 1179 1180 protected GuardingNode createNullCheck(ValueNode object, FixedNode before, LoweringTool tool) { 1181 if (StampTool.isPointerNonNull(object)) { 1182 return null; 1183 } 1184 return tool.createGuard(before, before.graph().unique(IsNullNode.create(object)), NullCheckException, InvalidateReprofile, SpeculationLog.NO_SPECULATION, true, null); 1185 } 1186 1187 protected ValueNode createNullCheckedValue(ValueNode object, FixedNode before, LoweringTool tool) { 1188 GuardingNode nullCheck = createNullCheck(object, before, tool); 1189 if (nullCheck == null) { 1190 return object; 1191 } 1192 return before.graph().maybeAddOrUnique(PiNode.create(object, (object.stamp(NodeView.DEFAULT)).join(StampFactory.objectNonNull()), (ValueNode) nullCheck)); 1193 } 1194 1195 @Override 1196 public ValueNode reconstructArrayIndex(JavaKind elementKind, AddressNode address) { 1197 StructuredGraph graph = address.graph(); 1198 ValueNode offset = ((OffsetAddressNode) address).getOffset(); 1199 1200 int base = metaAccess.getArrayBaseOffset(elementKind); 1201 ValueNode scaledIndex = graph.unique(new SubNode(offset, ConstantNode.forIntegerStamp(offset.stamp(NodeView.DEFAULT), base, graph))); 1202 1203 int shift = CodeUtil.log2(metaAccess.getArrayIndexScale(elementKind)); 1204 ValueNode ret = graph.unique(new RightShiftNode(scaledIndex, ConstantNode.forInt(shift, graph))); 1205 return IntegerConvertNode.convert(ret, StampFactory.forKind(JavaKind.Int), graph, NodeView.DEFAULT); 1206 } 1207 }