--- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.amd64/src/org/graalvm/compiler/asm/amd64/AMD64Assembler.java 2019-05-16 21:17:40.816187474 +0200 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.amd64/src/org/graalvm/compiler/asm/amd64/AMD64Assembler.java 2019-05-16 21:17:40.408184838 +0200 @@ -906,7 +906,8 @@ CPU_XMM(CPUFeature.AVX, null, CPU, null, XMM, null), AVX1_2_CPU_XMM(CPUFeature.AVX, CPUFeature.AVX2, CPU, null, XMM, null), BMI1(CPUFeature.BMI1, null, CPU, CPU, CPU, null), - BMI2(CPUFeature.BMI2, null, CPU, CPU, CPU, null); + BMI2(CPUFeature.BMI2, null, CPU, CPU, CPU, null), + FMA(CPUFeature.FMA, null, XMM, XMM, XMM, null); private final CPUFeature l128feature; private final CPUFeature l256feature; @@ -1308,6 +1309,8 @@ public static final VexRVMOp VPCMPGTW = new VexRVMOp("VPCMPGTW", P_66, M_0F, WIG, 0x65, VEXOpAssertion.AVX1_2); public static final VexRVMOp VPCMPGTD = new VexRVMOp("VPCMPGTD", P_66, M_0F, WIG, 0x66, VEXOpAssertion.AVX1_2); public static final VexRVMOp VPCMPGTQ = new VexRVMOp("VPCMPGTQ", P_66, M_0F38, WIG, 0x37, VEXOpAssertion.AVX1_2); + public static final VexRVMOp VFMADD231SS = new VexRVMOp("VFMADD231SS", P_66, M_0F38, W0, 0xB9, VEXOpAssertion.FMA); + public static final VexRVMOp VFMADD231SD = new VexRVMOp("VFMADD231SD", P_66, M_0F38, W1, 0xB9, VEXOpAssertion.FMA); // @formatter:on private VexRVMOp(String opcode, int pp, int mmmmm, int w, int op) { --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.amd64/src/org/graalvm/compiler/core/amd64/AMD64ArithmeticLIRGenerator.java 2019-05-16 21:17:41.380191119 +0200 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.amd64/src/org/graalvm/compiler/core/amd64/AMD64ArithmeticLIRGenerator.java 2019-05-16 21:17:40.976188508 +0200 @@ -57,6 +57,8 @@ import static org.graalvm.compiler.asm.amd64.AMD64Assembler.VexRVMOp.VADDSS; import static org.graalvm.compiler.asm.amd64.AMD64Assembler.VexRVMOp.VDIVSD; import static org.graalvm.compiler.asm.amd64.AMD64Assembler.VexRVMOp.VDIVSS; +import static org.graalvm.compiler.asm.amd64.AMD64Assembler.VexRVMOp.VFMADD231SD; +import static org.graalvm.compiler.asm.amd64.AMD64Assembler.VexRVMOp.VFMADD231SS; import static org.graalvm.compiler.asm.amd64.AMD64Assembler.VexRVMOp.VMULSD; import static org.graalvm.compiler.asm.amd64.AMD64Assembler.VexRVMOp.VMULSS; import static org.graalvm.compiler.asm.amd64.AMD64Assembler.VexRVMOp.VORPD; @@ -120,6 +122,7 @@ import org.graalvm.compiler.lir.amd64.AMD64MulDivOp; import org.graalvm.compiler.lir.amd64.AMD64ShiftOp; import org.graalvm.compiler.lir.amd64.AMD64SignExtendOp; +import org.graalvm.compiler.lir.amd64.AMD64Ternary; import org.graalvm.compiler.lir.amd64.AMD64Unary; import org.graalvm.compiler.lir.amd64.AMD64ZeroMemoryOp; import org.graalvm.compiler.lir.amd64.vector.AMD64VectorBinary; @@ -961,6 +964,22 @@ } return result; } + + @Override + public Variable emitFusedMultiplyAdd(Value a, Value b, Value c) { + Variable result = getLIRGen().newVariable(LIRKind.combine(a, b, c)); + assert ((AMD64Kind) a.getPlatformKind()).isXMM() && ((AMD64Kind) b.getPlatformKind()).isXMM() && ((AMD64Kind) c.getPlatformKind()).isXMM(); + assert a.getPlatformKind().equals(b.getPlatformKind()); + assert b.getPlatformKind().equals(c.getPlatformKind()); + + if (a.getPlatformKind() == AMD64Kind.DOUBLE) { + getLIRGen().append(new AMD64Ternary.ThreeOp(VFMADD231SD, AVXSize.XMM, result, asAllocatable(c), asAllocatable(a), asAllocatable(b))); + } else { + assert a.getPlatformKind() == AMD64Kind.SINGLE; + getLIRGen().append(new AMD64Ternary.ThreeOp(VFMADD231SS, AVXSize.XMM, result, asAllocatable(c), asAllocatable(a), asAllocatable(b))); + } + return result; + } @Override public Value emitCountLeadingZeros(Value value) { --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.graph/src/org/graalvm/compiler/graph/spi/Canonicalizable.java 2019-05-16 21:17:41.876194324 +0200 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.graph/src/org/graalvm/compiler/graph/spi/Canonicalizable.java 2019-05-16 21:17:41.476191739 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2011, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -155,4 +155,49 @@ */ Node maybeCommuteInputs(); } + + /** + * This sub-interface of {@link Canonicalizable} is intended for nodes that have exactly three + * inputs. It has an additional {@link #canonical(CanonicalizerTool, Node, Node, Node)} method + * that looks at the given inputs instead of the current inputs of the node - which can be used + * to ask "what if this input is changed to this node" - questions. + * + * @param the common supertype of all inputs of this node + */ + public interface Ternary extends Canonicalizable { + + /** + * Similar to {@link Canonicalizable#canonical(CanonicalizerTool)}, except that + * implementations should act as if the current input of the node was the given one, i.e., + * they should never look at the inputs via the this pointer. + */ + Node canonical(CanonicalizerTool tool, T forX, T forY, T forZ); + + /** + * Gets the current value of the input, so that calling + * {@link #canonical(CanonicalizerTool, Node, Node, Node)} with the value returned from this + * method should behave exactly like {@link Canonicalizable#canonical(CanonicalizerTool)}. + */ + T getX(); + + /** + * Gets the current value of the input, so that calling + * {@link #canonical(CanonicalizerTool, Node, Node, Node)} with the value returned from this + * method should behave exactly like {@link Canonicalizable#canonical(CanonicalizerTool)}. + */ + T getY(); + + /** + * Gets the current value of the input, so that calling + * {@link #canonical(CanonicalizerTool, Node, Node, Node)} with the value returned from this + * method should behave exactly like {@link Canonicalizable#canonical(CanonicalizerTool)}. + */ + T getZ(); + + @SuppressWarnings("unchecked") + @Override + default T canonical(CanonicalizerTool tool) { + return (T) canonical(tool, getX(), getY(), getZ()); + } + } } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotBackend.java 2019-05-16 21:17:42.388197632 +0200 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotBackend.java 2019-05-16 21:17:41.996195099 +0200 @@ -263,14 +263,19 @@ if (config.useCompressedClassPointers) { Register register = r10; - AMD64HotSpotMove.decodeKlassPointer(crb, asm, register, providers.getRegisters().getHeapBaseRegister(), src, config); + Register heapBase = providers.getRegisters().getHeapBaseRegister(); + AMD64HotSpotMove.decodeKlassPointer(crb, asm, register, heapBase, src, config); if (GeneratePIC.getValue(crb.getOptions())) { - asm.movq(providers.getRegisters().getHeapBaseRegister(), asm.getPlaceholder(-1)); + asm.movq(heapBase, asm.getPlaceholder(-1)); crb.recordMark(config.MARKID_NARROW_OOP_BASE_ADDRESS); } else { if (config.narrowKlassBase != 0) { // The heap base register was destroyed above, so restore it - asm.movq(providers.getRegisters().getHeapBaseRegister(), config.narrowOopBase); + if (config.narrowOopBase == 0L) { + asm.xorq(heapBase, heapBase); + } else { + asm.movq(heapBase, config.narrowOopBase); + } } } asm.cmpq(inlineCacheKlass, register); --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotBackendFactory.java 2019-05-16 21:17:42.904200967 +0200 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotBackendFactory.java 2019-05-16 21:17:42.504198381 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2012, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -161,7 +161,7 @@ HotSpotConstantReflectionProvider constantReflection, HotSpotHostForeignCallsProvider foreignCalls, HotSpotMetaAccessProvider metaAccess, HotSpotSnippetReflectionProvider snippetReflection, HotSpotReplacementsImpl replacements, HotSpotWordTypes wordTypes, OptionValues options) { Plugins plugins = HotSpotGraphBuilderPlugins.create(compilerConfiguration, config, wordTypes, metaAccess, constantReflection, snippetReflection, foreignCalls, replacements, options); - AMD64GraphBuilderPlugins.register(plugins, replacements.getDefaultReplacementBytecodeProvider(), (AMD64) target.arch, false, JAVA_SPECIFICATION_VERSION >= 9); + AMD64GraphBuilderPlugins.register(plugins, replacements.getDefaultReplacementBytecodeProvider(), (AMD64) target.arch, false, JAVA_SPECIFICATION_VERSION >= 9, config.useFMAIntrinsics); return plugins; } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/CheckGraalIntrinsics.java 2019-05-16 21:17:43.428204353 +0200 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/CheckGraalIntrinsics.java 2019-05-16 21:17:43.028201768 +0200 @@ -272,10 +272,6 @@ "jdk/jfr/internal/JVM.getClassId(Ljava/lang/Class;)J"); add(toBeInvestigated, - // HotSpot MacroAssembler-based intrinsic - "java/lang/Math.fma(DDD)D", - // HotSpot MacroAssembler-based intrinsic - "java/lang/Math.fma(FFF)F", // Just check if the argument is a compile time constant "java/lang/invoke/MethodHandleImpl.isCompileConstant(Ljava/lang/Object;)Z", // Only used as a marker for vectorization? @@ -371,6 +367,15 @@ add(ignore, "sun/security/provider/DigestBase.implCompressMultiBlock0([BII)I"); } + if (!config.useFMAIntrinsics) { + add(ignore, + "java/lang/Math.fma(DDD)D", + "java/lang/Math.fma(FFF)F"); + } else if (!(arch instanceof AMD64)) { + add(toBeInvestigated, + "java/lang/Math.fma(DDD)D", + "java/lang/Math.fma(FFF)F"); + } } if (isJDK10OrHigher()) { --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/WriteBarrierAdditionTest.java 2019-05-16 21:17:43.944207687 +0200 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/WriteBarrierAdditionTest.java 2019-05-16 21:17:43.544205102 +0200 @@ -55,6 +55,8 @@ import org.graalvm.compiler.phases.tiers.HighTierContext; import org.graalvm.compiler.phases.tiers.MidTierContext; import org.graalvm.compiler.replacements.NodeIntrinsificationProvider; +import org.graalvm.compiler.word.Word; +import jdk.internal.vm.compiler.word.WordFactory; import org.junit.Assert; import org.junit.Test; @@ -255,6 +257,34 @@ test2("testArrayCopy", src, dst, dst.length); } + public static class WordContainer { + public Word word; + } + + public static void testWordFieldSnippet() { + WordContainer wordContainer = new WordContainer(); + wordContainer.word = WordFactory.signed(42); + } + + @Test + public void testWordField() throws Exception { + testHelper("testWordFieldSnippet", 0); + } + + public static Word[] testWordArraySnippet(int length) { + Word fortyTwo = WordFactory.signed(42); + Word[] words = new Word[length]; + for (int i = 0; i < length; i++) { + words[i] = fortyTwo; + } + return words; + } + + @Test + public void testWordArray() throws Exception { + testHelper("testWordArraySnippet", 0); + } + public static Object testUnsafeLoad(Unsafe theUnsafe, Object a, Object b, Object c) throws Exception { final int offset = (c == null ? 0 : ((Integer) c).intValue()); final long displacement = (b == null ? 0 : ((Long) b).longValue()); @@ -315,9 +345,10 @@ JavaConstant constDisp = ((OffsetAddressNode) read.getAddress()).getOffset().asJavaConstant(); Assert.assertNotNull(constDisp); Assert.assertEquals(referentOffset(getMetaAccess()), constDisp.asLong()); - Assert.assertTrue(config.useG1GC); - Assert.assertEquals(BarrierType.PRECISE, read.getBarrierType()); - Assert.assertTrue(read.next() instanceof G1ReferentFieldReadBarrier); + Assert.assertEquals(BarrierType.WEAK_FIELD, read.getBarrierType()); + if (config.useG1GC) { + Assert.assertTrue(read.next() instanceof G1ReferentFieldReadBarrier); + } } } } catch (Throwable e) { --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/CompilationTask.java 2019-05-16 21:17:44.456210995 +0200 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/CompilationTask.java 2019-05-16 21:17:44.052208385 +0200 @@ -56,6 +56,7 @@ import jdk.vm.ci.hotspot.HotSpotJVMCIRuntime; import jdk.vm.ci.hotspot.HotSpotNmethod; import jdk.vm.ci.hotspot.HotSpotResolvedJavaMethod; +import jdk.vm.ci.meta.ResolvedJavaMethod; import jdk.vm.ci.runtime.JVMCICompiler; public class CompilationTask { @@ -169,7 +170,13 @@ } stats.finish(method, installedCode); if (result != null) { - return HotSpotCompilationRequestResult.success(result.getBytecodeSize() - method.getCodeSize()); + // For compilation of substitutions the method in the compilation request might be + // different than the actual method parsed. The root of the compilation will always + // be the first method in the methods list, so use that instead. + ResolvedJavaMethod rootMethod = result.getMethods()[0]; + int inlinedBytecodes = result.getBytecodeSize() - rootMethod.getCodeSize(); + assert inlinedBytecodes >= 0 : rootMethod + " " + method; + return HotSpotCompilationRequestResult.success(inlinedBytecodes); } return null; } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/GraalHotSpotVMConfig.java 2019-05-16 21:17:45.000214510 +0200 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/GraalHotSpotVMConfig.java 2019-05-16 21:17:44.588211848 +0200 @@ -108,6 +108,7 @@ private final boolean useMulAddIntrinsic = getFlag("UseMulAddIntrinsic", Boolean.class, false); private final boolean useSquareToLenIntrinsic = getFlag("UseSquareToLenIntrinsic", Boolean.class, false); public final boolean useVectorizedMismatchIntrinsic = getFlag("UseVectorizedMismatchIntrinsic", Boolean.class, false); + public final boolean useFMAIntrinsics = getFlag("UseFMA", Boolean.class, false); /* * These are methods because in some JDKs the flags are visible but the stubs themselves haven't --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/gc/g1/G1BarrierSet.java 2019-05-16 21:17:45.524217896 +0200 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/gc/g1/G1BarrierSet.java 2019-05-16 21:17:45.112215233 +0200 @@ -26,6 +26,7 @@ package org.graalvm.compiler.hotspot.gc.g1; import org.graalvm.compiler.debug.GraalError; +import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig; import org.graalvm.compiler.hotspot.gc.shared.BarrierSet; import org.graalvm.compiler.nodes.StructuredGraph; import org.graalvm.compiler.nodes.ValueNode; @@ -41,10 +42,13 @@ public class G1BarrierSet extends BarrierSet { + public G1BarrierSet(GraalHotSpotVMConfig vmConfig) { + super(vmConfig); + } + @Override public void addReadNodeBarriers(ReadNode node, StructuredGraph graph) { - if (node.getBarrierType() != HeapAccess.BarrierType.NONE) { - assert (node.getBarrierType() == HeapAccess.BarrierType.PRECISE); + if (node.getBarrierType() == HeapAccess.BarrierType.WEAK_FIELD) { G1ReferentFieldReadBarrier barrier = graph.add(new G1ReferentFieldReadBarrier(node.getAddress(), node, false)); graph.addAfterFixed(node, barrier); } @@ -57,15 +61,19 @@ case NONE: // nothing to do break; - case IMPRECISE: - case PRECISE: - boolean precise = barrierType == HeapAccess.BarrierType.PRECISE; - if (!node.getLocationIdentity().isInit()) { - // The pre barrier does nothing if the value being read is null, so it can - // be explicitly skipped when this is an initializing store. - addG1PreWriteBarrier(node, node.getAddress(), null, true, node.getNullCheck(), graph); + case FIELD: + case ARRAY: + case UNKNOWN: + boolean init = node.getLocationIdentity().isInit(); + if (!init || !getVMConfig().useDeferredInitBarriers) { + if (!init) { + // The pre barrier does nothing if the value being read is null, so it can + // be explicitly skipped when this is an initializing store. + addG1PreWriteBarrier(node, node.getAddress(), null, true, node.getNullCheck(), graph); + } + boolean precise = barrierType != HeapAccess.BarrierType.FIELD; + addG1PostWriteBarrier(node, node.getAddress(), node.value(), precise, graph); } - addG1PostWriteBarrier(node, node.getAddress(), node.value(), precise, graph); break; default: throw new GraalError("unexpected barrier type: " + barrierType); @@ -79,9 +87,10 @@ case NONE: // nothing to do break; - case IMPRECISE: - case PRECISE: - boolean precise = barrierType == HeapAccess.BarrierType.PRECISE; + case FIELD: + case ARRAY: + case UNKNOWN: + boolean precise = barrierType != HeapAccess.BarrierType.FIELD; addG1PreWriteBarrier(node, node.getAddress(), null, true, node.getNullCheck(), graph); addG1PostWriteBarrier(node, node.getAddress(), node.getNewValue(), precise, graph); break; @@ -97,9 +106,10 @@ case NONE: // nothing to do break; - case IMPRECISE: - case PRECISE: - boolean precise = barrierType == HeapAccess.BarrierType.PRECISE; + case FIELD: + case ARRAY: + case UNKNOWN: + boolean precise = barrierType != HeapAccess.BarrierType.FIELD; addG1PreWriteBarrier(node, node.getAddress(), node.getExpectedValue(), false, false, graph); addG1PostWriteBarrier(node, node.getAddress(), node.getNewValue(), precise, graph); break; --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/gc/shared/BarrierSet.java 2019-05-16 21:17:46.048221282 +0200 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/gc/shared/BarrierSet.java 2019-05-16 21:17:45.636218620 +0200 @@ -25,6 +25,7 @@ package org.graalvm.compiler.hotspot.gc.shared; +import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig; import org.graalvm.compiler.nodes.StructuredGraph; import org.graalvm.compiler.nodes.extended.ArrayRangeWrite; import org.graalvm.compiler.nodes.java.AbstractCompareAndSwapNode; @@ -33,6 +34,16 @@ import org.graalvm.compiler.nodes.memory.WriteNode; public abstract class BarrierSet { + private final GraalHotSpotVMConfig vmConfig; + + protected BarrierSet(GraalHotSpotVMConfig vmConfig) { + this.vmConfig = vmConfig; + } + + public final GraalHotSpotVMConfig getVMConfig() { + return vmConfig; + } + public abstract void addReadNodeBarriers(ReadNode node, StructuredGraph graph); public abstract void addWriteNodeBarriers(WriteNode node, StructuredGraph graph); --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/gc/shared/CardTableBarrierSet.java 2019-05-16 21:17:46.592224797 +0200 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/gc/shared/CardTableBarrierSet.java 2019-05-16 21:17:46.176222109 +0200 @@ -26,6 +26,7 @@ package org.graalvm.compiler.hotspot.gc.shared; import org.graalvm.compiler.debug.GraalError; +import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig; import org.graalvm.compiler.nodes.StructuredGraph; import org.graalvm.compiler.nodes.ValueNode; import org.graalvm.compiler.nodes.extended.ArrayRangeWrite; @@ -40,9 +41,13 @@ public class CardTableBarrierSet extends BarrierSet { + public CardTableBarrierSet(GraalHotSpotVMConfig vmConfig) { + super(vmConfig); + } + @Override public void addReadNodeBarriers(ReadNode node, StructuredGraph graph) { - assert node.getBarrierType() == HeapAccess.BarrierType.NONE : "Non precise read barrier has been attached to read node."; + // Nothing to do here. } @Override @@ -52,10 +57,14 @@ case NONE: // nothing to do break; - case IMPRECISE: - case PRECISE: - boolean precise = barrierType == HeapAccess.BarrierType.PRECISE; - addSerialPostWriteBarrier(node, node.getAddress(), node.value(), precise, graph); + case FIELD: + case ARRAY: + case UNKNOWN: + boolean precise = barrierType != HeapAccess.BarrierType.FIELD; + boolean init = node.getLocationIdentity().isInit(); + if (!init || !getVMConfig().useDeferredInitBarriers) { + addSerialPostWriteBarrier(node, node.getAddress(), node.value(), precise, graph); + } break; default: throw new GraalError("unexpected barrier type: " + barrierType); @@ -69,9 +78,10 @@ case NONE: // nothing to do break; - case IMPRECISE: - case PRECISE: - boolean precise = barrierType == HeapAccess.BarrierType.PRECISE; + case FIELD: + case ARRAY: + case UNKNOWN: + boolean precise = barrierType != HeapAccess.BarrierType.FIELD; addSerialPostWriteBarrier(node, node.getAddress(), node.getNewValue(), precise, graph); break; default: @@ -86,9 +96,10 @@ case NONE: // nothing to do break; - case IMPRECISE: - case PRECISE: - boolean precise = barrierType == HeapAccess.BarrierType.PRECISE; + case FIELD: + case ARRAY: + case UNKNOWN: + boolean precise = barrierType != HeapAccess.BarrierType.FIELD; addSerialPostWriteBarrier(node, node.getAddress(), node.getNewValue(), precise, graph); break; default: --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/DefaultHotSpotLoweringProvider.java 2019-05-16 21:17:47.132228287 +0200 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/DefaultHotSpotLoweringProvider.java 2019-05-16 21:17:46.720225624 +0200 @@ -583,16 +583,6 @@ graph.replaceFixed(storeHub, hub); } - @Override - public BarrierType fieldInitializationBarrier(JavaKind entryKind) { - return (entryKind == JavaKind.Object && !runtime.getVMConfig().useDeferredInitBarriers) ? BarrierType.IMPRECISE : BarrierType.NONE; - } - - @Override - public BarrierType arrayInitializationBarrier(JavaKind entryKind) { - return (entryKind == JavaKind.Object && !runtime.getVMConfig().useDeferredInitBarriers) ? BarrierType.PRECISE : BarrierType.NONE; - } - private void lowerOSRStartNode(OSRStartNode osrStart) { StructuredGraph graph = osrStart.graph(); if (graph.getGuardsStage() == StructuredGraph.GuardsStage.FIXED_DEOPTS) { @@ -783,12 +773,11 @@ @Override protected BarrierType fieldLoadBarrierType(ResolvedJavaField f) { HotSpotResolvedJavaField loadField = (HotSpotResolvedJavaField) f; - BarrierType barrierType = BarrierType.NONE; - if (runtime.getVMConfig().useG1GC && loadField.getJavaKind() == JavaKind.Object && metaAccess.lookupJavaType(Reference.class).equals(loadField.getDeclaringClass()) && + if (loadField.getJavaKind() == JavaKind.Object && metaAccess.lookupJavaType(Reference.class).equals(loadField.getDeclaringClass()) && loadField.getName().equals("referent")) { - barrierType = BarrierType.PRECISE; + return BarrierType.WEAK_FIELD; } - return barrierType; + return super.fieldLoadBarrierType(f); } @Override --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotObjdumpDisassemblerProvider.java 2019-05-16 21:17:47.668231749 +0200 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotObjdumpDisassemblerProvider.java 2019-05-16 21:17:47.260229114 +0200 @@ -39,6 +39,7 @@ import org.graalvm.compiler.code.CompilationResult.CodeAnnotation; import org.graalvm.compiler.code.DisassemblerProvider; import org.graalvm.compiler.serviceprovider.ServiceProvider; +import org.graalvm.util.CollectionsUtil; import jdk.vm.ci.code.CodeCacheProvider; import jdk.vm.ci.code.CodeUtil; @@ -56,28 +57,31 @@ import jdk.vm.ci.services.Services; /** - * This disassembles the code immediatly with objdump. + * A provider that uses the {@code GNU objdump} utility to disassemble code. */ @ServiceProvider(DisassemblerProvider.class) public class HotSpotObjdumpDisassemblerProvider extends HotSpotDisassemblerProvider { - /** - * Uses objdump to disassemble the compiled code. - */ + private final String objdump = getObjdump(); + @Override public String disassembleCompiledCode(CodeCacheProvider codeCache, CompilationResult compResult) { + if (objdump == null) { + return null; + } File tmp = null; try { tmp = File.createTempFile("compiledBinary", ".bin"); try (FileOutputStream fos = new FileOutputStream(tmp)) { fos.write(compResult.getTargetCode()); } + String[] cmdline; String arch = Services.getSavedProperties().get("os.arch"); - if (arch.equals("amd64")) { - cmdline = new String[]{"objdump", "-D", "-b", "binary", "-M", "x86-64", "-m", "i386", tmp.getAbsolutePath()}; + if (arch.equals("amd64") || arch.equals("x86_64")) { + cmdline = new String[]{objdump, "-D", "-b", "binary", "-M", "x86-64", "-m", "i386", tmp.getAbsolutePath()}; } else if (arch.equals("aarch64")) { - cmdline = new String[]{"objdump", "-D", "-b", "binary", "-m", "aarch64", tmp.getAbsolutePath()}; + cmdline = new String[]{objdump, "-D", "-b", "binary", "-m", "aarch64", tmp.getAbsolutePath()}; } else { return null; } @@ -116,43 +120,99 @@ Process proc = Runtime.getRuntime().exec(cmdline); InputStream is = proc.getInputStream(); + StringBuilder sb = new StringBuilder(); InputStreamReader isr = new InputStreamReader(is); - BufferedReader br = new BufferedReader(isr); - String line; - - StringBuilder sb = new StringBuilder(); - while ((line = br.readLine()) != null) { - Matcher m = p.matcher(line); - if (m.find()) { - int address = Integer.parseInt(m.group(2), 16); - String annotation = annotations.get(address); - if (annotation != null) { - annotation = annotation.replace("\n", "\n; "); - sb.append("; ").append(annotation).append('\n'); + try (BufferedReader br = new BufferedReader(isr)) { + String line; + while ((line = br.readLine()) != null) { + Matcher m = p.matcher(line); + if (m.find()) { + int address = Integer.parseInt(m.group(2), 16); + String annotation = annotations.get(address); + if (annotation != null) { + annotation = annotation.replace("\n", "\n; "); + sb.append("; ").append(annotation).append('\n'); + } + line = m.replaceAll("0x$1"); } - line = m.replaceAll("0x$1"); + sb.append(line).append("\n"); } - sb.append(line).append("\n"); } - BufferedReader ebr = new BufferedReader(new InputStreamReader(proc.getErrorStream())); - while ((line = ebr.readLine()) != null) { - System.err.println(line); + try (BufferedReader ebr = new BufferedReader(new InputStreamReader(proc.getErrorStream()))) { + String errLine = ebr.readLine(); + if (errLine != null) { + System.err.println("Error output from executing: " + CollectionsUtil.mapAndJoin(cmdline, e -> quoteShellArg(String.valueOf(e)), " ")); + System.err.println(errLine); + while ((errLine = ebr.readLine()) != null) { + System.err.println(errLine); + } + } } - ebr.close(); return sb.toString(); } catch (IOException e) { + e.printStackTrace(); + return null; + } finally { if (tmp != null) { tmp.delete(); } - e.printStackTrace(); - return null; } } + /** + * Pattern for a single shell command argument that does not need to quoted. + */ + private static final Pattern SAFE_SHELL_ARG = Pattern.compile("[A-Za-z0-9@%_\\-\\+=:,\\./]+"); + + /** + * Reliably quote a string as a single shell command argument. + */ + public static String quoteShellArg(String arg) { + if (arg.isEmpty()) { + return "\"\""; + } + Matcher m = SAFE_SHELL_ARG.matcher(arg); + if (m.matches()) { + return arg; + } + // See http://stackoverflow.com/a/1250279 + return "'" + arg.replace("'", "'\"'\"'") + "'"; + } + + /** + * Searches for a valid GNU objdump executable. + */ + private static String getObjdump() { + // On macOS, `brew install binutils` will provide + // an executable named gobjdump + for (String candidate : new String[]{"objdump", "gobjdump"}) { + try { + String[] cmd = {candidate, "--version"}; + Process proc = Runtime.getRuntime().exec(cmd); + InputStream is = proc.getInputStream(); + int exitValue = proc.waitFor(); + if (exitValue == 0) { + byte[] buf = new byte[is.available()]; + int pos = 0; + while (pos < buf.length) { + int read = is.read(buf, pos, buf.length - pos); + pos += read; + } + String output = new String(buf); + if (output.contains("GNU objdump")) { + return candidate; + } + } + } catch (IOException | InterruptedException e) { + } + } + return null; + } + private static void putAnnotation(Map annotations, int idx, String txt) { - String newAnnoation = annotations.getOrDefault(idx, "") + "\n" + txt; - annotations.put(idx, newAnnoation); + String newAnnotation = annotations.getOrDefault(idx, "") + "\n" + txt; + annotations.put(idx, newAnnotation); } @Override --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/phases/WriteBarrierAdditionPhase.java 2019-05-16 21:17:48.216235291 +0200 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/phases/WriteBarrierAdditionPhase.java 2019-05-16 21:17:47.804232628 +0200 @@ -77,17 +77,17 @@ private BarrierSet createBarrierSet(GraalHotSpotVMConfig config) { if (config.useG1GC) { - return createG1BarrierSet(); + return createG1BarrierSet(config); } else { - return createCardTableBarrierSet(); + return createCardTableBarrierSet(config); } } - protected BarrierSet createCardTableBarrierSet() { - return new CardTableBarrierSet(); + protected BarrierSet createCardTableBarrierSet(GraalHotSpotVMConfig config) { + return new CardTableBarrierSet(config); } - protected BarrierSet createG1BarrierSet() { - return new G1BarrierSet(); + protected BarrierSet createG1BarrierSet(GraalHotSpotVMConfig config) { + return new G1BarrierSet(config); } } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/phases/WriteBarrierVerificationPhase.java 2019-05-16 21:17:48.748238728 +0200 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/phases/WriteBarrierVerificationPhase.java 2019-05-16 21:17:48.324235988 +0200 @@ -44,6 +44,7 @@ import org.graalvm.compiler.nodes.extended.ArrayRangeWrite; import org.graalvm.compiler.nodes.java.LoweredAtomicReadAndWriteNode; import org.graalvm.compiler.nodes.java.LogicCompareAndSwapNode; +import org.graalvm.compiler.nodes.java.ValueCompareAndSwapNode; import org.graalvm.compiler.nodes.memory.FixedAccessNode; import org.graalvm.compiler.nodes.memory.HeapAccess; import org.graalvm.compiler.nodes.memory.HeapAccess.BarrierType; @@ -127,6 +128,9 @@ boolean validatePreBarrier = useG1GC() && (isObjectWrite(node) || !((ArrayRangeWrite) node).isInitialization()); if (node instanceof WriteNode) { WriteNode writeNode = (WriteNode) node; + if (config.useDeferredInitBarriers && writeNode.getLocationIdentity().isInit()) { + return true; + } if (writeNode.getLocationIdentity().isInit()) { validatePreBarrier = false; } @@ -191,7 +195,8 @@ } private static boolean validateBarrier(FixedAccessNode write, ObjectWriteBarrier barrier) { - assert write instanceof WriteNode || write instanceof LogicCompareAndSwapNode || write instanceof LoweredAtomicReadAndWriteNode : "Node must be of type requiring a write barrier " + write; + assert write instanceof WriteNode || write instanceof LogicCompareAndSwapNode || write instanceof ValueCompareAndSwapNode || + write instanceof LoweredAtomicReadAndWriteNode : "Node must be of type requiring a write barrier " + write; if (!barrier.usePrecise()) { if (barrier.getAddress() instanceof OffsetAddressNode && write.getAddress() instanceof OffsetAddressNode) { return GraphUtil.unproxify(((OffsetAddressNode) barrier.getAddress()).getBase()) == GraphUtil.unproxify(((OffsetAddressNode) write.getAddress()).getBase()); --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/UnsafeLoadSnippets.java 2019-05-16 21:17:49.284242192 +0200 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/UnsafeLoadSnippets.java 2019-05-16 21:17:48.876239555 +0200 @@ -50,7 +50,7 @@ public static Object lowerUnsafeLoad(Object object, long offset) { Object fixedObject = FixedValueAnchorNode.getObject(object); if (object instanceof java.lang.ref.Reference && referentOffset(INJECTED_METAACCESS) == offset) { - return Word.objectToTrackedPointer(fixedObject).readObject((int) offset, BarrierType.PRECISE); + return Word.objectToTrackedPointer(fixedObject).readObject((int) offset, BarrierType.WEAK_FIELD); } else { return Word.objectToTrackedPointer(fixedObject).readObject((int) offset, BarrierType.NONE); } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.java/src/org/graalvm/compiler/java/FrameStateBuilder.java 2019-05-16 21:17:49.816245630 +0200 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.java/src/org/graalvm/compiler/java/FrameStateBuilder.java 2019-05-16 21:17:49.404242967 +0200 @@ -743,6 +743,13 @@ } locals[i] = x; if (slotKind.needsTwoSlots()) { + if (i < locals.length - 2 && locals[i + 2] == TWO_SLOT_MARKER) { + /* + * Writing a two-slot marker to an index previously occupied by a two-slot value: + * clear the old marker of the second slot. + */ + locals[i + 2] = null; + } /* Writing a two-slot value: mark the second slot. */ locals[i + 1] = TWO_SLOT_MARKER; } else if (i < locals.length - 1 && locals[i + 1] == TWO_SLOT_MARKER) { --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/gen/ArithmeticLIRGeneratorTool.java 2019-05-16 21:17:50.352249093 +0200 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/gen/ArithmeticLIRGeneratorTool.java 2019-05-16 21:17:49.940246431 +0200 @@ -103,6 +103,11 @@ void emitStore(ValueKind kind, Value address, Value input, LIRFrameState state); @SuppressWarnings("unused") + default Value emitFusedMultiplyAdd(Value a, Value b, Value c) { + throw GraalError.unimplemented("No specialized implementation available"); + } + + @SuppressWarnings("unused") default Value emitMathLog(Value input, boolean base10) { throw GraalError.unimplemented("No specialized implementation available"); } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/memory/HeapAccess.java 2019-05-16 21:17:50.864252401 +0200 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/memory/HeapAccess.java 2019-05-16 21:17:50.444249688 +0200 @@ -34,17 +34,25 @@ */ enum BarrierType { /** - * Primitive stores which do not necessitate barriers. + * Primitive access which do not necessitate barriers. */ NONE, /** - * Array object stores which necessitate precise barriers. + * Array object access. */ - PRECISE, + ARRAY, /** - * Field object stores which necessitate imprecise barriers. + * Field object access. */ - IMPRECISE + FIELD, + /** + * Unknown (aka field or array) object access. + */ + UNKNOWN, + /** + * Weak field access (e.g. Hotspot's Reference.referent field). + */ + WEAK_FIELD } /** --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.amd64/src/org/graalvm/compiler/replacements/amd64/AMD64GraphBuilderPlugins.java 2019-05-16 21:17:51.384255761 +0200 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.amd64/src/org/graalvm/compiler/replacements/amd64/AMD64GraphBuilderPlugins.java 2019-05-16 21:17:50.976253125 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -59,6 +59,7 @@ import org.graalvm.compiler.replacements.nodes.BinaryMathIntrinsicNode; import org.graalvm.compiler.replacements.nodes.BinaryMathIntrinsicNode.BinaryOperation; import org.graalvm.compiler.replacements.nodes.BitCountNode; +import org.graalvm.compiler.replacements.nodes.FusedMultiplyAddNode; import org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode; import org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.UnaryOperation; @@ -70,7 +71,8 @@ public class AMD64GraphBuilderPlugins { - public static void register(Plugins plugins, BytecodeProvider replacementsBytecodeProvider, AMD64 arch, boolean explicitUnsafeNullChecks, boolean emitJDK9StringSubstitutions) { + public static void register(Plugins plugins, BytecodeProvider replacementsBytecodeProvider, AMD64 arch, boolean explicitUnsafeNullChecks, boolean emitJDK9StringSubstitutions, + boolean useFMAIntrinsics) { InvocationPlugins invocationPlugins = plugins.getInvocationPlugins(); invocationPlugins.defer(new Runnable() { @Override @@ -86,7 +88,7 @@ registerStringLatin1Plugins(invocationPlugins, replacementsBytecodeProvider); registerStringUTF16Plugins(invocationPlugins, replacementsBytecodeProvider); } - registerMathPlugins(invocationPlugins, arch, replacementsBytecodeProvider); + registerMathPlugins(invocationPlugins, useFMAIntrinsics, arch, replacementsBytecodeProvider); registerArraysEqualsPlugins(invocationPlugins, replacementsBytecodeProvider); } }); @@ -112,7 +114,9 @@ Class declaringClass = kind.toBoxedJavaClass(); Class type = kind.toJavaClass(); Registration r = new Registration(plugins, declaringClass, bytecodeProvider); + r.registerMethodSubstitution(substituteDeclaringClass, "numberOfLeadingZeros", type); if (arch.getFeatures().contains(AMD64.CPUFeature.LZCNT) && arch.getFlags().contains(AMD64.Flag.UseCountLeadingZerosInstruction)) { + r.setAllowOverwrite(true); r.register1("numberOfLeadingZeros", type, new InvocationPlugin() { @Override public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode value) { @@ -125,11 +129,13 @@ return true; } }); - } else { - r.registerMethodSubstitution(substituteDeclaringClass, "numberOfLeadingZeros", type); } + + r.registerMethodSubstitution(substituteDeclaringClass, "numberOfTrailingZeros", type); if (arch.getFeatures().contains(AMD64.CPUFeature.BMI1) && arch.getFlags().contains(AMD64.Flag.UseCountTrailingZerosInstruction)) { + r.setAllowOverwrite(true); r.register1("numberOfTrailingZeros", type, new InvocationPlugin() { + @Override public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode value) { ValueNode folded = AMD64CountTrailingZerosNode.tryFold(value); @@ -141,8 +147,6 @@ return true; } }); - } else { - r.registerMethodSubstitution(substituteDeclaringClass, "numberOfTrailingZeros", type); } if (arch.getFeatures().contains(AMD64.CPUFeature.POPCNT)) { @@ -154,9 +158,10 @@ } }); } + } - private static void registerMathPlugins(InvocationPlugins plugins, AMD64 arch, BytecodeProvider bytecodeProvider) { + private static void registerMathPlugins(InvocationPlugins plugins, boolean useFMAIntrinsics, AMD64 arch, BytecodeProvider bytecodeProvider) { Registration r = new Registration(plugins, Math.class, bytecodeProvider); registerUnaryMath(r, "log", LOG); registerUnaryMath(r, "log10", LOG10); @@ -171,6 +176,45 @@ registerRound(r, "ceil", RoundingMode.UP); registerRound(r, "floor", RoundingMode.DOWN); } + + if (useFMAIntrinsics && !Java8OrEarlier && arch.getFeatures().contains(CPUFeature.FMA)) { + registerFMA(r); + } + } + + private static void registerFMA(Registration r) { + r.register3("fma", + Double.TYPE, + Double.TYPE, + Double.TYPE, + new InvocationPlugin() { + @Override + public boolean apply(GraphBuilderContext b, + ResolvedJavaMethod targetMethod, + Receiver receiver, + ValueNode na, + ValueNode nb, + ValueNode nc) { + b.push(JavaKind.Double, b.append(new FusedMultiplyAddNode(na, nb, nc))); + return true; + } + }); + r.register3("fma", + Float.TYPE, + Float.TYPE, + Float.TYPE, + new InvocationPlugin() { + @Override + public boolean apply(GraphBuilderContext b, + ResolvedJavaMethod targetMethod, + Receiver receiver, + ValueNode na, + ValueNode nb, + ValueNode nc) { + b.push(JavaKind.Float, b.append(new FusedMultiplyAddNode(na, nb, nc))); + return true; + } + }); } private static void registerUnaryMath(Registration r, String name, UnaryOperation operation) { --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/RootMethodSubstitutionTest.java 2019-05-16 21:17:51.916259198 +0200 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/RootMethodSubstitutionTest.java 2019-05-16 21:17:51.508256563 +0200 @@ -35,6 +35,7 @@ import org.graalvm.compiler.core.test.GraalCompilerTest; import org.graalvm.compiler.debug.DebugContext; import org.graalvm.compiler.nodes.StructuredGraph; +import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugin; import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugins; import org.graalvm.compiler.nodes.graphbuilderconf.MethodSubstitutionPlugin; import org.graalvm.compiler.options.OptionValues; @@ -95,7 +96,11 @@ } } if (!original.isNative()) { - ret.add(new Object[]{original}); + // Make sure the plugin we found hasn't been overridden. + InvocationPlugin plugin = providers.getReplacements().getGraphBuilderPlugins().getInvocationPlugins().lookupInvocation(original); + if (plugin instanceof MethodSubstitutionPlugin) { + ret.add(new Object[]{original}); + } } } } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/DefaultJavaLoweringProvider.java 2019-05-16 21:17:52.456262687 +0200 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/DefaultJavaLoweringProvider.java 2019-05-16 21:17:52.040260000 +0200 @@ -48,6 +48,7 @@ import org.graalvm.compiler.api.replacements.SnippetReflectionProvider; import org.graalvm.compiler.core.common.LIRKind; import org.graalvm.compiler.core.common.spi.ForeignCallsProvider; +import org.graalvm.compiler.core.common.type.AbstractObjectStamp; import org.graalvm.compiler.core.common.type.IntegerStamp; import org.graalvm.compiler.core.common.type.ObjectStamp; import org.graalvm.compiler.core.common.type.Stamp; @@ -139,6 +140,7 @@ import org.graalvm.compiler.replacements.SnippetLowerableMemoryNode.SnippetLowering; import org.graalvm.compiler.replacements.nodes.BinaryMathIntrinsicNode; import org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode; +import org.graalvm.compiler.word.WordTypes; import jdk.internal.vm.compiler.word.LocationIdentity; import jdk.vm.ci.code.CodeUtil; @@ -164,6 +166,7 @@ protected final TargetDescription target; private final boolean useCompressedOops; private final ResolvedJavaType objectArrayType; + private final WordTypes wordTypes; private BoxingSnippets.Templates boxingSnippets; private ConstantStringIndexOfSnippets.Templates indexOfSnippets; @@ -174,6 +177,7 @@ this.target = target; this.useCompressedOops = useCompressedOops; this.objectArrayType = metaAccess.lookupJavaType(Object[].class); + this.wordTypes = new WordTypes(metaAccess, target.wordJavaKind); } public void initialize(OptionValues options, Iterable factories, SnippetCounter.Group.Factory factory, Providers providers, SnippetReflectionProvider snippetReflection) { @@ -513,7 +517,7 @@ AddressNode address = createArrayIndexAddress(graph, array, elementKind, storeIndexed.index(), boundsCheck); WriteNode memoryWrite = graph.add(new WriteNode(address, NamedLocationIdentity.getArrayLocation(elementKind), implicitStoreConvert(graph, elementKind, value), - arrayStoreBarrierType(storeIndexed.elementKind()))); + arrayStoreBarrierType(array, storeIndexed.elementKind()))); memoryWrite.setGuard(boundsCheck); if (condition != null) { tool.createGuard(storeIndexed, condition, DeoptimizationReason.ArrayStoreException, DeoptimizationAction.InvalidateReprofile); @@ -788,11 +792,11 @@ long offset = fieldOffset(field); if (offset >= 0) { address = createOffsetAddress(graph, newObject, offset); - barrierType = fieldInitializationBarrier(entryKind); + barrierType = fieldInitializationBarrier(field); } } else { address = createOffsetAddress(graph, newObject, metaAccess.getArrayBaseOffset(entryKind) + i * metaAccess.getArrayIndexScale(entryKind)); - barrierType = arrayInitializationBarrier(entryKind); + barrierType = arrayInitializationBarrier(newObject, entryKind); } if (address != null) { WriteNode write = new WriteNode(address, LocationIdentity.init(), implicitStoreConvert(graph, entryKind, value), barrierType); @@ -822,10 +826,10 @@ if (virtual instanceof VirtualInstanceNode) { VirtualInstanceNode virtualInstance = (VirtualInstanceNode) virtual; address = createFieldAddress(graph, newObject, virtualInstance.field(i)); - barrierType = BarrierType.IMPRECISE; + barrierType = fieldStoreBarrierType(virtualInstance.field(i)); } else { address = createArrayAddress(graph, newObject, virtual.entryKind(i), ConstantNode.forInt(i, graph)); - barrierType = BarrierType.PRECISE; + barrierType = arrayStoreBarrierType(newObject, virtual.entryKind(i)); } if (address != null) { WriteNode write = new WriteNode(address, LocationIdentity.init(), implicitStoreConvert(graph, JavaKind.Object, allocValue), barrierType); @@ -939,25 +943,50 @@ } protected BarrierType fieldStoreBarrierType(ResolvedJavaField field) { - if (field.getJavaKind() == JavaKind.Object) { - return BarrierType.IMPRECISE; + JavaKind fieldKind = wordTypes.asKind(field.getType()); + if (fieldKind == JavaKind.Object) { + return BarrierType.FIELD; } return BarrierType.NONE; } - protected BarrierType arrayStoreBarrierType(JavaKind elementKind) { - if (elementKind == JavaKind.Object) { - return BarrierType.PRECISE; + /** + * If the given value is indeed an array, and its elements are of a word type, return the + * correct word kind; in all other cases, return the defaultElementKind. This is needed for + * determining the correct write barrier type. + * + * @param array a value that is expected to have an array stamp + * @param defaultElementKind the array's element kind without taking word types into account + * @return the element kind of the array taking word types into account + */ + protected JavaKind maybeWordArrayElementKind(ValueNode array, JavaKind defaultElementKind) { + JavaKind elementKind = defaultElementKind; + Stamp arrayStamp = array.stamp(NodeView.DEFAULT); + if (arrayStamp instanceof AbstractObjectStamp && arrayStamp.hasValues()) { + ResolvedJavaType arrayType = ((AbstractObjectStamp) arrayStamp).type(); + if (arrayType != null && arrayType.getComponentType() != null) { + elementKind = wordTypes.asKind(arrayType.getComponentType()); + } + } + return elementKind; + } + + protected BarrierType arrayStoreBarrierType(ValueNode array, JavaKind elementKind) { + JavaKind kind = maybeWordArrayElementKind(array, elementKind); + if (kind == JavaKind.Object) { + return BarrierType.ARRAY; } return BarrierType.NONE; } - public BarrierType fieldInitializationBarrier(JavaKind entryKind) { - return entryKind == JavaKind.Object ? BarrierType.IMPRECISE : BarrierType.NONE; + public BarrierType fieldInitializationBarrier(ResolvedJavaField field) { + JavaKind fieldKind = wordTypes.asKind(field.getType()); + return fieldKind == JavaKind.Object ? BarrierType.FIELD : BarrierType.NONE; } - public BarrierType arrayInitializationBarrier(JavaKind entryKind) { - return entryKind == JavaKind.Object ? BarrierType.PRECISE : BarrierType.NONE; + public BarrierType arrayInitializationBarrier(ValueNode array, JavaKind entryKind) { + JavaKind kind = maybeWordArrayElementKind(array, entryKind); + return kind == JavaKind.Object ? BarrierType.ARRAY : BarrierType.NONE; } private BarrierType unsafeStoreBarrierType(RawStoreNode store) { @@ -972,10 +1001,14 @@ ResolvedJavaType type = StampTool.typeOrNull(object); // Array types must use a precise barrier, so if the type is unknown or is a supertype // of Object[] then treat it as an array. - if (type == null || type.isArray() || type.isAssignableFrom(objectArrayType)) { - return BarrierType.PRECISE; + if (type != null && type.isArray()) { + return arrayStoreBarrierType(object, JavaKind.Object); + } else if (type != null && wordTypes.isWord(type)) { + return BarrierType.NONE; + } else if (type == null || type.isAssignableFrom(objectArrayType)) { + return BarrierType.UNKNOWN; } else { - return BarrierType.IMPRECISE; + return BarrierType.FIELD; } } return BarrierType.NONE; --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.serviceprovider/src/org/graalvm/compiler/serviceprovider/GraalServices.java 2019-05-16 21:17:52.988266125 +0200 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.serviceprovider/src/org/graalvm/compiler/serviceprovider/GraalServices.java 2019-05-16 21:17:52.572263437 +0200 @@ -501,4 +501,22 @@ } return jmx.getInputArguments(); } + + /** + * Returns the fused multiply add of the three arguments; that is, returns the exact product of + * the first two arguments summed with the third argument and then rounded once to the nearest + * {@code float}. + */ + public static float fma(float a, float b, float c) { + return Math.fma(a, b, c); + } + + /** + * Returns the fused multiply add of the three arguments; that is, returns the exact product of + * the first two arguments summed with the third argument and then rounded once to the nearest + * {@code double}. + */ + public static double fma(double a, double b, double c) { + return Math.fma(a, b, c); + } } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.word/src/org/graalvm/compiler/word/WordOperationPlugin.java 2019-05-16 21:17:53.504269460 +0200 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.word/src/org/graalvm/compiler/word/WordOperationPlugin.java 2019-05-16 21:17:53.092266797 +0200 @@ -447,7 +447,7 @@ protected ValueNode readOp(GraphBuilderContext b, JavaKind readKind, AddressNode address, LocationIdentity location, Opcode op) { assert op == Opcode.READ_POINTER || op == Opcode.READ_OBJECT || op == Opcode.READ_BARRIERED; - final BarrierType barrier = (op == Opcode.READ_BARRIERED ? BarrierType.PRECISE : BarrierType.NONE); + final BarrierType barrier = (op == Opcode.READ_BARRIERED ? BarrierType.UNKNOWN : BarrierType.NONE); final boolean compressible = (op == Opcode.READ_OBJECT || op == Opcode.READ_BARRIERED); return readOp(b, readKind, address, location, barrier, compressible); @@ -465,7 +465,7 @@ protected void writeOp(GraphBuilderContext b, JavaKind writeKind, AddressNode address, LocationIdentity location, ValueNode value, Opcode op) { assert op == Opcode.WRITE_POINTER || op == Opcode.WRITE_OBJECT || op == Opcode.WRITE_BARRIERED || op == Opcode.INITIALIZE; - final BarrierType barrier = (op == Opcode.WRITE_BARRIERED ? BarrierType.PRECISE : BarrierType.NONE); + final BarrierType barrier = (op == Opcode.WRITE_BARRIERED ? BarrierType.UNKNOWN : BarrierType.NONE); final boolean compressible = (op == Opcode.WRITE_OBJECT || op == Opcode.WRITE_BARRIERED); assert op != Opcode.INITIALIZE || location.isInit() : "must use init location for initializing"; b.add(new JavaWriteNode(writeKind, address, location, value, barrier, compressible)); --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/TwoSlotMarkerClearingTest.java 2019-05-16 21:17:53.604270106 +0200 @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + + +package org.graalvm.compiler.core.test; + +import org.junit.Test; +import org.objectweb.asm.ClassWriter; +import org.objectweb.asm.MethodVisitor; +import org.objectweb.asm.Label; + +import jdk.vm.ci.meta.ResolvedJavaMethod; + +public class TwoSlotMarkerClearingTest extends CustomizedBytecodePatternTest { + + @Test + public void testTwoSlotMarkerClearing() throws ClassNotFoundException { + Class testClass = getClass("Test"); + ResolvedJavaMethod t1 = getResolvedJavaMethod(testClass, "t1"); + parseForCompile(t1); + ResolvedJavaMethod t2 = getResolvedJavaMethod(testClass, "t2"); + parseForCompile(t2); + } + + @Override + protected byte[] generateClass(String className) { + ClassWriter cw = new ClassWriter(ClassWriter.COMPUTE_FRAMES); + cw.visit(52, ACC_SUPER | ACC_PUBLIC, className, null, "java/lang/Object", null); + + String getDescriptor = "(" + "JII" + ")" + "I"; + MethodVisitor t1 = cw.visitMethod(ACC_PUBLIC | ACC_STATIC, "t1", getDescriptor, null, null); + t1.visitCode(); + t1.visitVarInsn(ILOAD, 2); + t1.visitVarInsn(ISTORE, 0); + t1.visitVarInsn(ILOAD, 0); + Label label = new Label(); + t1.visitJumpInsn(IFGE, label); + t1.visitVarInsn(ILOAD, 0); + t1.visitInsn(IRETURN); + t1.visitLabel(label); + t1.visitVarInsn(ILOAD, 3); + t1.visitInsn(IRETURN); + t1.visitMaxs(4, 1); + t1.visitEnd(); + + getDescriptor = "(" + "IJIJ" + ")" + "J"; + MethodVisitor t2 = cw.visitMethod(ACC_PUBLIC | ACC_STATIC, "t2", getDescriptor, null, null); + t2.visitCode(); + t2.visitVarInsn(LLOAD, 1); + t2.visitVarInsn(LSTORE, 0); + t2.visitVarInsn(ILOAD, 3); + Label label1 = new Label(); + t2.visitJumpInsn(IFGE, label1); + t2.visitVarInsn(LLOAD, 0); + t2.visitInsn(LRETURN); + t2.visitLabel(label1); + t2.visitVarInsn(LLOAD, 4); + t2.visitInsn(LRETURN); + t2.visitMaxs(6, 2); + t2.visitEnd(); + + cw.visitEnd(); + return cw.toByteArray(); + } +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.jdk9.test/src/org/graalvm/compiler/hotspot/jdk9/test/MathDoubleFMATest.java 2019-05-16 21:17:54.148273621 +0200 @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.hotspot.jdk9.test; + +import static org.junit.Assume.assumeTrue; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import org.graalvm.compiler.core.test.GraalCompilerTest; +import org.graalvm.compiler.test.AddExports; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameter; +import org.junit.runners.Parameterized.Parameters; + +import jdk.vm.ci.amd64.AMD64; + +@AddExports({"java.base/java.lang"}) +@RunWith(Parameterized.class) +public final class MathDoubleFMATest extends GraalCompilerTest { + + @Before + public void checkAMD64() { + assumeTrue("skipping AMD64 specific test", getTarget().arch instanceof AMD64); + } + + @Parameters(name = "{0}, {1}, {2}") + public static Collection data() { + double[] inputs = {0.0d, 1.0d, 4.0d, -0.0d, -1.0d, -4.0d, Double.MIN_VALUE, Double.MAX_VALUE, Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY, + Double.NaN, Double.longBitsToDouble(0xfff0000000000001L)}; + + List tests = new ArrayList<>(); + for (double a : inputs) { + for (double b : inputs) { + for (double c : inputs) { + tests.add(new Object[]{a, b, c}); + } + } + } + return tests; + } + + @Parameter(value = 0) public double input0; + @Parameter(value = 1) public double input1; + @Parameter(value = 2) public double input2; + + public static double fma(double a, double b, double c) { + return Math.fma(a, b, c); + } + + @Test + public void testFMA() { + test("fma", input0, input1, input2); + } + +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.jdk9.test/src/org/graalvm/compiler/hotspot/jdk9/test/MathFMAConstantInputTest.java 2019-05-16 21:17:54.688277110 +0200 @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.hotspot.jdk9.test; + +import static org.junit.Assume.assumeTrue; + +import org.graalvm.compiler.core.test.GraalCompilerTest; +import org.graalvm.compiler.test.AddExports; +import org.junit.Before; +import org.junit.Test; + +import jdk.vm.ci.amd64.AMD64; + +@AddExports({"java.base/java.lang"}) +public final class MathFMAConstantInputTest extends GraalCompilerTest { + + @Before + public void checkAMD64() { + assumeTrue("skipping AMD64 specific test", getTarget().arch instanceof AMD64); + } + + public static float floatFMA() { + return Math.fma(2.0f, 2.0f, 2.0f); + } + + @Test + public void testFloatFMA() { + test("floatFMA"); + } + + public static double doubleFMA() { + return Math.fma(2.0d, 2.0d, 2.0d); + } + + @Test + public void testDoubleFMA() { + test("doubleFMA"); + } + +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.jdk9.test/src/org/graalvm/compiler/hotspot/jdk9/test/MathFloatFMATest.java 2019-05-16 21:17:55.228280600 +0200 @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.hotspot.jdk9.test; + +import static org.junit.Assume.assumeTrue; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import org.graalvm.compiler.core.test.GraalCompilerTest; +import org.graalvm.compiler.test.AddExports; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameter; +import org.junit.runners.Parameterized.Parameters; + +import jdk.vm.ci.amd64.AMD64; + +@AddExports({"java.base/java.lang"}) +@RunWith(Parameterized.class) +public final class MathFloatFMATest extends GraalCompilerTest { + + @Before + public void checkAMD64() { + assumeTrue("skipping AMD64 specific test", getTarget().arch instanceof AMD64); + } + + @Parameters(name = "{0}, {1}, {2}") + public static Collection data() { + float[] inputs = {0.0f, 1.0f, 4.0f, -0.0f, -1.0f, 4.0f, Float.MIN_VALUE, Float.MAX_VALUE, Float.NEGATIVE_INFINITY, Float.POSITIVE_INFINITY, + Float.NaN, Float.intBitsToFloat(0xff800001)}; + + List tests = new ArrayList<>(); + for (float a : inputs) { + for (float b : inputs) { + for (float c : inputs) { + tests.add(new Object[]{a, b, c}); + } + } + } + return tests; + } + + @Parameter(value = 0) public float input0; + @Parameter(value = 1) public float input1; + @Parameter(value = 2) public float input2; + + public static float fma(float a, float b, float c) { + return Math.fma(a, b, c); + } + + @Test + public void testFMA() { + test("fma", input0, input1, input2); + } + +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64Ternary.java 2019-05-16 21:17:55.772284115 +0200 @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.lir.amd64; + +import static jdk.vm.ci.code.ValueUtil.asRegister; +import static jdk.vm.ci.code.ValueUtil.isRegister; +import static jdk.vm.ci.code.ValueUtil.isStackSlot; +import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.HINT; +import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.REG; +import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.STACK; + +import org.graalvm.compiler.asm.amd64.AMD64Address; +import org.graalvm.compiler.asm.amd64.AMD64Assembler.VexRVMOp; +import org.graalvm.compiler.asm.amd64.AMD64MacroAssembler; +import org.graalvm.compiler.asm.amd64.AVXKind.AVXSize; +import org.graalvm.compiler.lir.LIRInstructionClass; +import org.graalvm.compiler.lir.Opcode; +import org.graalvm.compiler.lir.asm.CompilationResultBuilder; + +import jdk.vm.ci.meta.AllocatableValue; + +/** + * AMD64 LIR instructions that have three inputs and one output. + */ +public class AMD64Ternary { + + /** + * Instruction that has two {@link AllocatableValue} operands. + */ + public static class ThreeOp extends AMD64LIRInstruction { + public static final LIRInstructionClass TYPE = LIRInstructionClass.create(ThreeOp.class); + + @Opcode private final VexRVMOp opcode; + private final AVXSize size; + + @Def({REG, HINT}) protected AllocatableValue result; + @Use({REG}) protected AllocatableValue x; + /** + * This argument must be Alive to ensure that result and y are not assigned to the same + * register, which would break the code generation by destroying y too early. + */ + @Alive({REG}) protected AllocatableValue y; + @Alive({REG, STACK}) protected AllocatableValue z; + + public ThreeOp(VexRVMOp opcode, AVXSize size, AllocatableValue result, AllocatableValue x, AllocatableValue y, AllocatableValue z) { + super(TYPE); + this.opcode = opcode; + this.size = size; + + this.result = result; + this.x = x; + this.y = y; + this.z = z; + } + + @Override + public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { + AMD64Move.move(crb, masm, result, x); + if (isRegister(z)) { + opcode.emit(masm, size, asRegister(result), asRegister(y), asRegister(z)); + } else { + assert isStackSlot(z); + opcode.emit(masm, size, asRegister(result), asRegister(y), (AMD64Address) crb.asAddress(z)); + } + } + } +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/TernaryNode.java 2019-05-16 21:17:56.308287577 +0200 @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.nodes.calc; + +import org.graalvm.compiler.core.common.type.Stamp; +import org.graalvm.compiler.graph.NodeClass; +import org.graalvm.compiler.graph.spi.Canonicalizable; +import org.graalvm.compiler.nodeinfo.NodeInfo; +import org.graalvm.compiler.nodes.NodeView; +import org.graalvm.compiler.nodes.ValueNode; + +/** + * The {@code TernaryNode} class is the base of arithmetic and logic operations with three inputs. + */ +@NodeInfo +public abstract class TernaryNode extends FloatingNode implements Canonicalizable.Ternary { + + public static final NodeClass TYPE = NodeClass.create(TernaryNode.class); + @Input protected ValueNode x; + @Input protected ValueNode y; + @Input protected ValueNode z; + + @Override + public ValueNode getX() { + return x; + } + + @Override + public ValueNode getY() { + return y; + } + + @Override + public ValueNode getZ() { + return z; + } + + public void setX(ValueNode x) { + updateUsages(this.x, x); + this.x = x; + } + + public void setY(ValueNode y) { + updateUsages(this.y, y); + this.y = y; + } + + public void setZ(ValueNode z) { + updateUsages(this.z, z); + this.z = z; + } + + /** + * Creates a new TernaryNode instance. + * + * @param stamp the result type of this instruction + * @param x the first input instruction + * @param y the second input instruction + * @param z the second input instruction + */ + protected TernaryNode(NodeClass c, Stamp stamp, ValueNode x, ValueNode y, ValueNode z) { + super(c, stamp); + this.x = x; + this.y = y; + this.z = z; + } + + @Override + public boolean inferStamp() { + return updateStamp(foldStamp(getX().stamp(NodeView.DEFAULT), getY().stamp(NodeView.DEFAULT), getZ().stamp(NodeView.DEFAULT))); + } + + /** + * Compute an improved stamp for this node using the passed in stamps. The stamps must be + * compatible with the current values of {@link #x}, {@link #y} and {@link #z}. This code is + * used to provide the default implementation of {@link #inferStamp()} and may be used by + * external optimizations. + * + * @param stampX + * @param stampY + * @param stampZ + */ + public abstract Stamp foldStamp(Stamp stampX, Stamp stampY, Stamp stampZ); +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/FusedMultiplyAddNode.java 2019-05-16 21:17:56.852291092 +0200 @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.replacements.nodes; + +import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_2; +import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_1; + +import org.graalvm.compiler.core.common.type.FloatStamp; +import org.graalvm.compiler.core.common.type.Stamp; +import org.graalvm.compiler.graph.NodeClass; +import org.graalvm.compiler.graph.spi.CanonicalizerTool; +import org.graalvm.compiler.lir.gen.ArithmeticLIRGeneratorTool; +import org.graalvm.compiler.nodeinfo.NodeInfo; +import org.graalvm.compiler.nodes.ConstantNode; +import org.graalvm.compiler.nodes.NodeView; +import org.graalvm.compiler.nodes.ValueNode; +import org.graalvm.compiler.nodes.calc.TernaryNode; +import org.graalvm.compiler.nodes.spi.ArithmeticLIRLowerable; +import org.graalvm.compiler.nodes.spi.NodeLIRBuilderTool; + +import jdk.vm.ci.meta.JavaConstant; +import jdk.vm.ci.meta.JavaKind; +import org.graalvm.compiler.serviceprovider.GraalServices; + +@NodeInfo(cycles = CYCLES_2, size = SIZE_1) +public final class FusedMultiplyAddNode extends TernaryNode implements ArithmeticLIRLowerable { + + public static final NodeClass TYPE = NodeClass.create(FusedMultiplyAddNode.class); + + public FusedMultiplyAddNode(ValueNode a, ValueNode b, ValueNode c) { + super(TYPE, computeStamp(a.stamp(NodeView.DEFAULT), b.stamp(NodeView.DEFAULT), c.stamp(NodeView.DEFAULT)), a, b, c); + assert a.getStackKind().isNumericFloat(); + assert b.getStackKind().isNumericFloat(); + assert c.getStackKind().isNumericFloat(); + } + + @Override + public Stamp foldStamp(Stamp stampX, Stamp stampY, Stamp stampZ) { + return computeStamp(stampX, stampY, stampZ); + } + + private static Stamp computeStamp(Stamp stampX, Stamp stampY, Stamp stampZ) { + Stamp m = FloatStamp.OPS.getMul().foldStamp(stampX, stampY); + return FloatStamp.OPS.getAdd().foldStamp(m, stampZ); + } + + @Override + public ValueNode canonical(CanonicalizerTool tool, ValueNode a, ValueNode b, ValueNode c) { + if (a.isConstant() && b.isConstant() && c.isConstant()) { + JavaConstant ca = a.asJavaConstant(); + JavaConstant cb = b.asJavaConstant(); + JavaConstant cc = c.asJavaConstant(); + + ValueNode res; + if (a.getStackKind() == JavaKind.Float) { + res = ConstantNode.forFloat(GraalServices.fma(ca.asFloat(), cb.asFloat(), cc.asFloat())); + } else { + assert a.getStackKind() == JavaKind.Double; + res = ConstantNode.forDouble(GraalServices.fma(ca.asDouble(), cb.asDouble(), cc.asDouble())); + } + return res; + } + return this; + } + + @Override + public void generate(NodeLIRBuilderTool builder, ArithmeticLIRGeneratorTool gen) { + builder.setResult(this, gen.emitFusedMultiplyAdd(builder.operand(getX()), builder.operand(getY()), builder.operand(getZ()))); + } +}