--- old/make/CompileJavaModules.gmk 2019-03-12 08:08:31.171154593 +0100 +++ new/make/CompileJavaModules.gmk 2019-03-12 08:08:30.803152197 +0100 @@ -461,6 +461,7 @@ org.graalvm.compiler.debug.test \ org.graalvm.compiler.graph.test \ org.graalvm.compiler.hotspot.amd64.test \ + org.graalvm.compiler.hotspot.jdk9.test \ org.graalvm.compiler.hotspot.lir.test \ org.graalvm.compiler.hotspot.sparc.test \ org.graalvm.compiler.hotspot.test \ --- old/make/test/JtregGraalUnit.gmk 2019-03-12 08:08:31.647157692 +0100 +++ new/make/test/JtregGraalUnit.gmk 2019-03-12 08:08:31.283155322 +0100 @@ -87,6 +87,7 @@ $(SRC_DIR)/org.graalvm.compiler.debug.test/src \ $(SRC_DIR)/org.graalvm.compiler.graph.test/src \ $(SRC_DIR)/org.graalvm.compiler.hotspot.amd64.test/src \ + $(SRC_DIR)/org.graalvm.compiler.hotspot.jdk9.test/src \ $(SRC_DIR)/org.graalvm.compiler.hotspot.lir.test/src \ $(SRC_DIR)/org.graalvm.compiler.hotspot.sparc.test/src \ $(SRC_DIR)/org.graalvm.compiler.hotspot.test/src \ --- old/src/jdk.aot/share/classes/jdk.tools.jaotc/src/jdk/tools/jaotc/Linker.java 2019-03-12 08:08:32.131160843 +0100 +++ new/src/jdk.aot/share/classes/jdk.tools.jaotc/src/jdk/tools/jaotc/Linker.java 2019-03-12 08:08:31.767158473 +0100 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -29,6 +29,10 @@ import java.io.File; import java.io.InputStream; import java.io.InputStreamReader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; import java.util.stream.Stream; final class Linker { @@ -46,9 +50,12 @@ return libraryFileName; } + private static Stream getLines(InputStream stream) { + return new BufferedReader(new InputStreamReader(stream)).lines(); + } + private static String getString(InputStream stream) { - BufferedReader br = new BufferedReader(new InputStreamReader(stream)); - Stream lines = br.lines(); + Stream lines = getLines(stream); StringBuilder sb = new StringBuilder(); lines.iterator().forEachRemaining(e -> sb.append(e)); return sb.toString(); @@ -150,9 +157,18 @@ } /** - * Search for Visual Studio link.exe Search Order is: VS2013, VS2015, VS2012. + * Search for Visual Studio link.exe Search Order is: VS2017+, VS2013, VS2015, VS2012. */ - private static String getWindowsLinkPath() { + private static String getWindowsLinkPath() throws Exception { + try { + Path vc141NewerLinker = getVC141AndNewerLinker(); + if (vc141NewerLinker != null) { + return vc141NewerLinker.toString(); + } + } catch (Exception e) { + e.printStackTrace(); + } + String link = "\\VC\\bin\\amd64\\link.exe"; /** @@ -183,10 +199,46 @@ return null; } + private static Path getVC141AndNewerLinker() throws Exception { + String programFilesX86 = System.getenv("ProgramFiles(x86)"); + if (programFilesX86 == null) { + throw new InternalError("Could not read the ProgramFiles(x86) environment variable"); + } + Path vswhere = Paths.get(programFilesX86 + "\\Microsoft Visual Studio\\Installer\\vswhere.exe"); + if (!Files.exists(vswhere)) { + return null; + } + + ProcessBuilder processBuilder = new ProcessBuilder(vswhere.toString(), "-requires", + "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", "-property", "installationPath", "-latest"); + processBuilder.redirectOutput(ProcessBuilder.Redirect.PIPE); + processBuilder.redirectError(ProcessBuilder.Redirect.PIPE); + Process process = processBuilder.start(); + final int exitCode = process.waitFor(); + if (exitCode != 0) { + String errorMessage = getString(process.getErrorStream()); + if (errorMessage.isEmpty()) { + errorMessage = getString(process.getInputStream()); + } + throw new InternalError(errorMessage); + } + + String installationPath = getLines(process.getInputStream()).findFirst().orElseThrow(() -> new InternalError("Unexpected empty output from vswhere")); + Path vcToolsVersionFilePath = Paths.get(installationPath, "VC\\Auxiliary\\Build\\Microsoft.VCToolsVersion.default.txt"); + List vcToolsVersionFileLines = Files.readAllLines(vcToolsVersionFilePath); + if (vcToolsVersionFileLines.isEmpty()) { + throw new InternalError(vcToolsVersionFilePath.toString() + " is empty"); + } + String vcToolsVersion = vcToolsVersionFileLines.get(0); + Path linkPath = Paths.get(installationPath, "VC\\Tools\\MSVC", vcToolsVersion, "bin\\Hostx64\\x64\\link.exe"); + if (!Files.exists(linkPath)) { + throw new InternalError("Linker at path " + linkPath.toString() + " does not exist"); + } + + return linkPath; + } + // @formatter:off (workaround for Eclipse formatting bug) - /** - * Visual Studio supported versions Search Order is: VS2013, VS2015, VS2012. - */ enum VSVERSIONS { VS2013("VS120COMNTOOLS", "C:\\Program Files (x86)\\Microsoft Visual Studio 12.0\\VC\\bin\\amd64\\link.exe"), VS2015("VS140COMNTOOLS", "C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\bin\\amd64\\link.exe"), --- old/src/jdk.aot/share/classes/jdk.tools.jaotc/src/jdk/tools/jaotc/Main.java 2019-03-12 08:08:32.623164046 +0100 +++ new/src/jdk.aot/share/classes/jdk.tools.jaotc/src/jdk/tools/jaotc/Main.java 2019-03-12 08:08:32.255161650 +0100 @@ -166,7 +166,7 @@ printer.printInfo(classesToCompile.size() + " classes found"); } - OptionValues graalOptions = HotSpotGraalOptionValues.HOTSPOT_OPTIONS; + OptionValues graalOptions = HotSpotGraalOptionValues.defaultOptions(); // Setting -Dgraal.TieredAOT overrides --compile-for-tiered if (!TieredAOT.hasBeenSet(graalOptions)) { graalOptions = new OptionValues(graalOptions, TieredAOT, options.tiered); --- old/src/jdk.aot/share/classes/jdk.tools.jaotc/src/jdk/tools/jaotc/collect/ClassSearch.java 2019-03-12 08:08:33.119167274 +0100 +++ new/src/jdk.aot/share/classes/jdk.tools.jaotc/src/jdk/tools/jaotc/collect/ClassSearch.java 2019-03-12 08:08:32.747164853 +0100 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -30,6 +30,7 @@ import java.util.ArrayList; import java.util.List; import java.util.function.BiConsumer; +import java.io.File; public final class ClassSearch { private final List providers = new ArrayList<>(); @@ -106,7 +107,7 @@ public static List makeList(String type, String argument) { List list = new ArrayList<>(); - String[] elements = argument.split(":"); + String[] elements = argument.split(File.pathSeparator); for (String element : elements) { list.add(new SearchFor(element, type)); } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.aarch64/src/org/graalvm/compiler/asm/aarch64/AArch64Assembler.java 2019-03-12 08:08:33.619170529 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.aarch64/src/org/graalvm/compiler/asm/aarch64/AArch64Assembler.java 2019-03-12 08:08:33.247168108 +0100 @@ -880,15 +880,15 @@ protected void tbnz(Register reg, int uimm6, int imm16, int pos) { assert reg.getRegisterCategory().equals(CPU); assert NumUtil.isUnsignedNbit(6, uimm6); - assert NumUtil.isSignedNbit(18, imm16); - assert (imm16 & 3) == 0; + assert NumUtil.isSignedNbit(16, imm16) : String.format("Offset value must fit in 16 bits signed: 0x%x", imm16); + assert (imm16 & 3) == 0 : String.format("Lower two bits must be zero: 0x%x", imm16 & 3); // size bit is overloaded as top bit of uimm6 bit index int size = (((uimm6 >> 5) & 1) == 0 ? 32 : 64); // remaining 5 bits are encoded lower down - int uimm5 = uimm6 >> 1; - int offset = (imm16 & NumUtil.getNbitNumberInt(16)) >> 2; + int uimm5 = uimm6 & 0x1F; + int imm14 = (imm16 & NumUtil.getNbitNumberInt(16)) >> 2; InstructionType type = generalFromSize(size); - int encoding = type.encoding | TBNZ.encoding | (uimm5 << 19) | (offset << 5) | rd(reg); + int encoding = type.encoding | TBNZ.encoding | (uimm5 << 19) | (imm14 << 5) | rd(reg); if (pos == -1) { emitInt(encoding); } else { @@ -907,15 +907,15 @@ protected void tbz(Register reg, int uimm6, int imm16, int pos) { assert reg.getRegisterCategory().equals(CPU); assert NumUtil.isUnsignedNbit(6, uimm6); - assert NumUtil.isSignedNbit(18, imm16); - assert (imm16 & 3) == 0; + assert NumUtil.isSignedNbit(16, imm16) : String.format("Offset value must fit in 16 bits signed: 0x%x", imm16); + assert (imm16 & 3) == 0 : String.format("Lower two bits must be zero: 0x%x", imm16 & 3); // size bit is overloaded as top bit of uimm6 bit index int size = (((uimm6 >> 5) & 1) == 0 ? 32 : 64); // remaining 5 bits are encoded lower down - int uimm5 = uimm6 >> 1; - int offset = (imm16 & NumUtil.getNbitNumberInt(16)) >> 2; + int uimm5 = uimm6 & 0x1F; + int imm14 = (imm16 & NumUtil.getNbitNumberInt(16)) >> 2; InstructionType type = generalFromSize(size); - int encoding = type.encoding | TBZ.encoding | (uimm5 << 19) | (offset << 5) | rd(reg); + int encoding = type.encoding | TBZ.encoding | (uimm5 << 19) | (imm14 << 5) | rd(reg); if (pos == -1) { emitInt(encoding); } else { --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.aarch64/src/org/graalvm/compiler/asm/aarch64/AArch64MacroAssembler.java 2019-03-12 08:08:34.151173993 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.aarch64/src/org/graalvm/compiler/asm/aarch64/AArch64MacroAssembler.java 2019-03-12 08:08:33.783171597 +0100 @@ -33,6 +33,9 @@ import static org.graalvm.compiler.asm.aarch64.AArch64MacroAssembler.AddressGenerationPlan.WorkPlan.ADD_TO_BASE; import static org.graalvm.compiler.asm.aarch64.AArch64MacroAssembler.AddressGenerationPlan.WorkPlan.ADD_TO_INDEX; import static org.graalvm.compiler.asm.aarch64.AArch64MacroAssembler.AddressGenerationPlan.WorkPlan.NO_WORK; + +import org.graalvm.compiler.asm.BranchTargetOutOfBoundsException; + import static jdk.vm.ci.aarch64.AArch64.CPU; import static jdk.vm.ci.aarch64.AArch64.r8; import static jdk.vm.ci.aarch64.AArch64.r9; @@ -1452,7 +1455,7 @@ * * @param cmp general purpose register. May not be null, zero-register or stackpointer. * @param uimm6 Unsigned 6-bit bit index. - * @param label Can only handle 21-bit word-aligned offsets for now. May be unbound. Non null. + * @param label Can only handle 16-bit word-aligned offsets for now. May be unbound. Non null. */ public void tbnz(Register cmp, int uimm6, Label label) { assert NumUtil.isUnsignedNbit(6, uimm6); @@ -1472,7 +1475,7 @@ * * @param cmp general purpose register. May not be null, zero-register or stackpointer. * @param uimm6 Unsigned 6-bit bit index. - * @param label Can only handle 21-bit word-aligned offsets for now. May be unbound. Non null. + * @param label Can only handle 16-bit word-aligned offsets for now. May be unbound. Non null. */ public void tbz(Register cmp, int uimm6, Label label) { assert NumUtil.isUnsignedNbit(6, uimm6); @@ -1681,6 +1684,9 @@ int sizeEncoding = information & NumUtil.getNbitNumberInt(6); int regEncoding = information >>> 6; Register reg = AArch64.cpuRegisters.get(regEncoding); + if (!NumUtil.isSignedNbit(16, branchOffset)) { + throw new BranchTargetOutOfBoundsException(true, "Branch target %d out of bounds", branchOffset); + } switch (type) { case BRANCH_BIT_NONZERO: super.tbnz(reg, sizeEncoding, branchOffset, branch); --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.amd64/src/org/graalvm/compiler/asm/amd64/AMD64Assembler.java 2019-03-12 08:08:34.675177403 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.amd64/src/org/graalvm/compiler/asm/amd64/AMD64Assembler.java 2019-03-12 08:08:34.299174955 +0100 @@ -683,7 +683,7 @@ emitImmediate(asm, size, imm); int nextInsnPos = asm.position(); if (annotateImm && asm.codePatchingAnnotationConsumer != null) { - asm.codePatchingAnnotationConsumer.accept(new ImmediateOperandAnnotation(insnPos, immPos, nextInsnPos - immPos, nextInsnPos)); + asm.codePatchingAnnotationConsumer.accept(new OperandDataAnnotation(insnPos, immPos, nextInsnPos - immPos, nextInsnPos)); } } @@ -700,7 +700,7 @@ emitImmediate(asm, size, imm); int nextInsnPos = asm.position(); if (annotateImm && asm.codePatchingAnnotationConsumer != null) { - asm.codePatchingAnnotationConsumer.accept(new ImmediateOperandAnnotation(insnPos, immPos, nextInsnPos - immPos, nextInsnPos)); + asm.codePatchingAnnotationConsumer.accept(new OperandDataAnnotation(insnPos, immPos, nextInsnPos - immPos, nextInsnPos)); } } } @@ -2023,7 +2023,7 @@ emitInt(imm32); int nextInsnPos = position(); if (annotateImm && codePatchingAnnotationConsumer != null) { - codePatchingAnnotationConsumer.accept(new ImmediateOperandAnnotation(insnPos, immPos, nextInsnPos - immPos, nextInsnPos)); + codePatchingAnnotationConsumer.accept(new OperandDataAnnotation(insnPos, immPos, nextInsnPos - immPos, nextInsnPos)); } } @@ -2201,10 +2201,11 @@ } public final void movswl(Register dst, AMD64Address src) { - prefix(src, dst); - emitByte(0x0F); - emitByte(0xBF); - emitOperandHelper(dst, src, 0); + AMD64RMOp.MOVSX.emit(this, DWORD, dst, src); + } + + public final void movswq(Register dst, AMD64Address src) { + AMD64RMOp.MOVSX.emit(this, QWORD, dst, src); } public final void movw(AMD64Address dst, int imm16) { @@ -2222,6 +2223,13 @@ emitOperandHelper(src, dst, 0); } + public final void movw(Register dst, AMD64Address src) { + emitByte(0x66); + prefix(src, dst); + emitByte(0x8B); + emitOperandHelper(dst, src, 0); + } + public final void movzbl(Register dst, AMD64Address src) { prefix(src, dst); emitByte(0x0F); @@ -2237,11 +2245,16 @@ AMD64RMOp.MOVZXB.emit(this, QWORD, dst, src); } + public final void movzbq(Register dst, AMD64Address src) { + AMD64RMOp.MOVZXB.emit(this, QWORD, dst, src); + } + public final void movzwl(Register dst, AMD64Address src) { - prefix(src, dst); - emitByte(0x0F); - emitByte(0xB7); - emitOperandHelper(dst, src, 0); + AMD64RMOp.MOVZX.emit(this, DWORD, dst, src); + } + + public final void movzwq(Register dst, AMD64Address src) { + AMD64RMOp.MOVZX.emit(this, QWORD, dst, src); } public final void negl(Register dst) { @@ -2557,16 +2570,63 @@ emitModRM(dst, src); } - // Insn: VPMOVZXBW xmm1, xmm2/m64 - - public final void pmovzxbw(Register dst, AMD64Address src) { + private void pmovSZx(Register dst, AMD64Address src, int op) { assert supports(CPUFeature.SSE4_1); assert inRC(XMM, dst); simdPrefix(dst, Register.None, src, PD, P_0F38, false); - emitByte(0x30); + emitByte(op); emitOperandHelper(dst, src, 0); } + public final void pmovsxbw(Register dst, AMD64Address src) { + pmovSZx(dst, src, 0x20); + } + + public final void pmovsxbd(Register dst, AMD64Address src) { + pmovSZx(dst, src, 0x21); + } + + public final void pmovsxbq(Register dst, AMD64Address src) { + pmovSZx(dst, src, 0x22); + } + + public final void pmovsxwd(Register dst, AMD64Address src) { + pmovSZx(dst, src, 0x23); + } + + public final void pmovsxwq(Register dst, AMD64Address src) { + pmovSZx(dst, src, 0x24); + } + + public final void pmovsxdq(Register dst, AMD64Address src) { + pmovSZx(dst, src, 0x25); + } + + // Insn: VPMOVZXBW xmm1, xmm2/m64 + public final void pmovzxbw(Register dst, AMD64Address src) { + pmovSZx(dst, src, 0x30); + } + + public final void pmovzxbd(Register dst, AMD64Address src) { + pmovSZx(dst, src, 0x31); + } + + public final void pmovzxbq(Register dst, AMD64Address src) { + pmovSZx(dst, src, 0x32); + } + + public final void pmovzxwd(Register dst, AMD64Address src) { + pmovSZx(dst, src, 0x33); + } + + public final void pmovzxwq(Register dst, AMD64Address src) { + pmovSZx(dst, src, 0x34); + } + + public final void pmovzxdq(Register dst, AMD64Address src) { + pmovSZx(dst, src, 0x35); + } + public final void pmovzxbw(Register dst, Register src) { assert supports(CPUFeature.SSE4_1); assert inRC(XMM, dst) && inRC(XMM, src); @@ -2881,6 +2941,10 @@ XOR.rmOp.emit(this, DWORD, dst, src); } + public final void xorq(Register dst, Register src) { + XOR.rmOp.emit(this, QWORD, dst, src); + } + public final void xorpd(Register dst, Register src) { SSEOp.XOR.emit(this, PD, dst, src); } @@ -3045,7 +3109,7 @@ emitLong(imm64); int nextInsnPos = position(); if (annotateImm && codePatchingAnnotationConsumer != null) { - codePatchingAnnotationConsumer.accept(new ImmediateOperandAnnotation(insnPos, immPos, nextInsnPos - immPos, nextInsnPos)); + codePatchingAnnotationConsumer.accept(new OperandDataAnnotation(insnPos, immPos, nextInsnPos - immPos, nextInsnPos)); } } @@ -3189,6 +3253,19 @@ emitModRM(5, dst); } + public final void sarq(Register dst, int imm8) { + assert isShiftCount(imm8 >> 1) : "illegal shift count"; + prefixq(dst); + if (imm8 == 1) { + emitByte(0xD1); + emitModRM(7, dst); + } else { + emitByte(0xC1); + emitModRM(7, dst); + emitByte(imm8); + } + } + public final void sbbq(Register dst, Register src) { SBB.rmOp.emit(this, QWORD, dst, src); } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.amd64/src/org/graalvm/compiler/asm/amd64/AMD64BaseAssembler.java 2019-03-12 08:08:35.223180970 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.amd64/src/org/graalvm/compiler/asm/amd64/AMD64BaseAssembler.java 2019-03-12 08:08:34.855178575 +0100 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -210,7 +210,7 @@ } } - public abstract static class OperandDataAnnotation extends CodeAnnotation { + public static class OperandDataAnnotation extends CodeAnnotation { /** * The position (bytes from the beginning of the method) of the operand. */ @@ -239,30 +239,10 @@ } } - /** - * Annotation that stores additional information about the displacement of a - * {@link Assembler#getPlaceholder placeholder address} that needs patching. - */ - protected static class AddressDisplacementAnnotation extends OperandDataAnnotation { - AddressDisplacementAnnotation(int instructionPosition, int operandPosition, int operandSize, int nextInstructionPosition) { - super(instructionPosition, operandPosition, operandSize, nextInstructionPosition); - } - } - - /** - * Annotation that stores additional information about the immediate operand, e.g., of a call - * instruction, that needs patching. - */ - protected static class ImmediateOperandAnnotation extends OperandDataAnnotation { - ImmediateOperandAnnotation(int instructionPosition, int operandPosition, int operandSize, int nextInstructionPosition) { - super(instructionPosition, operandPosition, operandSize, nextInstructionPosition); - } - } - protected void annotatePatchingImmediate(int operandOffset, int operandSize) { if (codePatchingAnnotationConsumer != null) { int pos = position(); - codePatchingAnnotationConsumer.accept(new ImmediateOperandAnnotation(pos, pos + operandOffset, operandSize, pos + operandOffset + operandSize)); + codePatchingAnnotationConsumer.accept(new OperandDataAnnotation(pos, pos + operandOffset, operandSize, pos + operandOffset + operandSize)); } } @@ -581,7 +561,7 @@ assert index.equals(Register.None) : "cannot use RIP relative addressing with index register"; emitByte(0x05 | regenc); if (codePatchingAnnotationConsumer != null && addr.instructionStartPosition >= 0) { - codePatchingAnnotationConsumer.accept(new AddressDisplacementAnnotation(addr.instructionStartPosition, position(), 4, position() + 4 + additionalInstructionSize)); + codePatchingAnnotationConsumer.accept(new OperandDataAnnotation(addr.instructionStartPosition, position(), 4, position() + 4 + additionalInstructionSize)); } emitInt(disp); } else if (base.isValid()) { --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.sparc/src/org/graalvm/compiler/asm/sparc/SPARCAssembler.java 2019-03-12 08:08:35.747184381 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.sparc/src/org/graalvm/compiler/asm/sparc/SPARCAssembler.java 2019-03-12 08:08:35.371181934 +0100 @@ -136,9 +136,9 @@ import java.util.Map; import org.graalvm.compiler.asm.Assembler; +import org.graalvm.compiler.asm.BranchTargetOutOfBoundsException; import org.graalvm.compiler.asm.Label; import org.graalvm.compiler.core.common.NumUtil; -import org.graalvm.compiler.core.common.PermanentBailoutException; import org.graalvm.compiler.debug.GraalError; import jdk.vm.ci.code.Register; @@ -1281,7 +1281,7 @@ public int setDisp(int inst, int d) { assert this.match(inst); if (!isValidDisp(d)) { - throw new PermanentBailoutException("Too large displacement 0x%x in field %s in instruction %s", d, this.disp, this); + throw new BranchTargetOutOfBoundsException(true, "Too large displacement 0x%x in field %s in instruction %s", d, this.disp, this); } return this.disp.setBits(inst, d); } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.test/src/org/graalvm/compiler/asm/test/AssemblerTest.java 2019-03-12 08:08:36.283187869 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.test/src/org/graalvm/compiler/asm/test/AssemblerTest.java 2019-03-12 08:08:35.911185448 +0100 @@ -32,6 +32,7 @@ import org.graalvm.compiler.code.CompilationResult; import org.graalvm.compiler.code.DisassemblerProvider; import org.graalvm.compiler.core.common.CompilationIdentifier; +import org.graalvm.compiler.core.gen.LIRGenerationProvider; import org.graalvm.compiler.core.target.Backend; import org.graalvm.compiler.debug.DebugContext; import org.graalvm.compiler.nodes.StructuredGraph; @@ -90,7 +91,7 @@ RegisterConfig registerConfig = codeCache.getRegisterConfig(); CompilationIdentifier compilationId = backend.getCompilationIdentifier(method); StructuredGraph graph = new StructuredGraph.Builder(options, debug).method(method).compilationId(compilationId).build(); - CallingConvention cc = backend.newLIRGenerationResult(compilationId, null, null, graph, null).getCallingConvention(); + CallingConvention cc = ((LIRGenerationProvider) backend).newLIRGenerationResult(compilationId, null, null, graph, null).getCallingConvention(); CompilationResult compResult = new CompilationResult(graph.compilationId()); byte[] targetCode = test.generateCode(compResult, codeCache.getTarget(), registerConfig, cc); --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.code/src/org/graalvm/compiler/code/CompilationResult.java 2019-03-12 08:08:36.771191046 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.code/src/org/graalvm/compiler/code/CompilationResult.java 2019-03-12 08:08:36.407188677 +0100 @@ -55,6 +55,7 @@ import jdk.vm.ci.meta.InvokeTarget; import jdk.vm.ci.meta.ResolvedJavaField; import jdk.vm.ci.meta.ResolvedJavaMethod; +import jdk.vm.ci.meta.SpeculationLog; /** * Represents the output from compiling a method, including the compiled machine code, associated @@ -221,6 +222,11 @@ private ResolvedJavaMethod[] methods; /** + * The {@link SpeculationLog} log used during compilation. + */ + private SpeculationLog speculationLog; + + /** * The list of fields that were accessed from the bytecodes. */ private ResolvedJavaField[] fields; @@ -373,6 +379,21 @@ } /** + * Sets the {@link SpeculationLog} log used during compilation. + */ + public void setSpeculationLog(SpeculationLog speculationLog) { + checkOpen(); + this.speculationLog = speculationLog; + } + + /** + * Gets the {@link SpeculationLog} log, if any, used during compilation. + */ + public SpeculationLog getSpeculationLog() { + return speculationLog; + } + + /** * Sets the fields that were referenced from the bytecodes that were used as input to the * compilation. * @@ -615,7 +636,7 @@ /** * @return the code annotations or {@code null} if there are none */ - public List getAnnotations() { + public List getCodeAnnotations() { if (annotations == null) { return Collections.emptyList(); } @@ -706,7 +727,8 @@ * Clears the information in this object pertaining to generating code. That is, the * {@linkplain #getMarks() marks}, {@linkplain #getInfopoints() infopoints}, * {@linkplain #getExceptionHandlers() exception handlers}, {@linkplain #getDataPatches() data - * patches} and {@linkplain #getAnnotations() annotations} recorded in this object are cleared. + * patches} and {@linkplain #getCodeAnnotations() annotations} recorded in this object are + * cleared. */ public void resetForEmittingCode() { checkOpen(); @@ -721,6 +743,14 @@ } } + public void clearInfopoints() { + infopoints.clear(); + } + + public void clearExceptionHandlers() { + exceptionHandlers.clear(); + } + private void checkOpen() { if (closed) { throw new IllegalStateException(); --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.code/src/org/graalvm/compiler/code/HexCodeFileDisassemblerProvider.java 2019-03-12 08:08:37.279194352 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.code/src/org/graalvm/compiler/code/HexCodeFileDisassemblerProvider.java 2019-03-12 08:08:36.907191931 +0100 @@ -78,7 +78,7 @@ long start = installedCode == null ? 0L : installedCode.getStart(); HexCodeFile hcf = new HexCodeFile(code, start, target.arch.getName(), target.wordSize * 8); if (compResult != null) { - HexCodeFile.addAnnotations(hcf, compResult.getAnnotations()); + HexCodeFile.addAnnotations(hcf, compResult.getCodeAnnotations()); addExceptionHandlersComment(compResult, hcf); Register fp = regConfig.getFrameRegister(); RefMapFormatter slotFormatter = new DefaultRefMapFormatter(target.wordSize, fp, 0); --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.aarch64/src/org/graalvm/compiler/core/aarch64/AArch64ArithmeticLIRGenerator.java 2019-03-12 08:08:37.771197553 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.aarch64/src/org/graalvm/compiler/core/aarch64/AArch64ArithmeticLIRGenerator.java 2019-03-12 08:08:37.403195159 +0100 @@ -485,26 +485,6 @@ } @Override - public Value emitMathLog(Value input, boolean base10) { - throw GraalError.unimplemented(); - } - - @Override - public Value emitMathCos(Value input) { - throw GraalError.unimplemented(); - } - - @Override - public Value emitMathSin(Value input) { - throw GraalError.unimplemented(); - } - - @Override - public Value emitMathTan(Value input) { - throw GraalError.unimplemented(); - } - - @Override public void emitCompareOp(AArch64Kind cmpKind, Variable left, Value right) { throw GraalError.unimplemented(); } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.aarch64/src/org/graalvm/compiler/core/aarch64/AArch64LIRGenerator.java 2019-03-12 08:08:38.259200730 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.aarch64/src/org/graalvm/compiler/core/aarch64/AArch64LIRGenerator.java 2019-03-12 08:08:37.883198282 +0100 @@ -52,6 +52,7 @@ import org.graalvm.compiler.lir.aarch64.AArch64Compare; import org.graalvm.compiler.lir.aarch64.AArch64ControlFlow; import org.graalvm.compiler.lir.aarch64.AArch64ControlFlow.BranchOp; +import org.graalvm.compiler.lir.aarch64.AArch64ControlFlow.CompareBranchZeroOp; import org.graalvm.compiler.lir.aarch64.AArch64ControlFlow.CondMoveOp; import org.graalvm.compiler.lir.aarch64.AArch64ControlFlow.CondSetOp; import org.graalvm.compiler.lir.aarch64.AArch64ControlFlow.StrategySwitchOp; @@ -257,6 +258,27 @@ @Override public void emitCompareBranch(PlatformKind cmpKind, Value left, Value right, Condition cond, boolean unorderedIsTrue, LabelRef trueDestination, LabelRef falseDestination, double trueDestinationProbability) { + if (cond == Condition.EQ) { + // emit cbz instruction for IsNullNode. + assert !LIRValueUtil.isNullConstant(left) : "emitNullCheckBranch()'s null input should be in right."; + if (LIRValueUtil.isNullConstant(right)) { + append(new CompareBranchZeroOp(asAllocatable(left), trueDestination, falseDestination, trueDestinationProbability)); + return; + } + + // emit cbz instruction for IntegerEquals when any of the inputs is zero. + AArch64Kind kind = (AArch64Kind) cmpKind; + if (kind.isInteger()) { + if (isIntConstant(left, 0)) { + append(new CompareBranchZeroOp(asAllocatable(right), trueDestination, falseDestination, trueDestinationProbability)); + return; + } else if (isIntConstant(right, 0)) { + append(new CompareBranchZeroOp(asAllocatable(left), trueDestination, falseDestination, trueDestinationProbability)); + return; + } + } + } + boolean mirrored = emitCompare(cmpKind, left, right, cond, unorderedIsTrue); Condition finalCondition = mirrored ? cond.mirror() : cond; boolean finalUnorderedIsTrue = mirrored ? !unorderedIsTrue : unorderedIsTrue; --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.aarch64/src/org/graalvm/compiler/core/aarch64/AArch64NodeMatchRules.java 2019-03-12 08:08:38.735203827 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.aarch64/src/org/graalvm/compiler/core/aarch64/AArch64NodeMatchRules.java 2019-03-12 08:08:38.367201432 +0100 @@ -37,11 +37,14 @@ import org.graalvm.compiler.core.match.MatchRule; import org.graalvm.compiler.graph.Node; import org.graalvm.compiler.lir.LIRFrameState; +import org.graalvm.compiler.lir.LabelRef; import org.graalvm.compiler.lir.Variable; import org.graalvm.compiler.lir.aarch64.AArch64ArithmeticOp; +import org.graalvm.compiler.lir.aarch64.AArch64ControlFlow; import org.graalvm.compiler.lir.gen.LIRGeneratorTool; import org.graalvm.compiler.nodes.ConstantNode; import org.graalvm.compiler.nodes.DeoptimizingNode; +import org.graalvm.compiler.nodes.IfNode; import org.graalvm.compiler.nodes.NodeView; import org.graalvm.compiler.nodes.ValueNode; import org.graalvm.compiler.nodes.calc.AddNode; @@ -178,6 +181,28 @@ return builder -> getArithmeticLIRGenerator().emitMSub(operand(a), operand(b), operand(c)); } + /** + * ((x & (1 << n)) == 0) -> tbz/tbnz n label. + */ + @MatchRule("(If (IntegerTest value Constant=a))") + public ComplexMatchResult testBitAndBranch(IfNode root, ValueNode value, ConstantNode a) { + if (value.getStackKind().isNumericInteger()) { + long constant = a.asJavaConstant().asLong(); + if (Long.bitCount(constant) == 1) { + int bitToTest = Long.numberOfTrailingZeros(constant); + return builder -> { + LabelRef trueDestination = getLIRBlock(root.trueSuccessor()); + LabelRef falseDestination = getLIRBlock(root.falseSuccessor()); + AllocatableValue src = moveSp(gen.asAllocatable(operand(value))); + double trueDestinationProbability = root.getTrueSuccessorProbability(); + gen.append(new AArch64ControlFlow.BitTestAndBranchOp(trueDestination, falseDestination, src, trueDestinationProbability, bitToTest)); + return null; + }; + } + } + return null; + } + @Override public AArch64LIRGenerator getLIRGeneratorTool() { return (AArch64LIRGenerator) gen; --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.amd64/src/org/graalvm/compiler/core/amd64/AMD64ArithmeticLIRGenerator.java 2019-03-12 08:08:39.239207107 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.amd64/src/org/graalvm/compiler/core/amd64/AMD64ArithmeticLIRGenerator.java 2019-03-12 08:08:38.867204686 +0100 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2009, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2009, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -80,13 +80,6 @@ import static org.graalvm.compiler.lir.LIRValueUtil.isJavaConstant; import static org.graalvm.compiler.lir.amd64.AMD64Arithmetic.DREM; import static org.graalvm.compiler.lir.amd64.AMD64Arithmetic.FREM; -import static org.graalvm.compiler.lir.amd64.AMD64MathIntrinsicBinaryOp.BinaryIntrinsicOpcode.POW; -import static org.graalvm.compiler.lir.amd64.AMD64MathIntrinsicUnaryOp.UnaryIntrinsicOpcode.COS; -import static org.graalvm.compiler.lir.amd64.AMD64MathIntrinsicUnaryOp.UnaryIntrinsicOpcode.EXP; -import static org.graalvm.compiler.lir.amd64.AMD64MathIntrinsicUnaryOp.UnaryIntrinsicOpcode.LOG; -import static org.graalvm.compiler.lir.amd64.AMD64MathIntrinsicUnaryOp.UnaryIntrinsicOpcode.LOG10; -import static org.graalvm.compiler.lir.amd64.AMD64MathIntrinsicUnaryOp.UnaryIntrinsicOpcode.SIN; -import static org.graalvm.compiler.lir.amd64.AMD64MathIntrinsicUnaryOp.UnaryIntrinsicOpcode.TAN; import org.graalvm.compiler.asm.amd64.AMD64Assembler.AMD64BinaryArithmetic; import org.graalvm.compiler.asm.amd64.AMD64Assembler.AMD64MIOp; @@ -116,8 +109,13 @@ import org.graalvm.compiler.lir.amd64.AMD64Binary; import org.graalvm.compiler.lir.amd64.AMD64BinaryConsumer; import org.graalvm.compiler.lir.amd64.AMD64ClearRegisterOp; -import org.graalvm.compiler.lir.amd64.AMD64MathIntrinsicBinaryOp; -import org.graalvm.compiler.lir.amd64.AMD64MathIntrinsicUnaryOp; +import org.graalvm.compiler.lir.amd64.AMD64MathCosOp; +import org.graalvm.compiler.lir.amd64.AMD64MathExpOp; +import org.graalvm.compiler.lir.amd64.AMD64MathLog10Op; +import org.graalvm.compiler.lir.amd64.AMD64MathLogOp; +import org.graalvm.compiler.lir.amd64.AMD64MathPowOp; +import org.graalvm.compiler.lir.amd64.AMD64MathSinOp; +import org.graalvm.compiler.lir.amd64.AMD64MathTanOp; import org.graalvm.compiler.lir.amd64.AMD64Move; import org.graalvm.compiler.lir.amd64.AMD64MulDivOp; import org.graalvm.compiler.lir.amd64.AMD64ShiftOp; @@ -127,7 +125,6 @@ import org.graalvm.compiler.lir.amd64.vector.AMD64VectorBinary.AVXBinaryOp; import org.graalvm.compiler.lir.amd64.vector.AMD64VectorUnary; import org.graalvm.compiler.lir.gen.ArithmeticLIRGenerator; -import org.graalvm.compiler.lir.gen.LIRGenerator; import jdk.vm.ci.amd64.AMD64; import jdk.vm.ci.amd64.AMD64.CPUFeature; @@ -152,41 +149,11 @@ private static final RegisterValue RCX_I = AMD64.rcx.asValue(LIRKind.value(AMD64Kind.DWORD)); - public AMD64ArithmeticLIRGenerator(AllocatableValue nullRegisterValue, Maths maths) { + public AMD64ArithmeticLIRGenerator(AllocatableValue nullRegisterValue) { this.nullRegisterValue = nullRegisterValue; - this.maths = maths == null ? new Maths() { - } : maths; } private final AllocatableValue nullRegisterValue; - private final Maths maths; - - /** - * Interface for emitting LIR for selected {@link Math} routines. A {@code null} return value - * for any method in this interface means the caller must emit the LIR itself. - */ - public interface Maths { - - @SuppressWarnings("unused") - default Variable emitLog(LIRGenerator gen, Value input, boolean base10) { - return null; - } - - @SuppressWarnings("unused") - default Variable emitCos(LIRGenerator gen, Value input) { - return null; - } - - @SuppressWarnings("unused") - default Variable emitSin(LIRGenerator gen, Value input) { - return null; - } - - @SuppressWarnings("unused") - default Variable emitTan(LIRGenerator gen, Value input) { - return null; - } - } @Override public Variable emitNegate(Value inputVal) { @@ -781,6 +748,7 @@ } } + @Override public Variable emitRor(Value a, Value b) { switch ((AMD64Kind) a.getPlatformKind()) { case DWORD: @@ -1103,65 +1071,36 @@ @Override public Value emitMathLog(Value input, boolean base10) { - LIRGenerator gen = getLIRGen(); - Variable result = maths.emitLog(gen, input, base10); - if (result == null) { - result = gen.newVariable(LIRKind.combine(input)); - AllocatableValue stackSlot = gen.getResult().getFrameMapBuilder().allocateSpillSlot(LIRKind.value(AMD64Kind.QWORD)); - gen.append(new AMD64MathIntrinsicUnaryOp(getAMD64LIRGen(), base10 ? LOG10 : LOG, result, asAllocatable(input), stackSlot)); + if (base10) { + return new AMD64MathLog10Op().emitLIRWrapper(getLIRGen(), input); + } else { + return new AMD64MathLogOp().emitLIRWrapper(getLIRGen(), input); } - return result; } @Override public Value emitMathCos(Value input) { - LIRGenerator gen = getLIRGen(); - Variable result = maths.emitCos(gen, input); - if (result == null) { - result = gen.newVariable(LIRKind.combine(input)); - AllocatableValue stackSlot = gen.getResult().getFrameMapBuilder().allocateSpillSlot(LIRKind.value(AMD64Kind.QWORD)); - gen.append(new AMD64MathIntrinsicUnaryOp(getAMD64LIRGen(), COS, result, asAllocatable(input), stackSlot)); - } - return result; + return new AMD64MathCosOp().emitLIRWrapper(getLIRGen(), input); } @Override public Value emitMathSin(Value input) { - LIRGenerator gen = getLIRGen(); - Variable result = maths.emitSin(gen, input); - if (result == null) { - result = gen.newVariable(LIRKind.combine(input)); - AllocatableValue stackSlot = gen.getResult().getFrameMapBuilder().allocateSpillSlot(LIRKind.value(AMD64Kind.QWORD)); - gen.append(new AMD64MathIntrinsicUnaryOp(getAMD64LIRGen(), SIN, result, asAllocatable(input), stackSlot)); - } - return result; + return new AMD64MathSinOp().emitLIRWrapper(getLIRGen(), input); } @Override public Value emitMathTan(Value input) { - LIRGenerator gen = getLIRGen(); - Variable result = maths.emitTan(gen, input); - if (result == null) { - result = gen.newVariable(LIRKind.combine(input)); - AllocatableValue stackSlot = gen.getResult().getFrameMapBuilder().allocateSpillSlot(LIRKind.value(AMD64Kind.QWORD)); - gen.append(new AMD64MathIntrinsicUnaryOp(getAMD64LIRGen(), TAN, result, asAllocatable(input), stackSlot)); - } - return result; + return new AMD64MathTanOp().emitLIRWrapper(getLIRGen(), input); } @Override public Value emitMathExp(Value input) { - Variable result = getLIRGen().newVariable(LIRKind.combine(input)); - AllocatableValue stackSlot = getLIRGen().getResult().getFrameMapBuilder().allocateSpillSlot(LIRKind.value(AMD64Kind.QWORD)); - getLIRGen().append(new AMD64MathIntrinsicUnaryOp(getAMD64LIRGen(), EXP, result, asAllocatable(input), stackSlot)); - return result; + return new AMD64MathExpOp().emitLIRWrapper(getLIRGen(), input); } @Override - public Value emitMathPow(Value input1, Value input2) { - Variable result = getLIRGen().newVariable(LIRKind.combine(input1)); - getLIRGen().append(new AMD64MathIntrinsicBinaryOp(getAMD64LIRGen(), POW, result, asAllocatable(input1), asAllocatable(input2))); - return result; + public Value emitMathPow(Value x, Value y) { + return new AMD64MathPowOp().emitLIRWrapper(getLIRGen(), x, y); } protected AMD64LIRGenerator getAMD64LIRGen() { --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.amd64/src/org/graalvm/compiler/core/amd64/AMD64LIRGenerator.java 2019-03-12 08:08:39.751210439 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.amd64/src/org/graalvm/compiler/core/amd64/AMD64LIRGenerator.java 2019-03-12 08:08:39.383208044 +0100 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2009, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2009, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -41,6 +41,8 @@ import static org.graalvm.compiler.lir.LIRValueUtil.isIntConstant; import static org.graalvm.compiler.lir.LIRValueUtil.isJavaConstant; +import java.util.Optional; + import org.graalvm.compiler.asm.amd64.AMD64Assembler.AMD64BinaryArithmetic; import org.graalvm.compiler.asm.amd64.AMD64Assembler.AMD64MIOp; import org.graalvm.compiler.asm.amd64.AMD64Assembler.AMD64RMOp; @@ -81,6 +83,7 @@ import org.graalvm.compiler.lir.amd64.AMD64ControlFlow.ReturnOp; import org.graalvm.compiler.lir.amd64.AMD64ControlFlow.StrategySwitchOp; import org.graalvm.compiler.lir.amd64.AMD64ControlFlow.TableSwitchOp; +import org.graalvm.compiler.lir.amd64.AMD64ControlFlow.HashTableSwitchOp; import org.graalvm.compiler.lir.amd64.AMD64LFenceOp; import org.graalvm.compiler.lir.amd64.AMD64Move; import org.graalvm.compiler.lir.amd64.AMD64Move.CompareAndSwapOp; @@ -93,6 +96,7 @@ import org.graalvm.compiler.lir.amd64.AMD64ZapStackOp; import org.graalvm.compiler.lir.gen.LIRGenerationResult; import org.graalvm.compiler.lir.gen.LIRGenerator; +import org.graalvm.compiler.lir.hashing.Hasher; import org.graalvm.compiler.phases.util.Providers; import jdk.vm.ci.amd64.AMD64; @@ -545,7 +549,14 @@ @Override public Variable emitArrayEquals(JavaKind kind, Value array1, Value array2, Value length, int constantLength, boolean directPointers) { Variable result = newVariable(LIRKind.value(AMD64Kind.DWORD)); - append(new AMD64ArrayEqualsOp(this, kind, result, array1, array2, asAllocatable(length), constantLength, directPointers, getMaxVectorSize())); + append(new AMD64ArrayEqualsOp(this, kind, kind, result, array1, array2, asAllocatable(length), constantLength, directPointers, getMaxVectorSize())); + return result; + } + + @Override + public Variable emitArrayEquals(JavaKind kind1, JavaKind kind2, Value array1, Value array2, Value length, int constantLength, boolean directPointers) { + Variable result = newVariable(LIRKind.value(AMD64Kind.DWORD)); + append(new AMD64ArrayEqualsOp(this, kind1, kind2, result, array1, array2, asAllocatable(length), constantLength, directPointers, getMaxVectorSize())); return result; } @@ -635,6 +646,19 @@ } @Override + protected Optional hasherFor(JavaConstant[] keyConstants, double minDensity) { + return Hasher.forKeys(keyConstants, minDensity); + } + + @Override + protected void emitHashTableSwitch(Hasher hasher, JavaConstant[] keys, LabelRef defaultTarget, LabelRef[] targets, Value value) { + Value index = hasher.hash(value, arithmeticLIRGen); + Variable scratch = newVariable(LIRKind.value(target().arch.getWordKind())); + Variable entryScratch = newVariable(LIRKind.value(target().arch.getWordKind())); + append(new HashTableSwitchOp(keys, defaultTarget, targets, value, index, scratch, entryScratch)); + } + + @Override public void emitPause() { append(new AMD64PauseOp()); } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/GraalOptions.java 2019-03-12 08:08:40.247213666 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/GraalOptions.java 2019-03-12 08:08:39.871211219 +0100 @@ -83,8 +83,8 @@ @Option(help = "", type = OptionType.Debug) public static final OptionKey EscapeAnalyzeOnly = new OptionKey<>(null); - @Option(help = "", type = OptionType.Expert) - public static final OptionKey MaximumEscapeAnalysisArrayLength = new OptionKey<>(32); + @Option(help = "The maximum length of an array that will be escape analyzed.", type = OptionType.Expert) + public static final OptionKey MaximumEscapeAnalysisArrayLength = new OptionKey<>(128); @Option(help = "", type = OptionType.Debug) public static final OptionKey PEAInliningHints = new OptionKey<>(false); @@ -142,7 +142,7 @@ public static final OptionKey VerifyPhases = new OptionKey<>(false); // Debug settings: - @Option(help = "", type = OptionType.Debug) + @Option(help = "Start tracing compiled GC barriers after N garbage collections (disabled if N <= 0).", type = OptionType.Debug) public static final OptionKey GCDebugStartCycle = new OptionKey<>(-1); @Option(help = "Perform platform dependent validation of the Java heap at returns", type = OptionType.Debug) @@ -206,9 +206,6 @@ @Option(help = "Generate position independent code", type = OptionType.Expert) public static final OptionKey GeneratePIC = new OptionKey<>(false); - @Option(help = "", type = OptionType.Expert) - public static final OptionKey CallArrayCopy = new OptionKey<>(true); - // Runtime settings @Option(help = "", type = OptionType.Expert) public static final OptionKey SupportJsrBytecodes = new OptionKey<>(true); --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/SpeculativeExecutionAttacksMitigations.java 2019-03-12 08:08:40.727216790 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/SpeculativeExecutionAttacksMitigations.java 2019-03-12 08:08:40.359214395 +0100 @@ -37,9 +37,9 @@ public static class Options { // @formatter:off - @Option(help = "Select a strategy to mitigate speculative execution attacks (e.g., SPECTRE)", type = OptionType.User) + @Option(help = "file:doc-files/MitigateSpeculativeExecutionAttacksHelp.txt") public static final EnumOptionKey MitigateSpeculativeExecutionAttacks = new EnumOptionKey<>(None); - @Option(help = "Use index masking after bounds check to mitigate speculative execution attacks", type = OptionType.User) + @Option(help = "Use index masking after bounds check to mitigate speculative execution attacks.", type = OptionType.User) public static final OptionKey UseIndexMasking = new OptionKey<>(false); // @formatter:on } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/cfg/AbstractBlockBase.java 2019-03-12 08:08:41.195219834 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/cfg/AbstractBlockBase.java 2019-03-12 08:08:40.827217441 +0100 @@ -24,6 +24,8 @@ package org.graalvm.compiler.core.common.cfg; +import java.util.Comparator; + public abstract class AbstractBlockBase> { protected int id; @@ -171,4 +173,13 @@ public int hashCode() { return id; } + + public static class BlockIdComparator implements Comparator> { + @Override + public int compare(AbstractBlockBase o1, AbstractBlockBase o2) { + return Integer.compare(o1.getId(), o2.getId()); + } + } + + public static final BlockIdComparator BLOCK_ID_COMPARATOR = new BlockIdComparator(); } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/cfg/CFGVerifier.java 2019-03-12 08:08:41.691223062 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/cfg/CFGVerifier.java 2019-03-12 08:08:41.315220615 +0100 @@ -121,7 +121,7 @@ } } - for (T block : loop.getExits()) { + for (T block : loop.getLoopExits()) { assert block.getId() >= loop.getHeader().getId(); Loop blockLoop = block.getLoop(); --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/cfg/Loop.java 2019-03-12 08:08:42.191226316 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/cfg/Loop.java 2019-03-12 08:08:41.819223895 +0100 @@ -25,19 +25,28 @@ package org.graalvm.compiler.core.common.cfg; +import static org.graalvm.compiler.core.common.cfg.AbstractBlockBase.BLOCK_ID_COMPARATOR; + import java.util.ArrayList; +import java.util.Collections; import java.util.List; public abstract class Loop> { private final Loop parent; - private final List> children; + private final ArrayList> children; private final int depth; private final int index; private final T header; - private final List blocks; - private final List exits; + private final ArrayList blocks; + private final ArrayList exits; + /** + * Natural exits, ignoring LoopExitNodes. + * + * @see #getNaturalExits() + */ + private final ArrayList naturalExits; protected Loop(Loop parent, int index, T header) { this.parent = parent; @@ -51,6 +60,7 @@ this.blocks = new ArrayList<>(); this.children = new ArrayList<>(); this.exits = new ArrayList<>(); + this.naturalExits = new ArrayList<>(); } public abstract long numBackedges(); @@ -84,12 +94,87 @@ return blocks; } - public List getExits() { + /** + * Returns the loop exits. + * + * This might be a conservative set: before framestate assignment it matches the LoopExitNodes + * even if earlier blocks could be considered as exits. After framestate assignments, this is + * the same as {@link #getNaturalExits()}. + * + *

+ * LoopExitNodes are inserted in the control-flow during parsing and are natural exits: they are + * the earliest block at which we are guaranteed to have exited the loop. However, after some + * transformations of the graph, the natural exit might go up but the LoopExitNodes are not + * updated. + *

+ * + *

+ * For example in: + * + * 

+     * for (int i = 0; i < N; i++) {
+     *     if (c) {
+     *         // Block 1
+     *         if (dummy) {
+     *             // ...
+     *         } else {
+     *             // ...
+     *         }
+     *         if (b) {
+     *             continue;
+     *         } else {
+     *             // Block 2
+     *             // LoopExitNode
+     *             break;
+     *         }
+     *     }
+     * }
+     *
+ * + * After parsing, the natural exits match the LoopExitNode: Block 2 is a natural exit and has a + * LoopExitNode. If the {@code b} condition gets canonicalized to {@code false}, the natural + * exit moves to Block 1 while the LoopExitNode remains in Block 2. In such a scenario, + * {@code getLoopExits()} will contain block 2 while {@link #getNaturalExits()} will contain + * block 1. + * + * + * @see #getNaturalExits() + */ + public List getLoopExits() { return exits; } - public void addExit(T t) { - exits.add(t); + public boolean isLoopExit(T block) { + assert isSorted(exits); + return Collections.binarySearch(exits, block, BLOCK_ID_COMPARATOR) >= 0; + } + + /** + * Returns the natural exit points: these are the earliest block that are guaranteed to never + * reach a back-edge. + * + * This can not be used in the context of preserving or using loop-closed form. + * + * @see #getLoopExits() + */ + public ArrayList getNaturalExits() { + return naturalExits; + } + + public boolean isNaturalExit(T block) { + assert isSorted(naturalExits); + return Collections.binarySearch(naturalExits, block, BLOCK_ID_COMPARATOR) >= 0; + } + + private static > boolean isSorted(List list) { + int lastId = Integer.MIN_VALUE; + for (AbstractBlockBase block : list) { + if (block.getId() < lastId) { + return false; + } + lastId = block.getId(); + } + return true; } /** @@ -115,4 +200,9 @@ public int hashCode() { return index + depth * 31; } + + @Override + public boolean equals(Object obj) { + return super.equals(obj); + } } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/type/AbstractObjectStamp.java 2019-03-12 08:08:42.687229542 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/type/AbstractObjectStamp.java 2019-03-12 08:08:42.307227070 +0100 @@ -48,6 +48,13 @@ this.exactType = exactType; } + @Override + public void accept(Visitor v) { + super.accept(v); + v.visitObject(type); + v.visitBoolean(exactType); + } + protected abstract AbstractObjectStamp copyWith(ResolvedJavaType newType, boolean newExactType, boolean newNonNull, boolean newAlwaysNull); @Override --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/type/AbstractPointerStamp.java 2019-03-12 08:08:43.183232769 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/type/AbstractPointerStamp.java 2019-03-12 08:08:42.811230349 +0100 @@ -36,6 +36,12 @@ private final boolean nonNull; private final boolean alwaysNull; + @Override + public void accept(Visitor v) { + v.visitBoolean(nonNull); + v.visitBoolean(alwaysNull); + } + protected AbstractPointerStamp(boolean nonNull, boolean alwaysNull) { this.nonNull = nonNull; this.alwaysNull = alwaysNull; --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/type/IllegalStamp.java 2019-03-12 08:08:43.683236022 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/type/IllegalStamp.java 2019-03-12 08:08:43.315233628 +0100 @@ -46,6 +46,10 @@ } @Override + public void accept(Visitor v) { + } + + @Override public JavaKind getStackKind() { return JavaKind.Illegal; } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/type/ObjectStamp.java 2019-03-12 08:08:44.163239145 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/type/ObjectStamp.java 2019-03-12 08:08:43.795236751 +0100 @@ -31,6 +31,7 @@ import jdk.vm.ci.meta.JavaConstant; import jdk.vm.ci.meta.MemoryAccessProvider; import jdk.vm.ci.meta.ResolvedJavaType; +import jdk.vm.ci.meta.UnresolvedJavaType; public class ObjectStamp extends AbstractObjectStamp { @@ -92,4 +93,48 @@ return null; } } + + /** + * Convert an ObjectStamp into a representation that can be resolved symbolically into the + * original stamp. + */ + @Override + public SymbolicJVMCIReference makeSymbolic() { + if (type() == null) { + return null; + } + return new SymbolicObjectStamp(this); + } + + static class SymbolicObjectStamp implements SymbolicJVMCIReference { + UnresolvedJavaType type; + private boolean exactType; + private boolean nonNull; + private boolean alwaysNull; + + SymbolicObjectStamp(ObjectStamp stamp) { + if (stamp.type() != null) { + type = UnresolvedJavaType.create(stamp.type().getName()); + } + exactType = stamp.isExactType(); + nonNull = stamp.nonNull(); + alwaysNull = stamp.alwaysNull(); + } + + @Override + public ObjectStamp resolve(ResolvedJavaType accessingClass) { + return new ObjectStamp(type != null ? type.resolve(accessingClass) : null, exactType, nonNull, alwaysNull); + } + + @Override + public String toString() { + return "SymbolicObjectStamp{" + + "declaringType=" + type + + ", exactType=" + exactType + + ", nonNull=" + nonNull + + ", alwaysNull=" + alwaysNull + + '}'; + } + } + } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/type/PrimitiveStamp.java 2019-03-12 08:08:44.663242398 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/type/PrimitiveStamp.java 2019-03-12 08:08:44.291239978 +0100 @@ -39,6 +39,11 @@ this.bits = bits; } + @Override + public void accept(Visitor v) { + v.visitInt(bits); + } + /** * The width in bits of the value described by this stamp. */ --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/type/Stamp.java 2019-03-12 08:08:45.159245625 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/type/Stamp.java 2019-03-12 08:08:44.787243205 +0100 @@ -26,6 +26,7 @@ import org.graalvm.compiler.core.common.LIRKind; import org.graalvm.compiler.core.common.spi.LIRKindTool; +import org.graalvm.compiler.serviceprovider.SpeculationReasonGroup.SpeculationContextObject; import jdk.vm.ci.meta.Constant; import jdk.vm.ci.meta.JavaKind; @@ -36,7 +37,7 @@ /** * A stamp is the basis for a type system. */ -public abstract class Stamp { +public abstract class Stamp implements SpeculationContextObject { protected Stamp() { } @@ -185,4 +186,15 @@ } return false; } + + /** + * Convert a Stamp into a representation that can be resolved symbolically into the original + * stamp. If this stamp contains no references to JVMCI types then simply return null. + */ + public SymbolicJVMCIReference makeSymbolic() { + return null; + } + + @Override + public abstract String toString(); } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/type/StampPair.java 2019-03-12 08:08:45.643248773 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/type/StampPair.java 2019-03-12 08:08:45.271246353 +0100 @@ -24,6 +24,8 @@ package org.graalvm.compiler.core.common.type; +import java.util.Objects; + /** * A pair of stamp with one being the stamp that can be trusted and the other one being a guess that * needs a dynamic check to be used. @@ -63,4 +65,22 @@ return trustedStamp + " (unchecked=" + uncheckedStamp + ")"; } } + + @Override + public int hashCode() { + return trustedStamp.hashCode() + 11 + (uncheckedStamp != null ? uncheckedStamp.hashCode() : 0); + + } + + @Override + public boolean equals(Object obj) { + if (obj == this) { + return true; + } + if (obj instanceof StampPair) { + StampPair other = (StampPair) obj; + return trustedStamp.equals(other.trustedStamp) && Objects.equals(uncheckedStamp, other.uncheckedStamp); + } + return false; + } } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/type/VoidStamp.java 2019-03-12 08:08:46.119251869 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/type/VoidStamp.java 2019-03-12 08:08:45.755249502 +0100 @@ -43,6 +43,10 @@ } @Override + public void accept(Visitor v) { + } + + @Override public Stamp unrestricted() { return this; } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/util/UnsignedLong.java 2019-03-12 08:08:46.611255070 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/util/UnsignedLong.java 2019-03-12 08:08:46.247252702 +0100 @@ -43,6 +43,10 @@ return Long.compareUnsigned(value, unsignedValue) < 0; } + public boolean isGreaterThan(long unsignedValue) { + return Long.compareUnsigned(value, unsignedValue) > 0; + } + public boolean isLessOrEqualTo(long unsignedValue) { return Long.compareUnsigned(value, unsignedValue) <= 0; } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/CheckGraalInvariants.java 2019-03-12 08:08:47.107258296 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/CheckGraalInvariants.java 2019-03-12 08:08:46.735255876 +0100 @@ -157,6 +157,10 @@ if (className.equals("org.graalvm.compiler.serviceprovider.GraalServices$Lazy")) { return false; } + } else { + if (className.equals("jdk.vm.ci.services.JVMCIClassLoaderFactory")) { + return false; + } } return true; } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/ConditionalEliminationTestBase.java 2019-03-12 08:08:47.599261496 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/ConditionalEliminationTestBase.java 2019-03-12 08:08:47.227259076 +0100 @@ -33,7 +33,7 @@ import org.graalvm.compiler.phases.OptimisticOptimizations.Optimization; import org.graalvm.compiler.phases.common.CanonicalizerPhase; import org.graalvm.compiler.phases.common.ConditionalEliminationPhase; -import org.graalvm.compiler.phases.common.ConvertDeoptimizeToGuardPhase; +import org.graalvm.compiler.loop.phases.ConvertDeoptimizeToGuardPhase; import org.graalvm.compiler.phases.common.IterativeConditionalEliminationPhase; import org.graalvm.compiler.phases.common.LoweringPhase; import org.graalvm.compiler.phases.schedule.SchedulePhase; --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/DumpPathTest.java 2019-03-12 08:08:48.083264644 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/DumpPathTest.java 2019-03-12 08:08:47.715262250 +0100 @@ -31,6 +31,7 @@ import jdk.internal.vm.compiler.collections.EconomicMap; import org.graalvm.compiler.debug.DebugOptions; +import org.graalvm.compiler.debug.DebugOptions.PrintGraphTarget; import org.graalvm.compiler.options.OptionKey; import org.graalvm.compiler.options.OptionValues; import org.junit.Test; @@ -52,7 +53,7 @@ String[] extensions = new String[]{".cfg", ".bgv", ".graph-strings"}; EconomicMap, Object> overrides = OptionValues.newOptionMap(); overrides.put(DebugOptions.DumpPath, dumpDirectoryPath.toString()); - overrides.put(DebugOptions.PrintGraphFile, true); + overrides.put(DebugOptions.PrintGraph, PrintGraphTarget.File); overrides.put(DebugOptions.PrintCanonicalGraphStrings, true); overrides.put(DebugOptions.Dump, "*"); --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/GraalCompilerTest.java 2019-03-12 08:08:48.587267922 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/GraalCompilerTest.java 2019-03-12 08:08:48.219265528 +0100 @@ -111,7 +111,7 @@ import org.graalvm.compiler.phases.Phase; import org.graalvm.compiler.phases.PhaseSuite; import org.graalvm.compiler.phases.common.CanonicalizerPhase; -import org.graalvm.compiler.phases.common.ConvertDeoptimizeToGuardPhase; +import org.graalvm.compiler.loop.phases.ConvertDeoptimizeToGuardPhase; import org.graalvm.compiler.phases.common.inlining.InliningPhase; import org.graalvm.compiler.phases.common.inlining.info.InlineInfo; import org.graalvm.compiler.phases.common.inlining.policy.GreedyInliningPolicy; @@ -899,11 +899,21 @@ return actual; } + private static final List> C2_OMIT_STACK_TRACE_IN_FAST_THROW_EXCEPTIONS = Arrays.asList( + ArithmeticException.class, + ArrayIndexOutOfBoundsException.class, + ArrayStoreException.class, + ClassCastException.class, + NullPointerException.class); + protected void assertEquals(Result expect, Result actual) { if (expect.exception != null) { Assert.assertTrue("expected " + expect.exception, actual.exception != null); Assert.assertEquals("Exception class", expect.exception.getClass(), actual.exception.getClass()); - Assert.assertEquals("Exception message", expect.exception.getMessage(), actual.exception.getMessage()); + // C2 can optimize out the stack trace and message in some cases + if (expect.exception.getMessage() != null || !C2_OMIT_STACK_TRACE_IN_FAST_THROW_EXCEPTIONS.contains(expect.exception.getClass())) { + Assert.assertEquals("Exception message", expect.exception.getMessage(), actual.exception.getMessage()); + } } else { if (actual.exception != null) { throw new AssertionError("expected " + expect.returnValue + " but got an exception", actual.exception); --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/GraphEncoderTest.java 2019-03-12 08:08:49.067271044 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/GraphEncoderTest.java 2019-03-12 08:08:48.699268651 +0100 @@ -30,16 +30,15 @@ import java.util.List; import java.util.Map; -import jdk.vm.ci.meta.ResolvedJavaMethod; - -import org.junit.Test; - import org.graalvm.compiler.nodes.EncodedGraph; import org.graalvm.compiler.nodes.GraphEncoder; import org.graalvm.compiler.nodes.StructuredGraph; import org.graalvm.compiler.nodes.StructuredGraph.AllowAssumptions; import org.graalvm.compiler.phases.common.CanonicalizerPhase; import org.graalvm.compiler.phases.tiers.PhaseContext; +import org.junit.Test; + +import jdk.vm.ci.meta.ResolvedJavaMethod; public class GraphEncoderTest extends GraalCompilerTest { @@ -80,7 +79,7 @@ for (StructuredGraph originalGraph : originalGraphs) { EncodedGraph encodedGraph = new EncodedGraph(encoder.getEncoding(), startOffsets.get(originalGraph), encoder.getObjects(), encoder.getNodeClasses(), originalGraph); - GraphEncoder.verifyEncoding(originalGraph, encodedGraph, getTarget().arch); + encoder.verifyEncoding(originalGraph, encodedGraph); } } } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/GuardPrioritiesTest.java 2019-03-12 08:08:49.547274166 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/GuardPrioritiesTest.java 2019-03-12 08:08:49.179271772 +0100 @@ -35,6 +35,7 @@ import org.graalvm.compiler.api.directives.GraalDirectives; import org.graalvm.compiler.core.common.GraalOptions; import org.graalvm.compiler.graph.iterators.NodeIterable; +import org.graalvm.compiler.loop.phases.ConvertDeoptimizeToGuardPhase; import org.graalvm.compiler.nodes.GuardNode; import org.graalvm.compiler.nodes.ParameterNode; import org.graalvm.compiler.nodes.StructuredGraph; @@ -42,7 +43,6 @@ import org.graalvm.compiler.nodes.calc.IsNullNode; import org.graalvm.compiler.nodes.spi.LoweringTool; import org.graalvm.compiler.phases.common.CanonicalizerPhase; -import org.graalvm.compiler.phases.common.ConvertDeoptimizeToGuardPhase; import org.graalvm.compiler.phases.common.FloatingReadPhase; import org.graalvm.compiler.phases.common.LoweringPhase; import org.graalvm.compiler.phases.schedule.SchedulePhase; --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/MemoryScheduleTest.java 2019-03-12 08:08:50.027277287 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/MemoryScheduleTest.java 2019-03-12 08:08:49.659274894 +0100 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2011, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/NestedLoopTest.java 2019-03-12 08:08:50.523280513 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/NestedLoopTest.java 2019-03-12 08:08:50.155278120 +0100 @@ -161,9 +161,9 @@ Assert.assertTrue(containsDirect(innerMostLoop, d, cfg)); Assert.assertTrue(contains(rootLoop, d, cfg)); Assert.assertTrue(contains(nestedLoop, d, cfg)); - Assert.assertEquals(rootExits, rootLoop.getExits().size()); - Assert.assertEquals(nestedExits, nestedLoop.getExits().size()); - Assert.assertEquals(innerExits, innerMostLoop.getExits().size()); + Assert.assertEquals(rootExits, rootLoop.getLoopExits().size()); + Assert.assertEquals(nestedExits, nestedLoop.getLoopExits().size()); + Assert.assertEquals(innerExits, innerMostLoop.getLoopExits().size()); debug.dump(DebugContext.BASIC_LEVEL, graph, "Graph"); } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/VerifyCallerSensitiveMethods.java 2019-03-12 08:08:51.015283713 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/VerifyCallerSensitiveMethods.java 2019-03-12 08:08:50.643281294 +0100 @@ -24,7 +24,7 @@ package org.graalvm.compiler.core.test; -import static org.graalvm.compiler.serviceprovider.GraalServices.Java8OrEarlier; +import static org.graalvm.compiler.serviceprovider.JavaVersionUtil.Java8OrEarlier; import java.lang.annotation.Annotation; --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/VerifyDebugUsage.java 2019-03-12 08:08:51.495286834 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/VerifyDebugUsage.java 2019-03-12 08:08:51.111284337 +0100 @@ -150,7 +150,8 @@ "org.graalvm.compiler.truffle.compiler.TruffleCompilerImpl.compilePEGraph", "org.graalvm.compiler.core.test.VerifyDebugUsageTest$ValidDumpUsagePhase.run", "org.graalvm.compiler.core.test.VerifyDebugUsageTest$InvalidConcatDumpUsagePhase.run", - "org.graalvm.compiler.core.test.VerifyDebugUsageTest$InvalidDumpUsagePhase.run")); + "org.graalvm.compiler.core.test.VerifyDebugUsageTest$InvalidDumpUsagePhase.run", + "org.graalvm.compiler.hotspot.SymbolicSnippetEncoder.verifySnippetEncodeDecode")); /** * The set of methods allowed to call a {@code Debug.dump(...)} method with the {@code level} @@ -165,7 +166,8 @@ "org.graalvm.compiler.core.GraalCompiler.emitFrontEnd", "org.graalvm.compiler.phases.BasePhase.dumpAfter", "org.graalvm.compiler.replacements.ReplacementsImpl$GraphMaker.makeGraph", - "org.graalvm.compiler.replacements.SnippetTemplate.instantiate")); + "org.graalvm.compiler.replacements.SnippetTemplate.instantiate", + "org.graalvm.compiler.hotspot.SymbolicSnippetEncoder.verifySnippetEncodeDecode")); private void verifyParameters(StructuredGraph callerGraph, MethodCallTargetNode debugCallTarget, List args, ResolvedJavaType stringType, int startArgIdx, int varArgsIndex) { --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/backend/BackendTest.java 2019-03-12 08:08:51.991290060 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/backend/BackendTest.java 2019-03-12 08:08:51.627287693 +0100 @@ -25,6 +25,7 @@ package org.graalvm.compiler.core.test.backend; import org.graalvm.compiler.core.GraalCompiler; +import org.graalvm.compiler.core.gen.LIRCompilerBackend; import org.graalvm.compiler.core.test.GraalCompilerTest; import org.graalvm.compiler.debug.DebugContext; import org.graalvm.compiler.lir.gen.LIRGenerationResult; @@ -52,7 +53,7 @@ throw debug.handle(e); } - LIRGenerationResult lirGen = GraalCompiler.emitLIR(getBackend(), graph, null, null, createLIRSuites(graph.getOptions())); + LIRGenerationResult lirGen = LIRCompilerBackend.emitLIR(getBackend(), graph, null, null, createLIRSuites(graph.getOptions())); return lirGen; } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/deopt/RethrowDeoptMaterializeTest.java 2019-03-12 08:08:52.479293233 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/deopt/RethrowDeoptMaterializeTest.java 2019-03-12 08:08:52.111290840 +0100 @@ -46,7 +46,7 @@ @SuppressWarnings("sync-override") @Override public final Throwable fillInStackTrace() { - return null; + return this; } } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/ea/EscapeAnalysisTest.java 2019-03-12 08:08:52.959296355 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/ea/EscapeAnalysisTest.java 2019-03-12 08:08:52.591293961 +0100 @@ -293,7 +293,7 @@ @SuppressWarnings("sync-override") @Override public final Throwable fillInStackTrace() { - return null; + return this; } } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core/src/org/graalvm/compiler/core/GraalCompiler.java 2019-03-12 08:08:53.439299476 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core/src/org/graalvm/compiler/core/GraalCompiler.java 2019-03-12 08:08:53.067297057 +0100 @@ -24,42 +24,18 @@ package org.graalvm.compiler.core; -import java.util.Collection; -import java.util.List; - -import jdk.internal.vm.compiler.collections.EconomicSet; import org.graalvm.compiler.code.CompilationResult; -import org.graalvm.compiler.core.LIRGenerationPhase.LIRGenerationContext; -import org.graalvm.compiler.core.common.GraalOptions; import org.graalvm.compiler.core.common.PermanentBailoutException; import org.graalvm.compiler.core.common.RetryableBailoutException; -import org.graalvm.compiler.core.common.alloc.ComputeBlockOrder; -import org.graalvm.compiler.core.common.alloc.RegisterAllocationConfig; -import org.graalvm.compiler.core.common.cfg.AbstractBlockBase; import org.graalvm.compiler.core.common.util.CompilationAlarm; import org.graalvm.compiler.core.target.Backend; -import org.graalvm.compiler.debug.CounterKey; import org.graalvm.compiler.debug.DebugCloseable; import org.graalvm.compiler.debug.DebugContext; -import org.graalvm.compiler.debug.GraalError; import org.graalvm.compiler.debug.MethodFilter; import org.graalvm.compiler.debug.TimerKey; -import org.graalvm.compiler.lir.LIR; -import org.graalvm.compiler.lir.alloc.OutOfRegistersException; -import org.graalvm.compiler.lir.asm.CompilationResultBuilder; import org.graalvm.compiler.lir.asm.CompilationResultBuilderFactory; -import org.graalvm.compiler.lir.framemap.FrameMap; -import org.graalvm.compiler.lir.framemap.FrameMapBuilder; -import org.graalvm.compiler.lir.gen.LIRGenerationResult; -import org.graalvm.compiler.lir.gen.LIRGeneratorTool; -import org.graalvm.compiler.lir.phases.AllocationPhase.AllocationContext; import org.graalvm.compiler.lir.phases.LIRSuites; -import org.graalvm.compiler.lir.phases.PostAllocationOptimizationPhase.PostAllocationOptimizationContext; -import org.graalvm.compiler.lir.phases.PreAllocationOptimizationPhase.PreAllocationOptimizationContext; import org.graalvm.compiler.nodes.StructuredGraph; -import org.graalvm.compiler.nodes.StructuredGraph.ScheduleResult; -import org.graalvm.compiler.nodes.cfg.Block; -import org.graalvm.compiler.nodes.spi.NodeLIRBuilderTool; import org.graalvm.compiler.phases.OptimisticOptimizations; import org.graalvm.compiler.phases.PhaseSuite; import org.graalvm.compiler.phases.common.DeadCodeEliminationPhase; @@ -70,17 +46,8 @@ import org.graalvm.compiler.phases.tiers.TargetProvider; import org.graalvm.compiler.phases.util.Providers; -import jdk.vm.ci.code.RegisterConfig; -import jdk.vm.ci.code.TargetDescription; -import jdk.vm.ci.code.site.ConstantReference; -import jdk.vm.ci.code.site.DataPatch; -import jdk.vm.ci.meta.Assumptions; -import jdk.vm.ci.meta.JavaConstant; -import jdk.vm.ci.meta.JavaKind; import jdk.vm.ci.meta.ProfilingInfo; -import jdk.vm.ci.meta.ResolvedJavaField; import jdk.vm.ci.meta.ResolvedJavaMethod; -import jdk.vm.ci.meta.VMConstant; /** * Static methods for orchestrating the compilation of a {@linkplain StructuredGraph graph}. @@ -89,9 +56,6 @@ private static final TimerKey CompilerTimer = DebugContext.timer("GraalCompiler").doc("Time spent in compilation (excludes code installation)."); private static final TimerKey FrontEnd = DebugContext.timer("FrontEnd").doc("Time spent processing HIR."); - private static final TimerKey EmitLIR = DebugContext.timer("EmitLIR").doc("Time spent generating LIR from HIR."); - private static final TimerKey EmitCode = DebugContext.timer("EmitCode").doc("Time spent generating machine code from LIR."); - private static final TimerKey BackEnd = DebugContext.timer("BackEnd").doc("Time spent in EmitLIR and EmitCode."); /** * Encapsulates all the inputs to a {@linkplain GraalCompiler#compile(Request) compilation}. @@ -178,7 +142,7 @@ assert !r.graph.isFrozen(); try (DebugContext.Scope s0 = debug.scope("GraalCompiler", r.graph, r.providers.getCodeCache()); DebugCloseable a = CompilerTimer.start(debug)) { emitFrontEnd(r.providers, r.backend, r.graph, r.graphBuilderSuite, r.optimisticOpts, r.profilingInfo, r.suites); - emitBackEnd(r.graph, null, r.installedCodeOwner, r.backend, r.compilationResult, r.factory, null, r.lirSuites); + r.backend.emitBackEnd(r.graph, null, r.installedCodeOwner, r.compilationResult, r.factory, null, r.lirSuites); if (r.verifySourcePositions) { assert r.graph.verifySourcePositions(true); } @@ -275,85 +239,6 @@ } } - @SuppressWarnings("try") - public static void emitBackEnd(StructuredGraph graph, Object stub, ResolvedJavaMethod installedCodeOwner, Backend backend, T compilationResult, - CompilationResultBuilderFactory factory, RegisterConfig registerConfig, LIRSuites lirSuites) { - DebugContext debug = graph.getDebug(); - try (DebugContext.Scope s = debug.scope("BackEnd", graph.getLastSchedule()); DebugCloseable a = BackEnd.start(debug)) { - LIRGenerationResult lirGen = null; - lirGen = emitLIR(backend, graph, stub, registerConfig, lirSuites); - try (DebugContext.Scope s2 = debug.scope("CodeGen", lirGen, lirGen.getLIR())) { - int bytecodeSize = graph.method() == null ? 0 : graph.getBytecodeSize(); - compilationResult.setHasUnsafeAccess(graph.hasUnsafeAccess()); - emitCode(backend, graph.getAssumptions(), graph.method(), graph.getMethods(), graph.getFields(), bytecodeSize, lirGen, compilationResult, installedCodeOwner, factory); - } catch (Throwable e) { - throw debug.handle(e); - } - } catch (Throwable e) { - throw debug.handle(e); - } finally { - graph.checkCancellation(); - } - } - - @SuppressWarnings("try") - public static LIRGenerationResult emitLIR(Backend backend, StructuredGraph graph, Object stub, RegisterConfig registerConfig, LIRSuites lirSuites) { - String registerPressure = GraalOptions.RegisterPressure.getValue(graph.getOptions()); - String[] allocationRestrictedTo = registerPressure == null ? null : registerPressure.split(","); - try { - return emitLIR0(backend, graph, stub, registerConfig, lirSuites, allocationRestrictedTo); - } catch (OutOfRegistersException e) { - if (allocationRestrictedTo != null) { - allocationRestrictedTo = null; - return emitLIR0(backend, graph, stub, registerConfig, lirSuites, allocationRestrictedTo); - } - /* If the re-execution fails we convert the exception into a "hard" failure */ - throw new GraalError(e); - } finally { - graph.checkCancellation(); - } - } - - @SuppressWarnings("try") - private static LIRGenerationResult emitLIR0(Backend backend, StructuredGraph graph, Object stub, RegisterConfig registerConfig, LIRSuites lirSuites, - String[] allocationRestrictedTo) { - DebugContext debug = graph.getDebug(); - try (DebugContext.Scope ds = debug.scope("EmitLIR"); DebugCloseable a = EmitLIR.start(debug)) { - assert !graph.hasValueProxies(); - ScheduleResult schedule = graph.getLastSchedule(); - Block[] blocks = schedule.getCFG().getBlocks(); - Block startBlock = schedule.getCFG().getStartBlock(); - assert startBlock != null; - assert startBlock.getPredecessorCount() == 0; - - AbstractBlockBase[] codeEmittingOrder = ComputeBlockOrder.computeCodeEmittingOrder(blocks.length, startBlock); - AbstractBlockBase[] linearScanOrder = ComputeBlockOrder.computeLinearScanOrder(blocks.length, startBlock); - LIR lir = new LIR(schedule.getCFG(), linearScanOrder, codeEmittingOrder, graph.getOptions(), graph.getDebug()); - - FrameMapBuilder frameMapBuilder = backend.newFrameMapBuilder(registerConfig); - LIRGenerationResult lirGenRes = backend.newLIRGenerationResult(graph.compilationId(), lir, frameMapBuilder, graph, stub); - LIRGeneratorTool lirGen = backend.newLIRGenerator(lirGenRes); - NodeLIRBuilderTool nodeLirGen = backend.newNodeLIRBuilder(graph, lirGen); - - // LIR generation - LIRGenerationContext context = new LIRGenerationContext(lirGen, nodeLirGen, graph, schedule); - new LIRGenerationPhase().apply(backend.getTarget(), lirGenRes, context); - - try (DebugContext.Scope s = debug.scope("LIRStages", nodeLirGen, lirGenRes, lir)) { - // Dump LIR along with HIR (the LIR is looked up from context) - debug.dump(DebugContext.BASIC_LEVEL, graph.getLastSchedule(), "After LIR generation"); - LIRGenerationResult result = emitLowLevel(backend.getTarget(), lirGenRes, lirGen, lirSuites, backend.newRegisterAllocationConfig(registerConfig, allocationRestrictedTo)); - return result; - } catch (Throwable e) { - throw debug.handle(e); - } - } catch (Throwable e) { - throw debug.handle(e); - } finally { - graph.checkCancellation(); - } - } - protected static String getCompilationUnitName(StructuredGraph graph, T compilationResult) { if (compilationResult != null && compilationResult.getName() != null) { return compilationResult.getName(); @@ -364,70 +249,4 @@ } return method.format("%H.%n(%p)"); } - - public static LIRGenerationResult emitLowLevel(TargetDescription target, LIRGenerationResult lirGenRes, LIRGeneratorTool lirGen, LIRSuites lirSuites, - RegisterAllocationConfig registerAllocationConfig) { - DebugContext debug = lirGenRes.getLIR().getDebug(); - PreAllocationOptimizationContext preAllocOptContext = new PreAllocationOptimizationContext(lirGen); - lirSuites.getPreAllocationOptimizationStage().apply(target, lirGenRes, preAllocOptContext); - debug.dump(DebugContext.BASIC_LEVEL, lirGenRes.getLIR(), "After PreAllocationOptimizationStage"); - - AllocationContext allocContext = new AllocationContext(lirGen.getSpillMoveFactory(), registerAllocationConfig); - lirSuites.getAllocationStage().apply(target, lirGenRes, allocContext); - debug.dump(DebugContext.BASIC_LEVEL, lirGenRes.getLIR(), "After AllocationStage"); - - PostAllocationOptimizationContext postAllocOptContext = new PostAllocationOptimizationContext(lirGen); - lirSuites.getPostAllocationOptimizationStage().apply(target, lirGenRes, postAllocOptContext); - debug.dump(DebugContext.BASIC_LEVEL, lirGenRes.getLIR(), "After PostAllocationOptimizationStage"); - - return lirGenRes; - } - - @SuppressWarnings("try") - public static void emitCode(Backend backend, Assumptions assumptions, ResolvedJavaMethod rootMethod, Collection inlinedMethods, EconomicSet accessedFields, - int bytecodeSize, LIRGenerationResult lirGenRes, - CompilationResult compilationResult, ResolvedJavaMethod installedCodeOwner, CompilationResultBuilderFactory factory) { - DebugContext debug = lirGenRes.getLIR().getDebug(); - try (DebugCloseable a = EmitCode.start(debug)) { - FrameMap frameMap = lirGenRes.getFrameMap(); - CompilationResultBuilder crb = backend.newCompilationResultBuilder(lirGenRes, frameMap, compilationResult, factory); - backend.emitCode(crb, lirGenRes.getLIR(), installedCodeOwner); - if (assumptions != null && !assumptions.isEmpty()) { - compilationResult.setAssumptions(assumptions.toArray()); - } - if (rootMethod != null) { - compilationResult.setMethods(rootMethod, inlinedMethods); - compilationResult.setFields(accessedFields); - compilationResult.setBytecodeSize(bytecodeSize); - } - crb.finish(); - if (debug.isCountEnabled()) { - List ldp = compilationResult.getDataPatches(); - JavaKind[] kindValues = JavaKind.values(); - CounterKey[] dms = new CounterKey[kindValues.length]; - for (int i = 0; i < dms.length; i++) { - dms[i] = DebugContext.counter("DataPatches-%s", kindValues[i]); - } - - for (DataPatch dp : ldp) { - JavaKind kind = JavaKind.Illegal; - if (dp.reference instanceof ConstantReference) { - VMConstant constant = ((ConstantReference) dp.reference).getConstant(); - if (constant instanceof JavaConstant) { - kind = ((JavaConstant) constant).getJavaKind(); - } - } - dms[kind.ordinal()].add(debug, 1); - } - - DebugContext.counter("CompilationResults").increment(debug); - DebugContext.counter("CodeBytesEmitted").add(debug, compilationResult.getTargetCodeSize()); - DebugContext.counter("InfopointsEmitted").add(debug, compilationResult.getInfopoints().size()); - DebugContext.counter("DataPatches").add(debug, ldp.size()); - DebugContext.counter("ExceptionHandlersEmitted").add(debug, compilationResult.getExceptionHandlers().size()); - } - - debug.dump(DebugContext.BASIC_LEVEL, compilationResult, "After code generation"); - } - } } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core/src/org/graalvm/compiler/core/gen/NodeLIRBuilder.java 2019-03-12 08:08:53.939302727 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core/src/org/graalvm/compiler/core/gen/NodeLIRBuilder.java 2019-03-12 08:08:53.571300334 +0100 @@ -768,4 +768,13 @@ public LIRGeneratorTool getLIRGeneratorTool() { return gen; } + + @Override + public void emitReadExceptionObject(ValueNode node) { + LIRGeneratorTool lirGenTool = getLIRGeneratorTool(); + Value returnRegister = lirGenTool.getRegisterConfig().getReturnRegister(node.getStackKind()).asValue( + LIRKind.fromJavaKind(lirGenTool.target().arch, node.getStackKind())); + lirGenTool.emitIncomingValues(new Value[]{returnRegister}); + setResult(node, lirGenTool.emitMove(returnRegister)); + } } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core/src/org/graalvm/compiler/core/phases/HighTier.java 2019-03-12 08:08:54.443306003 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core/src/org/graalvm/compiler/core/phases/HighTier.java 2019-03-12 08:08:54.063303532 +0100 @@ -47,7 +47,7 @@ import org.graalvm.compiler.options.OptionValues; import org.graalvm.compiler.phases.PhaseSuite; import org.graalvm.compiler.phases.common.CanonicalizerPhase; -import org.graalvm.compiler.phases.common.ConvertDeoptimizeToGuardPhase; +import org.graalvm.compiler.loop.phases.ConvertDeoptimizeToGuardPhase; import org.graalvm.compiler.phases.common.DeadCodeEliminationPhase; import org.graalvm.compiler.phases.common.IncrementalCanonicalizerPhase; import org.graalvm.compiler.phases.common.IterativeConditionalEliminationPhase; --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core/src/org/graalvm/compiler/core/phases/LowTier.java 2019-03-12 08:08:54.923309124 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core/src/org/graalvm/compiler/core/phases/LowTier.java 2019-03-12 08:08:54.555306732 +0100 @@ -39,6 +39,7 @@ import org.graalvm.compiler.phases.common.ExpandLogicPhase; import org.graalvm.compiler.phases.common.FixReadsPhase; import org.graalvm.compiler.phases.common.LoweringPhase; +import org.graalvm.compiler.phases.common.OptimizeDivPhase; import org.graalvm.compiler.phases.common.ProfileCompiledMethodsPhase; import org.graalvm.compiler.phases.common.PropagateDeoptimizeProbabilityPhase; import org.graalvm.compiler.phases.common.UseTrappingNullChecksPhase; @@ -72,6 +73,8 @@ appendPhase(new LoweringPhase(canonicalizer, LoweringTool.StandardLoweringStage.LOW_TIER)); + appendPhase(new OptimizeDivPhase()); + appendPhase(new ExpandLogicPhase()); appendPhase(new FixReadsPhase(true, new SchedulePhase(GraalOptions.StressTestEarlyReads.getValue(options) ? SchedulingStrategy.EARLIEST : SchedulingStrategy.LATEST_OUT_OF_LOOPS))); --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core/src/org/graalvm/compiler/core/target/Backend.java 2019-03-12 08:08:55.403312245 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core/src/org/graalvm/compiler/core/target/Backend.java 2019-03-12 08:08:55.031309826 +0100 @@ -26,24 +26,17 @@ import java.util.ArrayList; -import jdk.internal.vm.compiler.collections.EconomicSet; -import org.graalvm.compiler.asm.Assembler; import org.graalvm.compiler.code.CompilationResult; import org.graalvm.compiler.core.common.CompilationIdentifier; import org.graalvm.compiler.core.common.LIRKind; import org.graalvm.compiler.core.common.alloc.RegisterAllocationConfig; import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor; import org.graalvm.compiler.core.common.spi.ForeignCallsProvider; +import org.graalvm.compiler.core.gen.LIRCompilerBackend; import org.graalvm.compiler.debug.DebugContext; -import org.graalvm.compiler.lir.LIR; -import org.graalvm.compiler.lir.asm.CompilationResultBuilder; import org.graalvm.compiler.lir.asm.CompilationResultBuilderFactory; -import org.graalvm.compiler.lir.framemap.FrameMap; -import org.graalvm.compiler.lir.framemap.FrameMapBuilder; -import org.graalvm.compiler.lir.gen.LIRGenerationResult; -import org.graalvm.compiler.lir.gen.LIRGeneratorTool; +import org.graalvm.compiler.lir.phases.LIRSuites; import org.graalvm.compiler.nodes.StructuredGraph; -import org.graalvm.compiler.nodes.spi.NodeLIRBuilderTool; import org.graalvm.compiler.options.OptionValues; import org.graalvm.compiler.phases.tiers.SuitesProvider; import org.graalvm.compiler.phases.tiers.TargetProvider; @@ -54,7 +47,6 @@ import jdk.vm.ci.code.CompilationRequest; import jdk.vm.ci.code.CompiledCode; import jdk.vm.ci.code.InstalledCode; -import jdk.vm.ci.code.Register; import jdk.vm.ci.code.RegisterConfig; import jdk.vm.ci.code.TargetDescription; import jdk.vm.ci.code.ValueKindFactory; @@ -62,7 +54,6 @@ import jdk.vm.ci.meta.JavaKind; import jdk.vm.ci.meta.MetaAccessProvider; import jdk.vm.ci.meta.ResolvedJavaMethod; -import jdk.vm.ci.meta.SpeculationLog; /** * Represents a compiler backend for Graal. @@ -117,12 +108,6 @@ } /** - * The given registerConfig is optional, in case null is passed the default RegisterConfig from - * the CodeCacheProvider will be used. - */ - public abstract FrameMapBuilder newFrameMapBuilder(RegisterConfig registerConfig); - - /** * Creates a new configuration for register allocation. * * @param allocationRestrictedTo if not {@code null}, register allocation will be restricted to @@ -130,26 +115,6 @@ */ public abstract RegisterAllocationConfig newRegisterAllocationConfig(RegisterConfig registerConfig, String[] allocationRestrictedTo); - public abstract FrameMap newFrameMap(RegisterConfig registerConfig); - - public abstract LIRGeneratorTool newLIRGenerator(LIRGenerationResult lirGenRes); - - public abstract LIRGenerationResult newLIRGenerationResult(CompilationIdentifier compilationId, LIR lir, FrameMapBuilder frameMapBuilder, StructuredGraph graph, - Object stub); - - public abstract NodeLIRBuilderTool newNodeLIRBuilder(StructuredGraph graph, LIRGeneratorTool lirGen); - - /** - * Creates the assembler used to emit the machine code. - */ - protected abstract Assembler createAssembler(FrameMap frameMap); - - /** - * Creates the object used to fill in the details of a given compilation result. - */ - public abstract CompilationResultBuilder newCompilationResultBuilder(LIRGenerationResult lirGenResult, FrameMap frameMap, CompilationResult compilationResult, - CompilationResultBuilderFactory factory); - /** * Turns a Graal {@link CompilationResult} into a {@link CompiledCode} object that can be passed * to the VM for code installation. @@ -158,21 +123,28 @@ /** * @see #createInstalledCode(DebugContext, ResolvedJavaMethod, CompilationRequest, - * CompilationResult, SpeculationLog, InstalledCode, boolean, Object[]) + * CompilationResult, InstalledCode, boolean, Object[]) */ - public InstalledCode createInstalledCode(DebugContext debug, ResolvedJavaMethod method, CompilationResult compilationResult, - SpeculationLog speculationLog, InstalledCode predefinedInstalledCode, boolean isDefault) { - return createInstalledCode(debug, method, null, compilationResult, speculationLog, predefinedInstalledCode, isDefault, null); + public InstalledCode createInstalledCode(DebugContext debug, + ResolvedJavaMethod method, + CompilationResult compilationResult, + InstalledCode predefinedInstalledCode, + boolean isDefault) { + return createInstalledCode(debug, method, null, compilationResult, predefinedInstalledCode, isDefault, null); } /** * @see #createInstalledCode(DebugContext, ResolvedJavaMethod, CompilationRequest, - * CompilationResult, SpeculationLog, InstalledCode, boolean, Object[]) + * CompilationResult, InstalledCode, boolean, Object[]) */ @SuppressWarnings("try") - public InstalledCode createInstalledCode(DebugContext debug, ResolvedJavaMethod method, CompilationRequest compilationRequest, CompilationResult compilationResult, - SpeculationLog speculationLog, InstalledCode predefinedInstalledCode, boolean isDefault) { - return createInstalledCode(debug, method, compilationRequest, compilationResult, speculationLog, predefinedInstalledCode, isDefault, null); + public InstalledCode createInstalledCode(DebugContext debug, + ResolvedJavaMethod method, + CompilationRequest compilationRequest, + CompilationResult compilationResult, + InstalledCode predefinedInstalledCode, + boolean isDefault) { + return createInstalledCode(debug, method, compilationRequest, compilationResult, predefinedInstalledCode, isDefault, null); } /** @@ -185,7 +157,6 @@ * @param predefinedInstalledCode a pre-allocated {@link InstalledCode} object to use as a * reference to the installed code. If {@code null}, a new {@link InstalledCode} * object will be created. - * @param speculationLog the speculation log to be used * @param isDefault specifies if the installed code should be made the default implementation of * {@code compRequest.getMethod()}. The default implementation for a method is the * code executed for standard calls to the method. This argument is ignored if @@ -198,8 +169,13 @@ * {@link InstalledCode} object */ @SuppressWarnings("try") - public InstalledCode createInstalledCode(DebugContext debug, ResolvedJavaMethod method, CompilationRequest compilationRequest, CompilationResult compilationResult, - SpeculationLog speculationLog, InstalledCode predefinedInstalledCode, boolean isDefault, Object[] context) { + public InstalledCode createInstalledCode(DebugContext debug, + ResolvedJavaMethod method, + CompilationRequest compilationRequest, + CompilationResult compilationResult, + InstalledCode predefinedInstalledCode, + boolean isDefault, + Object[] context) { Object[] debugContext = context != null ? context : new Object[]{getProviders().getCodeCache(), method, compilationResult}; CodeInstallationTask[] tasks; synchronized (this) { @@ -215,7 +191,7 @@ try { preCodeInstallationTasks(tasks, compilationResult); CompiledCode compiledCode = createCompiledCode(method, compilationRequest, compilationResult, isDefault, debug.getOptions()); - installedCode = getProviders().getCodeCache().installCode(method, compiledCode, predefinedInstalledCode, speculationLog, isDefault); + installedCode = getProviders().getCodeCache().installCode(method, compiledCode, predefinedInstalledCode, compilationResult.getSpeculationLog(), isDefault); assert predefinedInstalledCode == null || installedCode == predefinedInstalledCode; } catch (Throwable t) { failCodeInstallationTasks(tasks, t); @@ -259,12 +235,15 @@ * @param method the method compiled to produce {@code compiledCode} or {@code null} if the * input to {@code compResult} was not a {@link ResolvedJavaMethod} * @param compilationRequest the request or {@code null} - * @param compilationResult the code to be compiled + * @param compilationResult the compiled code * @return a reference to the compiled and ready-to-run installed code * @throws BailoutException if the code installation failed */ - public InstalledCode addInstalledCode(DebugContext debug, ResolvedJavaMethod method, CompilationRequest compilationRequest, CompilationResult compilationResult) { - return createInstalledCode(debug, method, compilationRequest, compilationResult, null, null, false); + public InstalledCode addInstalledCode(DebugContext debug, + ResolvedJavaMethod method, + CompilationRequest compilationRequest, + CompilationResult compilationResult) { + return createInstalledCode(debug, method, compilationRequest, compilationResult, null, false); } /** @@ -273,30 +252,16 @@ * * @param method the method compiled to produce {@code compiledCode} or {@code null} if the * input to {@code compResult} was not a {@link ResolvedJavaMethod} - * @param compilationResult the code to be compiled + * @param compilationResult the compiled code * @return a reference to the compiled and ready-to-run installed code * @throws BailoutException if the code installation failed */ public InstalledCode createDefaultInstalledCode(DebugContext debug, ResolvedJavaMethod method, CompilationResult compilationResult) { - return createInstalledCode(debug, method, compilationResult, null, null, true); + System.out.println(compilationResult.getSpeculationLog()); + return createInstalledCode(debug, method, compilationResult, null, true); } /** - * Emits the code for a given graph. - * - * @param installedCodeOwner the method the compiled code will be associated with once - * installed. This argument can be null. - */ - public abstract void emitCode(CompilationResultBuilder crb, LIR lir, ResolvedJavaMethod installedCodeOwner); - - /** - * Translates a set of registers from the callee's perspective to the caller's perspective. This - * is needed for architectures where input/output registers are renamed during a call (e.g. - * register windows on SPARC). Registers which are not visible by the caller are removed. - */ - public abstract EconomicSet translateToCallerRegisters(EconomicSet calleeRegisters); - - /** * Gets the compilation id for a given {@link ResolvedJavaMethod}. Returns * {@code CompilationIdentifier#INVALID_COMPILATION_ID} in case there is no such id. * @@ -306,6 +271,15 @@ return CompilationIdentifier.INVALID_COMPILATION_ID; } + public void emitBackEnd(StructuredGraph graph, + Object stub, + ResolvedJavaMethod installedCodeOwner, + CompilationResult compilationResult, + CompilationResultBuilderFactory factory, + RegisterConfig config, LIRSuites lirSuites) { + LIRCompilerBackend.emitBackEnd(graph, stub, installedCodeOwner, this, compilationResult, factory, config, lirSuites); + } + /** * Encapsulates custom tasks done before and after code installation. */ --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.debug/src/org/graalvm/compiler/debug/DebugContext.java 2019-03-12 08:08:55.891315418 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.debug/src/org/graalvm/compiler/debug/DebugContext.java 2019-03-12 08:08:55.523313025 +0100 @@ -450,11 +450,11 @@ } } - public Path getDumpPath(String extension, boolean directory) { + public Path getDumpPath(String extension, boolean createMissingDirectory) { try { String id = description == null ? null : description.identifier; String label = description == null ? null : description.getLabel(); - Path result = PathUtilities.createUnique(immutable.options, DumpPath, id, label, extension, directory); + Path result = PathUtilities.createUnique(immutable.options, DumpPath, id, label, extension, createMissingDirectory); if (ShowDumpFiles.getValue(immutable.options)) { TTY.println("Dumping debug output to %s", result.toAbsolutePath().toString()); } @@ -807,8 +807,9 @@ return true; } return !currentScope.isTopLevel(); + } else { + return false; } - return immutable.scopesEnabled && currentScope == null; } class DisabledScope implements DebugContext.Scope { --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.debug/src/org/graalvm/compiler/debug/DebugOptions.java 2019-03-12 08:08:56.391318668 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.debug/src/org/graalvm/compiler/debug/DebugOptions.java 2019-03-12 08:08:56.023316276 +0100 @@ -30,6 +30,7 @@ import java.nio.file.Paths; import jdk.internal.vm.compiler.collections.EconomicMap; +import org.graalvm.compiler.options.EnumOptionKey; import org.graalvm.compiler.options.Option; import org.graalvm.compiler.options.OptionKey; import org.graalvm.compiler.options.OptionType; @@ -40,20 +41,28 @@ * Options that configure a {@link DebugContext} and related functionality. */ public class DebugOptions { - static class DeprecatedOptionKey extends OptionKey { - private final OptionKey replacement; - DeprecatedOptionKey(OptionKey replacement) { - super(replacement.getDefaultValue()); - this.replacement = replacement; - } - - @Override - protected void onValueUpdate(EconomicMap, Object> values, T oldValue, T newValue) { - // Ideally we'd use TTY here but it may not yet be initialized. - System.err.printf("Warning: the %s option is deprecated - use %s instead%n", getName(), replacement.getName()); - replacement.update(values, newValue); - } + /** + * Values for the {@link DebugOptions#PrintGraph} option denoting where graphs dumped as a + * result of the {@link DebugOptions#Dump} option are sent. + */ + public enum PrintGraphTarget { + /** + * Dump graphs to files. + */ + File, + + /** + * Dump graphs to the network. The network destination is specified by the + * {@link DebugOptions#PrintGraphHost} and {@link DebugOptions#PrintGraphPort} options. If a + * network connection cannot be opened, dumping falls back to {@link #File} dumping. + */ + Network, + + /** + * Do not dump graphs. + */ + Disable; } // @formatter:off @@ -118,7 +127,7 @@ public static final OptionKey LogVerbose = new OptionKey<>(false); @Option(help = "The directory where various Graal dump files are written.") - public static final OptionKey DumpPath = new OptionKey<>("dumps"); + public static final OptionKey DumpPath = new OptionKey<>("graal_dumps"); @Option(help = "Print the name of each dump file path as it's created.") public static final OptionKey ShowDumpFiles = new OptionKey<>(false); @@ -127,15 +136,30 @@ @Option(help = "Enable dumping LIR, register allocation and code generation info to the C1Visualizer.", type = OptionType.Debug) public static final OptionKey PrintBackendCFG = new OptionKey<>(true); - @Option(help = "Enable dumping to the IdealGraphVisualizer.", type = OptionType.Debug) - public static final OptionKey PrintGraph = new OptionKey<>(true); - @Option(help = "Print graphs to files instead of sending them over the network.", type = OptionType.Debug) - public static final OptionKey PrintGraphFile = new OptionKey<>(false); + @Option(help = "file:doc-files/PrintGraphHelp.txt", type = OptionType.Debug) + public static final EnumOptionKey PrintGraph = new EnumOptionKey<>(PrintGraphTarget.File); + + @Option(help = "Setting to true sets PrintGraph=file, setting to false sets PrintGraph=network", type = OptionType.Debug) + public static final OptionKey PrintGraphFile = new OptionKey(true) { + @Override + protected void onValueUpdate(EconomicMap, Object> values, Boolean oldValue, Boolean newValue) { + PrintGraphTarget v = PrintGraph.getValueOrDefault(values); + if (newValue.booleanValue()) { + if (v != PrintGraphTarget.File) { + PrintGraph.update(values, PrintGraphTarget.File); + } + } else { + if (v != PrintGraphTarget.Network) { + PrintGraph.update(values, PrintGraphTarget.Network); + } + } + } + }; @Option(help = "Host part of the address to which graphs are dumped.", type = OptionType.Debug) public static final OptionKey PrintGraphHost = new OptionKey<>("127.0.0.1"); @Option(help = "Port part of the address to which graphs are dumped in binary format.", type = OptionType.Debug) - public static final OptionKey PrintBinaryGraphPort = new OptionKey<>(4445); + public static final OptionKey PrintGraphPort = new OptionKey<>(4445); @Option(help = "Schedule graphs as they are dumped.", type = OptionType.Debug) public static final OptionKey PrintGraphWithSchedule = new OptionKey<>(false); --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.debug/src/org/graalvm/compiler/debug/IgvDumpChannel.java 2019-03-12 08:08:56.883321867 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.debug/src/org/graalvm/compiler/debug/IgvDumpChannel.java 2019-03-12 08:08:56.519319500 +0100 @@ -24,6 +24,10 @@ package org.graalvm.compiler.debug; +import static org.graalvm.compiler.debug.DebugOptions.PrintGraphHost; +import static org.graalvm.compiler.debug.DebugOptions.PrintGraphPort; + +import java.io.File; import java.io.IOException; import java.io.InterruptedIOException; import java.net.InetSocketAddress; @@ -35,10 +39,12 @@ import java.nio.file.Path; import java.nio.file.StandardOpenOption; import java.util.function.Supplier; -import static org.graalvm.compiler.debug.DebugOptions.PrintBinaryGraphPort; -import static org.graalvm.compiler.debug.DebugOptions.PrintGraphHost; + +import org.graalvm.compiler.debug.DebugOptions.PrintGraphTarget; import org.graalvm.compiler.options.OptionValues; +import jdk.vm.ci.common.NativeImageReinitialize; + final class IgvDumpChannel implements WritableByteChannel { private final Supplier pathProvider; private final OptionValues options; @@ -52,7 +58,8 @@ @Override public int write(ByteBuffer src) throws IOException { - return channel().write(src); + WritableByteChannel channel = channel(); + return channel == null ? 0 : channel.write(src); } @Override @@ -77,10 +84,13 @@ throw new IOException(); } if (sharedChannel == null) { - if (DebugOptions.PrintGraphFile.getValue(options)) { - sharedChannel = createFileChannel(pathProvider); - } else { + PrintGraphTarget target = DebugOptions.PrintGraph.getValue(options); + if (target == PrintGraphTarget.File) { + sharedChannel = createFileChannel(pathProvider, null); + } else if (target == PrintGraphTarget.Network) { sharedChannel = createNetworkChannel(pathProvider, options); + } else { + TTY.println("WARNING: Graph dumping requested but value of %s option is %s", DebugOptions.PrintGraph.getName(), PrintGraphTarget.Disable); } } return sharedChannel; @@ -88,10 +98,11 @@ private static WritableByteChannel createNetworkChannel(Supplier pathProvider, OptionValues options) throws IOException { String host = PrintGraphHost.getValue(options); - int port = PrintBinaryGraphPort.getValue(options); + int port = PrintGraphPort.getValue(options); try { WritableByteChannel channel = SocketChannel.open(new InetSocketAddress(host, port)); - TTY.println("Connected to the IGV on %s:%d", host, port); + String targetAnnouncement = String.format("Connected to the IGV on %s:%d", host, port); + maybeAnnounceTarget(targetAnnouncement); return channel; } catch (ClosedByInterruptException | InterruptedIOException e) { /* @@ -101,18 +112,39 @@ */ return null; } catch (IOException e) { - if (!DebugOptions.PrintGraphFile.hasBeenSet(options)) { - return createFileChannel(pathProvider); + String networkFailure = String.format("Could not connect to the IGV on %s:%d", host, port); + if (pathProvider != null) { + return createFileChannel(pathProvider, networkFailure); } else { - throw new IOException(String.format("Could not connect to the IGV on %s:%d", host, port), e); + throw new IOException(networkFailure, e); } } } - private static WritableByteChannel createFileChannel(Supplier pathProvider) throws IOException { + @NativeImageReinitialize private static String lastTargetAnnouncement; + + private static void maybeAnnounceTarget(String targetAnnouncement) { + if (!targetAnnouncement.equals(lastTargetAnnouncement)) { + // Ignore races - an extra announcement is ok + lastTargetAnnouncement = targetAnnouncement; + TTY.println(targetAnnouncement); + } + } + + private static WritableByteChannel createFileChannel(Supplier pathProvider, String networkFailure) throws IOException { Path path = pathProvider.get(); try { - return FileChannel.open(path, StandardOpenOption.WRITE, StandardOpenOption.CREATE); + FileChannel channel = FileChannel.open(path, StandardOpenOption.WRITE, StandardOpenOption.CREATE); + File dir = path.toFile(); + if (!dir.isDirectory()) { + dir = dir.getParentFile(); + } + if (networkFailure == null) { + maybeAnnounceTarget("Dumping IGV graphs in " + dir); + } else { + maybeAnnounceTarget(networkFailure + ". Dumping IGV graphs in " + dir); + } + return channel; } catch (IOException e) { throw new IOException(String.format("Failed to open %s to dump IGV graphs", path), e); } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.debug/src/org/graalvm/compiler/debug/PathUtilities.java 2019-03-12 08:08:57.375325065 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.debug/src/org/graalvm/compiler/debug/PathUtilities.java 2019-03-12 08:08:57.011322699 +0100 @@ -84,7 +84,7 @@ private static final String ELLIPSIS = "..."; - static Path createUnique(OptionValues options, OptionKey baseNameOption, String id, String label, String ext, boolean createDirectory) throws IOException { + static Path createUnique(OptionValues options, OptionKey baseNameOption, String id, String label, String ext, boolean createMissingDirectory) throws IOException { String uniqueTag = ""; int dumpCounter = 1; String prefix; @@ -118,7 +118,7 @@ Path dumpDir = DebugOptions.getDumpDirectory(options); Path result = Paths.get(dumpDir.toString(), fileName); try { - if (createDirectory) { + if (createMissingDirectory) { return Files.createDirectory(result); } else { try { --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.graph/src/org/graalvm/compiler/graph/NodeClass.java 2019-03-12 08:08:57.871328289 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.graph/src/org/graalvm/compiler/graph/NodeClass.java 2019-03-12 08:08:57.499325871 +0100 @@ -24,9 +24,7 @@ package org.graalvm.compiler.graph; -import static org.graalvm.compiler.core.common.Fields.translateInto; import static org.graalvm.compiler.debug.GraalError.shouldNotReachHere; -import static org.graalvm.compiler.graph.Edges.translateInto; import static org.graalvm.compiler.graph.Graph.isModificationCountsEnabled; import static org.graalvm.compiler.graph.InputEdges.translateInto; import static org.graalvm.compiler.graph.Node.WithAllEdges; --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.aarch64/src/org/graalvm/compiler/hotspot/aarch64/AArch64HotSpotBackend.java 2019-03-12 08:08:58.391331670 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.aarch64/src/org/graalvm/compiler/hotspot/aarch64/AArch64HotSpotBackend.java 2019-03-12 08:08:58.019329252 +0100 @@ -36,6 +36,7 @@ import jdk.internal.vm.compiler.collections.EconomicSet; import org.graalvm.compiler.asm.Assembler; +import org.graalvm.compiler.asm.BranchTargetOutOfBoundsException; import org.graalvm.compiler.asm.Label; import org.graalvm.compiler.asm.aarch64.AArch64Address; import org.graalvm.compiler.asm.aarch64.AArch64Assembler; @@ -47,6 +48,7 @@ import org.graalvm.compiler.core.common.LIRKind; import org.graalvm.compiler.core.common.alloc.RegisterAllocationConfig; import org.graalvm.compiler.core.common.spi.ForeignCallLinkage; +import org.graalvm.compiler.core.gen.LIRGenerationProvider; import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig; import org.graalvm.compiler.hotspot.HotSpotDataBuilder; import org.graalvm.compiler.hotspot.HotSpotGraalRuntimeProvider; @@ -86,21 +88,16 @@ /** * HotSpot AArch64 specific backend. */ -public class AArch64HotSpotBackend extends HotSpotHostBackend { +public class AArch64HotSpotBackend extends HotSpotHostBackend implements LIRGenerationProvider { public AArch64HotSpotBackend(GraalHotSpotVMConfig config, HotSpotGraalRuntimeProvider runtime, HotSpotProviders providers) { super(config, runtime, providers); } - @Override - public FrameMapBuilder newFrameMapBuilder(RegisterConfig registerConfig) { + private FrameMapBuilder newFrameMapBuilder(RegisterConfig registerConfig) { RegisterConfig registerConfigNonNull = registerConfig == null ? getCodeCache().getRegisterConfig() : registerConfig; - return new AArch64FrameMapBuilder(newFrameMap(registerConfigNonNull), getCodeCache(), registerConfigNonNull); - } - - @Override - public FrameMap newFrameMap(RegisterConfig registerConfig) { - return new AArch64FrameMap(getCodeCache(), registerConfig, this); + FrameMap frameMap = new AArch64FrameMap(getCodeCache(), registerConfigNonNull, this); + return new AArch64FrameMapBuilder(frameMap, getCodeCache(), registerConfigNonNull); } @Override @@ -109,8 +106,9 @@ } @Override - public LIRGenerationResult newLIRGenerationResult(CompilationIdentifier compilationId, LIR lir, FrameMapBuilder frameMapBuilder, StructuredGraph graph, Object stub) { - return new HotSpotLIRGenerationResult(compilationId, lir, frameMapBuilder, makeCallingConvention(graph, (Stub) stub), stub, config.requiresReservedStackCheck(graph.getMethods())); + public LIRGenerationResult newLIRGenerationResult(CompilationIdentifier compilationId, LIR lir, RegisterConfig registerConfig, StructuredGraph graph, Object stub) { + return new HotSpotLIRGenerationResult(compilationId, lir, newFrameMapBuilder(registerConfig), makeCallingConvention(graph, (Stub) stub), stub, + config.requiresReservedStackCheck(graph.getMethods())); } @Override @@ -219,18 +217,13 @@ } @Override - protected Assembler createAssembler(FrameMap frameMap) { - return new AArch64MacroAssembler(getTarget()); - } - - @Override public CompilationResultBuilder newCompilationResultBuilder(LIRGenerationResult lirGenRen, FrameMap frameMap, CompilationResult compilationResult, CompilationResultBuilderFactory factory) { HotSpotLIRGenerationResult gen = (HotSpotLIRGenerationResult) lirGenRen; LIR lir = gen.getLIR(); assert gen.getDeoptimizationRescueSlot() == null || frameMap.frameNeedsAllocating() : "method that can deoptimize must have a frame"; Stub stub = gen.getStub(); - Assembler masm = createAssembler(frameMap); + Assembler masm = new AArch64MacroAssembler(getTarget()); HotSpotFrameContext frameContext = new HotSpotFrameContext(stub != null); DataBuilder dataBuilder = new HotSpotDataBuilder(getCodeCache().getTarget()); @@ -252,11 +245,25 @@ @Override public void emitCode(CompilationResultBuilder crb, LIR lir, ResolvedJavaMethod installedCodeOwner) { + Label verifiedStub = new Label(); + crb.buildLabelOffsets(lir); + try { + emitCode(crb, lir, installedCodeOwner, verifiedStub); + } catch (BranchTargetOutOfBoundsException e) { + // A branch estimation was wrong, now retry with conservative label ranges, this + // should always work + crb.setConservativeLabelRanges(); + crb.resetForEmittingCode(); + lir.resetLabels(); + verifiedStub.reset(); + emitCode(crb, lir, installedCodeOwner, verifiedStub); + } + } + + private void emitCode(CompilationResultBuilder crb, LIR lir, ResolvedJavaMethod installedCodeOwner, Label verifiedStub) { AArch64MacroAssembler masm = (AArch64MacroAssembler) crb.asm; FrameMap frameMap = crb.frameMap; RegisterConfig regConfig = frameMap.getRegisterConfig(); - Label verifiedStub = new Label(); - emitCodePrefix(crb, installedCodeOwner, masm, regConfig, verifiedStub); emitCodeBody(crb, lir, masm); emitCodeSuffix(crb, masm, frameMap); --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.aarch64/src/org/graalvm/compiler/hotspot/aarch64/AArch64HotSpotJumpToExceptionHandlerInCallerOp.java 2019-03-12 08:08:58.859334712 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.aarch64/src/org/graalvm/compiler/hotspot/aarch64/AArch64HotSpotJumpToExceptionHandlerInCallerOp.java 2019-03-12 08:08:58.495332346 +0100 @@ -37,7 +37,7 @@ import org.graalvm.compiler.lir.LIRInstructionClass; import org.graalvm.compiler.lir.Opcode; import org.graalvm.compiler.lir.asm.CompilationResultBuilder; -import org.graalvm.compiler.serviceprovider.GraalServices; +import org.graalvm.compiler.serviceprovider.JavaVersionUtil; import jdk.vm.ci.code.Register; import jdk.vm.ci.meta.AllocatableValue; @@ -71,7 +71,7 @@ public void emitCode(CompilationResultBuilder crb, AArch64MacroAssembler masm) { leaveFrame(crb, masm, /* emitSafepoint */false, false); - if (GraalServices.JAVA_SPECIFICATION_VERSION < 8) { + if (JavaVersionUtil.JAVA_SPECIFICATION_VERSION < 8) { // Restore sp from fp if the exception PC is a method handle call site. try (ScratchRegister sc = masm.getScratchRegister()) { Register scratch = sc.getRegister(); --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotAddressLowering.java 2019-03-12 08:08:59.351337910 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotAddressLowering.java 2019-03-12 08:08:58.983335518 +0100 @@ -199,7 +199,7 @@ CountedLoopInfo countedLoopInfo = loop.counted(); IntegerStamp initStamp = (IntegerStamp) inductionVariable.initNode().stamp(NodeView.DEFAULT); if (initStamp.isPositive()) { - if (inductionVariable.isConstantExtremum()) { + if (inductionVariable.isConstantExtremum() && countedLoopInfo.counterNeverOverflows()) { long init = inductionVariable.constantInit(); long stride = inductionVariable.constantStride(); long extremum = inductionVariable.constantExtremum(); @@ -211,7 +211,9 @@ } } } - if (countedLoopInfo.getCounter() == inductionVariable && inductionVariable.direction() == InductionVariable.Direction.Up && countedLoopInfo.getOverFlowGuard() != null) { + if (countedLoopInfo.getCounter() == inductionVariable && + inductionVariable.direction() == InductionVariable.Direction.Up && + (countedLoopInfo.getOverFlowGuard() != null || countedLoopInfo.counterNeverOverflows())) { return graph.unique(new ZeroExtendNode(input, INT_BITS, ADDRESS_BITS, true)); } } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotBackend.java 2019-03-12 08:08:59.851341161 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotBackend.java 2019-03-12 08:08:59.479338742 +0100 @@ -43,7 +43,7 @@ import org.graalvm.compiler.core.common.CompilationIdentifier; import org.graalvm.compiler.core.common.LIRKind; import org.graalvm.compiler.core.common.alloc.RegisterAllocationConfig; -import org.graalvm.compiler.core.target.Backend; +import org.graalvm.compiler.core.gen.LIRGenerationProvider; import org.graalvm.compiler.debug.DebugContext; import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig; import org.graalvm.compiler.hotspot.HotSpotDataBuilder; @@ -85,21 +85,16 @@ /** * HotSpot AMD64 specific backend. */ -public class AMD64HotSpotBackend extends HotSpotHostBackend { +public class AMD64HotSpotBackend extends HotSpotHostBackend implements LIRGenerationProvider { public AMD64HotSpotBackend(GraalHotSpotVMConfig config, HotSpotGraalRuntimeProvider runtime, HotSpotProviders providers) { super(config, runtime, providers); } - @Override - public FrameMapBuilder newFrameMapBuilder(RegisterConfig registerConfig) { + private FrameMapBuilder newFrameMapBuilder(RegisterConfig registerConfig) { RegisterConfig registerConfigNonNull = registerConfig == null ? getCodeCache().getRegisterConfig() : registerConfig; - return new AMD64FrameMapBuilder(newFrameMap(registerConfigNonNull), getCodeCache(), registerConfigNonNull); - } - - @Override - public FrameMap newFrameMap(RegisterConfig registerConfig) { - return new AMD64FrameMap(getCodeCache(), registerConfig, this); + FrameMap frameMap = new AMD64FrameMap(getCodeCache(), registerConfigNonNull, this); + return new AMD64FrameMapBuilder(frameMap, getCodeCache(), registerConfigNonNull); } @Override @@ -108,8 +103,9 @@ } @Override - public LIRGenerationResult newLIRGenerationResult(CompilationIdentifier compilationId, LIR lir, FrameMapBuilder frameMapBuilder, StructuredGraph graph, Object stub) { - return new HotSpotLIRGenerationResult(compilationId, lir, frameMapBuilder, makeCallingConvention(graph, (Stub) stub), stub, config.requiresReservedStackCheck(graph.getMethods())); + public LIRGenerationResult newLIRGenerationResult(CompilationIdentifier compilationId, LIR lir, RegisterConfig registerConfig, StructuredGraph graph, Object stub) { + return new HotSpotLIRGenerationResult(compilationId, lir, newFrameMapBuilder(registerConfig), makeCallingConvention(graph, (Stub) stub), stub, + config.requiresReservedStackCheck(graph.getMethods())); } @Override @@ -196,11 +192,6 @@ } @Override - protected Assembler createAssembler(FrameMap frameMap) { - return new AMD64MacroAssembler(getTarget()); - } - - @Override public CompilationResultBuilder newCompilationResultBuilder(LIRGenerationResult lirGenRen, FrameMap frameMap, CompilationResult compilationResult, CompilationResultBuilderFactory factory) { // Omit the frame if the method: // - has no spill slots or other slots allocated during register allocation @@ -216,7 +207,7 @@ boolean omitFrame = CanOmitFrame.getValue(options) && !frameMap.frameNeedsAllocating() && !lir.hasArgInCallerFrame() && !gen.hasForeignCall(); Stub stub = gen.getStub(); - Assembler masm = createAssembler(frameMap); + Assembler masm = new AMD64MacroAssembler(getTarget()); HotSpotFrameContext frameContext = new HotSpotFrameContext(stub != null, omitFrame); DataBuilder dataBuilder = new HotSpotDataBuilder(getCodeCache().getTarget()); CompilationResultBuilder crb = factory.createBuilder(getCodeCache(), getForeignCalls(), frameMap, masm, dataBuilder, frameContext, options, debug, compilationResult, Register.None); @@ -258,7 +249,7 @@ /** * Emits the code prior to the verified entry point. * - * @param installedCodeOwner see {@link Backend#emitCode} + * @param installedCodeOwner see {@link LIRGenerationProvider#emitCode} */ public void emitCodePrefix(ResolvedJavaMethod installedCodeOwner, CompilationResultBuilder crb, AMD64MacroAssembler asm, RegisterConfig regConfig, Label verifiedEntry) { HotSpotProviders providers = getProviders(); @@ -309,14 +300,14 @@ /** * Emits the code which starts at the verified entry point. * - * @param installedCodeOwner see {@link Backend#emitCode} + * @param installedCodeOwner see {@link LIRGenerationProvider#emitCode} */ public void emitCodeBody(ResolvedJavaMethod installedCodeOwner, CompilationResultBuilder crb, LIR lir) { crb.emit(lir); } /** - * @param installedCodeOwner see {@link Backend#emitCode} + * @param installedCodeOwner see {@link LIRGenerationProvider#emitCode} */ public void emitCodeSuffix(ResolvedJavaMethod installedCodeOwner, CompilationResultBuilder crb, AMD64MacroAssembler asm, FrameMap frameMap) { HotSpotProviders providers = getProviders(); --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotBackendFactory.java 2019-03-12 08:09:00.339344332 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotBackendFactory.java 2019-03-12 08:08:59.971341940 +0100 @@ -25,7 +25,6 @@ package org.graalvm.compiler.hotspot.amd64; import static jdk.vm.ci.common.InitTimer.timer; -import static org.graalvm.compiler.hotspot.HotSpotBackend.Options.GraalArithmeticStubs; import java.util.ArrayList; import java.util.List; @@ -141,7 +140,7 @@ replacements = createReplacements(options, p, snippetReflection, bytecodeProvider); } try (InitTimer rt = timer("create GraphBuilderPhase plugins")) { - plugins = createGraphBuilderPlugins(compilerConfiguration, config, options, target, constantReflection, foreignCalls, metaAccess, snippetReflection, replacements, wordTypes); + plugins = createGraphBuilderPlugins(compilerConfiguration, config, target, constantReflection, foreignCalls, metaAccess, snippetReflection, replacements, wordTypes); replacements.setGraphBuilderPlugins(plugins); } try (InitTimer rt = timer("create Suites provider")) { @@ -157,11 +156,11 @@ } } - protected Plugins createGraphBuilderPlugins(CompilerConfiguration compilerConfiguration, GraalHotSpotVMConfig config, OptionValues options, TargetDescription target, + protected Plugins createGraphBuilderPlugins(CompilerConfiguration compilerConfiguration, GraalHotSpotVMConfig config, TargetDescription target, HotSpotConstantReflectionProvider constantReflection, HotSpotHostForeignCallsProvider foreignCalls, HotSpotMetaAccessProvider metaAccess, HotSpotSnippetReflectionProvider snippetReflection, HotSpotReplacementsImpl replacements, HotSpotWordTypes wordTypes) { Plugins plugins = HotSpotGraphBuilderPlugins.create(compilerConfiguration, config, wordTypes, metaAccess, constantReflection, snippetReflection, foreignCalls, replacements); - AMD64GraphBuilderPlugins.register(plugins, replacements.getDefaultReplacementBytecodeProvider(), (AMD64) target.arch, GraalArithmeticStubs.getValue(options), false); + AMD64GraphBuilderPlugins.register(plugins, replacements.getDefaultReplacementBytecodeProvider(), (AMD64) target.arch, false); return plugins; } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotForeignCallsProvider.java 2019-03-12 08:09:00.803347348 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotForeignCallsProvider.java 2019-03-12 08:09:00.439344982 +0100 @@ -30,6 +30,7 @@ import static jdk.vm.ci.meta.Value.ILLEGAL; import static org.graalvm.compiler.hotspot.HotSpotBackend.EXCEPTION_HANDLER; import static org.graalvm.compiler.hotspot.HotSpotBackend.EXCEPTION_HANDLER_IN_CALLER; +import static org.graalvm.compiler.hotspot.HotSpotBackend.Options.GraalArithmeticStubs; import static org.graalvm.compiler.hotspot.HotSpotForeignCallLinkage.JUMP_ADDRESS; import static org.graalvm.compiler.hotspot.HotSpotForeignCallLinkage.Reexecutability.REEXECUTABLE; import static org.graalvm.compiler.hotspot.HotSpotForeignCallLinkage.Reexecutability.REEXECUTABLE_ONLY_AFTER_EXCEPTION; @@ -38,10 +39,16 @@ import static org.graalvm.compiler.hotspot.HotSpotForeignCallLinkage.Transition.LEAF_NOFP; import static org.graalvm.compiler.hotspot.replacements.CRC32CSubstitutions.UPDATE_BYTES_CRC32C; import static org.graalvm.compiler.hotspot.replacements.CRC32Substitutions.UPDATE_BYTES_CRC32; +import static org.graalvm.compiler.replacements.nodes.BinaryMathIntrinsicNode.BinaryOperation.POW; +import static org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.UnaryOperation.COS; +import static org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.UnaryOperation.EXP; +import static org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.UnaryOperation.LOG; +import static org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.UnaryOperation.LOG10; +import static org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.UnaryOperation.SIN; +import static org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.UnaryOperation.TAN; import static jdk.internal.vm.compiler.word.LocationIdentity.any; import org.graalvm.compiler.core.common.LIRKind; -import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor; import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig; import org.graalvm.compiler.hotspot.HotSpotForeignCallLinkageImpl; import org.graalvm.compiler.hotspot.HotSpotGraalRuntimeProvider; @@ -62,14 +69,6 @@ public class AMD64HotSpotForeignCallsProvider extends HotSpotHostForeignCallsProvider { - public static final ForeignCallDescriptor ARITHMETIC_SIN_STUB = new ForeignCallDescriptor("arithmeticSinStub", double.class, double.class); - public static final ForeignCallDescriptor ARITHMETIC_COS_STUB = new ForeignCallDescriptor("arithmeticCosStub", double.class, double.class); - public static final ForeignCallDescriptor ARITHMETIC_TAN_STUB = new ForeignCallDescriptor("arithmeticTanStub", double.class, double.class); - public static final ForeignCallDescriptor ARITHMETIC_EXP_STUB = new ForeignCallDescriptor("arithmeticExpStub", double.class, double.class); - public static final ForeignCallDescriptor ARITHMETIC_POW_STUB = new ForeignCallDescriptor("arithmeticPowStub", double.class, double.class, double.class); - public static final ForeignCallDescriptor ARITHMETIC_LOG_STUB = new ForeignCallDescriptor("arithmeticLogStub", double.class, double.class); - public static final ForeignCallDescriptor ARITHMETIC_LOG10_STUB = new ForeignCallDescriptor("arithmeticLog10Stub", double.class, double.class); - private final Value[] nativeABICallerSaveRegisters; public AMD64HotSpotForeignCallsProvider(HotSpotJVMCIRuntime jvmciRuntime, HotSpotGraalRuntimeProvider runtime, MetaAccessProvider metaAccess, CodeCacheProvider codeCache, @@ -93,14 +92,6 @@ register(new HotSpotForeignCallLinkageImpl(EXCEPTION_HANDLER, 0L, PRESERVES_REGISTERS, LEAF_NOFP, REEXECUTABLE_ONLY_AFTER_EXCEPTION, exceptionCc, null, any())); register(new HotSpotForeignCallLinkageImpl(EXCEPTION_HANDLER_IN_CALLER, JUMP_ADDRESS, PRESERVES_REGISTERS, LEAF_NOFP, REEXECUTABLE_ONLY_AFTER_EXCEPTION, exceptionCc, null, any())); - link(new AMD64MathStub(ARITHMETIC_LOG_STUB, options, providers, registerStubCall(ARITHMETIC_LOG_STUB, LEAF, REEXECUTABLE, NO_LOCATIONS))); - link(new AMD64MathStub(ARITHMETIC_LOG10_STUB, options, providers, registerStubCall(ARITHMETIC_LOG10_STUB, LEAF, REEXECUTABLE, NO_LOCATIONS))); - link(new AMD64MathStub(ARITHMETIC_SIN_STUB, options, providers, registerStubCall(ARITHMETIC_SIN_STUB, LEAF, REEXECUTABLE, NO_LOCATIONS))); - link(new AMD64MathStub(ARITHMETIC_COS_STUB, options, providers, registerStubCall(ARITHMETIC_COS_STUB, LEAF, REEXECUTABLE, NO_LOCATIONS))); - link(new AMD64MathStub(ARITHMETIC_TAN_STUB, options, providers, registerStubCall(ARITHMETIC_TAN_STUB, LEAF, REEXECUTABLE, NO_LOCATIONS))); - link(new AMD64MathStub(ARITHMETIC_EXP_STUB, options, providers, registerStubCall(ARITHMETIC_EXP_STUB, LEAF, REEXECUTABLE, NO_LOCATIONS))); - link(new AMD64MathStub(ARITHMETIC_POW_STUB, options, providers, registerStubCall(ARITHMETIC_POW_STUB, LEAF, REEXECUTABLE, NO_LOCATIONS))); - if (config.useCRC32Intrinsics) { // This stub does callee saving registerForeignCall(UPDATE_BYTES_CRC32, config.updateBytesCRC32Stub, NativeCall, PRESERVES_REGISTERS, LEAF_NOFP, REEXECUTABLE_ONLY_AFTER_EXCEPTION, any()); @@ -138,4 +129,19 @@ return nativeABICallerSaveRegisters; } + @Override + protected void registerMathStubs(GraalHotSpotVMConfig hotSpotVMConfig, HotSpotProviders providers, OptionValues options) { + if (GraalArithmeticStubs.getValue(options)) { + link(new AMD64MathStub(SIN, options, providers, registerStubCall(SIN.foreignCallDescriptor, LEAF, REEXECUTABLE, NO_LOCATIONS))); + link(new AMD64MathStub(COS, options, providers, registerStubCall(COS.foreignCallDescriptor, LEAF, REEXECUTABLE, NO_LOCATIONS))); + link(new AMD64MathStub(TAN, options, providers, registerStubCall(TAN.foreignCallDescriptor, LEAF, REEXECUTABLE, NO_LOCATIONS))); + link(new AMD64MathStub(EXP, options, providers, registerStubCall(EXP.foreignCallDescriptor, LEAF, REEXECUTABLE, NO_LOCATIONS))); + link(new AMD64MathStub(LOG, options, providers, registerStubCall(LOG.foreignCallDescriptor, LEAF, REEXECUTABLE, NO_LOCATIONS))); + link(new AMD64MathStub(LOG10, options, providers, registerStubCall(LOG10.foreignCallDescriptor, LEAF, REEXECUTABLE, NO_LOCATIONS))); + link(new AMD64MathStub(POW, options, providers, registerStubCall(POW.foreignCallDescriptor, LEAF, REEXECUTABLE, NO_LOCATIONS))); + } else { + super.registerMathStubs(hotSpotVMConfig, providers, options); + } + } + } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotJumpToExceptionHandlerInCallerOp.java 2019-03-12 08:09:01.295350546 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotJumpToExceptionHandlerInCallerOp.java 2019-03-12 08:09:00.927348154 +0100 @@ -24,10 +24,10 @@ package org.graalvm.compiler.hotspot.amd64; -import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.REG; import static jdk.vm.ci.amd64.AMD64.rbp; import static jdk.vm.ci.amd64.AMD64.rsp; import static jdk.vm.ci.code.ValueUtil.asRegister; +import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.REG; import org.graalvm.compiler.asm.amd64.AMD64Address; import org.graalvm.compiler.asm.amd64.AMD64Assembler.ConditionFlag; @@ -35,7 +35,7 @@ import org.graalvm.compiler.lir.LIRInstructionClass; import org.graalvm.compiler.lir.Opcode; import org.graalvm.compiler.lir.asm.CompilationResultBuilder; -import org.graalvm.compiler.serviceprovider.GraalServices; +import org.graalvm.compiler.serviceprovider.JavaVersionUtil; import jdk.vm.ci.code.Register; import jdk.vm.ci.meta.AllocatableValue; @@ -71,7 +71,7 @@ // Discard the return address, thus completing restoration of caller frame masm.incrementq(rsp, 8); - if (GraalServices.JAVA_SPECIFICATION_VERSION < 8) { + if (JavaVersionUtil.JAVA_SPECIFICATION_VERSION < 8) { // Restore rsp from rbp if the exception PC is a method handle call site. AMD64Address dst = new AMD64Address(thread, isMethodHandleReturnOffset); masm.cmpl(dst, 0); --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotLIRGenerator.java 2019-03-12 08:09:01.787353743 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotLIRGenerator.java 2019-03-12 08:09:01.415351326 +0100 @@ -118,7 +118,7 @@ } private AMD64HotSpotLIRGenerator(HotSpotProviders providers, GraalHotSpotVMConfig config, LIRGenerationResult lirGenRes, BackupSlotProvider backupSlotProvider) { - this(new AMD64HotSpotLIRKindTool(), new AMD64ArithmeticLIRGenerator(null, new AMD64HotSpotMaths()), new AMD64HotSpotMoveFactory(backupSlotProvider), providers, config, lirGenRes); + this(new AMD64HotSpotLIRKindTool(), new AMD64ArithmeticLIRGenerator(null), new AMD64HotSpotMoveFactory(backupSlotProvider), providers, config, lirGenRes); } protected AMD64HotSpotLIRGenerator(LIRKindTool lirKindTool, AMD64ArithmeticLIRGenerator arithmeticLIRGen, MoveFactory moveFactory, HotSpotProviders providers, GraalHotSpotVMConfig config, --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotLoweringProvider.java 2019-03-12 08:09:02.239356681 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotLoweringProvider.java 2019-03-12 08:09:01.875354315 +0100 @@ -25,15 +25,8 @@ package org.graalvm.compiler.hotspot.amd64; import static org.graalvm.compiler.hotspot.HotSpotBackend.Options.GraalArithmeticStubs; -import static org.graalvm.compiler.hotspot.amd64.AMD64HotSpotForeignCallsProvider.ARITHMETIC_COS_STUB; -import static org.graalvm.compiler.hotspot.amd64.AMD64HotSpotForeignCallsProvider.ARITHMETIC_EXP_STUB; -import static org.graalvm.compiler.hotspot.amd64.AMD64HotSpotForeignCallsProvider.ARITHMETIC_LOG10_STUB; -import static org.graalvm.compiler.hotspot.amd64.AMD64HotSpotForeignCallsProvider.ARITHMETIC_LOG_STUB; -import static org.graalvm.compiler.hotspot.amd64.AMD64HotSpotForeignCallsProvider.ARITHMETIC_POW_STUB; -import static org.graalvm.compiler.hotspot.amd64.AMD64HotSpotForeignCallsProvider.ARITHMETIC_SIN_STUB; -import static org.graalvm.compiler.hotspot.amd64.AMD64HotSpotForeignCallsProvider.ARITHMETIC_TAN_STUB; -import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor; +import org.graalvm.compiler.api.replacements.Snippet; import org.graalvm.compiler.core.common.spi.ForeignCallsProvider; import org.graalvm.compiler.debug.DebugHandlersFactory; import org.graalvm.compiler.graph.Node; @@ -44,21 +37,26 @@ import org.graalvm.compiler.hotspot.meta.HotSpotRegistersProvider; import org.graalvm.compiler.hotspot.nodes.profiling.ProfileNode; import org.graalvm.compiler.hotspot.replacements.profiling.ProbabilisticProfileSnippets; +import org.graalvm.compiler.nodes.StructuredGraph; import org.graalvm.compiler.nodes.calc.FloatConvertNode; +import org.graalvm.compiler.nodes.extended.ForeignCallNode; import org.graalvm.compiler.nodes.spi.LoweringTool; import org.graalvm.compiler.options.OptionValues; import org.graalvm.compiler.replacements.amd64.AMD64ConvertSnippets; -import org.graalvm.compiler.replacements.nodes.BinaryMathIntrinsicNode.BinaryOperation; +import org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode; import org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.UnaryOperation; +import org.graalvm.compiler.serviceprovider.JavaVersionUtil; import jdk.vm.ci.code.TargetDescription; import jdk.vm.ci.hotspot.HotSpotConstantReflectionProvider; import jdk.vm.ci.meta.MetaAccessProvider; +import jdk.vm.ci.meta.ResolvedJavaMethod; public class AMD64HotSpotLoweringProvider extends DefaultHotSpotLoweringProvider { private AMD64ConvertSnippets.Templates convertSnippets; private ProbabilisticProfileSnippets.Templates profileSnippets; + private AMD64X87MathSnippets.Templates mathSnippets; public AMD64HotSpotLoweringProvider(HotSpotGraalRuntimeProvider runtime, MetaAccessProvider metaAccess, ForeignCallsProvider foreignCalls, HotSpotRegistersProvider registers, HotSpotConstantReflectionProvider constantReflection, TargetDescription target) { @@ -68,9 +66,10 @@ @Override public void initialize(OptionValues options, Iterable factories, HotSpotProviders providers, GraalHotSpotVMConfig config) { convertSnippets = new AMD64ConvertSnippets.Templates(options, factories, providers, providers.getSnippetReflection(), providers.getCodeCache().getTarget()); - profileSnippets = ProfileNode.Options.ProbabilisticProfiling.getValue(options) + profileSnippets = ProfileNode.Options.ProbabilisticProfiling.getValue(options) && !JavaVersionUtil.Java8OrEarlier ? new ProbabilisticProfileSnippets.Templates(options, factories, providers, providers.getCodeCache().getTarget()) : null; + mathSnippets = new AMD64X87MathSnippets.Templates(options, factories, providers, providers.getSnippetReflection(), providers.getCodeCache().getTarget()); super.initialize(options, factories, providers, config); } @@ -80,47 +79,49 @@ convertSnippets.lower((FloatConvertNode) n, tool); } else if (profileSnippets != null && n instanceof ProfileNode) { profileSnippets.lower((ProfileNode) n, tool); + } else if (n instanceof UnaryMathIntrinsicNode) { + lowerUnaryMath((UnaryMathIntrinsicNode) n, tool); } else { super.lower(n, tool); } } - @Override - protected ForeignCallDescriptor toForeignCall(UnaryOperation operation) { - if (GraalArithmeticStubs.getValue(runtime.getOptions())) { - switch (operation) { - case LOG: - return ARITHMETIC_LOG_STUB; - case LOG10: - return ARITHMETIC_LOG10_STUB; + private void lowerUnaryMath(UnaryMathIntrinsicNode math, LoweringTool tool) { + if (tool.getLoweringStage() == LoweringTool.StandardLoweringStage.HIGH_TIER) { + return; + } + StructuredGraph graph = math.graph(); + ResolvedJavaMethod method = graph.method(); + if (method != null) { + if (method.getAnnotation(Snippet.class) != null) { + // In the context of SnippetStub, i.e., Graal-generated stubs, use the LIR + // lowering to emit the stub assembly code instead of the Node lowering. + return; + } + } + if (!GraalArithmeticStubs.getValue(graph.getOptions())) { + switch (math.getOperation()) { case SIN: - return ARITHMETIC_SIN_STUB; case COS: - return ARITHMETIC_COS_STUB; case TAN: - return ARITHMETIC_TAN_STUB; - case EXP: - return ARITHMETIC_EXP_STUB; + // Math.sin(), .cos() and .tan() guarantee a value within 1 ULP of the exact + // result, but x87 trigonometric FPU instructions are only that accurate within + // [-pi/4, pi/4]. The snippets fall back to a foreign call to HotSpot stubs + // should the inputs outside of that interval. + mathSnippets.lower(math, tool); + return; + case LOG: + math.replaceAtUsages(graph.addOrUnique(new AMD64X87MathIntrinsicNode(math.getValue(), UnaryOperation.LOG))); + return; + case LOG10: + math.replaceAtUsages(graph.addOrUnique(new AMD64X87MathIntrinsicNode(math.getValue(), UnaryOperation.LOG10))); + return; } - } else if (operation == UnaryOperation.EXP) { - return operation.foreignCallDescriptor; } - // Lower only using LIRGenerator - return null; - } - @Override - protected ForeignCallDescriptor toForeignCall(BinaryOperation operation) { - if (GraalArithmeticStubs.getValue(runtime.getOptions())) { - switch (operation) { - case POW: - return ARITHMETIC_POW_STUB; - } - } else if (operation == BinaryOperation.POW) { - return operation.foreignCallDescriptor; - } - // Lower only using LIRGenerator - return null; + ForeignCallNode call = graph.add(new ForeignCallNode(foreignCalls, math.getOperation().foreignCallDescriptor, math.getValue())); + graph.addAfterFixed(tool.lastFixedNode(), call); + math.replaceAtUsages(call); } @Override --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64MathStub.java 2019-03-12 08:09:02.731359878 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64MathStub.java 2019-03-12 08:09:02.359357461 +0100 @@ -24,16 +24,8 @@ package org.graalvm.compiler.hotspot.amd64; -import static org.graalvm.compiler.hotspot.amd64.AMD64HotSpotForeignCallsProvider.ARITHMETIC_COS_STUB; -import static org.graalvm.compiler.hotspot.amd64.AMD64HotSpotForeignCallsProvider.ARITHMETIC_EXP_STUB; -import static org.graalvm.compiler.hotspot.amd64.AMD64HotSpotForeignCallsProvider.ARITHMETIC_LOG10_STUB; -import static org.graalvm.compiler.hotspot.amd64.AMD64HotSpotForeignCallsProvider.ARITHMETIC_LOG_STUB; -import static org.graalvm.compiler.hotspot.amd64.AMD64HotSpotForeignCallsProvider.ARITHMETIC_POW_STUB; -import static org.graalvm.compiler.hotspot.amd64.AMD64HotSpotForeignCallsProvider.ARITHMETIC_SIN_STUB; -import static org.graalvm.compiler.hotspot.amd64.AMD64HotSpotForeignCallsProvider.ARITHMETIC_TAN_STUB; - import org.graalvm.compiler.api.replacements.Snippet; -import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor; +import org.graalvm.compiler.debug.GraalError; import org.graalvm.compiler.hotspot.HotSpotForeignCallLinkage; import org.graalvm.compiler.hotspot.meta.HotSpotProviders; import org.graalvm.compiler.hotspot.stubs.SnippetStub; @@ -48,33 +40,38 @@ */ public class AMD64MathStub extends SnippetStub { - public AMD64MathStub(ForeignCallDescriptor descriptor, OptionValues options, HotSpotProviders providers, HotSpotForeignCallLinkage linkage) { - super(snippetName(descriptor), options, providers, linkage); + public AMD64MathStub(UnaryOperation operation, OptionValues options, HotSpotProviders providers, HotSpotForeignCallLinkage linkage) { + super(snippetName(operation), options, providers, linkage); } - private static String snippetName(ForeignCallDescriptor descriptor) { - if (descriptor == ARITHMETIC_LOG_STUB) { - return "log"; - } - if (descriptor == ARITHMETIC_LOG10_STUB) { - return "log10"; - } - if (descriptor == ARITHMETIC_SIN_STUB) { - return "sin"; - } - if (descriptor == ARITHMETIC_COS_STUB) { - return "cos"; - } - if (descriptor == ARITHMETIC_TAN_STUB) { - return "tan"; - } - if (descriptor == ARITHMETIC_EXP_STUB) { - return "exp"; + public AMD64MathStub(BinaryOperation operation, OptionValues options, HotSpotProviders providers, HotSpotForeignCallLinkage linkage) { + super(snippetName(operation), options, providers, linkage); + } + + private static String snippetName(UnaryOperation operation) { + switch (operation) { + case SIN: + return "sin"; + case COS: + return "cos"; + case TAN: + return "tan"; + case EXP: + return "exp"; + case LOG: + return "log"; + case LOG10: + return "log10"; + default: + throw GraalError.shouldNotReachHere("Unknown operation " + operation); } - if (descriptor == ARITHMETIC_POW_STUB) { + } + + private static String snippetName(BinaryOperation operation) { + if (operation == BinaryOperation.POW) { return "pow"; } - throw new InternalError("Unknown operation " + descriptor); + throw GraalError.shouldNotReachHere("Unknown operation " + operation); } @Snippet --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.lir.test/src/org/graalvm/compiler/hotspot/lir/test/BenchmarkCounterOverflowTest.java 2019-03-12 08:09:03.235363154 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.lir.test/src/org/graalvm/compiler/hotspot/lir/test/BenchmarkCounterOverflowTest.java 2019-03-12 08:09:02.859360711 +0100 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -27,9 +27,12 @@ import static org.graalvm.compiler.test.SubprocessUtil.getVMCommandLine; import static org.graalvm.compiler.test.SubprocessUtil.withoutDebuggerArguments; +import java.io.File; import java.util.ArrayList; import java.util.Iterator; import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.graalvm.compiler.api.directives.GraalDirectives; import org.graalvm.compiler.core.common.LIRKind; @@ -121,6 +124,7 @@ Assert.assertNotEquals("Expected non-zero exit status", 0, proc.exitCode); Iterator it = proc.output.iterator(); + boolean foundProblematicFrame = false; while (it.hasNext()) { String line = it.next(); if (line.contains("Problematic frame:")) { @@ -130,11 +134,34 @@ } line = it.next(); if (line.contains(BenchmarkCounterOverflowTest.class.getName() + ".test")) { - return; + foundProblematicFrame = true; + break; } Assert.fail("Unexpected stack trace: " + line); } } - Assert.fail(String.format("Could not find method in output:%n%s", proc)); + // find and delete hserr file + while (it.hasNext()) { + String line = it.next(); + if (line.contains("An error report file with more information is saved as:")) { + if (!it.hasNext()) { + // no more line + break; + } + line = it.next(); + Pattern pattern = Pattern.compile("^# (.*hs_err_pid.*log)$"); + Matcher matcher = pattern.matcher(line); + if (matcher.matches()) { + File hserrFile = new File(matcher.group(1)); + if (hserrFile.exists()) { + if (VERBOSE) { + System.out.println("Deleting error report file:" + hserrFile.getAbsolutePath()); + } + hserrFile.delete(); + } + } + } + } + Assert.assertTrue(String.format("Could not find method in output:%n%s", proc), foundProblematicFrame); } } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.sparc/src/org/graalvm/compiler/hotspot/sparc/SPARCHotSpotBackend.java 2019-03-12 08:09:03.727366351 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.sparc/src/org/graalvm/compiler/hotspot/sparc/SPARCHotSpotBackend.java 2019-03-12 08:09:03.351363908 +0100 @@ -63,6 +63,7 @@ import org.graalvm.compiler.core.common.CompilationIdentifier; import org.graalvm.compiler.core.common.alloc.RegisterAllocationConfig; import org.graalvm.compiler.core.common.cfg.AbstractBlockBase; +import org.graalvm.compiler.core.gen.LIRGenerationProvider; import org.graalvm.compiler.core.sparc.SPARCNodeMatchRules; import org.graalvm.compiler.debug.CounterKey; import org.graalvm.compiler.debug.DebugContext; @@ -109,7 +110,7 @@ /** * HotSpot SPARC specific backend. */ -public class SPARCHotSpotBackend extends HotSpotHostBackend { +public class SPARCHotSpotBackend extends HotSpotHostBackend implements LIRGenerationProvider { private static final SizeEstimateStatistics CONSTANT_ESTIMATED_STATS = new SizeEstimateStatistics("ESTIMATE"); private static final SizeEstimateStatistics CONSTANT_ACTUAL_STATS = new SizeEstimateStatistics("ACTUAL"); @@ -134,15 +135,10 @@ } } - @Override - public FrameMapBuilder newFrameMapBuilder(RegisterConfig registerConfig) { + private FrameMapBuilder newFrameMapBuilder(RegisterConfig registerConfig) { RegisterConfig registerConfigNonNull = registerConfig == null ? getCodeCache().getRegisterConfig() : registerConfig; - return new SPARCFrameMapBuilder(newFrameMap(registerConfigNonNull), getCodeCache(), registerConfigNonNull); - } - - @Override - public FrameMap newFrameMap(RegisterConfig registerConfig) { - return new SPARCFrameMap(getCodeCache(), registerConfig, this); + FrameMap frameMap = new SPARCFrameMap(getCodeCache(), registerConfigNonNull, this); + return new SPARCFrameMapBuilder(frameMap, getCodeCache(), registerConfigNonNull); } @Override @@ -151,8 +147,9 @@ } @Override - public LIRGenerationResult newLIRGenerationResult(CompilationIdentifier compilationId, LIR lir, FrameMapBuilder frameMapBuilder, StructuredGraph graph, Object stub) { - return new HotSpotLIRGenerationResult(compilationId, lir, frameMapBuilder, makeCallingConvention(graph, (Stub) stub), stub, config.requiresReservedStackCheck(graph.getMethods())); + public LIRGenerationResult newLIRGenerationResult(CompilationIdentifier compilationId, LIR lir, RegisterConfig registerConfig, StructuredGraph graph, Object stub) { + return new HotSpotLIRGenerationResult(compilationId, lir, newFrameMapBuilder(registerConfig), makeCallingConvention(graph, (Stub) stub), stub, + config.requiresReservedStackCheck(graph.getMethods())); } @Override @@ -227,18 +224,13 @@ } @Override - protected Assembler createAssembler(FrameMap frameMap) { - return new SPARCMacroAssembler(getTarget()); - } - - @Override public CompilationResultBuilder newCompilationResultBuilder(LIRGenerationResult lirGenRes, FrameMap frameMap, CompilationResult compilationResult, CompilationResultBuilderFactory factory) { HotSpotLIRGenerationResult gen = (HotSpotLIRGenerationResult) lirGenRes; LIR lir = gen.getLIR(); assert gen.getDeoptimizationRescueSlot() == null || frameMap.frameNeedsAllocating() : "method that can deoptimize must have a frame"; Stub stub = gen.getStub(); - Assembler masm = createAssembler(frameMap); + Assembler masm = new SPARCMacroAssembler(getTarget()); // On SPARC we always use stack frames. HotSpotFrameContext frameContext = new HotSpotFrameContext(stub != null); DataBuilder dataBuilder = new HotSpotDataBuilder(getCodeCache().getTarget()); --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.sparc/src/org/graalvm/compiler/hotspot/sparc/SPARCHotSpotMove.java 2019-03-12 08:09:04.223369575 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.sparc/src/org/graalvm/compiler/hotspot/sparc/SPARCHotSpotMove.java 2019-03-12 08:09:03.855367183 +0100 @@ -24,20 +24,21 @@ package org.graalvm.compiler.hotspot.sparc; +import static jdk.vm.ci.code.ValueUtil.asRegister; import static org.graalvm.compiler.asm.sparc.SPARCAssembler.BPR; import static org.graalvm.compiler.asm.sparc.SPARCAssembler.Annul.ANNUL; +import static org.graalvm.compiler.asm.sparc.SPARCAssembler.Annul.NOT_ANNUL; +import static org.graalvm.compiler.asm.sparc.SPARCAssembler.BranchPredict.PREDICT_NOT_TAKEN; import static org.graalvm.compiler.asm.sparc.SPARCAssembler.BranchPredict.PREDICT_TAKEN; +import static org.graalvm.compiler.asm.sparc.SPARCAssembler.RCondition.Rc_nz; import static org.graalvm.compiler.asm.sparc.SPARCAssembler.RCondition.Rc_z; import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.ILLEGAL; import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.REG; import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.STACK; import static org.graalvm.compiler.lir.sparc.SPARCMove.loadFromConstantTable; -import static jdk.vm.ci.code.ValueUtil.asRegister; import org.graalvm.compiler.asm.Label; import org.graalvm.compiler.asm.sparc.SPARCAddress; -import org.graalvm.compiler.asm.sparc.SPARCAssembler.CC; -import org.graalvm.compiler.asm.sparc.SPARCAssembler.ConditionFlag; import org.graalvm.compiler.asm.sparc.SPARCMacroAssembler; import org.graalvm.compiler.asm.sparc.SPARCMacroAssembler.ScratchRegister; import org.graalvm.compiler.core.common.CompressEncoding; @@ -53,6 +54,7 @@ import jdk.vm.ci.hotspot.HotSpotConstant; import jdk.vm.ci.meta.AllocatableValue; import jdk.vm.ci.meta.Constant; +import jdk.vm.ci.sparc.SPARC; public class SPARCHotSpotMove { @@ -150,17 +152,33 @@ if (encoding.hasBase()) { Register baseReg = asRegister(baseRegister); if (!nonNull) { - masm.cmp(inputRegister, baseReg); - masm.movcc(ConditionFlag.Equal, CC.Xcc, baseReg, resReg); - masm.sub(resReg, baseReg, resReg); + Label done = new Label(); + if (inputRegister.equals(resReg)) { + BPR.emit(masm, Rc_nz, ANNUL, PREDICT_TAKEN, inputRegister, done); + masm.sub(inputRegister, baseReg, resReg); + masm.bind(done); + if (encoding.getShift() != 0) { + masm.srlx(resReg, encoding.getShift(), resReg); + } + } else { + BPR.emit(masm, Rc_z, NOT_ANNUL, PREDICT_NOT_TAKEN, inputRegister, done); + masm.mov(SPARC.g0, resReg); + masm.sub(inputRegister, baseReg, resReg); + if (encoding.getShift() != 0) { + masm.srlx(resReg, encoding.getShift(), resReg); + } + masm.bind(done); + } } else { masm.sub(inputRegister, baseReg, resReg); + if (encoding.getShift() != 0) { + masm.srlx(resReg, encoding.getShift(), resReg); + } } + } else { if (encoding.getShift() != 0) { - masm.srlx(resReg, encoding.getShift(), resReg); + masm.srlx(inputRegister, encoding.getShift(), resReg); } - } else { - masm.srlx(inputRegister, encoding.getShift(), resReg); } } } @@ -196,7 +214,7 @@ public static void emitUncompressCode(SPARCMacroAssembler masm, Register inputRegister, Register resReg, Register baseReg, int shift, boolean nonNull) { Register secondaryInput; if (shift != 0) { - masm.sll(inputRegister, shift, resReg); + masm.sllx(inputRegister, shift, resReg); secondaryInput = resReg; } else { secondaryInput = inputRegister; @@ -207,7 +225,7 @@ masm.add(secondaryInput, baseReg, resReg); } else { Label done = new Label(); - BPR.emit(masm, Rc_z, ANNUL, PREDICT_TAKEN, secondaryInput, done); + BPR.emit(masm, Rc_nz, ANNUL, PREDICT_TAKEN, secondaryInput, done); masm.add(baseReg, secondaryInput, resReg); masm.bind(done); } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/CheckGraalIntrinsics.java 2019-03-12 08:09:04.703372694 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/CheckGraalIntrinsics.java 2019-03-12 08:09:04.331370276 +0100 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -47,7 +47,7 @@ import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugins; import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugins.Binding; import org.graalvm.compiler.runtime.RuntimeProvider; -import org.graalvm.compiler.serviceprovider.GraalServices; +import org.graalvm.compiler.serviceprovider.JavaVersionUtil; import org.graalvm.compiler.test.GraalTest; import org.junit.Test; @@ -257,7 +257,8 @@ "java/lang/reflect/Array.newArray(Ljava/lang/Class;I)Ljava/lang/Object;", // HotSpot MacroAssembler-based intrinsic "sun/nio/cs/ISO_8859_1$Encoder.encodeISOArray([CI[BII)I", - // Stub based intrinsics but implementation seems complex in C2 + // We have implemented implCompressMultiBlock0 on JDK9+. Does it worth + // backporting as corresponding HotSpot stubs are only generated on SPARC? "sun/security/provider/DigestBase.implCompressMultiBlock([BII)I"); // See JDK-8207146. @@ -271,18 +272,12 @@ "jdk/jfr/internal/JVM.getClassId(Ljava/lang/Class;)J"); add(toBeInvestigated, - // Some logic and a stub call - "com/sun/crypto/provider/CounterMode.implCrypt([BII[BI)I", - // Stub and very little logic - "com/sun/crypto/provider/GHASH.processBlocks([BII[J[J)V", // HotSpot MacroAssembler-based intrinsic "java/lang/Math.fma(DDD)D", // HotSpot MacroAssembler-based intrinsic "java/lang/Math.fma(FFF)F", // Just check if the argument is a compile time constant "java/lang/invoke/MethodHandleImpl.isCompileConstant(Ljava/lang/Object;)Z", - // Some logic and a runtime call - "java/util/ArraysSupport.vectorizedMismatch(Ljava/lang/Object;JLjava/lang/Object;JII)I", // Only used as a marker for vectorization? "java/util/stream/Streams$RangeIntSpliterator.forEachRemaining(Ljava/util/function/IntConsumer;)V", // Only implemented on non-AMD64 platforms (some logic and runtime call) @@ -295,9 +290,7 @@ // Control flow, deopts, and a cast "jdk/internal/util/Preconditions.checkIndex(IILjava/util/function/BiFunction;)I", // HotSpot MacroAssembler-based intrinsic - "sun/nio/cs/ISO_8859_1$Encoder.implEncodeISOArray([CI[BII)I", - // Runtime call and some complex compiler logic - "sun/security/provider/DigestBase.implCompressMultiBlock0([BII)I"); + "sun/nio/cs/ISO_8859_1$Encoder.implEncodeISOArray([CI[BII)I"); /* * Per default, all these operations are mapped to some generic method for which we @@ -344,27 +337,45 @@ // Compact string support - HotSpot MacroAssembler-based intrinsic or complex C2 logic. add(toBeInvestigated, "java/lang/StringCoding.hasNegatives([BII)Z", - "java/lang/StringCoding.implEncodeISOArray([BI[BII)I", + "java/lang/StringCoding.implEncodeISOArray([BI[BII)I"); + add(ignore, + // handled through an intrinsic for String.equals itself + "java/lang/StringLatin1.equals([B[B)Z", + + // handled by an intrinsic for StringLatin1.indexOf([BI[BII)I "java/lang/StringLatin1.indexOf([B[B)I", - "java/lang/StringUTF16.getChar([BI)C", - "java/lang/StringUTF16.getChars([BII[CI)V", + + // handled through an intrinsic for String.equals itself + "java/lang/StringUTF16.equals([B[B)Z", + + // handled by an intrinsic for StringUTF16.indexOfUnsafe "java/lang/StringUTF16.indexOf([BI[BII)I", "java/lang/StringUTF16.indexOf([B[B)I", + + // handled by an intrinsic for StringUTF16.indexOfCharUnsafe "java/lang/StringUTF16.indexOfChar([BIII)I", + + // handled by an intrinsic for StringUTF16.indexOfLatin1Unsafe "java/lang/StringUTF16.indexOfLatin1([BI[BII)I", - "java/lang/StringUTF16.indexOfLatin1([B[B)I", - "java/lang/StringUTF16.putChar([BII)V", - "java/lang/StringUTF16.toBytes([CII)[B"); - // These are handled through an intrinsic for String.equals itself - add(ignore, - "java/lang/StringLatin1.equals([B[B)Z", - "java/lang/StringUTF16.equals([B[B)Z"); + "java/lang/StringUTF16.indexOfLatin1([B[B)I"); + + if (!config.useAESCTRIntrinsics) { + add(ignore, + "com/sun/crypto/provider/CounterMode.implCrypt([BII[BI)I"); + } + if (!config.useGHASHIntrinsics()) { + add(ignore, + "com/sun/crypto/provider/GHASH.processBlocks([BII[J[J)V"); + } + if (!(config.useSHA1Intrinsics() || config.useSHA256Intrinsics() || config.useSHA512Intrinsics())) { + add(ignore, + "sun/security/provider/DigestBase.implCompressMultiBlock0([BII)I"); + } } if (isJDK10OrHigher()) { add(toBeInvestigated, - "java/lang/Math.multiplyHigh(JJ)J", - "jdk/internal/util/ArraysSupport.vectorizedMismatch(Ljava/lang/Object;JLjava/lang/Object;JII)I"); + "java/lang/Math.multiplyHigh(JJ)J"); } if (isJDK11OrHigher()) { @@ -401,10 +412,16 @@ // Can we implement these on non-AMD64 platforms? C2 seems to. add(toBeInvestigated, "java/lang/String.compareTo(Ljava/lang/String;)I", + "java/lang/StringLatin1.indexOf([B[B)I", "java/lang/StringLatin1.inflate([BI[BII)V", "java/lang/StringLatin1.inflate([BI[CII)V", "java/lang/StringUTF16.compress([BI[BII)I", "java/lang/StringUTF16.compress([CI[BII)I", + "java/lang/StringUTF16.indexOf([BI[BII)I", + "java/lang/StringUTF16.indexOf([B[B)I", + "java/lang/StringUTF16.indexOfChar([BIII)I", + "java/lang/StringUTF16.indexOfLatin1([BI[BII)I", + "java/lang/StringUTF16.indexOfLatin1([B[B)I", "jdk/internal/misc/Unsafe.compareAndExchangeByte(Ljava/lang/Object;JBB)B", "jdk/internal/misc/Unsafe.compareAndExchangeShort(Ljava/lang/Object;JSS)S", "jdk/internal/misc/Unsafe.compareAndSetByte(Ljava/lang/Object;JBB)Z", @@ -433,7 +450,9 @@ "jdk/internal/misc/Unsafe.getAndSet" + oopName + "(Ljava/lang/Object;JLjava/lang/Object;)Ljava/lang/Object;"); } add(toBeInvestigated, + "com/sun/crypto/provider/CounterMode.implCrypt([BII[BI)I", "java/lang/Thread.onSpinWait()V", + "java/util/ArraysSupport.vectorizedMismatch(Ljava/lang/Object;JLjava/lang/Object;JII)I", "jdk/internal/misc/Unsafe.getCharUnaligned(Ljava/lang/Object;J)C", "jdk/internal/misc/Unsafe.getIntUnaligned(Ljava/lang/Object;J)I", "jdk/internal/misc/Unsafe.getLongUnaligned(Ljava/lang/Object;J)J", @@ -443,6 +462,10 @@ "jdk/internal/misc/Unsafe.putLongUnaligned(Ljava/lang/Object;JJ)V", "jdk/internal/misc/Unsafe.putShortUnaligned(Ljava/lang/Object;JS)V"); } + if (isJDK10OrHigher()) { + add(toBeInvestigated, + "jdk/internal/util/ArraysSupport.vectorizedMismatch(Ljava/lang/Object;JLjava/lang/Object;JII)I"); + } } /* @@ -535,23 +558,23 @@ } private static boolean isJDK9OrHigher() { - return GraalServices.JAVA_SPECIFICATION_VERSION >= 9; + return JavaVersionUtil.JAVA_SPECIFICATION_VERSION >= 9; } private static boolean isJDK10OrHigher() { - return GraalServices.JAVA_SPECIFICATION_VERSION >= 10; + return JavaVersionUtil.JAVA_SPECIFICATION_VERSION >= 10; } private static boolean isJDK11OrHigher() { - return GraalServices.JAVA_SPECIFICATION_VERSION >= 11; + return JavaVersionUtil.JAVA_SPECIFICATION_VERSION >= 11; } private static boolean isJDK12OrHigher() { - return GraalServices.JAVA_SPECIFICATION_VERSION >= 12; + return JavaVersionUtil.JAVA_SPECIFICATION_VERSION >= 12; } private static boolean isJDK13OrHigher() { - return GraalServices.JAVA_SPECIFICATION_VERSION >= 13; + return JavaVersionUtil.JAVA_SPECIFICATION_VERSION >= 13; } public interface Refiner { --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/CompileTheWorld.java 2019-03-12 08:09:05.211375995 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/CompileTheWorld.java 2019-03-12 08:09:04.843373603 +0100 @@ -31,7 +31,7 @@ import static org.graalvm.compiler.core.test.ReflectionOptionDescriptors.extractEntries; import static org.graalvm.compiler.debug.MemUseTrackerKey.getCurrentThreadAllocatedBytes; import static org.graalvm.compiler.hotspot.test.CompileTheWorld.Options.DESCRIPTORS; -import static org.graalvm.compiler.serviceprovider.GraalServices.Java8OrEarlier; +import static org.graalvm.compiler.serviceprovider.JavaVersionUtil.Java8OrEarlier; import java.io.Closeable; import java.io.File; @@ -90,7 +90,7 @@ import org.graalvm.compiler.options.OptionKey; import org.graalvm.compiler.options.OptionValues; import org.graalvm.compiler.options.OptionsParser; -import org.graalvm.compiler.serviceprovider.GraalServices; +import org.graalvm.compiler.serviceprovider.JavaVersionUtil; import jdk.vm.ci.hotspot.HotSpotCodeCacheProvider; import jdk.vm.ci.hotspot.HotSpotCompilationRequest; @@ -110,7 +110,7 @@ /** * Magic token to denote that JDK classes are to be compiled. If - * {@link GraalServices#Java8OrEarlier}, then the classes in {@code rt.jar} are compiled. + * {@link JavaVersionUtil#Java8OrEarlier}, then the classes in {@code rt.jar} are compiled. * Otherwise the classes in the Java runtime image are compiled. */ public static final String SUN_BOOT_CLASS_PATH = "sun.boot.class.path"; --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/GraalOSRTest.java 2019-03-12 08:09:05.699379166 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/GraalOSRTest.java 2019-03-12 08:09:05.331376774 +0100 @@ -53,6 +53,11 @@ testOSR(getInitialOptions(), "testNonReduceLoop"); } + @Test + public void testOSR04() { + testOSR(getInitialOptions(), "testDeoptAfterCountedLoop"); + } + static int limit = 10000; public static int sideEffect; @@ -100,4 +105,14 @@ GraalDirectives.controlFlowAnchor(); return ret; } + + public static ReturnValue testDeoptAfterCountedLoop() { + long ret = 0; + for (int i = 0; GraalDirectives.injectBranchProbability(1, i < limit * limit); i++) { + GraalDirectives.blackhole(i); + ret = GraalDirectives.opaque(i); + } + GraalDirectives.controlFlowAnchor(); + return ret + 1 == limit * limit ? ReturnValue.SUCCESS : ReturnValue.FAILURE; + } } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/JVMCIInfopointErrorTest.java 2019-03-12 08:09:06.183382310 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/JVMCIInfopointErrorTest.java 2019-03-12 08:09:05.819379945 +0100 @@ -24,6 +24,7 @@ package org.graalvm.compiler.hotspot.test; +import static org.graalvm.compiler.debug.DebugOptions.DumpOnError; import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.REG; import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.STACK; import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_IGNORED; @@ -52,6 +53,7 @@ import org.graalvm.compiler.nodes.StructuredGraph; import org.graalvm.compiler.nodes.spi.LIRLowerable; import org.graalvm.compiler.nodes.spi.NodeLIRBuilderTool; +import org.graalvm.compiler.options.OptionValues; import org.junit.Test; import jdk.vm.ci.code.BytecodeFrame; @@ -139,6 +141,14 @@ test(getDebugContext(), spec); } + /** + * Avoids dumping during tests which are expected to fail. + */ + private void testNoDump(TestSpec spec) { + OptionValues options = new OptionValues(getInitialOptions(), DumpOnError, false); + test(getDebugContext(options, null, null), spec); + } + private void test(DebugContext debug, TestSpec spec) { ResolvedJavaMethod method = getResolvedJavaMethod("testMethod"); @@ -154,7 +164,7 @@ @Test(expected = Error.class) public void testInvalidShortOop() { - test((tool, state, safepoint) -> { + testNoDump((tool, state, safepoint) -> { PlatformKind kind = tool.target().arch.getPlatformKind(JavaKind.Short); LIRKind lirKind = LIRKind.reference(kind); @@ -167,7 +177,7 @@ @Test(expected = Error.class) public void testInvalidShortDerivedOop() { - test((tool, state, safepoint) -> { + testNoDump((tool, state, safepoint) -> { Variable baseOop = tool.newVariable(LIRKind.fromJavaKind(tool.target().arch, JavaKind.Object)); tool.append(new ValueDef(baseOop)); --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/OptionsInFileTest.java 2019-03-12 08:09:06.659385403 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/OptionsInFileTest.java 2019-03-12 08:09:06.291383013 +0100 @@ -37,6 +37,7 @@ import java.util.List; import org.graalvm.compiler.core.test.GraalCompilerTest; +import org.graalvm.compiler.debug.DebugOptions.PrintGraphTarget; import org.graalvm.compiler.test.SubprocessUtil; import org.graalvm.compiler.test.SubprocessUtil.Subprocess; import org.junit.Assert; @@ -54,12 +55,12 @@ try { Assert.assertFalse(methodFilterValue.equals(MethodFilter.getDefaultValue())); Assert.assertFalse(debugFilterValue.equals(Dump.getDefaultValue())); - Assert.assertTrue(PrintGraph.getDefaultValue()); + Assert.assertEquals(PrintGraphTarget.File, PrintGraph.getDefaultValue()); try (PrintStream out = new PrintStream(new FileOutputStream(optionsFile))) { out.println(MethodFilter.getName() + "=" + methodFilterValue); out.println(Dump.getName() + "=" + debugFilterValue); - out.println(PrintGraph.getName() + " = false"); + out.println(PrintGraph.getName() + " = Network"); } List vmArgs = withoutDebuggerArguments(getVMCommandLine()); @@ -70,7 +71,7 @@ String[] expected = { "graal.MethodFilter := \"a very unlikely method name\"", "graal.Dump := \"a very unlikely debug scope\"", - "graal.PrintGraph := false"}; + "graal.PrintGraph := Network"}; for (String line : proc.output) { for (int i = 0; i < expected.length; i++) { if (expected[i] != null && line.contains(expected[i])) { --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/ReservedStackAccessTest.java 2019-03-12 08:09:07.139388522 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/ReservedStackAccessTest.java 2019-03-12 08:09:06.771386131 +0100 @@ -73,6 +73,10 @@ vmArgs.add("-XX:+UseJVMCICompiler"); vmArgs.add("-Dgraal.Inline=false"); vmArgs.add("-XX:CompileCommand=exclude,java/util/concurrent/locks/AbstractOwnableSynchronizer.setExclusiveOwnerThread"); + + // Avoid SOE in HotSpotJVMCIRuntime.adjustCompilationLevel + vmArgs.add("-Dgraal.CompileGraalWithC1Only=false"); + Subprocess proc = SubprocessUtil.java(vmArgs, ReservedStackAccessTest.class.getName()); boolean passed = false; for (String line : proc.output) { @@ -81,9 +85,7 @@ } } if (!passed) { - for (String line : proc.output) { - System.err.println("" + line); - } + System.err.println(proc); } assertTrue(passed); } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/WriteBarrierAdditionTest.java 2019-03-12 08:09:07.631391719 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/WriteBarrierAdditionTest.java 2019-03-12 08:09:07.259389302 +0100 @@ -32,10 +32,10 @@ import org.graalvm.compiler.debug.DebugContext; import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig; import org.graalvm.compiler.hotspot.GraalHotSpotVMConfigBase; -import org.graalvm.compiler.hotspot.nodes.G1PostWriteBarrier; -import org.graalvm.compiler.hotspot.nodes.G1PreWriteBarrier; -import org.graalvm.compiler.hotspot.nodes.G1ReferentFieldReadBarrier; -import org.graalvm.compiler.hotspot.nodes.SerialWriteBarrier; +import org.graalvm.compiler.hotspot.gc.g1.G1PostWriteBarrier; +import org.graalvm.compiler.hotspot.gc.g1.G1PreWriteBarrier; +import org.graalvm.compiler.hotspot.gc.g1.G1ReferentFieldReadBarrier; +import org.graalvm.compiler.hotspot.gc.shared.SerialWriteBarrier; import org.graalvm.compiler.hotspot.phases.WriteBarrierAdditionPhase; import org.graalvm.compiler.nodes.StructuredGraph; import org.graalvm.compiler.nodes.StructuredGraph.AllowAssumptions; --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/WriteBarrierVerificationTest.java 2019-03-12 08:09:08.115394864 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/WriteBarrierVerificationTest.java 2019-03-12 08:09:07.739392420 +0100 @@ -32,12 +32,12 @@ import org.graalvm.compiler.debug.DebugContext.Scope; import org.graalvm.compiler.debug.DebugDumpScope; import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig; -import org.graalvm.compiler.hotspot.nodes.G1ArrayRangePostWriteBarrier; -import org.graalvm.compiler.hotspot.nodes.G1ArrayRangePreWriteBarrier; -import org.graalvm.compiler.hotspot.nodes.G1PostWriteBarrier; -import org.graalvm.compiler.hotspot.nodes.G1PreWriteBarrier; -import org.graalvm.compiler.hotspot.nodes.SerialArrayRangeWriteBarrier; -import org.graalvm.compiler.hotspot.nodes.SerialWriteBarrier; +import org.graalvm.compiler.hotspot.gc.g1.G1ArrayRangePostWriteBarrier; +import org.graalvm.compiler.hotspot.gc.g1.G1ArrayRangePreWriteBarrier; +import org.graalvm.compiler.hotspot.gc.g1.G1PostWriteBarrier; +import org.graalvm.compiler.hotspot.gc.g1.G1PreWriteBarrier; +import org.graalvm.compiler.hotspot.gc.shared.SerialArrayRangeWriteBarrier; +import org.graalvm.compiler.hotspot.gc.shared.SerialWriteBarrier; import org.graalvm.compiler.hotspot.phases.WriteBarrierAdditionPhase; import org.graalvm.compiler.hotspot.phases.WriteBarrierVerificationPhase; import org.graalvm.compiler.nodes.AbstractBeginNode; --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/CompilationTask.java 2019-03-12 08:09:08.603398034 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/CompilationTask.java 2019-03-12 08:09:08.239395669 +0100 @@ -368,8 +368,14 @@ installedCode = null; Object[] context = {new DebugDumpScope(getIdString(), true), codeCache, getMethod(), compResult}; try (DebugContext.Scope s = debug.scope("CodeInstall", context)) { - installedCode = (HotSpotInstalledCode) backend.createInstalledCode(debug, getRequest().getMethod(), getRequest(), compResult, - getRequest().getMethod().getSpeculationLog(), null, installAsDefault, context); + HotSpotCompilationRequest request = getRequest(); + installedCode = (HotSpotInstalledCode) backend.createInstalledCode(debug, + request.getMethod(), + request, + compResult, + null, + installAsDefault, + context); } catch (Throwable e) { throw debug.handle(e); } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/GraalHotSpotVMConfig.java 2019-03-12 08:09:09.107401308 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/GraalHotSpotVMConfig.java 2019-03-12 08:09:08.739398917 +0100 @@ -92,6 +92,7 @@ public final boolean useBiasedLocking = getFlag("UseBiasedLocking", Boolean.class); public final boolean usePopCountInstruction = getFlag("UsePopCountInstruction", Boolean.class); public final boolean useAESIntrinsics = getFlag("UseAESIntrinsics", Boolean.class); + public final boolean useAESCTRIntrinsics = getFlag("UseAESCTRIntrinsics", Boolean.class, false); public final boolean useCRC32Intrinsics = getFlag("UseCRC32Intrinsics", Boolean.class); public final boolean useCRC32CIntrinsics = versioned.useCRC32CIntrinsics; public final boolean threadLocalHandshakes = getFlag("ThreadLocalHandshakes", Boolean.class, false); @@ -100,10 +101,12 @@ private final boolean useSHA1Intrinsics = getFlag("UseSHA1Intrinsics", Boolean.class); private final boolean useSHA256Intrinsics = getFlag("UseSHA256Intrinsics", Boolean.class); private final boolean useSHA512Intrinsics = getFlag("UseSHA512Intrinsics", Boolean.class); + private final boolean useGHASHIntrinsics = getFlag("UseGHASHIntrinsics", Boolean.class, false); private final boolean useMontgomeryMultiplyIntrinsic = getFlag("UseMontgomeryMultiplyIntrinsic", Boolean.class, false); private final boolean useMontgomerySquareIntrinsic = getFlag("UseMontgomerySquareIntrinsic", Boolean.class, false); private final boolean useMulAddIntrinsic = getFlag("UseMulAddIntrinsic", Boolean.class, false); private final boolean useSquareToLenIntrinsic = getFlag("UseSquareToLenIntrinsic", Boolean.class, false); + public final boolean useVectorizedMismatchIntrinsic = getFlag("UseVectorizedMismatchIntrinsic", Boolean.class, false); /* * These are methods because in some JDKs the flags are visible but the stubs themselves haven't @@ -114,15 +117,19 @@ } public boolean useSHA1Intrinsics() { - return useSHA1Intrinsics && sha1ImplCompress != 0; + return useSHA1Intrinsics && sha1ImplCompress != 0 && sha1ImplCompressMultiBlock != 0; } public boolean useSHA256Intrinsics() { - return useSHA256Intrinsics && sha256ImplCompress != 0; + return useSHA256Intrinsics && sha256ImplCompress != 0 && sha256ImplCompressMultiBlock != 0; } public boolean useSHA512Intrinsics() { - return useSHA512Intrinsics && sha512ImplCompress != 0; + return useSHA512Intrinsics && sha512ImplCompress != 0 && sha512ImplCompressMultiBlock != 0; + } + + public boolean useGHASHIntrinsics() { + return useGHASHIntrinsics && ghashProcessBlocks != 0; } public boolean useMontgomeryMultiplyIntrinsic() { @@ -309,7 +316,17 @@ public final int jvmAccWrittenFlags = getConstant("JVM_ACC_WRITTEN_FLAGS", Integer.class); public final int jvmAccSynthetic = getConstant("JVM_ACC_SYNTHETIC", Integer.class); - public final int jvmciCompileStateCanPostOnExceptionsOffset = getFieldOffset("JVMCIEnv::_jvmti_can_post_on_exceptions", Integer.class, "jbyte", Integer.MIN_VALUE); + public final int jvmciCompileStateCanPostOnExceptionsOffset = getJvmciCompileStateCanPostOnExceptionsOffset(); + + // Integer.MIN_VALUE if not available + private int getJvmciCompileStateCanPostOnExceptionsOffset() { + int offset = getFieldOffset("JVMCICompileState::_jvmti_can_post_on_exceptions", Integer.class, "jbyte", Integer.MIN_VALUE); + if (offset == Integer.MIN_VALUE) { + // JDK 12 + offset = getFieldOffset("JVMCIEnv::_jvmti_can_post_on_exceptions", Integer.class, "jbyte", Integer.MIN_VALUE); + } + return offset; + } public final int threadTlabOffset = getFieldOffset("Thread::_tlab", Integer.class, "ThreadLocalAllocBuffer"); public final int javaThreadAnchorOffset = getFieldOffset("JavaThread::_anchor", Integer.class, "JavaFrameAnchor"); @@ -372,12 +389,29 @@ public final int pendingExceptionOffset = getFieldOffset("ThreadShadow::_pending_exception", Integer.class, "oop"); public final int pendingDeoptimizationOffset = getFieldOffset("JavaThread::_pending_deoptimization", Integer.class, "int"); - public final int pendingFailedSpeculationOffset = getFieldOffset("JavaThread::_pending_failed_speculation", Integer.class, "long"); public final int pendingTransferToInterpreterOffset = getFieldOffset("JavaThread::_pending_transfer_to_interpreter", Integer.class, "bool"); private final int javaFrameAnchorLastJavaSpOffset = getFieldOffset("JavaFrameAnchor::_last_Java_sp", Integer.class, "intptr_t*"); private final int javaFrameAnchorLastJavaPcOffset = getFieldOffset("JavaFrameAnchor::_last_Java_pc", Integer.class, "address"); + public final int pendingFailedSpeculationOffset; + { + String name = "JavaThread::_pending_failed_speculation"; + int offset = -1; + try { + offset = getFieldOffset(name, Integer.class, "jlong"); + } catch (JVMCIError e) { + try { + offset = getFieldOffset(name, Integer.class, "long"); + } catch (JVMCIError e2) { + } + } + if (offset == -1) { + throw new JVMCIError("cannot get offset of field " + name + " with type long or jlong"); + } + pendingFailedSpeculationOffset = offset; + } + public int threadLastJavaSpOffset() { return javaThreadAnchorOffset + javaFrameAnchorLastJavaSpOffset; } @@ -602,11 +636,11 @@ public final long crcTableAddress = getFieldValue("StubRoutines::_crc_table_adr", Long.class, "address"); public final long sha1ImplCompress = getFieldValue("StubRoutines::_sha1_implCompress", Long.class, "address", 0L); - public final long sha1ImplCompressMB = getFieldValue("StubRoutines::_sha1_implCompressMB", Long.class, "address", 0L); + public final long sha1ImplCompressMultiBlock = getFieldValue("StubRoutines::_sha1_implCompressMB", Long.class, "address", 0L); public final long sha256ImplCompress = getFieldValue("StubRoutines::_sha256_implCompress", Long.class, "address", 0L); - public final long sha256ImplCompressMB = getFieldValue("StubRoutines::_sha256_implCompressMB", Long.class, "address", 0L); + public final long sha256ImplCompressMultiBlock = getFieldValue("StubRoutines::_sha256_implCompressMB", Long.class, "address", 0L); public final long sha512ImplCompress = getFieldValue("StubRoutines::_sha512_implCompress", Long.class, "address", 0L); - public final long sha512ImplCompressMB = getFieldValue("StubRoutines::_sha512_implCompressMB", Long.class, "address", 0L); + public final long sha512ImplCompressMultiBlock = getFieldValue("StubRoutines::_sha512_implCompressMB", Long.class, "address", 0L); public final long multiplyToLen = getFieldValue("StubRoutines::_multiplyToLen", Long.class, "address", longRequiredOnAMD64); public final long counterModeAESCrypt = getFieldValue("StubRoutines::_counterMode_AESCrypt", Long.class, "address", 0L); --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotBackend.java 2019-03-12 08:09:09.579404375 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotBackend.java 2019-03-12 08:09:09.219402036 +0100 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2012, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -46,6 +46,7 @@ import org.graalvm.compiler.hotspot.replacements.AESCryptSubstitutions; import org.graalvm.compiler.hotspot.replacements.BigIntegerSubstitutions; import org.graalvm.compiler.hotspot.replacements.CipherBlockChainingSubstitutions; +import org.graalvm.compiler.hotspot.replacements.DigestBaseSubstitutions; import org.graalvm.compiler.hotspot.replacements.SHA2Substitutions; import org.graalvm.compiler.hotspot.replacements.SHA5Substitutions; import org.graalvm.compiler.hotspot.replacements.SHASubstitutions; @@ -71,6 +72,7 @@ import org.graalvm.compiler.options.OptionType; import org.graalvm.compiler.options.OptionValues; import org.graalvm.compiler.phases.tiers.SuitesProvider; +import org.graalvm.compiler.serviceprovider.JavaVersionUtil; import org.graalvm.compiler.word.Word; import jdk.internal.vm.compiler.word.Pointer; @@ -95,7 +97,7 @@ public static class Options { // @formatter:off @Option(help = "Use Graal arithmetic stubs instead of HotSpot stubs where possible") - public static final OptionKey GraalArithmeticStubs = new OptionKey<>(false); // GR-8276 + public static final OptionKey GraalArithmeticStubs = new OptionKey<>(JavaVersionUtil.JAVA_SPECIFICATION_VERSION >= 9); @Option(help = "Enables instruction profiling on assembler level. Valid values are a comma separated list of supported instructions." + " Compare with subclasses of Assembler.InstructionCounter.", type = OptionType.Debug) public static final OptionKey ASMInstructionProfiling = new OptionKey<>(null); @@ -261,6 +263,36 @@ @NodeIntrinsic(ForeignCallNode.class) private static native void sha5ImplCompressStub(@ConstantNodeParameter ForeignCallDescriptor descriptor, Word bufAddr, Object state); + /** + * @see DigestBaseSubstitutions#implCompressMultiBlock0 + */ + public static final ForeignCallDescriptor SHA_IMPL_COMPRESS_MB = new ForeignCallDescriptor("shaImplCompressMB", int.class, Word.class, Object.class, int.class, int.class); + + public static int shaImplCompressMBStub(Word bufAddr, Object stateAddr, int ofs, int limit) { + return shaImplCompressMBStub(HotSpotBackend.SHA_IMPL_COMPRESS_MB, bufAddr, stateAddr, ofs, limit); + } + + @NodeIntrinsic(ForeignCallNode.class) + private static native int shaImplCompressMBStub(@ConstantNodeParameter ForeignCallDescriptor descriptor, Word bufAddr, Object state, int ofs, int limit); + + public static final ForeignCallDescriptor SHA2_IMPL_COMPRESS_MB = new ForeignCallDescriptor("sha2ImplCompressMB", int.class, Word.class, Object.class, int.class, int.class); + + public static int sha2ImplCompressMBStub(Word bufAddr, Object stateAddr, int ofs, int limit) { + return sha2ImplCompressMBStub(HotSpotBackend.SHA2_IMPL_COMPRESS_MB, bufAddr, stateAddr, ofs, limit); + } + + @NodeIntrinsic(ForeignCallNode.class) + private static native int sha2ImplCompressMBStub(@ConstantNodeParameter ForeignCallDescriptor descriptor, Word bufAddr, Object state, int ofs, int limit); + + public static final ForeignCallDescriptor SHA5_IMPL_COMPRESS_MB = new ForeignCallDescriptor("sha5ImplCompressMB", int.class, Word.class, Object.class, int.class, int.class); + + public static int sha5ImplCompressMBStub(Word bufAddr, Object stateAddr, int ofs, int limit) { + return sha5ImplCompressMBStub(HotSpotBackend.SHA5_IMPL_COMPRESS_MB, bufAddr, stateAddr, ofs, limit); + } + + @NodeIntrinsic(ForeignCallNode.class) + private static native int sha5ImplCompressMBStub(@ConstantNodeParameter ForeignCallDescriptor descriptor, Word bufAddr, Object state, int ofs, int limit); + public static void unsafeArraycopy(Word srcAddr, Word dstAddr, Word size) { unsafeArraycopyStub(UNSAFE_ARRAYCOPY, srcAddr, dstAddr, size); } @@ -269,6 +301,37 @@ private static native void unsafeArraycopyStub(@ConstantNodeParameter ForeignCallDescriptor descriptor, Word srcAddr, Word dstAddr, Word size); /** + * Descriptor for {@code StubRoutines::_ghash_processBlocks}. + */ + public static final ForeignCallDescriptor GHASH_PROCESS_BLOCKS = new ForeignCallDescriptor("ghashProcessBlocks", void.class, Word.class, Word.class, Word.class, int.class); + + /** + * Descriptor for {@code StubRoutines::_counterMode_AESCrypt}. + */ + public static final ForeignCallDescriptor COUNTERMODE_IMPL_CRYPT = new ForeignCallDescriptor("counterModeAESCrypt", int.class, Word.class, Word.class, Word.class, Word.class, int.class, + Word.class, Word.class); + + public static int counterModeAESCrypt(Word srcAddr, Word dstAddr, Word kPtr, Word cntPtr, int len, Word encCntPtr, Word used) { + return counterModeAESCrypt(COUNTERMODE_IMPL_CRYPT, srcAddr, dstAddr, kPtr, cntPtr, len, encCntPtr, used); + } + + @NodeIntrinsic(ForeignCallNode.class) + private static native int counterModeAESCrypt(@ConstantNodeParameter ForeignCallDescriptor descriptor, Word srcAddr, Word dstAddr, Word kPtr, Word cntPtr, int len, Word encCntPtr, + Word used); + + /** + * Descriptor for {@code StubRoutines::_vectorizedMismatch}. + */ + public static final ForeignCallDescriptor VECTORIZED_MISMATCHED = new ForeignCallDescriptor("vectorizedMismatch", int.class, Word.class, Word.class, int.class, int.class); + + public static int vectorizedMismatch(Word aAddr, Word bAddr, int length, int log2ArrayIndexScale) { + return vectorizedMismatchStub(VECTORIZED_MISMATCHED, aAddr, bAddr, length, log2ArrayIndexScale); + } + + @NodeIntrinsic(ForeignCallNode.class) + private static native int vectorizedMismatchStub(@ConstantNodeParameter ForeignCallDescriptor descriptor, Word aAddr, Word bAddr, int length, int log2ArrayIndexScale); + + /** * @see VMErrorNode */ public static final ForeignCallDescriptor VM_ERROR = new ForeignCallDescriptor("vm_error", void.class, Object.class, Object.class, long.class); @@ -389,6 +452,13 @@ } /** + * Translates a set of registers from the callee's perspective to the caller's perspective. This + * is needed for architectures where input/output registers are renamed during a call (e.g. + * register windows on SPARC). Registers which are not visible by the caller are removed. + */ + protected abstract EconomicSet translateToCallerRegisters(EconomicSet calleeRegisters); + + /** * Updates a given stub with respect to the registers it destroys. *

* Any entry in {@code calleeSaveInfo} that {@linkplain SaveRegistersOp#supportsRemove() --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotCompiledCodeBuilder.java 2019-03-12 08:09:10.099407753 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotCompiledCodeBuilder.java 2019-03-12 08:09:09.719405284 +0100 @@ -88,7 +88,7 @@ ResolvedJavaMethod[] methods = compResult.getMethods(); - List annotations = compResult.getAnnotations(); + List annotations = compResult.getCodeAnnotations(); Comment[] comments = new Comment[annotations.size()]; if (!annotations.isEmpty()) { for (int i = 0; i < comments.length; i++) { @@ -129,16 +129,16 @@ boolean hasUnsafeAccess = compResult.hasUnsafeAccess(); int id; - long jvmciEnv; + long jvmciCompileState; if (compRequest != null) { id = compRequest.getId(); - jvmciEnv = compRequest.getJvmciEnv(); + jvmciCompileState = compRequest.getJvmciEnv(); } else { id = hsMethod.allocateCompileId(entryBCI); - jvmciEnv = 0L; + jvmciCompileState = 0L; } return new HotSpotCompiledNmethod(name, targetCode, targetCodeSize, sites, assumptions, methods, comments, dataSection, dataSectionAlignment, dataSectionPatches, isImmutablePIC, - totalFrameSize, customStackArea, hsMethod, entryBCI, id, jvmciEnv, hasUnsafeAccess); + totalFrameSize, customStackArea, hsMethod, entryBCI, id, jvmciCompileState, hasUnsafeAccess); } else { return new HotSpotCompiledCode(name, targetCode, targetCodeSize, sites, assumptions, methods, comments, dataSection, dataSectionAlignment, dataSectionPatches, isImmutablePIC, totalFrameSize, customStackArea); --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotDataBuilder.java 2019-03-12 08:09:10.591410949 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotDataBuilder.java 2019-03-12 08:09:10.219408532 +0100 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -51,12 +51,6 @@ } @Override - public boolean needDetailedPatchingInformation() { - /* The HotSpot VM finds operands that need patching by decoding the instruction. */ - return false; - } - - @Override public Data createDataItem(Constant constant) { if (JavaConstant.isNull(constant)) { boolean compressed = COMPRESSED_NULL.equals(constant); --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotGraalCompilerFactory.java 2019-03-12 08:09:11.063414015 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotGraalCompilerFactory.java 2019-03-12 08:09:10.699411651 +0100 @@ -25,6 +25,7 @@ package org.graalvm.compiler.hotspot; import static jdk.vm.ci.common.InitTimer.timer; +import static jdk.vm.ci.services.Services.IS_BUILDING_NATIVE_IMAGE; import static org.graalvm.compiler.hotspot.HotSpotGraalOptionValues.GRAAL_OPTION_PROPERTY_PREFIX; import java.io.PrintStream; @@ -71,7 +72,7 @@ public void onSelection() { JVMCIVersionCheck.check(false); assert options == null : "cannot select " + getClass() + " service more than once"; - options = HotSpotGraalOptionValues.HOTSPOT_OPTIONS; + options = HotSpotGraalOptionValues.defaultOptions(); initializeGraalCompilePolicyFields(options); isGraalPredicate = compileGraalWithC1Only ? new IsGraalPredicate() : null; /* @@ -80,6 +81,10 @@ */ adjustCompilationLevelInternal(Object.class, "hashCode", "()I", CompilationLevel.FullOptimization); adjustCompilationLevelInternal(Object.class, "hashCode", "()I", CompilationLevel.Simple); + if (IS_BUILDING_NATIVE_IMAGE) { + // Triggers initialization of all option descriptors + Options.CompileGraalWithC1Only.getName(); + } } private static void initializeGraalCompilePolicyFields(OptionValues options) { --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotGraalJVMCIServiceLocator.java 2019-03-12 08:09:11.559417237 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotGraalJVMCIServiceLocator.java 2019-03-12 08:09:11.187414820 +0100 @@ -26,6 +26,7 @@ import org.graalvm.compiler.serviceprovider.ServiceProvider; +import jdk.vm.ci.common.NativeImageReinitialize; import jdk.vm.ci.hotspot.HotSpotVMEventListener; import jdk.vm.ci.runtime.JVMCICompilerFactory; import jdk.vm.ci.services.JVMCIServiceLocator; @@ -51,7 +52,7 @@ return null; } - private HotSpotGraalRuntime graalRuntime; + @NativeImageReinitialize private HotSpotGraalRuntime graalRuntime; /** * Notifies this object of the compiler created via {@link HotSpotGraalJVMCIServiceLocator}. --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotGraalOptionValues.java 2019-03-12 08:09:12.027420277 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotGraalOptionValues.java 2019-03-12 08:09:11.659417887 +0100 @@ -40,9 +40,10 @@ import org.graalvm.compiler.options.OptionsParser; import jdk.vm.ci.common.InitTimer; +import jdk.vm.ci.common.NativeImageReinitialize; /** - * The {@link #HOTSPOT_OPTIONS} value contains the options values initialized in a HotSpot VM. The + * The {@link #defaultOptions()} method returns the options values initialized in a HotSpot VM. The * values are set via system properties with the {@value #GRAAL_OPTION_PROPERTY_PREFIX} prefix. */ public class HotSpotGraalOptionValues { @@ -71,7 +72,21 @@ return GRAAL_OPTION_PROPERTY_PREFIX + value.getName() + "=" + value.getValue(options); } - public static final OptionValues HOTSPOT_OPTIONS = initializeOptions(); + @NativeImageReinitialize private static volatile OptionValues hotspotOptions; + + public static OptionValues defaultOptions() { + OptionValues res = hotspotOptions; + if (res == null) { + synchronized (HotSpotGraalOptionValues.class) { + res = hotspotOptions; + if (res == null) { + res = initializeOptions(); + hotspotOptions = res; + } + } + } + return res; + } /** * Global options. The values for these options are initialized by parsing the file denoted by --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotGraalRuntime.java 2019-03-12 08:09:12.519423473 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotGraalRuntime.java 2019-03-12 08:09:12.147421057 +0100 @@ -95,6 +95,8 @@ */ public final class HotSpotGraalRuntime implements HotSpotGraalRuntimeProvider { + private static final boolean IS_AOT = Boolean.getBoolean("com.oracle.graalvm.isaot"); + private static boolean checkArrayIndexScaleInvariants(MetaAccessProvider metaAccess) { assert metaAccess.getArrayIndexScale(JavaKind.Byte) == 1; assert metaAccess.getArrayIndexScale(JavaKind.Boolean) == 1; @@ -159,9 +161,13 @@ compilerConfigurationName = compilerConfigurationFactory.getName(); compiler = new HotSpotGraalCompiler(jvmciRuntime, this, options); - management = GraalServices.loadSingle(HotSpotGraalManagementRegistration.class, false); - if (management != null) { - management.initialize(this); + if (IS_AOT) { + management = null; + } else { + management = GraalServices.loadSingle(HotSpotGraalManagementRegistration.class, false); + if (management != null) { + management.initialize(this); + } } BackendMap backendMap = compilerConfigurationFactory.createBackendMap(); @@ -571,7 +577,7 @@ EconomicMap, Object> extra = EconomicMap.create(); extra.put(DebugOptions.Dump, filter); extra.put(DebugOptions.PrintGraphHost, host); - extra.put(DebugOptions.PrintBinaryGraphPort, port); + extra.put(DebugOptions.PrintGraphPort, port); OptionValues compileOptions = new OptionValues(getOptions(), extra); compiler.compileMethod(new HotSpotCompilationRequest(hotSpotMethod, -1, 0L), false, compileOptions); } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotReplacementsImpl.java 2019-03-12 08:09:13.007426643 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotReplacementsImpl.java 2019-03-12 08:09:12.639424252 +0100 @@ -24,17 +24,34 @@ package org.graalvm.compiler.hotspot; +import static jdk.vm.ci.services.Services.IS_BUILDING_NATIVE_IMAGE; +import static jdk.vm.ci.services.Services.IS_IN_NATIVE_IMAGE; +import static org.graalvm.compiler.replacements.ReplacementsImpl.Options.UseEncodedSnippets; + +import java.util.Set; + +import jdk.internal.vm.compiler.collections.EconomicSet; import org.graalvm.compiler.api.replacements.SnippetReflectionProvider; import org.graalvm.compiler.bytecode.BytecodeProvider; +import org.graalvm.compiler.core.common.CompilationIdentifier; +import org.graalvm.compiler.debug.DebugContext; +import org.graalvm.compiler.debug.GraalError; +import org.graalvm.compiler.graph.NodeSourcePosition; import org.graalvm.compiler.hotspot.meta.HotSpotWordOperationPlugin; import org.graalvm.compiler.hotspot.word.HotSpotOperation; +import org.graalvm.compiler.nodes.Invoke; +import org.graalvm.compiler.nodes.StructuredGraph; +import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderContext; import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderPlugin; +import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugin; +import org.graalvm.compiler.nodes.graphbuilderconf.MethodSubstitutionPlugin; import org.graalvm.compiler.options.OptionValues; import org.graalvm.compiler.phases.util.Providers; import org.graalvm.compiler.printer.GraalDebugHandlersFactory; import org.graalvm.compiler.replacements.ReplacementsImpl; import jdk.vm.ci.code.TargetDescription; +import jdk.vm.ci.common.NativeImageReinitialize; import jdk.vm.ci.meta.ResolvedJavaMethod; /** @@ -46,21 +63,149 @@ super(options, new GraalDebugHandlersFactory(snippetReflection), providers, snippetReflection, bytecodeProvider, target); } + protected HotSpotReplacementsImpl(HotSpotReplacementsImpl replacements, Providers providers) { + super(replacements.options, new GraalDebugHandlersFactory(replacements.snippetReflection), providers, replacements.snippetReflection, + replacements.getDefaultReplacementBytecodeProvider(), replacements.target); + } + @Override public Class getIntrinsifyingPlugin(ResolvedJavaMethod method) { return method.getAnnotation(HotSpotOperation.class) != null ? HotSpotWordOperationPlugin.class : super.getIntrinsifyingPlugin(method); } + public void registerMethodSubstitution(ResolvedJavaMethod method, ResolvedJavaMethod original) { + if (!IS_IN_NATIVE_IMAGE) { + if (IS_BUILDING_NATIVE_IMAGE || UseEncodedSnippets.getValue(options)) { + synchronized (HotSpotReplacementsImpl.class) { + if (snippetEncoder == null) { + snippetEncoder = new SymbolicSnippetEncoder(this); + } + snippetEncoder.registerMethodSubstitution(method, original); + } + } + } + } + + @Override + public StructuredGraph getIntrinsicGraph(ResolvedJavaMethod method, CompilationIdentifier compilationId, DebugContext debug) { + if (IS_IN_NATIVE_IMAGE) { + HotSpotReplacementsImpl replacements = (HotSpotReplacementsImpl) providers.getReplacements(); + InvocationPlugin plugin = replacements.getGraphBuilderPlugins().getInvocationPlugins().lookupInvocation(method); + if (plugin instanceof MethodSubstitutionPlugin) { + MethodSubstitutionPlugin msp = (MethodSubstitutionPlugin) plugin; + return replacements.getMethodSubstitution(msp, method); + } + return null; + } + return super.getIntrinsicGraph(method, compilationId, debug); + } + + @Override + public void notifyNotInlined(GraphBuilderContext b, ResolvedJavaMethod method, Invoke invoke) { + if (b.parsingIntrinsic() && snippetEncoder != null) { + if (getIntrinsifyingPlugin(method) != null) { + snippetEncoder.addDelayedInvocationPluginMethod(method); + return; + } + } + super.notifyNotInlined(b, method, invoke); + } + + // When assertions are enabled, these fields are used to ensure all snippets are + // registered during Graal initialization which in turn ensures that native image + // building will not miss any snippets. + @NativeImageReinitialize private EconomicSet registeredSnippets = EconomicSet.create(); private boolean snippetRegistrationClosed; @Override public void registerSnippet(ResolvedJavaMethod method, ResolvedJavaMethod original, Object receiver, boolean trackNodeSourcePosition) { - assert !snippetRegistrationClosed; - super.registerSnippet(method, original, receiver, trackNodeSourcePosition); + if (!IS_IN_NATIVE_IMAGE) { + assert !snippetRegistrationClosed : "Cannot register snippet after registration is closed: " + method.format("%H.%n(%p)"); + assert registeredSnippets.add(method) : "Cannot register snippet twice: " + method.format("%H.%n(%p)"); + if (IS_BUILDING_NATIVE_IMAGE || UseEncodedSnippets.getValue(options)) { + synchronized (HotSpotReplacementsImpl.class) { + if (snippetEncoder == null) { + snippetEncoder = new SymbolicSnippetEncoder(this); + } + snippetEncoder.registerSnippet(method, original, receiver, trackNodeSourcePosition); + } + } + } } @Override public void closeSnippetRegistration() { snippetRegistrationClosed = true; } + + static SymbolicSnippetEncoder.EncodedSnippets getEncodedSnippets() { + return encodedSnippets; + } + + public Set getSnippetMethods() { + if (snippetEncoder != null) { + return snippetEncoder.getSnippetMethods(); + } + return null; + } + + static void setEncodedSnippets(SymbolicSnippetEncoder.EncodedSnippets encodedSnippets) { + HotSpotReplacementsImpl.encodedSnippets = encodedSnippets; + } + + public boolean encode() { + SymbolicSnippetEncoder encoder = HotSpotReplacementsImpl.snippetEncoder; + if (encoder != null) { + return encoder.encode(); + } + return false; + } + + private static volatile SymbolicSnippetEncoder.EncodedSnippets encodedSnippets; + + @NativeImageReinitialize static SymbolicSnippetEncoder snippetEncoder; + + @Override + public StructuredGraph getSnippet(ResolvedJavaMethod method, ResolvedJavaMethod recursiveEntry, Object[] args, boolean trackNodeSourcePosition, NodeSourcePosition replaceePosition) { + StructuredGraph graph = getEncodedSnippet(method, args); + if (graph != null) { + return graph; + } + + assert !IS_IN_NATIVE_IMAGE : "should be using encoded snippets"; + return super.getSnippet(method, recursiveEntry, args, trackNodeSourcePosition, replaceePosition); + } + + public StructuredGraph getEncodedSnippet(ResolvedJavaMethod method, Object[] args) { + if (IS_IN_NATIVE_IMAGE || UseEncodedSnippets.getValue(options)) { + synchronized (HotSpotReplacementsImpl.class) { + if (!IS_IN_NATIVE_IMAGE && UseEncodedSnippets.getValue(options)) { + snippetEncoder.encode(); + } + + if (getEncodedSnippets() == null) { + throw GraalError.shouldNotReachHere("encoded snippets not found"); + } + StructuredGraph graph = getEncodedSnippets().getEncodedSnippet(method, this, args); + if (graph == null) { + throw GraalError.shouldNotReachHere("snippet not found: " + method.format("%H.%n(%p)")); + } + return graph; + } + } else if (registeredSnippets != null) { + assert registeredSnippets.contains(method) : "Asking for snippet method that was never registered: " + method.format("%H.%n(%p)"); + } + return null; + } + + public StructuredGraph getMethodSubstitution(MethodSubstitutionPlugin plugin, ResolvedJavaMethod original) { + if (IS_IN_NATIVE_IMAGE || UseEncodedSnippets.getValue(options)) { + if (getEncodedSnippets() == null) { + throw GraalError.shouldNotReachHere("encoded snippets not found"); + } + return getEncodedSnippets().getMethodSubstitutionGraph(plugin, original, this); + } + return null; + } + } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotTTYStreamProvider.java 2019-03-12 08:09:13.479429709 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotTTYStreamProvider.java 2019-03-12 08:09:13.119427370 +0100 @@ -25,7 +25,7 @@ package org.graalvm.compiler.hotspot; import static org.graalvm.compiler.hotspot.HotSpotGraalOptionValues.GRAAL_OPTION_PROPERTY_PREFIX; -import static org.graalvm.compiler.hotspot.HotSpotGraalOptionValues.HOTSPOT_OPTIONS; +import static org.graalvm.compiler.hotspot.HotSpotGraalOptionValues.defaultOptions; import java.io.FileNotFoundException; import java.io.FileOutputStream; @@ -59,7 +59,7 @@ @Override public PrintStream getStream() { - return Options.LogFile.getStream(HOTSPOT_OPTIONS); + return Options.LogFile.getStream(defaultOptions()); } /** --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/JVMCIVersionCheck.java 2019-03-12 08:09:13.963432852 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/JVMCIVersionCheck.java 2019-03-12 08:09:13.591430436 +0100 @@ -38,8 +38,9 @@ */ class JVMCIVersionCheck { + // 0.55 introduces new HotSpotSpeculationLog API private static final int JVMCI8_MIN_MAJOR_VERSION = 0; - private static final int JVMCI8_MIN_MINOR_VERSION = 46; + private static final int JVMCI8_MIN_MINOR_VERSION = 55; private static void failVersionCheck(boolean exit, String reason, Object... args) { Formatter errorMessage = new Formatter().format(reason, args); --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/DefaultHotSpotLoweringProvider.java 2019-03-12 08:09:14.455436048 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/DefaultHotSpotLoweringProvider.java 2019-03-12 08:09:14.075433579 +0100 @@ -24,6 +24,7 @@ package org.graalvm.compiler.hotspot.meta; +import static jdk.vm.ci.services.Services.IS_IN_NATIVE_IMAGE; import static org.graalvm.compiler.core.common.GraalOptions.AlwaysInlineVTableStubs; import static org.graalvm.compiler.core.common.GraalOptions.InlineVTableStubs; import static org.graalvm.compiler.core.common.GraalOptions.OmitHotExceptionStacktrace; @@ -58,17 +59,17 @@ import org.graalvm.compiler.graph.NodeInputList; import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig; import org.graalvm.compiler.hotspot.HotSpotGraalRuntimeProvider; +import org.graalvm.compiler.hotspot.gc.g1.G1ArrayRangePostWriteBarrier; +import org.graalvm.compiler.hotspot.gc.g1.G1ArrayRangePreWriteBarrier; +import org.graalvm.compiler.hotspot.gc.g1.G1PostWriteBarrier; +import org.graalvm.compiler.hotspot.gc.g1.G1PreWriteBarrier; +import org.graalvm.compiler.hotspot.gc.g1.G1ReferentFieldReadBarrier; +import org.graalvm.compiler.hotspot.gc.shared.SerialArrayRangeWriteBarrier; +import org.graalvm.compiler.hotspot.gc.shared.SerialWriteBarrier; import org.graalvm.compiler.hotspot.nodes.BeginLockScopeNode; -import org.graalvm.compiler.hotspot.nodes.G1ArrayRangePostWriteBarrier; -import org.graalvm.compiler.hotspot.nodes.G1ArrayRangePreWriteBarrier; -import org.graalvm.compiler.hotspot.nodes.G1PostWriteBarrier; -import org.graalvm.compiler.hotspot.nodes.G1PreWriteBarrier; -import org.graalvm.compiler.hotspot.nodes.G1ReferentFieldReadBarrier; import org.graalvm.compiler.hotspot.nodes.HotSpotCompressionNode; import org.graalvm.compiler.hotspot.nodes.HotSpotDirectCallTargetNode; import org.graalvm.compiler.hotspot.nodes.HotSpotIndirectCallTargetNode; -import org.graalvm.compiler.hotspot.nodes.SerialArrayRangeWriteBarrier; -import org.graalvm.compiler.hotspot.nodes.SerialWriteBarrier; import org.graalvm.compiler.hotspot.nodes.aot.InitializeKlassNode; import org.graalvm.compiler.hotspot.nodes.aot.ResolveConstantNode; import org.graalvm.compiler.hotspot.nodes.aot.ResolveDynamicConstantNode; @@ -162,6 +163,7 @@ import org.graalvm.compiler.replacements.arraycopy.ArrayCopySnippets; import org.graalvm.compiler.replacements.arraycopy.ArrayCopyWithSlowPathNode; import org.graalvm.compiler.replacements.nodes.AssertionNode; +import org.graalvm.compiler.serviceprovider.JavaVersionUtil; import jdk.internal.vm.compiler.word.LocationIdentity; import jdk.vm.ci.code.TargetDescription; @@ -225,7 +227,9 @@ stringToBytesSnippets = new StringToBytesSnippets.Templates(options, factories, providers, target); hashCodeSnippets = new HashCodeSnippets.Templates(options, factories, providers, target); resolveConstantSnippets = new ResolveConstantSnippets.Templates(options, factories, providers, target); - profileSnippets = new ProfileSnippets.Templates(options, factories, providers, target); + if (!JavaVersionUtil.Java8OrEarlier) { + profileSnippets = new ProfileSnippets.Templates(options, factories, providers, target); + } objectCloneSnippets = new ObjectCloneSnippets.Templates(options, factories, providers, target); foreignCallSnippets = new ForeignCallSnippets.Templates(options, factories, providers, target); } @@ -683,6 +687,9 @@ } private void throwCachedException(BytecodeExceptionNode node) { + if (IS_IN_NATIVE_IMAGE) { + throw new InternalError("Can't throw exception from SVM object"); + } Throwable exception = Exceptions.cachedExceptions.get(node.getExceptionKind()); assert exception != null; --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotClassInitializationPlugin.java 2019-03-12 08:09:14.951439269 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotClassInitializationPlugin.java 2019-03-12 08:09:14.587436905 +0100 @@ -24,9 +24,7 @@ package org.graalvm.compiler.hotspot.meta; -import java.lang.invoke.MethodHandle; -import java.lang.invoke.MethodHandles; -import java.lang.invoke.MethodType; +import java.lang.reflect.Method; import java.util.function.Supplier; import org.graalvm.compiler.core.common.type.ObjectStamp; @@ -88,15 +86,16 @@ } private static final Class hscp; - private static final MethodHandle loadReferencedTypeIIZMH; + private static final Method loadReferencedTypeIIZMH; static { - MethodHandle m = null; + Method m = null; Class c = null; try { c = Class.forName("jdk.vm.ci.hotspot.HotSpotConstantPool").asSubclass(ConstantPool.class); - m = MethodHandles.lookup().findVirtual(c, "loadReferencedType", MethodType.methodType(void.class, int.class, int.class, boolean.class)); + m = c.getDeclaredMethod("loadReferencedType", int.class, int.class, boolean.class); } catch (Exception e) { + throw GraalError.shouldNotReachHere(e); } loadReferencedTypeIIZMH = m; hscp = c; --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotGraalConstantFieldProvider.java 2019-03-12 08:09:15.451442516 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotGraalConstantFieldProvider.java 2019-03-12 08:09:15.087440152 +0100 @@ -24,6 +24,8 @@ package org.graalvm.compiler.hotspot.meta; +import static jdk.vm.ci.services.Services.IS_BUILDING_NATIVE_IMAGE; +import static jdk.vm.ci.services.Services.IS_IN_NATIVE_IMAGE; import static org.graalvm.compiler.core.common.GraalOptions.ImmutableCode; import java.util.ArrayList; @@ -63,7 +65,7 @@ private volatile List nonEmbeddableFields; protected boolean isEmbeddableField(ResolvedJavaField field) { - if (nonEmbeddableFields == null) { + if (!IS_IN_NATIVE_IMAGE && (IS_BUILDING_NATIVE_IMAGE || nonEmbeddableFields == null)) { synchronized (this) { if (nonEmbeddableFields == null) { List fields = new ArrayList<>(); --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotGraphBuilderPlugins.java 2019-03-12 08:09:15.923445582 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotGraphBuilderPlugins.java 2019-03-12 08:09:15.555443192 +0100 @@ -25,10 +25,11 @@ package org.graalvm.compiler.hotspot.meta; import static org.graalvm.compiler.core.common.GraalOptions.GeneratePIC; +import static org.graalvm.compiler.hotspot.HotSpotBackend.GHASH_PROCESS_BLOCKS; import static org.graalvm.compiler.hotspot.meta.HotSpotAOTProfilingPlugin.Options.TieredAOT; import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.JAVA_THREAD_THREAD_OBJECT_LOCATION; import static org.graalvm.compiler.java.BytecodeParserOptions.InlineDuringParsing; -import static org.graalvm.compiler.serviceprovider.GraalServices.Java8OrEarlier; +import static org.graalvm.compiler.serviceprovider.JavaVersionUtil.Java8OrEarlier; import java.lang.invoke.ConstantCallSite; import java.lang.invoke.MutableCallSite; @@ -46,12 +47,15 @@ import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig; import org.graalvm.compiler.hotspot.nodes.CurrentJavaThreadNode; import org.graalvm.compiler.hotspot.replacements.AESCryptSubstitutions; +import org.graalvm.compiler.hotspot.replacements.ArraysSupportSubstitutions; import org.graalvm.compiler.hotspot.replacements.BigIntegerSubstitutions; import org.graalvm.compiler.hotspot.replacements.CRC32CSubstitutions; import org.graalvm.compiler.hotspot.replacements.CRC32Substitutions; import org.graalvm.compiler.hotspot.replacements.CallSiteTargetNode; import org.graalvm.compiler.hotspot.replacements.CipherBlockChainingSubstitutions; import org.graalvm.compiler.hotspot.replacements.ClassGetHubNode; +import org.graalvm.compiler.hotspot.replacements.CounterModeSubstitutions; +import org.graalvm.compiler.hotspot.replacements.DigestBaseSubstitutions; import org.graalvm.compiler.hotspot.replacements.HotSpotArraySubstitutions; import org.graalvm.compiler.hotspot.replacements.HotSpotClassSubstitutions; import org.graalvm.compiler.hotspot.replacements.IdentityHashCodeNode; @@ -62,8 +66,10 @@ import org.graalvm.compiler.hotspot.replacements.SHA2Substitutions; import org.graalvm.compiler.hotspot.replacements.SHA5Substitutions; import org.graalvm.compiler.hotspot.replacements.SHASubstitutions; +import org.graalvm.compiler.hotspot.replacements.StringUTF16Substitutions; import org.graalvm.compiler.hotspot.replacements.ThreadSubstitutions; import org.graalvm.compiler.hotspot.word.HotSpotWordTypes; +import org.graalvm.compiler.nodes.ComputeObjectAddressNode; import org.graalvm.compiler.nodes.ConstantNode; import org.graalvm.compiler.nodes.NamedLocationIdentity; import org.graalvm.compiler.nodes.NodeView; @@ -71,6 +77,7 @@ import org.graalvm.compiler.nodes.calc.AddNode; import org.graalvm.compiler.nodes.calc.IntegerConvertNode; import org.graalvm.compiler.nodes.calc.LeftShiftNode; +import org.graalvm.compiler.nodes.extended.ForeignCallNode; import org.graalvm.compiler.nodes.graphbuilderconf.ForeignCallPlugin; import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderConfiguration.Plugins; import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderContext; @@ -163,9 +170,13 @@ registerCRC32CPlugins(invocationPlugins, config, replacementBytecodeProvider); registerBigIntegerPlugins(invocationPlugins, config, replacementBytecodeProvider); registerSHAPlugins(invocationPlugins, config, replacementBytecodeProvider); + registerGHASHPlugins(invocationPlugins, config, metaAccess, foreignCalls); + registerCounterModePlugins(invocationPlugins, config, replacementBytecodeProvider); registerUnsafePlugins(invocationPlugins, replacementBytecodeProvider); StandardGraphBuilderPlugins.registerInvocationPlugins(metaAccess, snippetReflection, invocationPlugins, replacementBytecodeProvider, true, false); registerArrayPlugins(invocationPlugins, replacementBytecodeProvider); + registerStringPlugins(invocationPlugins, replacementBytecodeProvider); + registerArraysSupportPlugins(invocationPlugins, config, replacementBytecodeProvider); for (NodeIntrinsicPluginFactory factory : GraalServices.load(NodeIntrinsicPluginFactory.class)) { factory.registerPlugins(invocationPlugins, nodeIntrinsificationProvider); @@ -388,6 +399,14 @@ r.registerMethodSubstitution(HotSpotArraySubstitutions.class, "newInstance", Class.class, int.class); } + private static void registerStringPlugins(InvocationPlugins plugins, BytecodeProvider bytecodeProvider) { + if (!Java8OrEarlier) { + final Registration utf16r = new Registration(plugins, "java.lang.StringUTF16", bytecodeProvider); + utf16r.registerMethodSubstitution(StringUTF16Substitutions.class, "toBytes", char[].class, int.class, int.class); + utf16r.registerMethodSubstitution(StringUTF16Substitutions.class, "getChars", byte[].class, int.class, int.class, char[].class, int.class); + } + } + private static void registerThreadPlugins(InvocationPlugins plugins, MetaAccessProvider metaAccess, WordTypes wordTypes, GraalHotSpotVMConfig config, BytecodeProvider bytecodeProvider) { Registration r = new Registration(plugins, Thread.class, bytecodeProvider); r.register0("currentThread", new InvocationPlugin() { @@ -477,23 +496,73 @@ } private static void registerSHAPlugins(InvocationPlugins plugins, GraalHotSpotVMConfig config, BytecodeProvider bytecodeProvider) { - if (config.useSHA1Intrinsics()) { + boolean useSha1 = config.useSHA1Intrinsics(); + boolean useSha256 = config.useSHA256Intrinsics(); + boolean useSha512 = config.useSHA512Intrinsics(); + + if (!Java8OrEarlier && (useSha1 || useSha256 || useSha512)) { + Registration r = new Registration(plugins, "sun.security.provider.DigestBase", bytecodeProvider); + r.registerMethodSubstitution(DigestBaseSubstitutions.class, "implCompressMultiBlock0", Receiver.class, byte[].class, int.class, int.class); + } + + if (useSha1) { assert config.sha1ImplCompress != 0L; Registration r = new Registration(plugins, "sun.security.provider.SHA", bytecodeProvider); r.registerMethodSubstitution(SHASubstitutions.class, SHASubstitutions.implCompressName, "implCompress0", Receiver.class, byte[].class, int.class); } - if (config.useSHA256Intrinsics()) { + if (useSha256) { assert config.sha256ImplCompress != 0L; Registration r = new Registration(plugins, "sun.security.provider.SHA2", bytecodeProvider); r.registerMethodSubstitution(SHA2Substitutions.class, SHA2Substitutions.implCompressName, "implCompress0", Receiver.class, byte[].class, int.class); } - if (config.useSHA512Intrinsics()) { + if (useSha512) { assert config.sha512ImplCompress != 0L; Registration r = new Registration(plugins, "sun.security.provider.SHA5", bytecodeProvider); r.registerMethodSubstitution(SHA5Substitutions.class, SHA5Substitutions.implCompressName, "implCompress0", Receiver.class, byte[].class, int.class); } } + private static void registerGHASHPlugins(InvocationPlugins plugins, GraalHotSpotVMConfig config, MetaAccessProvider metaAccess, ForeignCallsProvider foreignCalls) { + if (config.useGHASHIntrinsics()) { + assert config.ghashProcessBlocks != 0L; + Registration r = new Registration(plugins, "com.sun.crypto.provider.GHASH"); + r.register5("processBlocks", + byte[].class, + int.class, + int.class, + long[].class, + long[].class, + new InvocationPlugin() { + @Override + public boolean apply(GraphBuilderContext b, + ResolvedJavaMethod targetMethod, + Receiver receiver, + ValueNode data, + ValueNode inOffset, + ValueNode blocks, + ValueNode state, + ValueNode hashSubkey) { + int longArrayBaseOffset = metaAccess.getArrayBaseOffset(JavaKind.Long); + int byteArrayBaseOffset = metaAccess.getArrayBaseOffset(JavaKind.Byte); + ValueNode dataOffset = AddNode.create(ConstantNode.forInt(byteArrayBaseOffset), inOffset, NodeView.DEFAULT); + ComputeObjectAddressNode dataAddress = b.add(new ComputeObjectAddressNode(data, dataOffset)); + ComputeObjectAddressNode stateAddress = b.add(new ComputeObjectAddressNode(state, ConstantNode.forInt(longArrayBaseOffset))); + ComputeObjectAddressNode hashSubkeyAddress = b.add(new ComputeObjectAddressNode(hashSubkey, ConstantNode.forInt(longArrayBaseOffset))); + b.add(new ForeignCallNode(foreignCalls, GHASH_PROCESS_BLOCKS, stateAddress, hashSubkeyAddress, dataAddress, blocks)); + return true; + } + }); + } + } + + private static void registerCounterModePlugins(InvocationPlugins plugins, GraalHotSpotVMConfig config, BytecodeProvider bytecodeProvider) { + if (config.useAESCTRIntrinsics) { + assert config.counterModeAESCrypt != 0L; + Registration r = new Registration(plugins, "com.sun.crypto.provider.CounterMode", bytecodeProvider); + r.registerMethodSubstitution(CounterModeSubstitutions.class, "implCrypt", Receiver.class, byte[].class, int.class, int.class, byte[].class, int.class); + } + } + private static void registerCRC32Plugins(InvocationPlugins plugins, GraalHotSpotVMConfig config, BytecodeProvider bytecodeProvider) { if (config.useCRC32Intrinsics) { Registration r = new Registration(plugins, CRC32.class, bytecodeProvider); @@ -515,4 +584,11 @@ r.registerMethodSubstitution(CRC32CSubstitutions.class, "updateDirectByteBuffer", int.class, long.class, int.class, int.class); } } + + private static void registerArraysSupportPlugins(InvocationPlugins plugins, GraalHotSpotVMConfig config, BytecodeProvider bytecodeProvider) { + if (config.useVectorizedMismatchIntrinsic) { + Registration r = new Registration(plugins, "jdk.internal.util.ArraysSupport", bytecodeProvider); + r.registerMethodSubstitution(ArraysSupportSubstitutions.class, "vectorizedMismatch", Object.class, long.class, Object.class, long.class, int.class, int.class); + } + } } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotHostForeignCallsProvider.java 2019-03-12 08:09:16.423448828 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotHostForeignCallsProvider.java 2019-03-12 08:09:16.051446413 +0100 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2013, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -29,6 +29,7 @@ import static org.graalvm.compiler.core.target.Backend.ARITHMETIC_DREM; import static org.graalvm.compiler.core.target.Backend.ARITHMETIC_FREM; import static org.graalvm.compiler.hotspot.HotSpotBackend.BACKEDGE_EVENT; +import static org.graalvm.compiler.hotspot.HotSpotBackend.COUNTERMODE_IMPL_CRYPT; import static org.graalvm.compiler.hotspot.HotSpotBackend.DECRYPT; import static org.graalvm.compiler.hotspot.HotSpotBackend.DECRYPT_BLOCK; import static org.graalvm.compiler.hotspot.HotSpotBackend.DECRYPT_BLOCK_WITH_ORIGINAL_KEY; @@ -36,6 +37,7 @@ import static org.graalvm.compiler.hotspot.HotSpotBackend.ENCRYPT; import static org.graalvm.compiler.hotspot.HotSpotBackend.ENCRYPT_BLOCK; import static org.graalvm.compiler.hotspot.HotSpotBackend.EXCEPTION_HANDLER; +import static org.graalvm.compiler.hotspot.HotSpotBackend.GHASH_PROCESS_BLOCKS; import static org.graalvm.compiler.hotspot.HotSpotBackend.IC_MISS_HANDLER; import static org.graalvm.compiler.hotspot.HotSpotBackend.INITIALIZE_KLASS_BY_SYMBOL; import static org.graalvm.compiler.hotspot.HotSpotBackend.INVOCATION_EVENT; @@ -54,10 +56,14 @@ import static org.graalvm.compiler.hotspot.HotSpotBackend.RESOLVE_METHOD_BY_SYMBOL_AND_LOAD_COUNTERS; import static org.graalvm.compiler.hotspot.HotSpotBackend.RESOLVE_STRING_BY_SYMBOL; import static org.graalvm.compiler.hotspot.HotSpotBackend.SHA2_IMPL_COMPRESS; +import static org.graalvm.compiler.hotspot.HotSpotBackend.SHA2_IMPL_COMPRESS_MB; import static org.graalvm.compiler.hotspot.HotSpotBackend.SHA5_IMPL_COMPRESS; +import static org.graalvm.compiler.hotspot.HotSpotBackend.SHA5_IMPL_COMPRESS_MB; import static org.graalvm.compiler.hotspot.HotSpotBackend.SHA_IMPL_COMPRESS; +import static org.graalvm.compiler.hotspot.HotSpotBackend.SHA_IMPL_COMPRESS_MB; import static org.graalvm.compiler.hotspot.HotSpotBackend.SQUARE_TO_LEN; import static org.graalvm.compiler.hotspot.HotSpotBackend.UNWIND_EXCEPTION_TO_CALLER; +import static org.graalvm.compiler.hotspot.HotSpotBackend.VECTORIZED_MISMATCHED; import static org.graalvm.compiler.hotspot.HotSpotBackend.VM_ERROR; import static org.graalvm.compiler.hotspot.HotSpotBackend.WRONG_METHOD_HANDLER; import static org.graalvm.compiler.hotspot.HotSpotForeignCallLinkage.Reexecutability.NOT_REEXECUTABLE; @@ -169,23 +175,23 @@ return uninitObjectArraycopyDescriptors[aligned ? 1 : 0][disjoint ? 1 : 0]; } if (killAny) { - assert kind == JavaKind.Object; - return objectArraycopyDescriptorsKillAny[aligned ? 1 : 0][disjoint ? 1 : 0]; + return arraycopyDescriptorsKillAny[aligned ? 1 : 0][disjoint ? 1 : 0].get(kind); } return arraycopyDescriptors[aligned ? 1 : 0][disjoint ? 1 : 0].get(kind); } - @SuppressWarnings({"unchecked"}) private static final EnumMap[][] arraycopyDescriptors = (EnumMap[][]) new EnumMap[2][2]; + @SuppressWarnings("unchecked") private static final EnumMap[][] arraycopyDescriptors = (EnumMap[][]) new EnumMap[2][2]; + @SuppressWarnings("unchecked") private static final EnumMap[][] arraycopyDescriptorsKillAny = (EnumMap[][]) new EnumMap[2][2]; private static final ForeignCallDescriptor[][] uninitObjectArraycopyDescriptors = new ForeignCallDescriptor[2][2]; private static final ForeignCallDescriptor[] checkcastArraycopyDescriptors = new ForeignCallDescriptor[2]; - private static ForeignCallDescriptor[][] objectArraycopyDescriptorsKillAny = new ForeignCallDescriptor[2][2]; static { // Populate the EnumMap instances for (int i = 0; i < arraycopyDescriptors.length; i++) { for (int j = 0; j < arraycopyDescriptors[i].length; j++) { arraycopyDescriptors[i][j] = new EnumMap<>(JavaKind.class); + arraycopyDescriptorsKillAny[i][j] = new EnumMap<>(JavaKind.class); } } } @@ -199,13 +205,15 @@ if (uninit) { assert kind == JavaKind.Object; uninitObjectArraycopyDescriptors[aligned ? 1 : 0][disjoint ? 1 : 0] = desc; + } else if (killAny) { + arraycopyDescriptorsKillAny[aligned ? 1 : 0][disjoint ? 1 : 0].put(kind, desc); } else { arraycopyDescriptors[aligned ? 1 : 0][disjoint ? 1 : 0].put(kind, desc); } } private ForeignCallDescriptor buildDescriptor(JavaKind kind, boolean aligned, boolean disjoint, boolean uninit, boolean killAny, long routine) { - assert !killAny || kind == JavaKind.Object; + assert !uninit || kind == JavaKind.Object; String name = kind + (aligned ? "Aligned" : "") + (disjoint ? "Disjoint" : "") + (uninit ? "Uninit" : "") + "Arraycopy" + (killAny ? "KillAny" : ""); ForeignCallDescriptor desc = new ForeignCallDescriptor(name, void.class, Word.class, Word.class, Word.class); LocationIdentity killed = killAny ? LocationIdentity.any() : NamedLocationIdentity.getArrayLocation(kind); @@ -253,11 +261,12 @@ registerArraycopyDescriptor(descMap, kind, false, true, uninit, false, disjointRoutine); registerArraycopyDescriptor(descMap, kind, true, true, uninit, false, alignedDisjointRoutine); - if (kind == JavaKind.Object && !uninit) { - objectArraycopyDescriptorsKillAny[0][0] = buildDescriptor(kind, false, false, uninit, true, routine); - objectArraycopyDescriptorsKillAny[1][0] = buildDescriptor(kind, true, false, uninit, true, alignedRoutine); - objectArraycopyDescriptorsKillAny[0][1] = buildDescriptor(kind, false, true, uninit, true, disjointRoutine); - objectArraycopyDescriptorsKillAny[1][1] = buildDescriptor(kind, true, true, uninit, true, alignedDisjointRoutine); + if (!uninit) { + EconomicMap killAnyDescMap = EconomicMap.create(); + registerArraycopyDescriptor(killAnyDescMap, kind, false, false, uninit, true, routine); + registerArraycopyDescriptor(killAnyDescMap, kind, true, false, uninit, true, alignedRoutine); + registerArraycopyDescriptor(killAnyDescMap, kind, false, true, uninit, true, disjointRoutine); + registerArraycopyDescriptor(killAnyDescMap, kind, true, true, uninit, true, alignedDisjointRoutine); } } @@ -275,13 +284,9 @@ registerForeignCall(JAVA_TIME_MILLIS, c.javaTimeMillisAddress, NativeCall, DESTROYS_REGISTERS, LEAF_NOFP, REEXECUTABLE, NO_LOCATIONS); registerForeignCall(JAVA_TIME_NANOS, c.javaTimeNanosAddress, NativeCall, DESTROYS_REGISTERS, LEAF_NOFP, REEXECUTABLE, NO_LOCATIONS); - registerForeignCall(SIN.foreignCallDescriptor, c.arithmeticSinAddress, NativeCall, DESTROYS_REGISTERS, LEAF, REEXECUTABLE, NO_LOCATIONS); - registerForeignCall(COS.foreignCallDescriptor, c.arithmeticCosAddress, NativeCall, DESTROYS_REGISTERS, LEAF, REEXECUTABLE, NO_LOCATIONS); - registerForeignCall(TAN.foreignCallDescriptor, c.arithmeticTanAddress, NativeCall, DESTROYS_REGISTERS, LEAF, REEXECUTABLE, NO_LOCATIONS); - registerForeignCall(EXP.foreignCallDescriptor, c.arithmeticExpAddress, NativeCall, DESTROYS_REGISTERS, LEAF, REEXECUTABLE, NO_LOCATIONS); - registerForeignCall(LOG.foreignCallDescriptor, c.arithmeticLogAddress, NativeCall, DESTROYS_REGISTERS, LEAF, REEXECUTABLE, NO_LOCATIONS); - registerForeignCall(LOG10.foreignCallDescriptor, c.arithmeticLog10Address, NativeCall, DESTROYS_REGISTERS, LEAF, REEXECUTABLE, NO_LOCATIONS); - registerForeignCall(POW.foreignCallDescriptor, c.arithmeticPowAddress, NativeCall, DESTROYS_REGISTERS, LEAF, REEXECUTABLE, NO_LOCATIONS); + + registerMathStubs(c, providers, options); + registerForeignCall(ARITHMETIC_FREM, c.fremAddress, NativeCall, DESTROYS_REGISTERS, LEAF, REEXECUTABLE, NO_LOCATIONS); registerForeignCall(ARITHMETIC_DREM, c.dremAddress, NativeCall, DESTROYS_REGISTERS, LEAF, REEXECUTABLE, NO_LOCATIONS); @@ -377,14 +382,21 @@ registerForeignCall(MULTIPLY_TO_LEN, c.multiplyToLen, NativeCall, DESTROYS_REGISTERS, LEAF_NOFP, REEXECUTABLE_ONLY_AFTER_EXCEPTION, NamedLocationIdentity.getArrayLocation(JavaKind.Int)); } + if (c.useSHA1Intrinsics()) { - registerForeignCall(SHA_IMPL_COMPRESS, c.sha1ImplCompress, NativeCall, DESTROYS_REGISTERS, LEAF_NOFP, REEXECUTABLE_ONLY_AFTER_EXCEPTION, NamedLocationIdentity.any()); + registerForeignCall(SHA_IMPL_COMPRESS, c.sha1ImplCompress, NativeCall, DESTROYS_REGISTERS, LEAF, REEXECUTABLE_ONLY_AFTER_EXCEPTION, NamedLocationIdentity.any()); + registerForeignCall(SHA_IMPL_COMPRESS_MB, c.sha1ImplCompressMultiBlock, NativeCall, DESTROYS_REGISTERS, LEAF, REEXECUTABLE_ONLY_AFTER_EXCEPTION, NamedLocationIdentity.any()); } if (c.useSHA256Intrinsics()) { - registerForeignCall(SHA2_IMPL_COMPRESS, c.sha256ImplCompress, NativeCall, DESTROYS_REGISTERS, LEAF_NOFP, REEXECUTABLE_ONLY_AFTER_EXCEPTION, NamedLocationIdentity.any()); + registerForeignCall(SHA2_IMPL_COMPRESS, c.sha256ImplCompress, NativeCall, DESTROYS_REGISTERS, LEAF, REEXECUTABLE_ONLY_AFTER_EXCEPTION, NamedLocationIdentity.any()); + registerForeignCall(SHA2_IMPL_COMPRESS_MB, c.sha256ImplCompressMultiBlock, NativeCall, DESTROYS_REGISTERS, LEAF, REEXECUTABLE_ONLY_AFTER_EXCEPTION, NamedLocationIdentity.any()); } if (c.useSHA512Intrinsics()) { - registerForeignCall(SHA5_IMPL_COMPRESS, c.sha512ImplCompress, NativeCall, DESTROYS_REGISTERS, LEAF_NOFP, REEXECUTABLE_ONLY_AFTER_EXCEPTION, NamedLocationIdentity.any()); + registerForeignCall(SHA5_IMPL_COMPRESS, c.sha512ImplCompress, NativeCall, DESTROYS_REGISTERS, LEAF, REEXECUTABLE_ONLY_AFTER_EXCEPTION, NamedLocationIdentity.any()); + registerForeignCall(SHA5_IMPL_COMPRESS_MB, c.sha512ImplCompressMultiBlock, NativeCall, DESTROYS_REGISTERS, LEAF, REEXECUTABLE_ONLY_AFTER_EXCEPTION, NamedLocationIdentity.any()); + } + if (c.useGHASHIntrinsics()) { + registerForeignCall(GHASH_PROCESS_BLOCKS, c.ghashProcessBlocks, NativeCall, DESTROYS_REGISTERS, LEAF, REEXECUTABLE_ONLY_AFTER_EXCEPTION, NamedLocationIdentity.any()); } if (c.useMulAddIntrinsic()) { registerForeignCall(MUL_ADD, c.mulAdd, NativeCall, DESTROYS_REGISTERS, LEAF_NOFP, REEXECUTABLE_ONLY_AFTER_EXCEPTION, NamedLocationIdentity.getArrayLocation(JavaKind.Int)); @@ -409,11 +421,11 @@ */ try { // These stubs do callee saving - registerForeignCall(ENCRYPT_BLOCK, c.aescryptEncryptBlockStub, NativeCall, PRESERVES_REGISTERS, LEAF_NOFP, REEXECUTABLE_ONLY_AFTER_EXCEPTION, + registerForeignCall(ENCRYPT_BLOCK, c.aescryptEncryptBlockStub, NativeCall, PRESERVES_REGISTERS, LEAF, REEXECUTABLE_ONLY_AFTER_EXCEPTION, NamedLocationIdentity.getArrayLocation(JavaKind.Byte)); - registerForeignCall(DECRYPT_BLOCK, c.aescryptDecryptBlockStub, NativeCall, PRESERVES_REGISTERS, LEAF_NOFP, REEXECUTABLE_ONLY_AFTER_EXCEPTION, + registerForeignCall(DECRYPT_BLOCK, c.aescryptDecryptBlockStub, NativeCall, PRESERVES_REGISTERS, LEAF, REEXECUTABLE_ONLY_AFTER_EXCEPTION, NamedLocationIdentity.getArrayLocation(JavaKind.Byte)); - registerForeignCall(DECRYPT_BLOCK_WITH_ORIGINAL_KEY, c.aescryptDecryptBlockStub, NativeCall, PRESERVES_REGISTERS, LEAF_NOFP, REEXECUTABLE_ONLY_AFTER_EXCEPTION, + registerForeignCall(DECRYPT_BLOCK_WITH_ORIGINAL_KEY, c.aescryptDecryptBlockStub, NativeCall, PRESERVES_REGISTERS, LEAF, REEXECUTABLE_ONLY_AFTER_EXCEPTION, NamedLocationIdentity.getArrayLocation(JavaKind.Byte)); } catch (GraalError e) { if (!(e.getCause() instanceof ClassNotFoundException)) { @@ -422,11 +434,11 @@ } try { // These stubs do callee saving - registerForeignCall(ENCRYPT, c.cipherBlockChainingEncryptAESCryptStub, NativeCall, PRESERVES_REGISTERS, LEAF_NOFP, REEXECUTABLE_ONLY_AFTER_EXCEPTION, + registerForeignCall(ENCRYPT, c.cipherBlockChainingEncryptAESCryptStub, NativeCall, PRESERVES_REGISTERS, LEAF, REEXECUTABLE_ONLY_AFTER_EXCEPTION, NamedLocationIdentity.getArrayLocation(JavaKind.Byte)); - registerForeignCall(DECRYPT, c.cipherBlockChainingDecryptAESCryptStub, NativeCall, PRESERVES_REGISTERS, LEAF_NOFP, REEXECUTABLE_ONLY_AFTER_EXCEPTION, + registerForeignCall(DECRYPT, c.cipherBlockChainingDecryptAESCryptStub, NativeCall, PRESERVES_REGISTERS, LEAF, REEXECUTABLE_ONLY_AFTER_EXCEPTION, NamedLocationIdentity.getArrayLocation(JavaKind.Byte)); - registerForeignCall(DECRYPT_WITH_ORIGINAL_KEY, c.cipherBlockChainingDecryptAESCryptStub, NativeCall, PRESERVES_REGISTERS, LEAF_NOFP, REEXECUTABLE_ONLY_AFTER_EXCEPTION, + registerForeignCall(DECRYPT_WITH_ORIGINAL_KEY, c.cipherBlockChainingDecryptAESCryptStub, NativeCall, PRESERVES_REGISTERS, LEAF, REEXECUTABLE_ONLY_AFTER_EXCEPTION, NamedLocationIdentity.getArrayLocation(JavaKind.Byte)); } catch (GraalError e) { if (!(e.getCause() instanceof ClassNotFoundException)) { @@ -434,10 +446,34 @@ } } } + + if (c.useAESCTRIntrinsics) { + assert (c.counterModeAESCrypt != 0L); + registerForeignCall(COUNTERMODE_IMPL_CRYPT, c.counterModeAESCrypt, NativeCall, PRESERVES_REGISTERS, LEAF, REEXECUTABLE_ONLY_AFTER_EXCEPTION, + NamedLocationIdentity.any()); + } + + if (c.useVectorizedMismatchIntrinsic) { + assert (c.vectorizedMismatch != 0L); + registerForeignCall(VECTORIZED_MISMATCHED, c.vectorizedMismatch, NativeCall, PRESERVES_REGISTERS, LEAF, REEXECUTABLE_ONLY_AFTER_EXCEPTION, + NamedLocationIdentity.any()); + + } } public HotSpotForeignCallLinkage getForeignCall(ForeignCallDescriptor descriptor) { assert foreignCalls != null : descriptor; return foreignCalls.get(descriptor); } + + @SuppressWarnings("unused") + protected void registerMathStubs(GraalHotSpotVMConfig hotSpotVMConfig, HotSpotProviders providers, OptionValues options) { + registerForeignCall(SIN.foreignCallDescriptor, hotSpotVMConfig.arithmeticSinAddress, NativeCall, DESTROYS_REGISTERS, LEAF, REEXECUTABLE, NO_LOCATIONS); + registerForeignCall(COS.foreignCallDescriptor, hotSpotVMConfig.arithmeticCosAddress, NativeCall, DESTROYS_REGISTERS, LEAF, REEXECUTABLE, NO_LOCATIONS); + registerForeignCall(TAN.foreignCallDescriptor, hotSpotVMConfig.arithmeticTanAddress, NativeCall, DESTROYS_REGISTERS, LEAF, REEXECUTABLE, NO_LOCATIONS); + registerForeignCall(EXP.foreignCallDescriptor, hotSpotVMConfig.arithmeticExpAddress, NativeCall, DESTROYS_REGISTERS, LEAF, REEXECUTABLE, NO_LOCATIONS); + registerForeignCall(LOG.foreignCallDescriptor, hotSpotVMConfig.arithmeticLogAddress, NativeCall, DESTROYS_REGISTERS, LEAF, REEXECUTABLE, NO_LOCATIONS); + registerForeignCall(LOG10.foreignCallDescriptor, hotSpotVMConfig.arithmeticLog10Address, NativeCall, DESTROYS_REGISTERS, LEAF, REEXECUTABLE, NO_LOCATIONS); + registerForeignCall(POW.foreignCallDescriptor, hotSpotVMConfig.arithmeticPowAddress, NativeCall, DESTROYS_REGISTERS, LEAF, REEXECUTABLE, NO_LOCATIONS); + } } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotNodePlugin.java 2019-03-12 08:09:16.919452049 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotNodePlugin.java 2019-03-12 08:09:16.543449608 +0100 @@ -28,6 +28,8 @@ import static jdk.vm.ci.meta.DeoptimizationReason.TransferToInterpreter; import static org.graalvm.compiler.core.common.GraalOptions.ImmutableCode; +import java.lang.reflect.Field; + import org.graalvm.compiler.core.common.CompilationIdentifier; import org.graalvm.compiler.core.common.type.StampFactory; import org.graalvm.compiler.core.common.type.StampPair; @@ -66,8 +68,6 @@ import jdk.vm.ci.meta.ResolvedJavaField; import jdk.vm.ci.meta.ResolvedJavaMethod; import jdk.vm.ci.meta.ResolvedJavaType; - -import java.lang.reflect.Field; import sun.misc.Unsafe; /** @@ -243,10 +243,9 @@ } private static final LocationIdentity JAVA_THREAD_SHOULD_POST_ON_EXCEPTIONS_FLAG_LOCATION = NamedLocationIdentity.mutable("JavaThread::_should_post_on_exceptions_flag"); + static final Unsafe UNSAFE = initUnsafe(); - private static final Unsafe UNSAFE = initUnsafe(); - - private static Unsafe initUnsafe() { + static Unsafe initUnsafe() { try { // Fast path when we are trusted. return Unsafe.getUnsafe(); --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotUnsafeSubstitutions.java 2019-03-12 08:09:17.387455089 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotUnsafeSubstitutions.java 2019-03-12 08:09:17.023452725 +0100 @@ -24,7 +24,7 @@ package org.graalvm.compiler.hotspot.meta; -import static org.graalvm.compiler.serviceprovider.GraalServices.Java8OrEarlier; +import static org.graalvm.compiler.serviceprovider.JavaVersionUtil.Java8OrEarlier; import org.graalvm.compiler.api.replacements.ClassSubstitution; import org.graalvm.compiler.api.replacements.MethodSubstitution; --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotWordOperationPlugin.java 2019-03-12 08:09:17.879458284 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotWordOperationPlugin.java 2019-03-12 08:09:17.503455842 +0100 @@ -121,7 +121,7 @@ ValueNode pointer = args[0]; assert pointer.stamp(NodeView.DEFAULT) instanceof MetaspacePointerStamp; - LogicNode isNull = b.addWithInputs(IsNullNode.create(pointer)); + LogicNode isNull = b.add(IsNullNode.create(pointer)); b.addPush(returnKind, ConditionalNode.create(isNull, b.add(forBoolean(true)), b.add(forBoolean(false)), NodeView.DEFAULT)); break; --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/nodes/AllocaNode.java 2019-03-12 08:09:18.371461478 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/nodes/AllocaNode.java 2019-03-12 08:09:18.003459089 +0100 @@ -74,7 +74,7 @@ @Override public void generate(NodeLIRBuilderTool gen) { - VirtualStackSlot array = gen.getLIRGeneratorTool().getResult().getFrameMapBuilder().allocateStackSlots(slots, objects, null); + VirtualStackSlot array = gen.getLIRGeneratorTool().allocateStackSlots(slots, objects, null); Value result = gen.getLIRGeneratorTool().emitAddress(array); gen.setResult(this, result); } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/nodes/DimensionsNode.java 2019-03-12 08:09:18.843464544 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/nodes/DimensionsNode.java 2019-03-12 08:09:18.479462180 +0100 @@ -64,7 +64,7 @@ int size = rank * 4; int wordSize = lirGen.target().wordSize; int slots = roundUp(size, wordSize) / wordSize; - VirtualStackSlot array = lirGen.getResult().getFrameMapBuilder().allocateStackSlots(slots, new BitSet(0), null); + VirtualStackSlot array = lirGen.allocateStackSlots(slots, new BitSet(0), null); Value result = lirGen.emitAddress(array); gen.setResult(this, result); } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/nodes/type/KlassPointerStamp.java 2019-03-12 08:09:19.343467790 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/nodes/type/KlassPointerStamp.java 2019-03-12 08:09:18.971465374 +0100 @@ -72,6 +72,13 @@ } @Override + public void accept(Visitor v) { + super.accept(v); + v.visitLong(encoding.getBase()); + v.visitInt(encoding.getShift()); + } + + @Override protected AbstractPointerStamp copyWith(boolean newNonNull, boolean newAlwaysNull) { return new KlassPointerStamp(newNonNull, newAlwaysNull, encoding); } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/phases/OnStackReplacementPhase.java 2019-03-12 08:09:19.823470907 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/phases/OnStackReplacementPhase.java 2019-03-12 08:09:19.459468543 +0100 @@ -36,6 +36,7 @@ import org.graalvm.compiler.debug.GraalError; import org.graalvm.compiler.graph.Node; import org.graalvm.compiler.graph.iterators.NodeIterable; +import org.graalvm.compiler.loop.LoopEx; import org.graalvm.compiler.loop.LoopsData; import org.graalvm.compiler.loop.phases.LoopTransformations; import org.graalvm.compiler.nodeinfo.InputType; @@ -71,6 +72,7 @@ import org.graalvm.compiler.options.OptionValues; import org.graalvm.compiler.phases.Phase; import org.graalvm.compiler.phases.common.DeadCodeEliminationPhase; +import org.graalvm.compiler.serviceprovider.SpeculationReasonGroup; import jdk.vm.ci.meta.DeoptimizationAction; import jdk.vm.ci.meta.DeoptimizationReason; @@ -87,7 +89,7 @@ "if there is no mature profile available for the rest of the method.", type = OptionType.Debug) public static final OptionKey DeoptAfterOSR = new OptionKey<>(true); @Option(help = "Support OSR compilations with locks. If DeoptAfterOSR is true we can per definition not have " + - "unbalaced enter/extis mappings. If DeoptAfterOSR is false insert artificial monitor enters after " + + "unbalanced enter/exits mappings. If DeoptAfterOSR is false insert artificial monitor enters after " + "the OSRStart to have balanced enter/exits in the graph.", type = OptionType.Debug) public static final OptionKey SupportOSRWithLocks = new OptionKey<>(true); // @formatter:on @@ -99,6 +101,8 @@ return Options.SupportOSRWithLocks.getValue(options); } + private static final SpeculationReasonGroup OSR_LOCAL_SPECULATIONS = new SpeculationReasonGroup("OSRLocal", int.class, Stamp.class, int.class); + @Override @SuppressWarnings("try") protected void run(StructuredGraph graph) { @@ -152,7 +156,9 @@ l = l.getParent(); } - LoopTransformations.peel(loops.loop(l)); + LoopEx loop = loops.loop(l); + loop.loopBegin().markOsrLoop(); + LoopTransformations.peel(loop); osr.replaceAtUsages(InputType.Guard, AbstractBeginNode.prevBegin((FixedNode) osr.predecessor())); for (Node usage : osr.usages().snapshot()) { EntryProxyNode proxy = (EntryProxyNode) usage; @@ -179,7 +185,7 @@ final int locksSize = osrState.locksSize(); for (int i = 0; i < localsSize + locksSize; i++) { - ValueNode value = null; + ValueNode value; if (i >= localsSize) { value = osrState.lockAt(i - localsSize); } else { @@ -201,7 +207,7 @@ osrLocal = graph.addOrUnique(new OSRLocalNode(i, unrestrictedStamp)); } // Speculate on the OSRLocal stamps that could be more precise. - OSRLocalSpeculationReason reason = new OSRLocalSpeculationReason(osrState.bci, narrowedStamp, i); + SpeculationReason reason = OSR_LOCAL_SPECULATIONS.createSpeculationReason(osrState.bci, narrowedStamp, i); if (graph.getSpeculationLog().maySpeculate(reason) && osrLocal instanceof OSRLocalNode && value.getStackKind().equals(JavaKind.Object) && !narrowedStamp.isUnrestricted()) { // Add guard. LogicNode check = graph.addOrUniqueWithInputs(InstanceOfNode.createHelper((ObjectStamp) narrowedStamp, osrLocal, null, null)); @@ -305,30 +311,4 @@ public float codeSizeIncrease() { return 5.0f; } - - private static class OSRLocalSpeculationReason implements SpeculationReason { - private int bci; - private Stamp speculatedStamp; - private int localIndex; - - OSRLocalSpeculationReason(int bci, Stamp speculatedStamp, int localIndex) { - this.bci = bci; - this.speculatedStamp = speculatedStamp; - this.localIndex = localIndex; - } - - @Override - public boolean equals(Object obj) { - if (obj instanceof OSRLocalSpeculationReason) { - OSRLocalSpeculationReason that = (OSRLocalSpeculationReason) obj; - return this.bci == that.bci && this.speculatedStamp.equals(that.speculatedStamp) && this.localIndex == that.localIndex; - } - return false; - } - - @Override - public int hashCode() { - return (bci << 16) ^ speculatedStamp.hashCode() ^ localIndex; - } - } } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/phases/WriteBarrierAdditionPhase.java 2019-03-12 08:09:20.315474101 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/phases/WriteBarrierAdditionPhase.java 2019-03-12 08:09:19.947471712 +0100 @@ -25,35 +25,25 @@ package org.graalvm.compiler.hotspot.phases; import org.graalvm.compiler.debug.DebugCloseable; -import org.graalvm.compiler.debug.GraalError; import org.graalvm.compiler.graph.Node; import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig; -import org.graalvm.compiler.hotspot.nodes.G1ArrayRangePostWriteBarrier; -import org.graalvm.compiler.hotspot.nodes.G1ArrayRangePreWriteBarrier; -import org.graalvm.compiler.hotspot.nodes.G1PostWriteBarrier; -import org.graalvm.compiler.hotspot.nodes.G1PreWriteBarrier; -import org.graalvm.compiler.hotspot.nodes.G1ReferentFieldReadBarrier; -import org.graalvm.compiler.hotspot.nodes.SerialArrayRangeWriteBarrier; -import org.graalvm.compiler.hotspot.nodes.SerialWriteBarrier; +import org.graalvm.compiler.hotspot.gc.g1.G1BarrierSet; +import org.graalvm.compiler.hotspot.gc.shared.BarrierSet; +import org.graalvm.compiler.hotspot.gc.shared.CardTableBarrierSet; import org.graalvm.compiler.nodes.StructuredGraph; -import org.graalvm.compiler.nodes.ValueNode; import org.graalvm.compiler.nodes.extended.ArrayRangeWrite; import org.graalvm.compiler.nodes.java.AbstractCompareAndSwapNode; import org.graalvm.compiler.nodes.java.LoweredAtomicReadAndWriteNode; -import org.graalvm.compiler.nodes.memory.FixedAccessNode; -import org.graalvm.compiler.nodes.memory.HeapAccess.BarrierType; import org.graalvm.compiler.nodes.memory.ReadNode; import org.graalvm.compiler.nodes.memory.WriteNode; -import org.graalvm.compiler.nodes.memory.address.AddressNode; -import org.graalvm.compiler.nodes.type.StampTool; import org.graalvm.compiler.phases.Phase; public class WriteBarrierAdditionPhase extends Phase { - private GraalHotSpotVMConfig config; + private BarrierSet barrierSet; public WriteBarrierAdditionPhase(GraalHotSpotVMConfig config) { - this.config = config; + this.barrierSet = createBarrierSet(config); } @SuppressWarnings("try") @@ -62,141 +52,42 @@ for (Node n : graph.getNodes()) { try (DebugCloseable scope = n.graph().withNodeSourcePosition(n)) { if (n instanceof ReadNode) { - addReadNodeBarriers((ReadNode) n, graph); + barrierSet.addReadNodeBarriers((ReadNode) n, graph); } else if (n instanceof WriteNode) { - addWriteNodeBarriers((WriteNode) n, graph); + barrierSet.addWriteNodeBarriers((WriteNode) n, graph); } else if (n instanceof LoweredAtomicReadAndWriteNode) { LoweredAtomicReadAndWriteNode loweredAtomicReadAndWriteNode = (LoweredAtomicReadAndWriteNode) n; - addAtomicReadWriteNodeBarriers(loweredAtomicReadAndWriteNode, graph); + barrierSet.addAtomicReadWriteNodeBarriers(loweredAtomicReadAndWriteNode, graph); } else if (n instanceof AbstractCompareAndSwapNode) { - addCASBarriers((AbstractCompareAndSwapNode) n, graph); + barrierSet.addCASBarriers((AbstractCompareAndSwapNode) n, graph); } else if (n instanceof ArrayRangeWrite) { ArrayRangeWrite node = (ArrayRangeWrite) n; if (node.writesObjectArray()) { - addArrayRangeBarriers(node, graph); + barrierSet.addArrayRangeBarriers(node, graph); } } } } } - private void addReadNodeBarriers(ReadNode node, StructuredGraph graph) { - if (node.getBarrierType() == BarrierType.PRECISE) { - assert config.useG1GC; - G1ReferentFieldReadBarrier barrier = graph.add(new G1ReferentFieldReadBarrier(node.getAddress(), node, false)); - graph.addAfterFixed(node, barrier); - } else { - assert node.getBarrierType() == BarrierType.NONE : "Non precise read barrier has been attached to read node."; - } - } - - protected static void addG1PreWriteBarrier(FixedAccessNode node, AddressNode address, ValueNode value, boolean doLoad, boolean nullCheck, StructuredGraph graph) { - G1PreWriteBarrier preBarrier = graph.add(new G1PreWriteBarrier(address, value, doLoad, nullCheck)); - preBarrier.setStateBefore(node.stateBefore()); - node.setNullCheck(false); - node.setStateBefore(null); - graph.addBeforeFixed(node, preBarrier); - } - - protected void addG1PostWriteBarrier(FixedAccessNode node, AddressNode address, ValueNode value, boolean precise, StructuredGraph graph) { - final boolean alwaysNull = StampTool.isPointerAlwaysNull(value); - graph.addAfterFixed(node, graph.add(new G1PostWriteBarrier(address, value, precise, alwaysNull))); - } - - protected void addSerialPostWriteBarrier(FixedAccessNode node, AddressNode address, ValueNode value, boolean precise, StructuredGraph graph) { - final boolean alwaysNull = StampTool.isPointerAlwaysNull(value); - if (alwaysNull) { - // Serial barrier isn't needed for null value - return; - } - graph.addAfterFixed(node, graph.add(new SerialWriteBarrier(address, precise))); - } - - private void addWriteNodeBarriers(WriteNode node, StructuredGraph graph) { - BarrierType barrierType = node.getBarrierType(); - switch (barrierType) { - case NONE: - // nothing to do - break; - case IMPRECISE: - case PRECISE: - boolean precise = barrierType == BarrierType.PRECISE; - if (config.useG1GC) { - if (!node.getLocationIdentity().isInit()) { - // The pre barrier does nothing if the value being read is null, so it can - // be explicitly skipped when this is an initializing store. - addG1PreWriteBarrier(node, node.getAddress(), null, true, node.getNullCheck(), graph); - } - addG1PostWriteBarrier(node, node.getAddress(), node.value(), precise, graph); - } else { - addSerialPostWriteBarrier(node, node.getAddress(), node.value(), precise, graph); - } - break; - default: - throw new GraalError("unexpected barrier type: " + barrierType); - } - } - - private void addAtomicReadWriteNodeBarriers(LoweredAtomicReadAndWriteNode node, StructuredGraph graph) { - BarrierType barrierType = node.getBarrierType(); - switch (barrierType) { - case NONE: - // nothing to do - break; - case IMPRECISE: - case PRECISE: - boolean precise = barrierType == BarrierType.PRECISE; - if (config.useG1GC) { - addG1PreWriteBarrier(node, node.getAddress(), null, true, node.getNullCheck(), graph); - addG1PostWriteBarrier(node, node.getAddress(), node.getNewValue(), precise, graph); - } else { - addSerialPostWriteBarrier(node, node.getAddress(), node.getNewValue(), precise, graph); - } - break; - default: - throw new GraalError("unexpected barrier type: " + barrierType); - } - } - - private void addCASBarriers(AbstractCompareAndSwapNode node, StructuredGraph graph) { - BarrierType barrierType = node.getBarrierType(); - switch (barrierType) { - case NONE: - // nothing to do - break; - case IMPRECISE: - case PRECISE: - boolean precise = barrierType == BarrierType.PRECISE; - if (config.useG1GC) { - addG1PreWriteBarrier(node, node.getAddress(), node.getExpectedValue(), false, false, graph); - addG1PostWriteBarrier(node, node.getAddress(), node.getNewValue(), precise, graph); - } else { - addSerialPostWriteBarrier(node, node.getAddress(), node.getNewValue(), precise, graph); - } - break; - default: - throw new GraalError("unexpected barrier type: " + barrierType); - } + @Override + public boolean checkContract() { + return false; } - private void addArrayRangeBarriers(ArrayRangeWrite write, StructuredGraph graph) { + private BarrierSet createBarrierSet(GraalHotSpotVMConfig config) { if (config.useG1GC) { - if (!write.isInitialization()) { - // The pre barrier does nothing if the value being read is null, so it can - // be explicitly skipped when this is an initializing store. - G1ArrayRangePreWriteBarrier g1ArrayRangePreWriteBarrier = graph.add(new G1ArrayRangePreWriteBarrier(write.getAddress(), write.getLength(), write.getElementStride())); - graph.addBeforeFixed(write.asNode(), g1ArrayRangePreWriteBarrier); - } - G1ArrayRangePostWriteBarrier g1ArrayRangePostWriteBarrier = graph.add(new G1ArrayRangePostWriteBarrier(write.getAddress(), write.getLength(), write.getElementStride())); - graph.addAfterFixed(write.asNode(), g1ArrayRangePostWriteBarrier); + return createG1BarrierSet(); } else { - SerialArrayRangeWriteBarrier serialArrayRangeWriteBarrier = graph.add(new SerialArrayRangeWriteBarrier(write.getAddress(), write.getLength(), write.getElementStride())); - graph.addAfterFixed(write.asNode(), serialArrayRangeWriteBarrier); + return createCardTableBarrierSet(); } } - @Override - public boolean checkContract() { - return false; + protected BarrierSet createCardTableBarrierSet() { + return new CardTableBarrierSet(); + } + + protected BarrierSet createG1BarrierSet() { + return new G1BarrierSet(); } } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/phases/WriteBarrierVerificationPhase.java 2019-03-12 08:09:20.795477217 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/phases/WriteBarrierVerificationPhase.java 2019-03-12 08:09:20.435474880 +0100 @@ -31,10 +31,10 @@ import org.graalvm.compiler.graph.Node; import org.graalvm.compiler.graph.NodeFlood; import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig; -import org.graalvm.compiler.hotspot.nodes.ArrayRangeWriteBarrier; -import org.graalvm.compiler.hotspot.nodes.G1PostWriteBarrier; -import org.graalvm.compiler.hotspot.nodes.ObjectWriteBarrier; -import org.graalvm.compiler.hotspot.nodes.SerialWriteBarrier; +import org.graalvm.compiler.hotspot.gc.g1.G1PostWriteBarrier; +import org.graalvm.compiler.hotspot.gc.shared.ArrayRangeWriteBarrier; +import org.graalvm.compiler.hotspot.gc.shared.ObjectWriteBarrier; +import org.graalvm.compiler.hotspot.gc.shared.SerialWriteBarrier; import org.graalvm.compiler.nodeinfo.Verbosity; import org.graalvm.compiler.nodes.DeoptimizingNode; import org.graalvm.compiler.nodes.FixedWithNextNode; --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/HotSpotClassSubstitutions.java 2019-03-12 08:09:21.311480568 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/HotSpotClassSubstitutions.java 2019-03-12 08:09:20.935478126 +0100 @@ -25,6 +25,7 @@ package org.graalvm.compiler.hotspot.replacements; import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFIG; +import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfigBase.INJECTED_METAACCESS; import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.ARRAY_KLASS_COMPONENT_MIRROR; import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.KLASS_ACCESS_FLAGS_LOCATION; import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.KLASS_MODIFIER_FLAGS_LOCATION; @@ -38,11 +39,16 @@ import java.lang.reflect.Modifier; import org.graalvm.compiler.api.replacements.ClassSubstitution; +import org.graalvm.compiler.api.replacements.Fold; import org.graalvm.compiler.api.replacements.MethodSubstitution; import org.graalvm.compiler.hotspot.word.KlassPointer; +import org.graalvm.compiler.nodes.ConstantNode; import org.graalvm.compiler.nodes.PiNode; import org.graalvm.compiler.nodes.SnippetAnchorNode; +import jdk.vm.ci.meta.MetaAccessProvider; +import jdk.vm.ci.meta.ResolvedJavaType; + // JaCoCo Exclude /** @@ -92,6 +98,11 @@ return klass.isNull(); } + @Fold + public static ResolvedJavaType getObjectType(@Fold.InjectedParameter MetaAccessProvider metaAccesss) { + return metaAccesss.lookupJavaType(Object.class); + } + @MethodSubstitution(isStatic = false) public static Class getSuperclass(final Class thisObj) { KlassPointer klass = ClassGetHubNode.readClass(thisObj); @@ -100,7 +111,7 @@ int accessFlags = klassNonNull.readInt(klassAccessFlagsOffset(INJECTED_VMCONFIG), KLASS_ACCESS_FLAGS_LOCATION); if ((accessFlags & Modifier.INTERFACE) == 0) { if (klassIsArray(klassNonNull)) { - return Object.class; + return ConstantNode.forClass(getObjectType(INJECTED_METAACCESS)); } else { KlassPointer superKlass = klassNonNull.readKlassPointer(klassSuperKlassOffset(INJECTED_VMCONFIG), KLASS_SUPER_KLASS_LOCATION); if (superKlass.isNull()) { --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/HotSpotReplacementsUtil.java 2019-03-12 08:09:21.803483762 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/HotSpotReplacementsUtil.java 2019-03-12 08:09:21.435481373 +0100 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2012, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -138,10 +138,12 @@ } } - public static ResolvedJavaType methodHolderClass(@Fold.InjectedParameter IntrinsicContext context) { + @Fold + public static ResolvedJavaType methodHolderClass(@InjectedParameter IntrinsicContext context) { return context.getOriginalMethod().getDeclaringClass(); } + @Fold static ResolvedJavaType getType(@Fold.InjectedParameter IntrinsicContext context, String typeName) { try { UnresolvedJavaType unresolved = UnresolvedJavaType.create(typeName); @@ -151,6 +153,7 @@ } } + @Fold static int getFieldOffset(ResolvedJavaType type, String fieldName) { for (ResolvedJavaField field : type.getInstanceFields(true)) { if (field.getName().equals(fieldName)) { @@ -584,7 +587,7 @@ * Calls {@link #arrayAllocationSize(int, int, int, int)} using an injected VM configuration * object. */ - public static int arrayAllocationSize(int length, int headerSize, int log2ElementSize) { + public static long arrayAllocationSize(int length, int headerSize, int log2ElementSize) { return arrayAllocationSize(length, headerSize, log2ElementSize, objectAlignment(INJECTED_VMCONFIG)); } @@ -600,9 +603,9 @@ * requirement} * @return the size of the memory chunk */ - public static int arrayAllocationSize(int length, int headerSize, int log2ElementSize, int alignment) { - int size = (length << log2ElementSize) + headerSize + (alignment - 1); - int mask = ~(alignment - 1); + public static long arrayAllocationSize(int length, int headerSize, int log2ElementSize, int alignment) { + long size = ((long) length << log2ElementSize) + headerSize + (alignment - 1); + long mask = ~(alignment - 1); return size & mask; } @@ -730,7 +733,7 @@ } public static KlassPointer loadKlassFromObject(Object object, int offset, LocationIdentity identity) { - ReplacementsUtil.staticAssert(offset != hubOffset(INJECTED_VMCONFIG), "Use loadHubIntrinsic instead of loadWordFromObject"); + ReplacementsUtil.staticAssert(offset != hubOffset(INJECTED_VMCONFIG), "Use loadHubIntrinsic instead of loadKlassFromObject"); return loadKlassFromObjectIntrinsic(object, offset, identity, getWordKind()); } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/NewObjectSnippets.java 2019-03-12 08:09:22.299486982 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/NewObjectSnippets.java 2019-03-12 08:09:21.931484593 +0100 @@ -209,19 +209,19 @@ } @Snippet - public static Object allocateInstance(@ConstantParameter int size, KlassPointer hub, Word prototypeMarkWord, @ConstantParameter boolean fillContents, + public static Object allocateInstance(@ConstantParameter long size, KlassPointer hub, Word prototypeMarkWord, @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean constantSize, @ConstantParameter String typeContext, @ConstantParameter Counters counters) { return piCastToSnippetReplaceeStamp(allocateInstanceHelper(size, hub, prototypeMarkWord, fillContents, threadRegister, constantSize, typeContext, counters)); } - public static Object allocateInstanceHelper(int size, KlassPointer hub, Word prototypeMarkWord, boolean fillContents, + public static Object allocateInstanceHelper(long size, KlassPointer hub, Word prototypeMarkWord, boolean fillContents, Register threadRegister, boolean constantSize, String typeContext, Counters counters) { Object result; Word thread = registerAsWord(threadRegister); Word top = readTlabTop(thread); Word end = readTlabEnd(thread); - Word newTop = top.add(size); + Word newTop = top.add(WordFactory.unsigned(size)); if (useTLAB(INJECTED_VMCONFIG) && probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) { writeTlabTop(thread, newTop); emitPrefetchAllocate(newTop, false); @@ -252,7 +252,7 @@ private static native Object newInstanceOrNull(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub); @Snippet - public static Object allocateInstancePIC(@ConstantParameter int size, KlassPointer hub, Word prototypeMarkWord, @ConstantParameter boolean fillContents, + public static Object allocateInstancePIC(@ConstantParameter long size, KlassPointer hub, Word prototypeMarkWord, @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean constantSize, @ConstantParameter String typeContext, @ConstantParameter Counters counters) { // Klass must be initialized by the time the first instance is allocated, therefore we can @@ -316,7 +316,7 @@ @ConstantParameter Counters counters) { // Primitive array types are eagerly pre-resolved. We can use a floating load. KlassPointer picHub = LoadConstantIndirectlyNode.loadKlass(hub); - return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false, counters); + return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, counters); } @Snippet @@ -325,7 +325,7 @@ @ConstantParameter Counters counters) { // Array type would be resolved by dominating resolution. KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub); - return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false, counters); + return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, counters); } @Snippet @@ -348,7 +348,6 @@ threadRegister, maybeUnroll, typeContext, - false, counters); return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length); @@ -364,14 +363,14 @@ } private static Object allocateArrayImpl(KlassPointer hub, int length, Word prototypeMarkWord, int headerSize, int log2ElementSize, boolean fillContents, Register threadRegister, - boolean maybeUnroll, String typeContext, boolean skipNegativeCheck, Counters counters) { + boolean maybeUnroll, String typeContext, Counters counters) { Object result; - int allocationSize = arrayAllocationSize(length, headerSize, log2ElementSize); + long allocationSize = arrayAllocationSize(length, headerSize, log2ElementSize); Word thread = registerAsWord(threadRegister); Word top = readTlabTop(thread); Word end = readTlabEnd(thread); - Word newTop = top.add(allocationSize); - if (probability(FREQUENT_PROBABILITY, skipNegativeCheck || belowThan(length, MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH)) && useTLAB(INJECTED_VMCONFIG) && + Word newTop = top.add(WordFactory.unsigned(allocationSize)); + if (probability(FREQUENT_PROBABILITY, belowThan(length, MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH)) && useTLAB(INJECTED_VMCONFIG) && probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) { writeTlabTop(thread, newTop); emitPrefetchAllocate(newTop, true); @@ -486,7 +485,7 @@ int headerSize = (layoutHelper >> layoutHelperHeaderSizeShift(INJECTED_VMCONFIG)) & layoutHelperHeaderSizeMask(INJECTED_VMCONFIG); int log2ElementSize = (layoutHelper >> layoutHelperLog2ElementSizeShift(INJECTED_VMCONFIG)) & layoutHelperLog2ElementSizeMask(INJECTED_VMCONFIG); - Object result = allocateArrayImpl(nonNullKlass, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, false, "dynamic type", true, counters); + Object result = allocateArrayImpl(nonNullKlass, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, false, "dynamic type", counters); return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length); } @@ -540,11 +539,11 @@ * @param startOffset offset to begin zeroing. May not be word aligned. * @param manualUnroll maximally unroll zeroing */ - private static void zeroMemory(int size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) { + private static void zeroMemory(long size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) { fillMemory(0, size, memory, constantSize, startOffset, manualUnroll, counters); } - private static void fillMemory(long value, int size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) { + private static void fillMemory(long value, long size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) { ReplacementsUtil.runtimeAssert((size & 0x7) == 0, "unaligned object size"); int offset = startOffset; if ((offset & 0x7) != 0) { @@ -598,14 +597,14 @@ * @param startOffset offset to begin zeroing. May not be word aligned. * @param manualUnroll maximally unroll zeroing */ - private static void fillWithGarbage(int size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) { + private static void fillWithGarbage(long size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) { fillMemory(0xfefefefefefefefeL, size, memory, constantSize, startOffset, manualUnroll, counters); } /** * Formats some allocated memory with an object header and zeroes out the rest. */ - private static Object formatObject(KlassPointer hub, int size, Word memory, Word compileTimePrototypeMarkWord, boolean fillContents, boolean constantSize, Counters counters) { + private static Object formatObject(KlassPointer hub, long size, Word memory, Word compileTimePrototypeMarkWord, boolean fillContents, boolean constantSize, Counters counters) { Word prototypeMarkWord = useBiasedLocking(INJECTED_VMCONFIG) ? hub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION) : compileTimePrototypeMarkWord; initializeObjectHeader(memory, prototypeMarkWord, hub); if (fillContents) { @@ -632,7 +631,7 @@ /** * Formats some allocated memory with an object header and zeroes out the rest. */ - private static Object formatArray(KlassPointer hub, int allocationSize, int length, int headerSize, Word memory, Word prototypeMarkWord, boolean fillContents, boolean maybeUnroll, + private static Object formatArray(KlassPointer hub, long allocationSize, int length, int headerSize, Word memory, Word prototypeMarkWord, boolean fillContents, boolean maybeUnroll, Counters counters) { memory.writeInt(arrayLengthOffset(INJECTED_VMCONFIG), length, LocationIdentity.init()); /* @@ -699,7 +698,7 @@ HotSpotResolvedObjectType type = (HotSpotResolvedObjectType) newInstanceNode.instanceClass(); assert !type.isArray(); ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), type.klass(), providers.getMetaAccess(), graph); - int size = instanceSize(type); + long size = instanceSize(type); OptionValues localOptions = graph.getOptions(); SnippetInfo snippet = GeneratePIC.getValue(localOptions) ? allocateInstancePIC : allocateInstance; @@ -824,8 +823,8 @@ template(newmultiarrayNode, args).instantiate(providers.getMetaAccess(), newmultiarrayNode, DEFAULT_REPLACER, args); } - private static int instanceSize(HotSpotResolvedObjectType type) { - int size = type.instanceSize(); + private static long instanceSize(HotSpotResolvedObjectType type) { + long size = type.instanceSize(); assert size >= 0; return size; } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/SHA2Substitutions.java 2019-03-12 08:09:22.771490046 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/SHA2Substitutions.java 2019-03-12 08:09:22.399487631 +0100 @@ -26,7 +26,7 @@ import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_METAACCESS; import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfigBase.INJECTED_INTRINSIC_CONTEXT; -import static org.graalvm.compiler.serviceprovider.GraalServices.Java8OrEarlier; +import static org.graalvm.compiler.serviceprovider.JavaVersionUtil.Java8OrEarlier; import org.graalvm.compiler.api.replacements.ClassSubstitution; import org.graalvm.compiler.api.replacements.Fold; --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/SHA5Substitutions.java 2019-03-12 08:09:23.211492902 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/SHA5Substitutions.java 2019-03-12 08:09:22.843490514 +0100 @@ -26,14 +26,17 @@ import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfigBase.INJECTED_INTRINSIC_CONTEXT; import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfigBase.INJECTED_METAACCESS; -import static org.graalvm.compiler.serviceprovider.GraalServices.Java8OrEarlier; +import static org.graalvm.compiler.serviceprovider.JavaVersionUtil.Java8OrEarlier; import org.graalvm.compiler.api.replacements.ClassSubstitution; +import org.graalvm.compiler.api.replacements.Fold; +import org.graalvm.compiler.api.replacements.Fold.InjectedParameter; import org.graalvm.compiler.api.replacements.MethodSubstitution; import org.graalvm.compiler.hotspot.HotSpotBackend; import org.graalvm.compiler.nodes.ComputeObjectAddressNode; import org.graalvm.compiler.nodes.PiNode; import org.graalvm.compiler.nodes.extended.RawLoadNode; +import org.graalvm.compiler.nodes.graphbuilderconf.IntrinsicContext; import org.graalvm.compiler.replacements.ReplacementsUtil; import org.graalvm.compiler.word.Word; import jdk.internal.vm.compiler.word.LocationIdentity; @@ -49,14 +52,15 @@ @MethodSubstitution(isStatic = false) static void implCompress0(Object receiver, byte[] buf, int ofs) { Object realReceiver = PiNode.piCastNonNull(receiver, HotSpotReplacementsUtil.methodHolderClass(INJECTED_INTRINSIC_CONTEXT)); - Object state = RawLoadNode.load(realReceiver, stateOffset(), JavaKind.Object, LocationIdentity.any()); + Object state = RawLoadNode.load(realReceiver, stateOffset(INJECTED_INTRINSIC_CONTEXT), JavaKind.Object, LocationIdentity.any()); Word bufAddr = WordFactory.unsigned(ComputeObjectAddressNode.get(buf, ReplacementsUtil.getArrayBaseOffset(INJECTED_METAACCESS, JavaKind.Byte) + ofs)); Word stateAddr = WordFactory.unsigned(ComputeObjectAddressNode.get(state, ReplacementsUtil.getArrayBaseOffset(INJECTED_METAACCESS, JavaKind.Int))); HotSpotBackend.sha5ImplCompressStub(bufAddr, stateAddr); } - static long stateOffset() { - return HotSpotReplacementsUtil.getFieldOffset(HotSpotReplacementsUtil.methodHolderClass(INJECTED_INTRINSIC_CONTEXT), "state"); + @Fold + static long stateOffset(@InjectedParameter IntrinsicContext context) { + return HotSpotReplacementsUtil.getFieldOffset(HotSpotReplacementsUtil.methodHolderClass(context), "state"); } } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/SHASubstitutions.java 2019-03-12 08:09:23.655495785 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/SHASubstitutions.java 2019-03-12 08:09:23.287493396 +0100 @@ -26,14 +26,17 @@ import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfigBase.INJECTED_INTRINSIC_CONTEXT; import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfigBase.INJECTED_METAACCESS; -import static org.graalvm.compiler.serviceprovider.GraalServices.Java8OrEarlier; +import static org.graalvm.compiler.serviceprovider.JavaVersionUtil.Java8OrEarlier; import org.graalvm.compiler.api.replacements.ClassSubstitution; +import org.graalvm.compiler.api.replacements.Fold; +import org.graalvm.compiler.api.replacements.Fold.InjectedParameter; import org.graalvm.compiler.api.replacements.MethodSubstitution; import org.graalvm.compiler.hotspot.HotSpotBackend; import org.graalvm.compiler.nodes.ComputeObjectAddressNode; import org.graalvm.compiler.nodes.PiNode; import org.graalvm.compiler.nodes.extended.RawLoadNode; +import org.graalvm.compiler.nodes.graphbuilderconf.IntrinsicContext; import org.graalvm.compiler.replacements.ReplacementsUtil; import org.graalvm.compiler.word.Word; import jdk.internal.vm.compiler.word.LocationIdentity; @@ -49,14 +52,15 @@ @MethodSubstitution(isStatic = false) static void implCompress0(Object receiver, byte[] buf, int ofs) { Object realReceiver = PiNode.piCastNonNull(receiver, HotSpotReplacementsUtil.methodHolderClass(INJECTED_INTRINSIC_CONTEXT)); - Object state = RawLoadNode.load(realReceiver, stateOffset(), JavaKind.Object, LocationIdentity.any()); + Object state = RawLoadNode.load(realReceiver, stateOffset(INJECTED_INTRINSIC_CONTEXT), JavaKind.Object, LocationIdentity.any()); Word bufAddr = WordFactory.unsigned(ComputeObjectAddressNode.get(buf, ReplacementsUtil.getArrayBaseOffset(INJECTED_METAACCESS, JavaKind.Byte) + ofs)); Word stateAddr = WordFactory.unsigned(ComputeObjectAddressNode.get(state, ReplacementsUtil.getArrayBaseOffset(INJECTED_METAACCESS, JavaKind.Int))); HotSpotBackend.shaImplCompressStub(bufAddr, stateAddr); } - static long stateOffset() { - return HotSpotReplacementsUtil.getFieldOffset(HotSpotReplacementsUtil.methodHolderClass(INJECTED_INTRINSIC_CONTEXT), "state"); + @Fold + static long stateOffset(@InjectedParameter IntrinsicContext context) { + return HotSpotReplacementsUtil.getFieldOffset(HotSpotReplacementsUtil.methodHolderClass(context), "state"); } } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/WriteBarrierSnippets.java 2019-03-12 08:09:24.059498408 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/WriteBarrierSnippets.java 2019-03-12 08:09:23.727496252 +0100 @@ -54,17 +54,17 @@ import org.graalvm.compiler.graph.Node.ConstantNodeParameter; import org.graalvm.compiler.graph.Node.NodeIntrinsic; import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig; +import org.graalvm.compiler.hotspot.gc.g1.G1ArrayRangePostWriteBarrier; +import org.graalvm.compiler.hotspot.gc.g1.G1ArrayRangePreWriteBarrier; +import org.graalvm.compiler.hotspot.gc.g1.G1PostWriteBarrier; +import org.graalvm.compiler.hotspot.gc.g1.G1PreWriteBarrier; +import org.graalvm.compiler.hotspot.gc.g1.G1ReferentFieldReadBarrier; +import org.graalvm.compiler.hotspot.gc.shared.SerialArrayRangeWriteBarrier; +import org.graalvm.compiler.hotspot.gc.shared.SerialWriteBarrier; import org.graalvm.compiler.hotspot.meta.HotSpotProviders; import org.graalvm.compiler.hotspot.meta.HotSpotRegistersProvider; -import org.graalvm.compiler.hotspot.nodes.G1ArrayRangePostWriteBarrier; -import org.graalvm.compiler.hotspot.nodes.G1ArrayRangePreWriteBarrier; -import org.graalvm.compiler.hotspot.nodes.G1PostWriteBarrier; -import org.graalvm.compiler.hotspot.nodes.G1PreWriteBarrier; -import org.graalvm.compiler.hotspot.nodes.G1ReferentFieldReadBarrier; import org.graalvm.compiler.hotspot.nodes.GraalHotSpotVMConfigNode; import org.graalvm.compiler.hotspot.nodes.HotSpotCompressionNode; -import org.graalvm.compiler.hotspot.nodes.SerialArrayRangeWriteBarrier; -import org.graalvm.compiler.hotspot.nodes.SerialWriteBarrier; import org.graalvm.compiler.hotspot.nodes.VMErrorNode; import org.graalvm.compiler.nodes.NamedLocationIdentity; import org.graalvm.compiler.nodes.NodeView; --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/stubs/OutOfBoundsExceptionStub.java 2019-03-12 08:09:24.471501082 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/stubs/OutOfBoundsExceptionStub.java 2019-03-12 08:09:24.135498901 +0100 @@ -34,7 +34,7 @@ import org.graalvm.compiler.hotspot.meta.HotSpotProviders; import org.graalvm.compiler.hotspot.nodes.AllocaNode; import org.graalvm.compiler.options.OptionValues; -import org.graalvm.compiler.serviceprovider.GraalServices; +import org.graalvm.compiler.serviceprovider.JavaVersionUtil; import org.graalvm.compiler.word.Word; import jdk.vm.ci.code.Register; @@ -48,7 +48,7 @@ } // JDK-8201593: Print array length in ArrayIndexOutOfBoundsException. - private static final boolean PRINT_LENGTH_IN_EXCEPTION = GraalServices.JAVA_SPECIFICATION_VERSION >= 11; + private static final boolean PRINT_LENGTH_IN_EXCEPTION = JavaVersionUtil.JAVA_SPECIFICATION_VERSION >= 11; private static final int MAX_INT_STRING_SIZE = Integer.toString(Integer.MIN_VALUE).length(); private static final String STR_INDEX = "Index "; private static final String STR_OUTOFBOUNDSFORLENGTH = " out of bounds for length "; --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/stubs/Stub.java 2019-03-12 08:09:24.875503704 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/stubs/Stub.java 2019-03-12 08:09:24.543501549 +0100 @@ -25,7 +25,6 @@ package org.graalvm.compiler.hotspot.stubs; import static java.util.Collections.singletonList; -import static org.graalvm.compiler.core.GraalCompiler.emitBackEnd; import static org.graalvm.compiler.core.GraalCompiler.emitFrontEnd; import static org.graalvm.compiler.core.common.GraalOptions.GeneratePIC; import static org.graalvm.compiler.debug.DebugContext.DEFAULT_LOG_STREAM; @@ -236,7 +235,7 @@ Suites suites = createSuites(); emitFrontEnd(providers, backend, graph, providers.getSuites().getDefaultGraphBuilderSuite(), OptimisticOptimizations.ALL, DefaultProfilingInfo.get(TriState.UNKNOWN), suites); LIRSuites lirSuites = createLIRSuites(); - emitBackEnd(graph, Stub.this, getInstalledCodeOwner(), backend, compResult, CompilationResultBuilderFactory.Default, getRegisterConfig(), lirSuites); + backend.emitBackEnd(graph, Stub.this, getInstalledCodeOwner(), compResult, CompilationResultBuilderFactory.Default, getRegisterConfig(), lirSuites); assert checkStubInvariants(compResult); } catch (Throwable e) { throw debug.handle(e); --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.java/src/org/graalvm/compiler/java/BytecodeParser.java 2019-03-12 08:09:25.283506353 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.java/src/org/graalvm/compiler/java/BytecodeParser.java 2019-03-12 08:09:24.947504172 +0100 @@ -38,6 +38,7 @@ import static jdk.vm.ci.meta.DeoptimizationReason.UnreachedCode; import static jdk.vm.ci.meta.DeoptimizationReason.Unresolved; import static jdk.vm.ci.runtime.JVMCICompiler.INVOCATION_ENTRY_BCI; +import static jdk.vm.ci.services.Services.IS_BUILDING_NATIVE_IMAGE; import static org.graalvm.compiler.bytecode.Bytecodes.AALOAD; import static org.graalvm.compiler.bytecode.Bytecodes.AASTORE; import static org.graalvm.compiler.bytecode.Bytecodes.ACONST_NULL; @@ -270,6 +271,7 @@ import jdk.internal.vm.compiler.collections.EconomicMap; import jdk.internal.vm.compiler.collections.Equivalence; +import org.graalvm.compiler.api.replacements.Fold; import org.graalvm.compiler.api.replacements.Snippet; import org.graalvm.compiler.bytecode.Bytecode; import org.graalvm.compiler.bytecode.BytecodeDisassembler; @@ -424,7 +426,7 @@ import org.graalvm.compiler.options.OptionValues; import org.graalvm.compiler.phases.OptimisticOptimizations; import org.graalvm.compiler.phases.util.ValueMergeUtil; -import org.graalvm.compiler.serviceprovider.GraalServices; +import org.graalvm.compiler.serviceprovider.JavaVersionUtil; import jdk.internal.vm.compiler.word.LocationIdentity; import jdk.vm.ci.code.BailoutException; @@ -1290,8 +1292,8 @@ return graph.addOrUniqueWithInputs(x); } - protected ValueNode genIfNode(LogicNode condition, FixedNode falseSuccessor, FixedNode trueSuccessor, double d) { - return new IfNode(condition, falseSuccessor, trueSuccessor, d); + protected ValueNode genIfNode(LogicNode condition, FixedNode trueSuccessor, FixedNode falseSuccessor, double d) { + return new IfNode(condition, trueSuccessor, falseSuccessor, d); } protected void genThrow() { @@ -1646,11 +1648,11 @@ private boolean forceInliningEverything; @Override - public void handleReplacedInvoke(InvokeKind invokeKind, ResolvedJavaMethod targetMethod, ValueNode[] args, boolean inlineEverything) { + public Invoke handleReplacedInvoke(InvokeKind invokeKind, ResolvedJavaMethod targetMethod, ValueNode[] args, boolean inlineEverything) { boolean previous = forceInliningEverything; forceInliningEverything = previous || inlineEverything; try { - appendInvoke(invokeKind, targetMethod, args); + return appendInvoke(invokeKind, targetMethod, args); } finally { forceInliningEverything = previous; } @@ -1720,7 +1722,6 @@ } } if (invokeKind.isDirect()) { - inlineInfo = tryInline(args, targetMethod); if (inlineInfo == SUCCESSFULLY_INLINED) { return null; @@ -2320,7 +2321,7 @@ } return false; } - if (canInlinePartialIntrinsicExit() && InlinePartialIntrinsicExitDuringParsing.getValue(options)) { + if (canInlinePartialIntrinsicExit() && InlinePartialIntrinsicExitDuringParsing.getValue(options) && !IS_BUILDING_NATIVE_IMAGE) { // Otherwise inline the original method. Any frame state created // during the inlining will exclude frame(s) in the // intrinsic method (see FrameStateBuilder.create(int bci)). @@ -2540,6 +2541,7 @@ protected void genReturn(ValueNode returnVal, JavaKind returnKind) { if (parsingIntrinsic() && returnVal != null) { + if (returnVal instanceof StateSplit) { StateSplit stateSplit = (StateSplit) returnVal; FrameState stateAfter = stateSplit.stateAfter(); @@ -2548,7 +2550,20 @@ if (stateAfter.bci == BytecodeFrame.AFTER_BCI) { assert stateAfter.usages().count() == 1; assert stateAfter.usages().first() == stateSplit; - stateAfter.replaceAtUsages(graph.add(new FrameState(BytecodeFrame.AFTER_BCI, returnVal))); + FrameState state; + if (returnVal.getStackKind() == JavaKind.Illegal) { + // This should only occur when Fold and NodeIntrinsic plugins are + // deferred. Their return value might not be a Java type and in that + // case this can't be the final AFTER_BCI so just create a FrameState + // without a return value on the top of stack. + assert stateSplit instanceof Invoke; + ResolvedJavaMethod targetMethod = ((Invoke) stateSplit).getTargetMethod(); + assert targetMethod != null && (targetMethod.getAnnotation(Fold.class) != null || targetMethod.getAnnotation(Node.NodeIntrinsic.class) != null); + state = new FrameState(BytecodeFrame.AFTER_BCI); + } else { + state = new FrameState(BytecodeFrame.AFTER_BCI, returnVal); + } + stateAfter.replaceAtUsages(graph.add(state)); GraphUtil.killWithUnusedFloatingInputs(stateAfter); } else { /* @@ -3395,26 +3410,44 @@ condition = genUnique(condition); } - NodeSourcePosition currentPosition = graph.currentNodeSourcePosition(); + BciBlock deoptBlock = null; + BciBlock noDeoptBlock = null; if (isNeverExecutedCode(probability)) { - NodeSourcePosition survivingSuccessorPosition = graph.trackNodeSourcePosition() - ? new NodeSourcePosition(currentPosition.getCaller(), currentPosition.getMethod(), falseBlock.startBci) - : null; - append(new FixedGuardNode(condition, UnreachedCode, InvalidateReprofile, true, survivingSuccessorPosition)); - if (profilingPlugin != null && profilingPlugin.shouldProfile(this, method)) { - profilingPlugin.profileGoto(this, method, bci(), falseBlock.startBci, stateBefore); - } - appendGoto(falseBlock); - return; + deoptBlock = trueBlock; + noDeoptBlock = falseBlock; } else if (isNeverExecutedCode(1 - probability)) { - NodeSourcePosition survivingSuccessorPosition = graph.trackNodeSourcePosition() - ? new NodeSourcePosition(currentPosition.getCaller(), currentPosition.getMethod(), trueBlock.startBci) - : null; - append(new FixedGuardNode(condition, UnreachedCode, InvalidateReprofile, false, survivingSuccessorPosition)); - if (profilingPlugin != null && profilingPlugin.shouldProfile(this, method)) { - profilingPlugin.profileGoto(this, method, bci(), trueBlock.startBci, stateBefore); + deoptBlock = falseBlock; + noDeoptBlock = trueBlock; + } + + if (deoptBlock != null) { + NodeSourcePosition currentPosition = graph.currentNodeSourcePosition(); + NodeSourcePosition survivingSuccessorPosition = null; + if (graph.trackNodeSourcePosition()) { + survivingSuccessorPosition = new NodeSourcePosition(currentPosition.getCaller(), currentPosition.getMethod(), noDeoptBlock.startBci); + } + boolean negated = deoptBlock == trueBlock; + if (!isPotentialCountedLoopExit(condition, deoptBlock)) { + if (profilingPlugin != null && profilingPlugin.shouldProfile(this, method)) { + profilingPlugin.profileGoto(this, method, bci(), noDeoptBlock.startBci, stateBefore); + } + append(new FixedGuardNode(condition, UnreachedCode, InvalidateReprofile, negated, survivingSuccessorPosition)); + appendGoto(noDeoptBlock); + } else { + this.controlFlowSplit = true; + FixedNode noDeoptSuccessor = createTarget(noDeoptBlock, frameState, false, true); + DeoptimizeNode deopt = graph.add(new DeoptimizeNode(InvalidateReprofile, UnreachedCode)); + /* + * We do not want to `checkLoopExit` here: otherwise the deopt will go to the + * deoptBlock's BCI, skipping the branch in the interpreter, and the profile + * will never see that the branch is taken. This can lead to deopt loops or OSR + * failure. + */ + FixedNode deoptSuccessor = BeginNode.begin(deopt); + ValueNode ifNode = genIfNode(condition, negated ? deoptSuccessor : noDeoptSuccessor, negated ? noDeoptSuccessor : deoptSuccessor, negated ? 1 - probability : probability); + postProcessIfNode(ifNode); + append(ifNode); } - appendGoto(trueBlock); return; } @@ -3442,6 +3475,16 @@ } } + public boolean isPotentialCountedLoopExit(LogicNode condition, BciBlock target) { + if (currentBlock != null) { + long exits = currentBlock.loops & ~target.loops; + if (exits != 0) { + return condition instanceof CompareNode; + } + } + return false; + } + /** * Hook for subclasses to decide whether the IfNode probability should be complemented during * conversion to Graal IR. @@ -3953,7 +3996,7 @@ private String unresolvedMethodAssertionMessage(JavaMethod result) { String message = result.format("%H.%n(%P)%R"); - if (GraalServices.Java8OrEarlier) { + if (JavaVersionUtil.Java8OrEarlier) { JavaType declaringClass = result.getDeclaringClass(); String className = declaringClass.getName(); switch (className) { @@ -4214,13 +4257,10 @@ return; } - ResolvedJavaType[] skippedExceptionTypes = this.graphBuilderConfig.getSkippedExceptionTypes(); - if (skippedExceptionTypes != null) { - for (ResolvedJavaType exceptionType : skippedExceptionTypes) { - if (exceptionType.isAssignableFrom(resolvedType)) { - append(new DeoptimizeNode(DeoptimizationAction.InvalidateRecompile, RuntimeConstraint)); - return; - } + for (ResolvedJavaType exceptionType : this.graphBuilderConfig.getSkippedExceptionTypes()) { + if (exceptionType.isAssignableFrom(resolvedType)) { + append(new DeoptimizeNode(DeoptimizationAction.InvalidateRecompile, RuntimeConstraint)); + return; } } @@ -4513,9 +4553,13 @@ * Javac does not allow use of "$assertionsDisabled" for a field name but Eclipse does, in * which case a suffix is added to the generated field. */ - if ((parsingIntrinsic() || graphBuilderConfig.omitAssertions()) && resolvedField.isSynthetic() && resolvedField.getName().startsWith("$assertionsDisabled")) { - frameState.push(field.getJavaKind(), ConstantNode.forBoolean(true, graph)); - return; + if (resolvedField.isSynthetic() && resolvedField.getName().startsWith("$assertionsDisabled")) { + if (parsingIntrinsic()) { + throw new GraalError("Cannot use an assertion within the context of an intrinsic."); + } else if (graphBuilderConfig.omitAssertions()) { + frameState.push(field.getJavaKind(), ConstantNode.forBoolean(true, graph)); + return; + } } ResolvedJavaType holder = resolvedField.getDeclaringClass(); --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.aarch64/src/org/graalvm/compiler/lir/aarch64/AArch64Call.java 2019-03-12 08:09:25.755509417 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.aarch64/src/org/graalvm/compiler/lir/aarch64/AArch64Call.java 2019-03-12 08:09:25.427507288 +0100 @@ -40,6 +40,7 @@ import org.graalvm.compiler.lir.LIRFrameState; import org.graalvm.compiler.lir.LIRInstructionClass; import org.graalvm.compiler.lir.Opcode; +import org.graalvm.compiler.lir.StandardOp.LabelHoldingOp; import org.graalvm.compiler.lir.asm.CompilationResultBuilder; import jdk.vm.ci.code.Register; @@ -126,7 +127,7 @@ } } - public abstract static class ForeignCallOp extends CallOp { + public abstract static class ForeignCallOp extends CallOp implements LabelHoldingOp { protected final ForeignCallLinkage callTarget; protected final Label label; @@ -147,6 +148,11 @@ } protected abstract void emitCall(CompilationResultBuilder crb, AArch64MacroAssembler masm); + + @Override + public Label getLabel() { + return label; + } } @Opcode("NEAR_FOREIGN_CALL") --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.aarch64/src/org/graalvm/compiler/lir/aarch64/AArch64ControlFlow.java 2019-03-12 08:09:26.179512169 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.aarch64/src/org/graalvm/compiler/lir/aarch64/AArch64ControlFlow.java 2019-03-12 08:09:25.831509910 +0100 @@ -31,6 +31,7 @@ import java.util.function.Function; +import jdk.vm.ci.meta.AllocatableValue; import org.graalvm.compiler.asm.Label; import org.graalvm.compiler.core.common.NumUtil; import org.graalvm.compiler.asm.aarch64.AArch64Assembler; @@ -59,53 +60,21 @@ public class AArch64ControlFlow { - /** - * Compares integer register to 0 and branches if condition is true. Condition may only be equal - * or non-equal. - */ - // TODO (das) where do we need this? - // public static class CompareAndBranchOp extends AArch64LIRInstruction implements - // StandardOp.BranchOp { - // private final ConditionFlag condition; - // private final LabelRef destination; - // @Use({REG}) private Value x; - // - // public CompareAndBranchOp(Condition condition, LabelRef destination, Value x) { - // assert condition == Condition.EQ || condition == Condition.NE; - // assert ARMv8.isGpKind(x.getKind()); - // this.condition = condition == Condition.EQ ? ConditionFlag.EQ : ConditionFlag.NE; - // this.destination = destination; - // this.x = x; - // } - // - // @Override - // public void emitCode(CompilationResultBuilder crb, ARMv8MacroAssembler masm) { - // int size = ARMv8.bitsize(x.getKind()); - // if (condition == ConditionFlag.EQ) { - // masm.cbz(size, asRegister(x), destination.label()); - // } else { - // masm.cbnz(size, asRegister(x), destination.label()); - // } - // } - // } - - public static class BranchOp extends AArch64BlockEndOp implements StandardOp.BranchOp { - public static final LIRInstructionClass TYPE = LIRInstructionClass.create(BranchOp.class); - - private final AArch64Assembler.ConditionFlag condition; + public abstract static class AbstractBranchOp extends AArch64BlockEndOp implements StandardOp.BranchOp { private final LabelRef trueDestination; private final LabelRef falseDestination; private final double trueDestinationProbability; - public BranchOp(AArch64Assembler.ConditionFlag condition, LabelRef trueDestination, LabelRef falseDestination, double trueDestinationProbability) { - super(TYPE); - this.condition = condition; + private AbstractBranchOp(LIRInstructionClass c, LabelRef trueDestination, LabelRef falseDestination, double trueDestinationProbability) { + super(c); this.trueDestination = trueDestination; this.falseDestination = falseDestination; this.trueDestinationProbability = trueDestinationProbability; } + protected abstract void emitBranch(CompilationResultBuilder crb, AArch64MacroAssembler masm, LabelRef target, boolean negate); + @Override public void emitCode(CompilationResultBuilder crb, AArch64MacroAssembler masm) { /* @@ -115,18 +84,104 @@ * executing two instructions instead of one. */ if (crb.isSuccessorEdge(trueDestination)) { - masm.branchConditionally(condition.negate(), falseDestination.label()); + emitBranch(crb, masm, falseDestination, true); } else if (crb.isSuccessorEdge(falseDestination)) { - masm.branchConditionally(condition, trueDestination.label()); + emitBranch(crb, masm, trueDestination, false); } else if (trueDestinationProbability < 0.5) { - masm.branchConditionally(condition.negate(), falseDestination.label()); + emitBranch(crb, masm, falseDestination, true); masm.jmp(trueDestination.label()); } else { - masm.branchConditionally(condition, trueDestination.label()); + emitBranch(crb, masm, trueDestination, false); masm.jmp(falseDestination.label()); } } + } + + public static class BranchOp extends AbstractBranchOp implements StandardOp.BranchOp { + public static final LIRInstructionClass TYPE = LIRInstructionClass.create(BranchOp.class); + + private final AArch64Assembler.ConditionFlag condition; + + public BranchOp(AArch64Assembler.ConditionFlag condition, LabelRef trueDestination, LabelRef falseDestination, double trueDestinationProbability) { + super(TYPE, trueDestination, falseDestination, trueDestinationProbability); + this.condition = condition; + } + + @Override + protected void emitBranch(CompilationResultBuilder crb, AArch64MacroAssembler masm, LabelRef target, boolean negate) { + AArch64Assembler.ConditionFlag finalCond = negate ? condition.negate() : condition; + masm.branchConditionally(finalCond, target.label()); + } + } + + public static class CompareBranchZeroOp extends AbstractBranchOp implements StandardOp.BranchOp { + public static final LIRInstructionClass TYPE = LIRInstructionClass.create(CompareBranchZeroOp.class); + + @Use(REG) private AllocatableValue value; + + public CompareBranchZeroOp(AllocatableValue value, LabelRef trueDestination, LabelRef falseDestination, double trueDestinationProbability) { + super(TYPE, trueDestination, falseDestination, trueDestinationProbability); + this.value = value; + } + @Override + protected void emitBranch(CompilationResultBuilder crb, AArch64MacroAssembler masm, LabelRef target, boolean negate) { + AArch64Kind kind = (AArch64Kind) this.value.getPlatformKind(); + assert kind.isInteger(); + int size = kind.getSizeInBytes() * Byte.SIZE; + + if (negate) { + masm.cbnz(size, asRegister(this.value), target.label()); + } else { + masm.cbz(size, asRegister(this.value), target.label()); + } + } + } + + public static class BitTestAndBranchOp extends AbstractBranchOp implements StandardOp.BranchOp { + public static final LIRInstructionClass TYPE = LIRInstructionClass.create(BitTestAndBranchOp.class); + + @Use protected AllocatableValue value; + private final int index; + + public BitTestAndBranchOp(LabelRef trueDestination, LabelRef falseDestination, AllocatableValue value, double trueDestinationProbability, int index) { + super(TYPE, trueDestination, falseDestination, trueDestinationProbability); + this.value = value; + this.index = index; + } + + @Override + protected void emitBranch(CompilationResultBuilder crb, AArch64MacroAssembler masm, LabelRef target, boolean negate) { + ConditionFlag cond = negate ? ConditionFlag.NE : ConditionFlag.EQ; + Label label = target.label(); + boolean isFarBranch; + + if (label.isBound()) { + isFarBranch = NumUtil.isSignedNbit(18, masm.position() - label.position()); + } else { + // Max range of tbz is +-2^13 instructions. We estimate that each LIR instruction + // emits 2 AArch64 instructions on average. Thus we test for maximum 2^12 LIR + // instruction offset. + int maxLIRDistance = (1 << 12); + isFarBranch = !crb.labelWithinRange(this, label, maxLIRDistance); + } + + if (isFarBranch) { + cond = cond.negate(); + label = new Label(); + } + + if (cond == ConditionFlag.EQ) { + masm.tbz(asRegister(value), index, label); + } else { + masm.tbnz(asRegister(value), index, label); + } + + if (isFarBranch) { + masm.jmp(target.label()); + masm.bind(label); + } + } } @Opcode("CMOVE") --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.aarch64/src/org/graalvm/compiler/lir/aarch64/AArch64Move.java 2019-03-12 08:09:26.599514895 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.aarch64/src/org/graalvm/compiler/lir/aarch64/AArch64Move.java 2019-03-12 08:09:26.275512792 +0100 @@ -24,13 +24,6 @@ package org.graalvm.compiler.lir.aarch64; -import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.COMPOSITE; -import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.HINT; -import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.REG; -import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.STACK; -import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.UNINITIALIZED; -import static org.graalvm.compiler.lir.LIRValueUtil.asJavaConstant; -import static org.graalvm.compiler.lir.LIRValueUtil.isJavaConstant; import static jdk.vm.ci.aarch64.AArch64.sp; import static jdk.vm.ci.aarch64.AArch64.zr; import static jdk.vm.ci.code.ValueUtil.asAllocatableValue; @@ -38,6 +31,13 @@ import static jdk.vm.ci.code.ValueUtil.asStackSlot; import static jdk.vm.ci.code.ValueUtil.isRegister; import static jdk.vm.ci.code.ValueUtil.isStackSlot; +import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.COMPOSITE; +import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.HINT; +import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.REG; +import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.STACK; +import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.UNINITIALIZED; +import static org.graalvm.compiler.lir.LIRValueUtil.asJavaConstant; +import static org.graalvm.compiler.lir.LIRValueUtil.isJavaConstant; import org.graalvm.compiler.asm.aarch64.AArch64Address; import org.graalvm.compiler.asm.aarch64.AArch64MacroAssembler; @@ -56,6 +56,7 @@ import org.graalvm.compiler.lir.asm.CompilationResultBuilder; import jdk.vm.ci.aarch64.AArch64Kind; +import jdk.vm.ci.code.MemoryBarriers; import jdk.vm.ci.code.Register; import jdk.vm.ci.code.StackSlot; import jdk.vm.ci.meta.AllocatableValue; @@ -212,14 +213,20 @@ // a compiler bug which warns that crb is unused, and also // warns that @SuppressWarnings("unused") is unnecessary. public void emitCode(@SuppressWarnings("all") CompilationResultBuilder crb, AArch64MacroAssembler masm) { - // As I understand it load acquire/store release have the same semantics as on IA64 - // and allow us to handle LoadStore, LoadLoad and StoreStore without an explicit - // barrier. - // But Graal support to figure out if a load/store is volatile is non-existant so for - // now just use memory barriers everywhere. - // if ((barrier & MemoryBarriers.STORE_LOAD) != 0) { - masm.dmb(AArch64MacroAssembler.BarrierKind.ANY_ANY); - // } + assert barriers >= MemoryBarriers.LOAD_LOAD && barriers <= (MemoryBarriers.STORE_STORE | MemoryBarriers.STORE_LOAD | MemoryBarriers.LOAD_STORE | MemoryBarriers.LOAD_LOAD); + switch (barriers) { + case MemoryBarriers.STORE_STORE: + masm.dmb(AArch64MacroAssembler.BarrierKind.STORE_STORE); + break; + case MemoryBarriers.LOAD_LOAD: + case MemoryBarriers.LOAD_STORE: + case MemoryBarriers.LOAD_LOAD | MemoryBarriers.LOAD_STORE: + masm.dmb(AArch64MacroAssembler.BarrierKind.LOAD_LOAD); + break; + default: + masm.dmb(AArch64MacroAssembler.BarrierKind.ANY_ANY); + break; + } } } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64ArrayEqualsOp.java 2019-03-12 08:09:27.043517777 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64ArrayEqualsOp.java 2019-03-12 08:09:26.715515648 +0100 @@ -41,7 +41,7 @@ import org.graalvm.compiler.asm.amd64.AMD64MacroAssembler; import org.graalvm.compiler.asm.amd64.AVXKind; import org.graalvm.compiler.core.common.LIRKind; -import org.graalvm.compiler.core.common.NumUtil; +import org.graalvm.compiler.debug.GraalError; import org.graalvm.compiler.lir.LIRInstructionClass; import org.graalvm.compiler.lir.Opcode; import org.graalvm.compiler.lir.asm.CompilationResultBuilder; @@ -51,18 +51,29 @@ import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.ILLEGAL; import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.REG; +import java.util.Objects; + /** * Emits code which compares two arrays of the same length. If the CPU supports any vector * instructions specialized code is emitted to leverage these instructions. + * + * This op can also compare arrays of different integer types (e.g. {@code byte[]} and + * {@code char[]}) with on-the-fly sign- or zero-extension. If one of the given arrays is a + * {@code char[]} array, the smaller elements are zero-extended, otherwise they are sign-extended. */ @Opcode("ARRAY_EQUALS") public final class AMD64ArrayEqualsOp extends AMD64LIRInstruction { public static final LIRInstructionClass TYPE = LIRInstructionClass.create(AMD64ArrayEqualsOp.class); - private final JavaKind kind; - private final int arrayBaseOffset; - private final int arrayIndexScale; - private final int constantByteLength; + private final JavaKind kind1; + private final JavaKind kind2; + private final int arrayBaseOffset1; + private final int arrayBaseOffset2; + private final Scale arrayIndexScale1; + private final Scale arrayIndexScale2; + private final AVXKind.AVXSize vectorSize; + private final int constantLength; + private final boolean signExtend; @Def({REG}) private Value resultValue; @Alive({REG}) private Value array1Value; @@ -81,20 +92,21 @@ @Temp({REG, ILLEGAL}) private Value vectorTemp3; @Temp({REG, ILLEGAL}) private Value vectorTemp4; - public AMD64ArrayEqualsOp(LIRGeneratorTool tool, JavaKind kind, Value result, Value array1, Value array2, Value length, + public AMD64ArrayEqualsOp(LIRGeneratorTool tool, JavaKind kind1, JavaKind kind2, Value result, Value array1, Value array2, Value length, int constantLength, boolean directPointers, int maxVectorSize) { super(TYPE); - this.kind = kind; - - this.arrayBaseOffset = directPointers ? 0 : tool.getProviders().getMetaAccess().getArrayBaseOffset(kind); - this.arrayIndexScale = tool.getProviders().getMetaAccess().getArrayIndexScale(kind); - - if (constantLength >= 0 && arrayIndexScale > 1) { - // scale length - this.constantByteLength = constantLength << NumUtil.log2Ceil(arrayIndexScale); - } else { - this.constantByteLength = constantLength; - } + this.kind1 = kind1; + this.kind2 = kind2; + this.signExtend = kind1 != JavaKind.Char && kind2 != JavaKind.Char; + + assert kind1.isNumericInteger() && kind2.isNumericInteger() || kind1 == kind2; + + this.arrayBaseOffset1 = directPointers ? 0 : tool.getProviders().getMetaAccess().getArrayBaseOffset(kind1); + this.arrayBaseOffset2 = directPointers ? 0 : tool.getProviders().getMetaAccess().getArrayBaseOffset(kind2); + this.arrayIndexScale1 = Objects.requireNonNull(Scale.fromInt(tool.getProviders().getMetaAccess().getArrayIndexScale(kind1))); + this.arrayIndexScale2 = Objects.requireNonNull(Scale.fromInt(tool.getProviders().getMetaAccess().getArrayIndexScale(kind2))); + this.vectorSize = ((AMD64) tool.target().arch).getFeatures().contains(CPUFeature.AVX2) && (maxVectorSize < 0 || maxVectorSize >= 32) ? AVXKind.AVXSize.YMM : AVXKind.AVXSize.XMM; + this.constantLength = constantLength; this.resultValue = result; this.array1Value = array1; @@ -107,10 +119,10 @@ this.temp3 = tool.newVariable(LIRKind.value(tool.target().arch.getWordKind())); this.temp4 = tool.newVariable(LIRKind.value(tool.target().arch.getWordKind())); - this.temp5 = kind.isNumericFloat() ? tool.newVariable(LIRKind.value(tool.target().arch.getWordKind())) : Value.ILLEGAL; - if (kind == JavaKind.Float) { + this.temp5 = kind1.isNumericFloat() || kind1 != kind2 ? tool.newVariable(LIRKind.value(tool.target().arch.getWordKind())) : Value.ILLEGAL; + if (kind1 == JavaKind.Float) { this.tempXMM = tool.newVariable(LIRKind.value(AMD64Kind.SINGLE)); - } else if (kind == JavaKind.Double) { + } else if (kind1 == JavaKind.Double) { this.tempXMM = tool.newVariable(LIRKind.value(AMD64Kind.DOUBLE)); } else { this.tempXMM = Value.ILLEGAL; @@ -119,7 +131,7 @@ // We only need the vector temporaries if we generate SSE code. if (supportsSSE41(tool.target())) { if (canGenerateConstantLengthCompare(tool.target())) { - LIRKind lirKind = LIRKind.value(supportsAVX2(tool.target()) && (maxVectorSize < 0 || maxVectorSize >= 32) ? AMD64Kind.V256_BYTE : AMD64Kind.V128_BYTE); + LIRKind lirKind = LIRKind.value(vectorSize == AVXKind.AVXSize.YMM ? AMD64Kind.V256_BYTE : AMD64Kind.V128_BYTE); this.vectorTemp1 = tool.newVariable(lirKind); this.vectorTemp2 = tool.newVariable(lirKind); this.vectorTemp3 = tool.newVariable(lirKind); @@ -139,7 +151,7 @@ } private boolean canGenerateConstantLengthCompare(TargetDescription target) { - return constantByteLength >= 0 && kind.isNumericInteger() && supportsSSE41(target); + return constantLength >= 0 && kind1.isNumericInteger() && (kind1 == kind2 || getElementsPerVector(AVXKind.AVXSize.XMM) <= constantLength) && supportsSSE41(target); } @Override @@ -153,26 +165,19 @@ Label done = new Label(); // Load array base addresses. - masm.leaq(array1, new AMD64Address(asRegister(array1Value), arrayBaseOffset)); - masm.leaq(array2, new AMD64Address(asRegister(array2Value), arrayBaseOffset)); + masm.leaq(array1, new AMD64Address(asRegister(array1Value), arrayBaseOffset1)); + masm.leaq(array2, new AMD64Address(asRegister(array2Value), arrayBaseOffset2)); if (canGenerateConstantLengthCompare(crb.target)) { - emitConstantLengthArrayCompareBytes(masm, array1, array2, asRegister(temp3), asRegister(temp4), - new Register[]{asRegister(vectorTemp1), asRegister(vectorTemp2), asRegister(vectorTemp3), asRegister(vectorTemp4)}, - falseLabel, constantByteLength, AVXKind.getRegisterSize(vectorTemp1).getBytes()); + emitConstantLengthArrayCompareBytes(crb, masm, array1, array2, asRegister(temp3), asRegister(temp4), + new Register[]{asRegister(vectorTemp1), asRegister(vectorTemp2), asRegister(vectorTemp3), asRegister(vectorTemp4)}, falseLabel); } else { Register length = asRegister(temp3); - - // Get array length in bytes. + // Get array length. masm.movl(length, asRegister(lengthValue)); - - if (arrayIndexScale > 1) { - masm.shll(length, NumUtil.log2Ceil(arrayIndexScale)); // scale length - } - - masm.movl(result, length); // copy - - emitArrayCompare(crb, masm, kind, result, array1, array2, length, temp4, temp5, tempXMM, vectorTemp1, vectorTemp2, trueLabel, falseLabel); + // copy + masm.movl(result, length); + emitArrayCompare(crb, masm, result, array1, array2, length, trueLabel, falseLabel); } // Return true @@ -188,19 +193,18 @@ masm.bind(done); } - private static void emitArrayCompare(CompilationResultBuilder crb, AMD64MacroAssembler masm, JavaKind kind, + private void emitArrayCompare(CompilationResultBuilder crb, AMD64MacroAssembler masm, Register result, Register array1, Register array2, Register length, - Value temp4, Value temp5, Value tempXMM, Value vectorTemp1, Value vectorTemp2, Label trueLabel, Label falseLabel) { - if (supportsAVX2(crb.target)) { - emitAVXCompare(crb, masm, kind, result, array1, array2, length, temp4, temp5, tempXMM, vectorTemp1, vectorTemp2, trueLabel, falseLabel); - } else if (supportsSSE41(crb.target)) { - // this code is used for AVX as well because our backend correctly ensures that - // VEX-prefixed instructions are emitted if AVX is supported - emitSSE41Compare(crb, masm, kind, result, array1, array2, length, temp4, temp5, tempXMM, vectorTemp1, vectorTemp2, trueLabel, falseLabel); + if (supportsSSE41(crb.target)) { + emitVectorCompare(crb, masm, result, array1, array2, length, trueLabel, falseLabel); + } + if (kind1 == kind2) { + emit8ByteCompare(crb, masm, result, array1, array2, length, trueLabel, falseLabel); + emitTailCompares(masm, result, array1, array2, length, trueLabel, falseLabel); + } else { + emitDifferentKindsElementWiseCompare(crb, masm, result, array1, array2, length, trueLabel, falseLabel); } - emit8ByteCompare(crb, masm, kind, result, array1, array2, length, temp4, tempXMM, trueLabel, falseLabel); - emitTailCompares(masm, kind, result, array1, array2, length, temp4, tempXMM, trueLabel, falseLabel); } /** @@ -215,49 +219,44 @@ } /** - * Vector size used in {@link #emitSSE41Compare}. + * Emits code that uses SSE4.1/AVX1 128-bit (16-byte) or AVX2 256-bit (32-byte) vector compares. */ - private static final int SSE4_1_VECTOR_SIZE = 16; - - /** - * Emits code that uses SSE4.1 128-bit (16-byte) vector compares. - */ - private static void emitSSE41Compare(CompilationResultBuilder crb, AMD64MacroAssembler masm, JavaKind kind, + private void emitVectorCompare(CompilationResultBuilder crb, AMD64MacroAssembler masm, Register result, Register array1, Register array2, Register length, - Value temp4, Value temp5, Value tempXMM, Value vectorTemp1, Value vectorTemp2, Label trueLabel, Label falseLabel) { assert supportsSSE41(crb.target); Register vector1 = asRegister(vectorTemp1); Register vector2 = asRegister(vectorTemp2); + int elementsPerVector = getElementsPerVector(vectorSize); + Label loop = new Label(); Label compareTail = new Label(); - boolean requiresNaNCheck = kind.isNumericFloat(); + boolean requiresNaNCheck = kind1.isNumericFloat(); Label loopCheck = new Label(); Label nanCheck = new Label(); // Compare 16-byte vectors - masm.andl(result, SSE4_1_VECTOR_SIZE - 1); // tail count (in bytes) - masm.andl(length, ~(SSE4_1_VECTOR_SIZE - 1)); // vector count (in bytes) + masm.andl(result, elementsPerVector - 1); // tail count + masm.andl(length, ~(elementsPerVector - 1)); // vector count masm.jcc(ConditionFlag.Zero, compareTail); - masm.leaq(array1, new AMD64Address(array1, length, Scale.Times1, 0)); - masm.leaq(array2, new AMD64Address(array2, length, Scale.Times1, 0)); + masm.leaq(array1, new AMD64Address(array1, length, arrayIndexScale1, 0)); + masm.leaq(array2, new AMD64Address(array2, length, arrayIndexScale2, 0)); masm.negq(length); // Align the main loop masm.align(crb.target.wordSize * 2); masm.bind(loop); - masm.movdqu(vector1, new AMD64Address(array1, length, Scale.Times1, 0)); - masm.movdqu(vector2, new AMD64Address(array2, length, Scale.Times1, 0)); - masm.pxor(vector1, vector2); - masm.ptest(vector1, vector1); + emitVectorLoad1(masm, vector1, array1, length, 0, vectorSize); + emitVectorLoad2(masm, vector2, array2, length, 0, vectorSize); + emitVectorCmp(masm, vector1, vector2, vectorSize); masm.jcc(ConditionFlag.NotZero, requiresNaNCheck ? nanCheck : falseLabel); masm.bind(loopCheck); - masm.addq(length, SSE4_1_VECTOR_SIZE); + masm.addq(length, elementsPerVector); masm.jcc(ConditionFlag.NotZero, loop); masm.testl(result, result); @@ -267,7 +266,7 @@ Label unalignedCheck = new Label(); masm.jmpb(unalignedCheck); masm.bind(nanCheck); - emitFloatCompareWithinRange(crb, masm, kind, array1, array2, length, temp4, temp5, tempXMM, 0, falseLabel, SSE4_1_VECTOR_SIZE); + emitFloatCompareWithinRange(crb, masm, array1, array2, length, 0, falseLabel, elementsPerVector); masm.jmpb(loopCheck); masm.bind(unalignedCheck); } @@ -276,13 +275,12 @@ * Compare the remaining bytes with an unaligned memory load aligned to the end of the * array. */ - masm.movdqu(vector1, new AMD64Address(array1, result, Scale.Times1, -SSE4_1_VECTOR_SIZE)); - masm.movdqu(vector2, new AMD64Address(array2, result, Scale.Times1, -SSE4_1_VECTOR_SIZE)); - masm.pxor(vector1, vector2); - masm.ptest(vector1, vector1); + emitVectorLoad1(masm, vector1, array1, result, scaleDisplacement1(-vectorSize.getBytes()), vectorSize); + emitVectorLoad2(masm, vector2, array2, result, scaleDisplacement2(-vectorSize.getBytes()), vectorSize); + emitVectorCmp(masm, vector1, vector2, vectorSize); if (requiresNaNCheck) { masm.jcc(ConditionFlag.Zero, trueLabel); - emitFloatCompareWithinRange(crb, masm, kind, array1, array2, result, temp4, temp5, tempXMM, -SSE4_1_VECTOR_SIZE, falseLabel, SSE4_1_VECTOR_SIZE); + emitFloatCompareWithinRange(crb, masm, array1, array2, result, -vectorSize.getBytes(), falseLabel, elementsPerVector); } else { masm.jcc(ConditionFlag.NotZero, falseLabel); } @@ -292,90 +290,159 @@ masm.movl(length, result); } - /** - * Returns if the underlying AMD64 architecture supports AVX instructions. - * - * @param target target description of the underlying architecture - * @return true if the underlying architecture supports AVX - */ - private static boolean supportsAVX2(TargetDescription target) { - AMD64 arch = (AMD64) target.arch; - return arch.getFeatures().contains(CPUFeature.AVX2); + private int getElementsPerVector(AVXKind.AVXSize vSize) { + return vSize.getBytes() >> Math.max(arrayIndexScale1.log2, arrayIndexScale2.log2); } - /** - * Vector size used in {@link #emitAVXCompare}. - */ - private static final int AVX_VECTOR_SIZE = 32; + private void emitVectorLoad1(AMD64MacroAssembler asm, Register dst, Register src, int displacement, AVXKind.AVXSize size) { + emitVectorLoad1(asm, dst, src, Register.None, displacement, size); + } - private static void emitAVXCompare(CompilationResultBuilder crb, AMD64MacroAssembler masm, JavaKind kind, Register result, - Register array1, Register array2, Register length, - Value temp4, Value temp5, Value tempXMM, Value vectorTemp1, Value vectorTemp2, - Label trueLabel, Label falseLabel) { - assert supportsAVX2(crb.target); + private void emitVectorLoad2(AMD64MacroAssembler asm, Register dst, Register src, int displacement, AVXKind.AVXSize size) { + emitVectorLoad2(asm, dst, src, Register.None, displacement, size); + } - Register vector1 = asRegister(vectorTemp1); - Register vector2 = asRegister(vectorTemp2); + private void emitVectorLoad1(AMD64MacroAssembler asm, Register dst, Register src, Register index, int displacement, AVXKind.AVXSize size) { + emitVectorLoad(asm, dst, src, index, displacement, arrayIndexScale1, arrayIndexScale2, size); + } - Label loop = new Label(); - Label compareTail = new Label(); + private void emitVectorLoad2(AMD64MacroAssembler asm, Register dst, Register src, Register index, int displacement, AVXKind.AVXSize size) { + emitVectorLoad(asm, dst, src, index, displacement, arrayIndexScale2, arrayIndexScale1, size); + } - boolean requiresNaNCheck = kind.isNumericFloat(); - Label loopCheck = new Label(); - Label nanCheck = new Label(); + private void emitVectorLoad(AMD64MacroAssembler asm, Register dst, Register src, Register index, int displacement, Scale ownScale, Scale otherScale, AVXKind.AVXSize size) { + AMD64Address address = new AMD64Address(src, index, ownScale, displacement); + if (ownScale.value < otherScale.value) { + if (size == AVXKind.AVXSize.YMM) { + getAVX2LoadAndExtendOp(ownScale, otherScale, signExtend).emit(asm, size, dst, address); + } else { + loadAndExtendSSE(asm, dst, address, ownScale, otherScale, signExtend); + } + } else { + if (size == AVXKind.AVXSize.YMM) { + asm.vmovdqu(dst, address); + } else { + asm.movdqu(dst, address); + } + } + } - // Compare 32-byte vectors - masm.andl(result, AVX_VECTOR_SIZE - 1); // tail count (in bytes) - masm.andl(length, ~(AVX_VECTOR_SIZE - 1)); // vector count (in bytes) - masm.jcc(ConditionFlag.Zero, compareTail); + private int scaleDisplacement1(int displacement) { + return scaleDisplacement(displacement, arrayIndexScale1, arrayIndexScale2); + } - masm.leaq(array1, new AMD64Address(array1, length, Scale.Times1, 0)); - masm.leaq(array2, new AMD64Address(array2, length, Scale.Times1, 0)); - masm.negq(length); + private int scaleDisplacement2(int displacement) { + return scaleDisplacement(displacement, arrayIndexScale2, arrayIndexScale1); + } - // Align the main loop - masm.align(crb.target.wordSize * 2); - masm.bind(loop); - masm.vmovdqu(vector1, new AMD64Address(array1, length, Scale.Times1, 0)); - masm.vmovdqu(vector2, new AMD64Address(array2, length, Scale.Times1, 0)); - masm.vpxor(vector1, vector1, vector2); - masm.vptest(vector1, vector1); - masm.jcc(ConditionFlag.NotZero, requiresNaNCheck ? nanCheck : falseLabel); + private static int scaleDisplacement(int displacement, Scale ownScale, Scale otherScale) { + if (ownScale.value < otherScale.value) { + return displacement >> (otherScale.log2 - ownScale.log2); + } + return displacement; + } - masm.bind(loopCheck); - masm.addq(length, AVX_VECTOR_SIZE); - masm.jcc(ConditionFlag.NotZero, loop); + private static AMD64Assembler.VexRMOp getAVX2LoadAndExtendOp(Scale ownScale, Scale otherScale, boolean signExtend) { + switch (ownScale) { + case Times1: + switch (otherScale) { + case Times2: + return signExtend ? AMD64Assembler.VexRMOp.VPMOVSXBW : AMD64Assembler.VexRMOp.VPMOVZXBW; + case Times4: + return signExtend ? AMD64Assembler.VexRMOp.VPMOVSXBD : AMD64Assembler.VexRMOp.VPMOVZXBD; + case Times8: + return signExtend ? AMD64Assembler.VexRMOp.VPMOVSXBQ : AMD64Assembler.VexRMOp.VPMOVZXBQ; + } + throw GraalError.shouldNotReachHere(); + case Times2: + switch (otherScale) { + case Times4: + return signExtend ? AMD64Assembler.VexRMOp.VPMOVSXWD : AMD64Assembler.VexRMOp.VPMOVZXWD; + case Times8: + return signExtend ? AMD64Assembler.VexRMOp.VPMOVSXWQ : AMD64Assembler.VexRMOp.VPMOVZXWQ; + } + throw GraalError.shouldNotReachHere(); + case Times4: + return signExtend ? AMD64Assembler.VexRMOp.VPMOVSXDQ : AMD64Assembler.VexRMOp.VPMOVZXDQ; + } + throw GraalError.shouldNotReachHere(); + } + + private static void loadAndExtendSSE(AMD64MacroAssembler asm, Register dst, AMD64Address src, Scale ownScale, Scale otherScale, boolean signExtend) { + switch (ownScale) { + case Times1: + switch (otherScale) { + case Times2: + if (signExtend) { + asm.pmovsxbw(dst, src); + } else { + asm.pmovzxbw(dst, src); + } + return; + case Times4: + if (signExtend) { + asm.pmovsxbd(dst, src); + } else { + asm.pmovzxbd(dst, src); + } + return; + case Times8: + if (signExtend) { + asm.pmovsxbq(dst, src); + } else { + asm.pmovzxbq(dst, src); + } + return; + } + throw GraalError.shouldNotReachHere(); + case Times2: + switch (otherScale) { + case Times4: + if (signExtend) { + asm.pmovsxwd(dst, src); + } else { + asm.pmovzxwd(dst, src); + } + return; + case Times8: + if (signExtend) { + asm.pmovsxwq(dst, src); + } else { + asm.pmovzxwq(dst, src); + } + return; + } + throw GraalError.shouldNotReachHere(); + case Times4: + if (signExtend) { + asm.pmovsxdq(dst, src); + } else { + asm.pmovzxdq(dst, src); + } + return; + } + throw GraalError.shouldNotReachHere(); + } - masm.testl(result, result); - masm.jcc(ConditionFlag.Zero, trueLabel); + private static void emitVectorCmp(AMD64MacroAssembler masm, Register vector1, Register vector2, AVXKind.AVXSize size) { + emitVectorXor(masm, vector1, vector2, size); + emitVectorTest(masm, vector1, size); + } - if (requiresNaNCheck) { - Label unalignedCheck = new Label(); - masm.jmpb(unalignedCheck); - masm.bind(nanCheck); - emitFloatCompareWithinRange(crb, masm, kind, array1, array2, length, temp4, temp5, tempXMM, 0, falseLabel, AVX_VECTOR_SIZE); - masm.jmpb(loopCheck); - masm.bind(unalignedCheck); + private static void emitVectorXor(AMD64MacroAssembler masm, Register vector1, Register vector2, AVXKind.AVXSize size) { + if (size == AVXKind.AVXSize.YMM) { + masm.vpxor(vector1, vector1, vector2); + } else { + masm.pxor(vector1, vector2); } + } - /* - * Compare the remaining bytes with an unaligned memory load aligned to the end of the - * array. - */ - masm.vmovdqu(vector1, new AMD64Address(array1, result, Scale.Times1, -AVX_VECTOR_SIZE)); - masm.vmovdqu(vector2, new AMD64Address(array2, result, Scale.Times1, -AVX_VECTOR_SIZE)); - masm.vpxor(vector1, vector1, vector2); - masm.vptest(vector1, vector1); - if (requiresNaNCheck) { - masm.jcc(ConditionFlag.Zero, trueLabel); - emitFloatCompareWithinRange(crb, masm, kind, array1, array2, result, temp4, temp5, tempXMM, -AVX_VECTOR_SIZE, falseLabel, AVX_VECTOR_SIZE); + private static void emitVectorTest(AMD64MacroAssembler masm, Register vector1, AVXKind.AVXSize size) { + if (size == AVXKind.AVXSize.YMM) { + masm.vptest(vector1, vector1); } else { - masm.jcc(ConditionFlag.NotZero, falseLabel); + masm.ptest(vector1, vector1); } - masm.jmp(trueLabel); - - masm.bind(compareTail); - masm.movl(length, result); } /** @@ -386,34 +453,37 @@ /** * Emits code that uses 8-byte vector compares. */ - private static void emit8ByteCompare(CompilationResultBuilder crb, AMD64MacroAssembler masm, JavaKind kind, Register result, Register array1, Register array2, Register length, Value temp4, - Value tempXMM, Label trueLabel, Label falseLabel) { + private void emit8ByteCompare(CompilationResultBuilder crb, AMD64MacroAssembler masm, + Register result, Register array1, Register array2, Register length, Label trueLabel, Label falseLabel) { + assert kind1 == kind2; Label loop = new Label(); Label compareTail = new Label(); - boolean requiresNaNCheck = kind.isNumericFloat(); + int elementsPerVector = 8 >> arrayIndexScale1.log2; + + boolean requiresNaNCheck = kind1.isNumericFloat(); Label loopCheck = new Label(); Label nanCheck = new Label(); Register temp = asRegister(temp4); - masm.andl(result, VECTOR_SIZE - 1); // tail count (in bytes) - masm.andl(length, ~(VECTOR_SIZE - 1)); // vector count (in bytes) + masm.andl(result, elementsPerVector - 1); // tail count + masm.andl(length, ~(elementsPerVector - 1)); // vector count masm.jcc(ConditionFlag.Zero, compareTail); - masm.leaq(array1, new AMD64Address(array1, length, Scale.Times1, 0)); - masm.leaq(array2, new AMD64Address(array2, length, Scale.Times1, 0)); + masm.leaq(array1, new AMD64Address(array1, length, arrayIndexScale1, 0)); + masm.leaq(array2, new AMD64Address(array2, length, arrayIndexScale2, 0)); masm.negq(length); // Align the main loop masm.align(crb.target.wordSize * 2); masm.bind(loop); - masm.movq(temp, new AMD64Address(array1, length, Scale.Times1, 0)); - masm.cmpq(temp, new AMD64Address(array2, length, Scale.Times1, 0)); + masm.movq(temp, new AMD64Address(array1, length, arrayIndexScale1, 0)); + masm.cmpq(temp, new AMD64Address(array2, length, arrayIndexScale2, 0)); masm.jcc(ConditionFlag.NotEqual, requiresNaNCheck ? nanCheck : falseLabel); masm.bind(loopCheck); - masm.addq(length, VECTOR_SIZE); + masm.addq(length, elementsPerVector); masm.jccb(ConditionFlag.NotZero, loop); masm.testl(result, result); @@ -425,8 +495,8 @@ masm.jmpb(unalignedCheck); masm.bind(nanCheck); // At most two iterations, unroll in the emitted code. - for (int offset = 0; offset < VECTOR_SIZE; offset += kind.getByteCount()) { - emitFloatCompare(masm, kind, array1, array2, length, temp4, tempXMM, offset, falseLabel, kind.getByteCount() == VECTOR_SIZE); + for (int offset = 0; offset < VECTOR_SIZE; offset += kind1.getByteCount()) { + emitFloatCompare(masm, array1, array2, length, offset, falseLabel, kind1.getByteCount() == VECTOR_SIZE); } masm.jmpb(loopCheck); masm.bind(unalignedCheck); @@ -436,13 +506,13 @@ * Compare the remaining bytes with an unaligned memory load aligned to the end of the * array. */ - masm.movq(temp, new AMD64Address(array1, result, Scale.Times1, -VECTOR_SIZE)); - masm.cmpq(temp, new AMD64Address(array2, result, Scale.Times1, -VECTOR_SIZE)); + masm.movq(temp, new AMD64Address(array1, result, arrayIndexScale1, -VECTOR_SIZE)); + masm.cmpq(temp, new AMD64Address(array2, result, arrayIndexScale2, -VECTOR_SIZE)); if (requiresNaNCheck) { masm.jcc(ConditionFlag.Equal, trueLabel); // At most two iterations, unroll in the emitted code. - for (int offset = 0; offset < VECTOR_SIZE; offset += kind.getByteCount()) { - emitFloatCompare(masm, kind, array1, array2, result, temp4, tempXMM, -VECTOR_SIZE + offset, falseLabel, kind.getByteCount() == VECTOR_SIZE); + for (int offset = 0; offset < VECTOR_SIZE; offset += kind1.getByteCount()) { + emitFloatCompare(masm, array1, array2, result, -VECTOR_SIZE + offset, falseLabel, kind1.getByteCount() == VECTOR_SIZE); } } else { masm.jccb(ConditionFlag.NotEqual, falseLabel); @@ -456,34 +526,35 @@ /** * Emits code to compare the remaining 1 to 4 bytes. */ - private static void emitTailCompares(AMD64MacroAssembler masm, JavaKind kind, Register result, Register array1, Register array2, Register length, Value temp4, Value tempXMM, - Label trueLabel, Label falseLabel) { + private void emitTailCompares(AMD64MacroAssembler masm, + Register result, Register array1, Register array2, Register length, Label trueLabel, Label falseLabel) { + assert kind1 == kind2; Label compare2Bytes = new Label(); Label compare1Byte = new Label(); Register temp = asRegister(temp4); - if (kind.getByteCount() <= 4) { + if (kind1.getByteCount() <= 4) { // Compare trailing 4 bytes, if any. - masm.testl(result, 4); + masm.testl(result, arrayIndexScale1.log2 == 0 ? 4 : 4 >> arrayIndexScale1.log2); masm.jccb(ConditionFlag.Zero, compare2Bytes); masm.movl(temp, new AMD64Address(array1, 0)); masm.cmpl(temp, new AMD64Address(array2, 0)); - if (kind == JavaKind.Float) { + if (kind1 == JavaKind.Float) { masm.jccb(ConditionFlag.Equal, trueLabel); - emitFloatCompare(masm, kind, array1, array2, Register.None, temp4, tempXMM, 0, falseLabel, true); + emitFloatCompare(masm, array1, array2, Register.None, 0, falseLabel, true); masm.jmpb(trueLabel); } else { masm.jccb(ConditionFlag.NotEqual, falseLabel); } - if (kind.getByteCount() <= 2) { + if (kind1.getByteCount() <= 2) { // Move array pointers forward. masm.leaq(array1, new AMD64Address(array1, 4)); masm.leaq(array2, new AMD64Address(array2, 4)); // Compare trailing 2 bytes, if any. masm.bind(compare2Bytes); - masm.testl(result, 2); + masm.testl(result, arrayIndexScale1.log2 == 0 ? 2 : 2 >> arrayIndexScale1.log2); masm.jccb(ConditionFlag.Zero, compare1Byte); masm.movzwl(temp, new AMD64Address(array1, 0)); masm.movzwl(length, new AMD64Address(array2, 0)); @@ -491,7 +562,7 @@ masm.jccb(ConditionFlag.NotEqual, falseLabel); // The one-byte tail compare is only required for boolean and byte arrays. - if (kind.getByteCount() <= 1) { + if (kind1.getByteCount() <= 1) { // Move array pointers forward before we compare the last trailing byte. masm.leaq(array1, new AMD64Address(array1, 2)); masm.leaq(array2, new AMD64Address(array2, 2)); @@ -513,28 +584,82 @@ } } + private void emitDifferentKindsElementWiseCompare(CompilationResultBuilder crb, AMD64MacroAssembler masm, + Register result, Register array1, Register array2, Register length, Label trueLabel, Label falseLabel) { + assert kind1 != kind2; + assert kind1.isNumericInteger() && kind2.isNumericInteger(); + Label loop = new Label(); + Label compareTail = new Label(); + + int elementsPerLoopIteration = 4; + + Register tmp1 = asRegister(temp4); + Register tmp2 = asRegister(temp5); + + masm.andl(result, elementsPerLoopIteration - 1); // tail count + masm.andl(length, ~(elementsPerLoopIteration - 1)); // bulk loop count + masm.jcc(ConditionFlag.Zero, compareTail); + + masm.leaq(array1, new AMD64Address(array1, length, arrayIndexScale1, 0)); + masm.leaq(array2, new AMD64Address(array2, length, arrayIndexScale2, 0)); + masm.negq(length); + + // clear comparison registers because of the missing movzlq instruction + masm.xorq(tmp1, tmp1); + masm.xorq(tmp2, tmp2); + + // Align the main loop + masm.align(crb.target.wordSize * 2); + masm.bind(loop); + for (int i = 0; i < elementsPerLoopIteration; i++) { + emitMovBytes(masm, tmp1, new AMD64Address(array1, length, arrayIndexScale1, i << arrayIndexScale1.log2), kind1.getByteCount()); + emitMovBytes(masm, tmp2, new AMD64Address(array2, length, arrayIndexScale2, i << arrayIndexScale2.log2), kind2.getByteCount()); + masm.cmpq(tmp1, tmp2); + masm.jcc(ConditionFlag.NotEqual, falseLabel); + } + masm.addq(length, elementsPerLoopIteration); + masm.jccb(ConditionFlag.NotZero, loop); + + masm.bind(compareTail); + masm.testl(result, result); + masm.jcc(ConditionFlag.Zero, trueLabel); + for (int i = 0; i < elementsPerLoopIteration - 1; i++) { + emitMovBytes(masm, tmp1, new AMD64Address(array1, length, arrayIndexScale1, 0), kind1.getByteCount()); + emitMovBytes(masm, tmp2, new AMD64Address(array2, length, arrayIndexScale2, 0), kind2.getByteCount()); + masm.cmpq(tmp1, tmp2); + masm.jcc(ConditionFlag.NotEqual, falseLabel); + if (i < elementsPerLoopIteration - 2) { + masm.incrementq(length, 1); + masm.decrementq(result, 1); + masm.jcc(ConditionFlag.Zero, trueLabel); + } else { + masm.jmpb(trueLabel); + } + } + } + /** * Emits code to fall through if {@code src} is NaN, otherwise jump to {@code branchOrdered}. */ - private static void emitNaNCheck(AMD64MacroAssembler masm, JavaKind kind, Value tempXMM, AMD64Address src, Label branchIfNonNaN) { - assert kind.isNumericFloat(); + private void emitNaNCheck(AMD64MacroAssembler masm, AMD64Address src, Label branchIfNonNaN) { + assert kind1.isNumericFloat(); Register tempXMMReg = asRegister(tempXMM); - if (kind == JavaKind.Float) { + if (kind1 == JavaKind.Float) { masm.movflt(tempXMMReg, src); } else { masm.movdbl(tempXMMReg, src); } - SSEOp.UCOMIS.emit(masm, kind == JavaKind.Float ? OperandSize.PS : OperandSize.PD, tempXMMReg, tempXMMReg); + SSEOp.UCOMIS.emit(masm, kind1 == JavaKind.Float ? OperandSize.PS : OperandSize.PD, tempXMMReg, tempXMMReg); masm.jcc(ConditionFlag.NoParity, branchIfNonNaN); } /** * Emits code to compare if two floats are bitwise equal or both NaN. */ - private static void emitFloatCompare(AMD64MacroAssembler masm, JavaKind kind, Register base1, Register base2, Register index, Value temp4, Value tempXMM, int offset, Label falseLabel, + private void emitFloatCompare(AMD64MacroAssembler masm, Register base1, Register base2, Register index, int offset, Label falseLabel, boolean skipBitwiseCompare) { - AMD64Address address1 = new AMD64Address(base1, index, Scale.Times1, offset); - AMD64Address address2 = new AMD64Address(base2, index, Scale.Times1, offset); + AMD64Address address1 = new AMD64Address(base1, index, arrayIndexScale1, offset); + AMD64Address address2 = new AMD64Address(base2, index, arrayIndexScale2, offset); Label bitwiseEqual = new Label(); @@ -542,7 +667,7 @@ // Bitwise compare Register temp = asRegister(temp4); - if (kind == JavaKind.Float) { + if (kind1 == JavaKind.Float) { masm.movl(temp, address1); masm.cmpl(temp, address2); } else { @@ -552,8 +677,8 @@ masm.jccb(ConditionFlag.Equal, bitwiseEqual); } - emitNaNCheck(masm, kind, tempXMM, address1, falseLabel); - emitNaNCheck(masm, kind, tempXMM, address2, falseLabel); + emitNaNCheck(masm, address1, falseLabel); + emitNaNCheck(masm, address2, falseLabel); masm.bind(bitwiseEqual); } @@ -561,9 +686,9 @@ /** * Emits code to compare float equality within a range. */ - private static void emitFloatCompareWithinRange(CompilationResultBuilder crb, AMD64MacroAssembler masm, JavaKind kind, Register base1, Register base2, Register index, Value temp4, Value temp5, - Value tempXMM, int offset, Label falseLabel, int range) { - assert kind.isNumericFloat(); + private void emitFloatCompareWithinRange(CompilationResultBuilder crb, AMD64MacroAssembler masm, + Register base1, Register base2, Register index, int offset, Label falseLabel, int range) { + assert kind1.isNumericFloat(); Label loop = new Label(); Register i = asRegister(temp5); @@ -572,9 +697,9 @@ // Align the main loop masm.align(crb.target.wordSize * 2); masm.bind(loop); - emitFloatCompare(masm, kind, base1, base2, index, temp4, tempXMM, offset, falseLabel, kind.getByteCount() == range); - masm.addq(index, kind.getByteCount()); - masm.addq(i, kind.getByteCount()); + emitFloatCompare(masm, base1, base2, index, offset, falseLabel, range == 1); + masm.incrementq(index, 1); + masm.incrementq(i, 1); masm.jccb(ConditionFlag.NotZero, loop); // Floats within the range are equal, revert change to the register index masm.subq(index, range); @@ -585,189 +710,104 @@ * {@code arrayPtr1[0..nBytes]} and {@code arrayPtr2[0..nBytes]}. If they match, execution * continues directly after the emitted code block, otherwise we jump to {@code noMatch}. */ - private static void emitConstantLengthArrayCompareBytes( + private void emitConstantLengthArrayCompareBytes( + CompilationResultBuilder crb, AMD64MacroAssembler asm, Register arrayPtr1, Register arrayPtr2, Register tmp1, Register tmp2, Register[] tmpVectors, - Label noMatch, - int nBytes, - int bytesPerVector) { - assert bytesPerVector >= 16; - if (nBytes == 0) { + Label noMatch) { + if (constantLength == 0) { // do nothing return; } - if (nBytes < 16) { + AVXKind.AVXSize vSize = vectorSize; + if (constantLength < getElementsPerVector(vectorSize)) { + vSize = AVXKind.AVXSize.XMM; + } + int elementsPerVector = getElementsPerVector(vSize); + if (elementsPerVector > constantLength) { + assert kind1 == kind2; + int byteLength = constantLength << arrayIndexScale1.log2; // array is shorter than any vector register, use regular CMP instructions - int movSize = (nBytes < 2) ? 1 : ((nBytes < 4) ? 2 : ((nBytes < 8) ? 4 : 8)); + int movSize = (byteLength < 2) ? 1 : ((byteLength < 4) ? 2 : ((byteLength < 8) ? 4 : 8)); emitMovBytes(asm, tmp1, new AMD64Address(arrayPtr1), movSize); emitMovBytes(asm, tmp2, new AMD64Address(arrayPtr2), movSize); emitCmpBytes(asm, tmp1, tmp2, movSize); asm.jcc(AMD64Assembler.ConditionFlag.NotEqual, noMatch); - if (nBytes > movSize) { - emitMovBytes(asm, tmp1, new AMD64Address(arrayPtr1, nBytes - movSize), movSize); - emitMovBytes(asm, tmp2, new AMD64Address(arrayPtr2, nBytes - movSize), movSize); + if (byteLength > movSize) { + emitMovBytes(asm, tmp1, new AMD64Address(arrayPtr1, byteLength - movSize), movSize); + emitMovBytes(asm, tmp2, new AMD64Address(arrayPtr2, byteLength - movSize), movSize); emitCmpBytes(asm, tmp1, tmp2, movSize); asm.jcc(AMD64Assembler.ConditionFlag.NotEqual, noMatch); } - } else if (nBytes < 32 && bytesPerVector >= 32) { - // we could use YMM registers, but the array is too short, force XMM registers - int bytesPerXMMVector = AVXKind.AVXSize.XMM.getBytes(); - AMD64Assembler.VexMoveOp.VMOVDQU.emit(asm, AVXKind.AVXSize.XMM, tmpVectors[0], new AMD64Address(arrayPtr1)); - AMD64Assembler.VexMoveOp.VMOVDQU.emit(asm, AVXKind.AVXSize.XMM, tmpVectors[1], new AMD64Address(arrayPtr2)); - AMD64Assembler.VexRVMOp.VPXOR.emit(asm, AVXKind.AVXSize.XMM, tmpVectors[0], tmpVectors[0], tmpVectors[1]); - if (nBytes > bytesPerXMMVector) { - AMD64Assembler.VexMoveOp.VMOVDQU.emit(asm, AVXKind.AVXSize.XMM, tmpVectors[2], new AMD64Address(arrayPtr1, nBytes - bytesPerXMMVector)); - AMD64Assembler.VexMoveOp.VMOVDQU.emit(asm, AVXKind.AVXSize.XMM, tmpVectors[3], new AMD64Address(arrayPtr2, nBytes - bytesPerXMMVector)); - AMD64Assembler.VexRVMOp.VPXOR.emit(asm, AVXKind.AVXSize.XMM, tmpVectors[2], tmpVectors[2], tmpVectors[3]); - AMD64Assembler.VexRMOp.VPTEST.emit(asm, AVXKind.AVXSize.XMM, tmpVectors[2], tmpVectors[2]); + } else { + int elementsPerVectorLoop = 2 * elementsPerVector; + int tailCount = constantLength & (elementsPerVectorLoop - 1); + int vectorCount = constantLength & ~(elementsPerVectorLoop - 1); + int bytesPerVector = vSize.getBytes(); + if (vectorCount > 0) { + Label loopBegin = new Label(); + asm.leaq(arrayPtr1, new AMD64Address(arrayPtr1, vectorCount << arrayIndexScale1.log2)); + asm.leaq(arrayPtr2, new AMD64Address(arrayPtr2, vectorCount << arrayIndexScale2.log2)); + asm.movq(tmp1, -vectorCount); + asm.align(crb.target.wordSize * 2); + asm.bind(loopBegin); + emitVectorLoad1(asm, tmpVectors[0], arrayPtr1, tmp1, 0, vSize); + emitVectorLoad2(asm, tmpVectors[1], arrayPtr2, tmp1, 0, vSize); + emitVectorLoad1(asm, tmpVectors[2], arrayPtr1, tmp1, scaleDisplacement1(bytesPerVector), vSize); + emitVectorLoad2(asm, tmpVectors[3], arrayPtr2, tmp1, scaleDisplacement2(bytesPerVector), vSize); + emitVectorXor(asm, tmpVectors[0], tmpVectors[1], vSize); + emitVectorXor(asm, tmpVectors[2], tmpVectors[3], vSize); + emitVectorTest(asm, tmpVectors[0], vSize); asm.jcc(AMD64Assembler.ConditionFlag.NotZero, noMatch); - } - AMD64Assembler.VexRMOp.VPTEST.emit(asm, AVXKind.AVXSize.XMM, tmpVectors[0], tmpVectors[0]); - asm.jcc(AMD64Assembler.ConditionFlag.NotZero, noMatch); - } else if (bytesPerVector >= 32) { - // AVX2 supported, use YMM vectors - assert asm.supports(CPUFeature.AVX2); - int loopCount = nBytes / (bytesPerVector * 2); - int rest = nBytes % (bytesPerVector * 2); - if (loopCount > 0) { - if (0 < rest && rest < bytesPerVector) { - loopCount--; - } - if (loopCount > 0) { - if (loopCount > 1) { - asm.movl(tmp1, loopCount); - } - Label loopBegin = new Label(); - asm.bind(loopBegin); - asm.vmovdqu(tmpVectors[0], new AMD64Address(arrayPtr1)); - asm.vmovdqu(tmpVectors[1], new AMD64Address(arrayPtr2)); - asm.vmovdqu(tmpVectors[2], new AMD64Address(arrayPtr1, bytesPerVector)); - asm.vmovdqu(tmpVectors[3], new AMD64Address(arrayPtr2, bytesPerVector)); - asm.vpxor(tmpVectors[0], tmpVectors[0], tmpVectors[1]); - asm.vpxor(tmpVectors[2], tmpVectors[2], tmpVectors[3]); - asm.vptest(tmpVectors[0], tmpVectors[0]); - asm.jcc(AMD64Assembler.ConditionFlag.NotZero, noMatch); - asm.vptest(tmpVectors[2], tmpVectors[2]); - asm.jcc(AMD64Assembler.ConditionFlag.NotZero, noMatch); - asm.addq(arrayPtr1, bytesPerVector * 2); - asm.addq(arrayPtr2, bytesPerVector * 2); - if (loopCount > 1) { - asm.decrementl(tmp1); - asm.jcc(AMD64Assembler.ConditionFlag.NotZero, loopBegin); - } - } - if (0 < rest && rest < bytesPerVector) { - asm.vmovdqu(tmpVectors[0], new AMD64Address(arrayPtr1)); - asm.vmovdqu(tmpVectors[1], new AMD64Address(arrayPtr2)); - asm.vmovdqu(tmpVectors[2], new AMD64Address(arrayPtr1, bytesPerVector)); - asm.vmovdqu(tmpVectors[3], new AMD64Address(arrayPtr2, bytesPerVector)); - asm.vpxor(tmpVectors[0], tmpVectors[0], tmpVectors[1]); - asm.vpxor(tmpVectors[2], tmpVectors[2], tmpVectors[3]); - asm.vptest(tmpVectors[0], tmpVectors[0]); - asm.jcc(AMD64Assembler.ConditionFlag.NotZero, noMatch); - asm.vptest(tmpVectors[2], tmpVectors[2]); - asm.jcc(AMD64Assembler.ConditionFlag.NotZero, noMatch); - asm.vmovdqu(tmpVectors[0], new AMD64Address(arrayPtr1, bytesPerVector + rest)); - asm.vmovdqu(tmpVectors[1], new AMD64Address(arrayPtr2, bytesPerVector + rest)); - asm.vpxor(tmpVectors[0], tmpVectors[0], tmpVectors[1]); - asm.vptest(tmpVectors[0], tmpVectors[0]); - asm.jcc(AMD64Assembler.ConditionFlag.NotZero, noMatch); - } - } - if (rest >= bytesPerVector) { - asm.vmovdqu(tmpVectors[0], new AMD64Address(arrayPtr1)); - asm.vmovdqu(tmpVectors[1], new AMD64Address(arrayPtr2)); - asm.vpxor(tmpVectors[0], tmpVectors[0], tmpVectors[1]); - if (rest > bytesPerVector) { - asm.vmovdqu(tmpVectors[2], new AMD64Address(arrayPtr1, rest - bytesPerVector)); - asm.vmovdqu(tmpVectors[3], new AMD64Address(arrayPtr2, rest - bytesPerVector)); - asm.vpxor(tmpVectors[2], tmpVectors[2], tmpVectors[3]); - asm.vptest(tmpVectors[2], tmpVectors[2]); - asm.jcc(AMD64Assembler.ConditionFlag.NotZero, noMatch); - } - asm.vptest(tmpVectors[0], tmpVectors[0]); + emitVectorTest(asm, tmpVectors[2], vSize); asm.jcc(AMD64Assembler.ConditionFlag.NotZero, noMatch); + asm.addq(tmp1, elementsPerVectorLoop); + asm.jcc(AMD64Assembler.ConditionFlag.NotZero, loopBegin); } - } else { - // on AVX or SSE, use XMM vectors - int loopCount = nBytes / (bytesPerVector * 2); - int rest = nBytes % (bytesPerVector * 2); - if (loopCount > 0) { - if (0 < rest && rest < bytesPerVector) { - loopCount--; - } - if (loopCount > 0) { - if (loopCount > 1) { - asm.movl(tmp1, loopCount); - } - Label loopBegin = new Label(); - asm.bind(loopBegin); - asm.movdqu(tmpVectors[0], new AMD64Address(arrayPtr1)); - asm.movdqu(tmpVectors[1], new AMD64Address(arrayPtr2)); - asm.movdqu(tmpVectors[2], new AMD64Address(arrayPtr1, bytesPerVector)); - asm.movdqu(tmpVectors[3], new AMD64Address(arrayPtr2, bytesPerVector)); - asm.pxor(tmpVectors[0], tmpVectors[1]); - asm.pxor(tmpVectors[2], tmpVectors[3]); - asm.ptest(tmpVectors[0], tmpVectors[0]); + if (tailCount > 0) { + emitVectorLoad1(asm, tmpVectors[0], arrayPtr1, (tailCount << arrayIndexScale1.log2) - scaleDisplacement1(bytesPerVector), vSize); + emitVectorLoad2(asm, tmpVectors[1], arrayPtr2, (tailCount << arrayIndexScale2.log2) - scaleDisplacement2(bytesPerVector), vSize); + emitVectorXor(asm, tmpVectors[0], tmpVectors[1], vSize); + if (tailCount > elementsPerVector) { + emitVectorLoad1(asm, tmpVectors[2], arrayPtr1, 0, vSize); + emitVectorLoad2(asm, tmpVectors[3], arrayPtr2, 0, vSize); + emitVectorXor(asm, tmpVectors[2], tmpVectors[3], vSize); + emitVectorTest(asm, tmpVectors[2], vSize); asm.jcc(AMD64Assembler.ConditionFlag.NotZero, noMatch); - asm.ptest(tmpVectors[2], tmpVectors[2]); - asm.jcc(AMD64Assembler.ConditionFlag.NotZero, noMatch); - asm.addq(arrayPtr1, bytesPerVector * 2); - asm.addq(arrayPtr2, bytesPerVector * 2); - if (loopCount > 1) { - asm.decrementl(tmp1); - asm.jcc(AMD64Assembler.ConditionFlag.NotZero, loopBegin); - } } - if (0 < rest && rest < bytesPerVector) { - asm.movdqu(tmpVectors[0], new AMD64Address(arrayPtr1)); - asm.movdqu(tmpVectors[1], new AMD64Address(arrayPtr2)); - asm.movdqu(tmpVectors[2], new AMD64Address(arrayPtr1, bytesPerVector)); - asm.movdqu(tmpVectors[3], new AMD64Address(arrayPtr2, bytesPerVector)); - asm.pxor(tmpVectors[0], tmpVectors[1]); - asm.pxor(tmpVectors[2], tmpVectors[3]); - asm.ptest(tmpVectors[0], tmpVectors[0]); - asm.jcc(AMD64Assembler.ConditionFlag.NotZero, noMatch); - asm.ptest(tmpVectors[2], tmpVectors[2]); - asm.jcc(AMD64Assembler.ConditionFlag.NotZero, noMatch); - asm.movdqu(tmpVectors[0], new AMD64Address(arrayPtr1, bytesPerVector + rest)); - asm.movdqu(tmpVectors[1], new AMD64Address(arrayPtr2, bytesPerVector + rest)); - asm.pxor(tmpVectors[0], tmpVectors[1]); - asm.ptest(tmpVectors[0], tmpVectors[0]); - asm.jcc(AMD64Assembler.ConditionFlag.NotZero, noMatch); - } - } - if (rest >= bytesPerVector) { - asm.movdqu(tmpVectors[0], new AMD64Address(arrayPtr1)); - asm.movdqu(tmpVectors[1], new AMD64Address(arrayPtr2)); - asm.pxor(tmpVectors[0], tmpVectors[1]); - if (rest > bytesPerVector) { - asm.movdqu(tmpVectors[2], new AMD64Address(arrayPtr1, rest - bytesPerVector)); - asm.movdqu(tmpVectors[3], new AMD64Address(arrayPtr2, rest - bytesPerVector)); - asm.pxor(tmpVectors[2], tmpVectors[3]); - asm.ptest(tmpVectors[2], tmpVectors[2]); - asm.jcc(AMD64Assembler.ConditionFlag.NotZero, noMatch); - } - asm.ptest(tmpVectors[0], tmpVectors[0]); + emitVectorTest(asm, tmpVectors[0], vSize); asm.jcc(AMD64Assembler.ConditionFlag.NotZero, noMatch); } } } - private static void emitMovBytes(AMD64MacroAssembler asm, Register dst, AMD64Address src, int size) { + private void emitMovBytes(AMD64MacroAssembler asm, Register dst, AMD64Address src, int size) { switch (size) { case 1: - asm.movzbl(dst, src); + if (signExtend) { + asm.movsbq(dst, src); + } else { + asm.movzbq(dst, src); + } break; case 2: - asm.movzwl(dst, src); + if (signExtend) { + asm.movswq(dst, src); + } else { + asm.movzwq(dst, src); + } break; case 4: - asm.movl(dst, src); + if (signExtend) { + asm.movslq(dst, src); + } else { + // there is no movzlq + asm.movl(dst, src); + } break; case 8: asm.movq(dst, src); --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64ControlFlow.java 2019-03-12 08:09:27.523520892 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64ControlFlow.java 2019-03-12 08:09:27.195518763 +0100 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2011, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -283,12 +283,10 @@ masm.jmp(scratchReg); // Inserting padding so that jump table address is 4-byte aligned - if ((masm.position() & 0x3) != 0) { - masm.nop(4 - (masm.position() & 0x3)); - } + masm.align(4); // Patch LEA instruction above now that we know the position of the jump table - // TODO this is ugly and should be done differently + // this is ugly but there is no better way to do this given the assembler API final int jumpTablePos = masm.position(); final int leaDisplacementPosition = afterLea - 4; masm.emitInt(jumpTablePos - afterLea, leaDisplacementPosition); @@ -313,6 +311,99 @@ crb.compilationResult.addAnnotation(jt); } } + + public static final class HashTableSwitchOp extends AMD64BlockEndOp { + public static final LIRInstructionClass TYPE = LIRInstructionClass.create(HashTableSwitchOp.class); + private final JavaConstant[] keys; + private final LabelRef defaultTarget; + private final LabelRef[] targets; + @Alive protected Value value; + @Alive protected Value hash; + @Temp({REG}) protected Value entryScratch; + @Temp({REG}) protected Value scratch; + + public HashTableSwitchOp(final JavaConstant[] keys, final LabelRef defaultTarget, LabelRef[] targets, Value value, Value hash, Variable scratch, Variable entryScratch) { + super(TYPE); + this.keys = keys; + this.defaultTarget = defaultTarget; + this.targets = targets; + this.value = value; + this.hash = hash; + this.scratch = scratch; + this.entryScratch = entryScratch; + } + + @Override + public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { + Register valueReg = asRegister(value, AMD64Kind.DWORD); + Register indexReg = asRegister(hash, AMD64Kind.DWORD); + Register scratchReg = asRegister(scratch, AMD64Kind.QWORD); + Register entryScratchReg = asRegister(entryScratch, AMD64Kind.QWORD); + + // Set scratch to address of jump table + masm.leaq(scratchReg, new AMD64Address(AMD64.rip, 0)); + final int afterLea = masm.position(); + + // When the default target is set, the jump table contains entries with two DWORDS: + // the original key before hashing and the label jump address + if (defaultTarget != null) { + + // Move the table entry (two DWORDs) into a QWORD + masm.movq(entryScratchReg, new AMD64Address(scratchReg, indexReg, Scale.Times8, 0)); + + // Jump to the default target if the first DWORD (original key) doesn't match the + // current key. Accounts for hash collisions with unknown keys + masm.cmpl(entryScratchReg, valueReg); + masm.jcc(ConditionFlag.NotEqual, defaultTarget.label()); + + // Shift to the second DWORD + masm.sarq(entryScratchReg, 32); + } else { + + // The jump table has a single DWORD with the label address if there's no + // default target + masm.movslq(entryScratchReg, new AMD64Address(scratchReg, indexReg, Scale.Times4, 0)); + } + masm.addq(scratchReg, entryScratchReg); + masm.jmp(scratchReg); + + // Inserting padding so that jump the table address is aligned + if (defaultTarget != null) { + masm.align(8); + } else { + masm.align(4); + } + + // Patch LEA instruction above now that we know the position of the jump table + // this is ugly but there is no better way to do this given the assembler API + final int jumpTablePos = masm.position(); + final int leaDisplacementPosition = afterLea - 4; + masm.emitInt(jumpTablePos - afterLea, leaDisplacementPosition); + + // Emit jump table entries + for (int i = 0; i < targets.length; i++) { + + Label label = targets[i].label(); + + if (defaultTarget != null) { + masm.emitInt(keys[i].asInt()); + } + if (label.isBound()) { + int imm32 = label.position() - jumpTablePos; + masm.emitInt(imm32); + } else { + int offsetToJumpTableBase = masm.position() - jumpTablePos; + label.addPatchAt(masm.position()); + masm.emitByte(0); // pseudo-opcode for jump table entry + masm.emitShort(offsetToJumpTableBase); + masm.emitByte(0); // padding to make jump table entry 4 bytes wide + } + } + + JumpTable jt = new JumpTable(jumpTablePos, keys[0].asInt(), keys[keys.length - 1].asInt(), 4); + crb.compilationResult.addAnnotation(jt); + } + } @Opcode("SETcc") public static final class CondSetOp extends AMD64LIRInstruction { --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathIntrinsicBinaryOp.java 2019-03-12 08:09:27.967523774 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathIntrinsicBinaryOp.java 2019-03-12 08:09:27.639521645 +0100 @@ -24,2043 +24,52 @@ package org.graalvm.compiler.lir.amd64; -import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.ILLEGAL; -import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.REG; -import static jdk.vm.ci.code.ValueUtil.asRegister; +import static jdk.vm.ci.amd64.AMD64.xmm0; +import static jdk.vm.ci.amd64.AMD64.xmm1; +import static org.graalvm.compiler.lir.amd64.AMD64HotSpotHelper.registersToValues; -import org.graalvm.compiler.asm.Label; -import org.graalvm.compiler.asm.amd64.AMD64Address; -import org.graalvm.compiler.asm.amd64.AMD64Address.Scale; -import org.graalvm.compiler.asm.amd64.AMD64Assembler.ConditionFlag; -import org.graalvm.compiler.asm.amd64.AMD64MacroAssembler; import org.graalvm.compiler.core.common.LIRKind; -import org.graalvm.compiler.debug.GraalError; import org.graalvm.compiler.lir.LIRInstructionClass; -import org.graalvm.compiler.lir.Opcode; -import org.graalvm.compiler.lir.asm.ArrayDataPointerConstant; -import org.graalvm.compiler.lir.asm.CompilationResultBuilder; -import org.graalvm.compiler.lir.gen.LIRGeneratorTool; +import org.graalvm.compiler.lir.Variable; +import org.graalvm.compiler.lir.gen.LIRGenerator; -import jdk.vm.ci.amd64.AMD64; import jdk.vm.ci.amd64.AMD64Kind; import jdk.vm.ci.code.Register; -import jdk.vm.ci.meta.AllocatableValue; +import jdk.vm.ci.code.RegisterValue; import jdk.vm.ci.meta.Value; -public final class AMD64MathIntrinsicBinaryOp extends AMD64LIRInstruction { - public static final LIRInstructionClass TYPE = LIRInstructionClass.create(AMD64MathIntrinsicBinaryOp.class); - - public enum BinaryIntrinsicOpcode { - POW - } - - @Opcode private final BinaryIntrinsicOpcode opcode; - @Def protected Value result; - @Use protected Value input; - @Use protected Value secondInput; - @Temp({REG, ILLEGAL}) protected Value xmm1Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value xmm2Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value xmm3Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value xmm4Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value xmm5Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value xmm6Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value xmm7Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value xmm8Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value xmm9Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value xmm10Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value gpr1Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value gpr2Temp = Value.ILLEGAL; - @Temp protected AllocatableValue rcxTemp; - @Temp({REG, ILLEGAL}) protected Value gpr4Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value gpr5Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value gpr6Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value gpr7Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value gpr8Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value gpr9Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value gpr10Temp = Value.ILLEGAL; - - CompilationResultBuilder internalCrb; - - public AMD64MathIntrinsicBinaryOp(LIRGeneratorTool tool, BinaryIntrinsicOpcode opcode, Value result, Value input, Value alternateInput) { - super(TYPE); - this.opcode = opcode; - this.result = result; - this.input = input; - this.secondInput = alternateInput; - if (opcode == BinaryIntrinsicOpcode.POW) { - this.gpr1Temp = tool.newVariable(LIRKind.value(AMD64Kind.QWORD)); - this.gpr2Temp = tool.newVariable(LIRKind.value(AMD64Kind.QWORD)); - this.rcxTemp = AMD64.rcx.asValue(LIRKind.value(AMD64Kind.QWORD)); - this.gpr4Temp = tool.newVariable(LIRKind.value(AMD64Kind.QWORD)); - this.gpr5Temp = tool.newVariable(LIRKind.value(AMD64Kind.QWORD)); - this.gpr6Temp = tool.newVariable(LIRKind.value(AMD64Kind.QWORD)); - this.gpr7Temp = tool.newVariable(LIRKind.value(AMD64Kind.QWORD)); - this.gpr8Temp = tool.newVariable(LIRKind.value(AMD64Kind.QWORD)); - - this.xmm1Temp = tool.newVariable(LIRKind.value(AMD64Kind.DOUBLE)); - this.xmm2Temp = tool.newVariable(LIRKind.value(AMD64Kind.DOUBLE)); - this.xmm3Temp = tool.newVariable(LIRKind.value(AMD64Kind.DOUBLE)); - this.xmm4Temp = tool.newVariable(LIRKind.value(AMD64Kind.DOUBLE)); - this.xmm5Temp = tool.newVariable(LIRKind.value(AMD64Kind.DOUBLE)); - this.xmm6Temp = tool.newVariable(LIRKind.value(AMD64Kind.DOUBLE)); - this.xmm7Temp = tool.newVariable(LIRKind.value(AMD64Kind.DOUBLE)); - this.xmm8Temp = tool.newVariable(LIRKind.value(AMD64Kind.DOUBLE)); - this.xmm9Temp = tool.newVariable(LIRKind.value(AMD64Kind.DOUBLE)); - this.xmm10Temp = tool.newVariable(LIRKind.value(AMD64Kind.DOUBLE)); - } - } - - private void setCrb(CompilationResultBuilder crb) { - internalCrb = crb; - } - - private AMD64Address externalAddress(ArrayDataPointerConstant curPtr) { - return (AMD64Address) internalCrb.recordDataReferenceInCode(curPtr); - } - - @Override - public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { - switch (opcode) { - case POW: - powIntrinsic(asRegister(result, AMD64Kind.DOUBLE), asRegister(input, AMD64Kind.DOUBLE), asRegister(secondInput, AMD64Kind.DOUBLE), crb, masm); - break; - default: - throw GraalError.shouldNotReachHere(); - } - } - - /* - * Copyright (c) 2014, 2016, Intel Corporation. All rights reserved. Intel Math Library (LIBM) - * Source Code - * - * ALGORITHM DESCRIPTION - POW() --------------------- - * - * Let x=2^k * mx, mx in [1,2) - * - * log2(x) calculation: - * - * Get B~1/mx based on the output of rcpps instruction (B0) B = int((B0*LH*2^9+0.5))/2^9 LH is a - * short approximation for log2(e) - * - * Reduced argument, scaled by LH: r=B*mx-LH (computed accurately in high and low parts) - * - * log2(x) result: k - log2(B) + p(r) p(r) is a degree 8 polynomial -log2(B) read from data - * table (high, low parts) log2(x) is formed from high and low parts For |x| in [1-1/32, - * 1+1/16), a slower but more accurate computation based om the same table design is performed. - * - * Main path is taken if | floor(log2(|log2(|x|)|) + floor(log2|y|) | < 8, to filter out all - * potential OF/UF cases. exp2(y*log2(x)) is computed using an 8-bit index table and a degree 5 - * polynomial - * - * Special cases: pow(-0,y) = -INF and raises the divide-by-zero exception for y an odd integer - * < 0. pow(-0,y) = +INF and raises the divide-by-zero exception for y < 0 and not an odd - * integer. pow(-0,y) = -0 for y an odd integer > 0. pow(-0,y) = +0 for y > 0 and not an odd - * integer. pow(-1,-INF) = NaN. pow(+1,y) = NaN for any y, even a NaN. pow(x,-0) = 1 for any x, - * even a NaN. pow(x,y) = a NaN and raises the invalid exception for finite x < 0 and finite - * non-integer y. pow(x,-INF) = +INF for |x|<1. pow(x,-INF) = +0 for |x|>1. pow(x,+INF) = +0 for - * |x|<1. pow(x,+INF) = +INF for |x|>1. pow(-INF,y) = -0 for y an odd integer < 0. pow(-INF,y) = - * +0 for y < 0 and not an odd integer. pow(-INF,y) = -INF for y an odd integer > 0. pow(-INF,y) - * = +INF for y > 0 and not an odd integer. pow(+INF,y) = +0 for y <0. pow(+INF,y) = +INF for y - * >0. - * - */ - - private static int[] highSigMask = { - 0x00000000, 0xfffff800, 0x00000000, 0xfffff800 - }; - - private static int[] logTwoE = { - 0x00000000, 0x3ff72000, 0x161bb241, 0xbf5dabe1 - }; - - private static int[] highmaskY = { - 0x00000000, 0xfffffff8, 0x00000000, 0xffffffff - }; - - private static int[] tExp = { - 0x00000000, 0x3ff00000, 0x00000000, 0x3b700000, 0xfa5abcbf, - 0x3ff00b1a, 0xa7609f71, 0xbc84f6b2, 0xa9fb3335, 0x3ff0163d, - 0x9ab8cdb7, 0x3c9b6129, 0x143b0281, 0x3ff02168, 0x0fc54eb6, - 0xbc82bf31, 0x3e778061, 0x3ff02c9a, 0x535b085d, 0xbc719083, - 0x2e11bbcc, 0x3ff037d4, 0xeeade11a, 0x3c656811, 0xe86e7f85, - 0x3ff04315, 0x1977c96e, 0xbc90a31c, 0x72f654b1, 0x3ff04e5f, - 0x3aa0d08c, 0x3c84c379, 0xd3158574, 0x3ff059b0, 0xa475b465, - 0x3c8d73e2, 0x0e3c1f89, 0x3ff0650a, 0x5799c397, 0xbc95cb7b, - 0x29ddf6de, 0x3ff0706b, 0xe2b13c27, 0xbc8c91df, 0x2b72a836, - 0x3ff07bd4, 0x54458700, 0x3c832334, 0x18759bc8, 0x3ff08745, - 0x4bb284ff, 0x3c6186be, 0xf66607e0, 0x3ff092bd, 0x800a3fd1, - 0xbc968063, 0xcac6f383, 0x3ff09e3e, 0x18316136, 0x3c914878, - 0x9b1f3919, 0x3ff0a9c7, 0x873d1d38, 0x3c85d16c, 0x6cf9890f, - 0x3ff0b558, 0x4adc610b, 0x3c98a62e, 0x45e46c85, 0x3ff0c0f1, - 0x06d21cef, 0x3c94f989, 0x2b7247f7, 0x3ff0cc92, 0x16e24f71, - 0x3c901edc, 0x23395dec, 0x3ff0d83b, 0xe43f316a, 0xbc9bc14d, - 0x32d3d1a2, 0x3ff0e3ec, 0x27c57b52, 0x3c403a17, 0x5fdfa9c5, - 0x3ff0efa5, 0xbc54021b, 0xbc949db9, 0xaffed31b, 0x3ff0fb66, - 0xc44ebd7b, 0xbc6b9bed, 0x28d7233e, 0x3ff10730, 0x1692fdd5, - 0x3c8d46eb, 0xd0125b51, 0x3ff11301, 0x39449b3a, 0xbc96c510, - 0xab5e2ab6, 0x3ff11edb, 0xf703fb72, 0xbc9ca454, 0xc06c31cc, - 0x3ff12abd, 0xb36ca5c7, 0xbc51b514, 0x14f204ab, 0x3ff136a8, - 0xba48dcf0, 0xbc67108f, 0xaea92de0, 0x3ff1429a, 0x9af1369e, - 0xbc932fbf, 0x934f312e, 0x3ff14e95, 0x39bf44ab, 0xbc8b91e8, - 0xc8a58e51, 0x3ff15a98, 0xb9eeab0a, 0x3c82406a, 0x5471c3c2, - 0x3ff166a4, 0x82ea1a32, 0x3c58f23b, 0x3c7d517b, 0x3ff172b8, - 0xb9d78a76, 0xbc819041, 0x8695bbc0, 0x3ff17ed4, 0xe2ac5a64, - 0x3c709e3f, 0x388c8dea, 0x3ff18af9, 0xd1970f6c, 0xbc911023, - 0x58375d2f, 0x3ff19726, 0x85f17e08, 0x3c94aadd, 0xeb6fcb75, - 0x3ff1a35b, 0x7b4968e4, 0x3c8e5b4c, 0xf8138a1c, 0x3ff1af99, - 0xa4b69280, 0x3c97bf85, 0x84045cd4, 0x3ff1bbe0, 0x352ef607, - 0xbc995386, 0x95281c6b, 0x3ff1c82f, 0x8010f8c9, 0x3c900977, - 0x3168b9aa, 0x3ff1d487, 0x00a2643c, 0x3c9e016e, 0x5eb44027, - 0x3ff1e0e7, 0x088cb6de, 0xbc96fdd8, 0x22fcd91d, 0x3ff1ed50, - 0x027bb78c, 0xbc91df98, 0x8438ce4d, 0x3ff1f9c1, 0xa097af5c, - 0xbc9bf524, 0x88628cd6, 0x3ff2063b, 0x814a8495, 0x3c8dc775, - 0x3578a819, 0x3ff212be, 0x2cfcaac9, 0x3c93592d, 0x917ddc96, - 0x3ff21f49, 0x9494a5ee, 0x3c82a97e, 0xa27912d1, 0x3ff22bdd, - 0x5577d69f, 0x3c8d34fb, 0x6e756238, 0x3ff2387a, 0xb6c70573, - 0x3c99b07e, 0xfb82140a, 0x3ff2451f, 0x911ca996, 0x3c8acfcc, - 0x4fb2a63f, 0x3ff251ce, 0xbef4f4a4, 0x3c8ac155, 0x711ece75, - 0x3ff25e85, 0x4ac31b2c, 0x3c93e1a2, 0x65e27cdd, 0x3ff26b45, - 0x9940e9d9, 0x3c82bd33, 0x341ddf29, 0x3ff2780e, 0x05f9e76c, - 0x3c9e067c, 0xe1f56381, 0x3ff284df, 0x8c3f0d7e, 0xbc9a4c3a, - 0x7591bb70, 0x3ff291ba, 0x28401cbd, 0xbc82cc72, 0xf51fdee1, - 0x3ff29e9d, 0xafad1255, 0x3c8612e8, 0x66d10f13, 0x3ff2ab8a, - 0x191690a7, 0xbc995743, 0xd0dad990, 0x3ff2b87f, 0xd6381aa4, - 0xbc410adc, 0x39771b2f, 0x3ff2c57e, 0xa6eb5124, 0xbc950145, - 0xa6e4030b, 0x3ff2d285, 0x54db41d5, 0x3c900247, 0x1f641589, - 0x3ff2df96, 0xfbbce198, 0x3c9d16cf, 0xa93e2f56, 0x3ff2ecaf, - 0x45d52383, 0x3c71ca0f, 0x4abd886b, 0x3ff2f9d2, 0x532bda93, - 0xbc653c55, 0x0a31b715, 0x3ff306fe, 0xd23182e4, 0x3c86f46a, - 0xedeeb2fd, 0x3ff31432, 0xf3f3fcd1, 0x3c8959a3, 0xfc4cd831, - 0x3ff32170, 0x8e18047c, 0x3c8a9ce7, 0x3ba8ea32, 0x3ff32eb8, - 0x3cb4f318, 0xbc9c45e8, 0xb26416ff, 0x3ff33c08, 0x843659a6, - 0x3c932721, 0x66e3fa2d, 0x3ff34962, 0x930881a4, 0xbc835a75, - 0x5f929ff1, 0x3ff356c5, 0x5c4e4628, 0xbc8b5cee, 0xa2de883b, - 0x3ff36431, 0xa06cb85e, 0xbc8c3144, 0x373aa9cb, 0x3ff371a7, - 0xbf42eae2, 0xbc963aea, 0x231e754a, 0x3ff37f26, 0x9eceb23c, - 0xbc99f5ca, 0x6d05d866, 0x3ff38cae, 0x3c9904bd, 0xbc9e958d, - 0x1b7140ef, 0x3ff39a40, 0xfc8e2934, 0xbc99a9a5, 0x34e59ff7, - 0x3ff3a7db, 0xd661f5e3, 0xbc75e436, 0xbfec6cf4, 0x3ff3b57f, - 0xe26fff18, 0x3c954c66, 0xc313a8e5, 0x3ff3c32d, 0x375d29c3, - 0xbc9efff8, 0x44ede173, 0x3ff3d0e5, 0x8c284c71, 0x3c7fe8d0, - 0x4c123422, 0x3ff3dea6, 0x11f09ebc, 0x3c8ada09, 0xdf1c5175, - 0x3ff3ec70, 0x7b8c9bca, 0xbc8af663, 0x04ac801c, 0x3ff3fa45, - 0xf956f9f3, 0xbc97d023, 0xc367a024, 0x3ff40822, 0xb6f4d048, - 0x3c8bddf8, 0x21f72e2a, 0x3ff4160a, 0x1c309278, 0xbc5ef369, - 0x2709468a, 0x3ff423fb, 0xc0b314dd, 0xbc98462d, 0xd950a897, - 0x3ff431f5, 0xe35f7999, 0xbc81c7dd, 0x3f84b9d4, 0x3ff43ffa, - 0x9704c003, 0x3c8880be, 0x6061892d, 0x3ff44e08, 0x04ef80d0, - 0x3c489b7a, 0x42a7d232, 0x3ff45c20, 0x82fb1f8e, 0xbc686419, - 0xed1d0057, 0x3ff46a41, 0xd1648a76, 0x3c9c944b, 0x668b3237, - 0x3ff4786d, 0xed445733, 0xbc9c20f0, 0xb5c13cd0, 0x3ff486a2, - 0xb69062f0, 0x3c73c1a3, 0xe192aed2, 0x3ff494e1, 0x5e499ea0, - 0xbc83b289, 0xf0d7d3de, 0x3ff4a32a, 0xf3d1be56, 0x3c99cb62, - 0xea6db7d7, 0x3ff4b17d, 0x7f2897f0, 0xbc8125b8, 0xd5362a27, - 0x3ff4bfda, 0xafec42e2, 0x3c7d4397, 0xb817c114, 0x3ff4ce41, - 0x690abd5d, 0x3c905e29, 0x99fddd0d, 0x3ff4dcb2, 0xbc6a7833, - 0x3c98ecdb, 0x81d8abff, 0x3ff4eb2d, 0x2e5d7a52, 0xbc95257d, - 0x769d2ca7, 0x3ff4f9b2, 0xd25957e3, 0xbc94b309, 0x7f4531ee, - 0x3ff50841, 0x49b7465f, 0x3c7a249b, 0xa2cf6642, 0x3ff516da, - 0x69bd93ef, 0xbc8f7685, 0xe83f4eef, 0x3ff5257d, 0x43efef71, - 0xbc7c998d, 0x569d4f82, 0x3ff5342b, 0x1db13cad, 0xbc807abe, - 0xf4f6ad27, 0x3ff542e2, 0x192d5f7e, 0x3c87926d, 0xca5d920f, - 0x3ff551a4, 0xefede59b, 0xbc8d689c, 0xdde910d2, 0x3ff56070, - 0x168eebf0, 0xbc90fb6e, 0x36b527da, 0x3ff56f47, 0x011d93ad, - 0x3c99bb2c, 0xdbe2c4cf, 0x3ff57e27, 0x8a57b9c4, 0xbc90b98c, - 0xd497c7fd, 0x3ff58d12, 0x5b9a1de8, 0x3c8295e1, 0x27ff07cc, - 0x3ff59c08, 0xe467e60f, 0xbc97e2ce, 0xdd485429, 0x3ff5ab07, - 0x054647ad, 0x3c96324c, 0xfba87a03, 0x3ff5ba11, 0x4c233e1a, - 0xbc9b77a1, 0x8a5946b7, 0x3ff5c926, 0x816986a2, 0x3c3c4b1b, - 0x90998b93, 0x3ff5d845, 0xa8b45643, 0xbc9cd6a7, 0x15ad2148, - 0x3ff5e76f, 0x3080e65e, 0x3c9ba6f9, 0x20dceb71, 0x3ff5f6a3, - 0xe3cdcf92, 0xbc89eadd, 0xb976dc09, 0x3ff605e1, 0x9b56de47, - 0xbc93e242, 0xe6cdf6f4, 0x3ff6152a, 0x4ab84c27, 0x3c9e4b3e, - 0xb03a5585, 0x3ff6247e, 0x7e40b497, 0xbc9383c1, 0x1d1929fd, - 0x3ff633dd, 0xbeb964e5, 0x3c984710, 0x34ccc320, 0x3ff64346, - 0x759d8933, 0xbc8c483c, 0xfebc8fb7, 0x3ff652b9, 0xc9a73e09, - 0xbc9ae3d5, 0x82552225, 0x3ff66238, 0x87591c34, 0xbc9bb609, - 0xc70833f6, 0x3ff671c1, 0x586c6134, 0xbc8e8732, 0xd44ca973, - 0x3ff68155, 0x44f73e65, 0x3c6038ae, 0xb19e9538, 0x3ff690f4, - 0x9aeb445d, 0x3c8804bd, 0x667f3bcd, 0x3ff6a09e, 0x13b26456, - 0xbc9bdd34, 0xfa75173e, 0x3ff6b052, 0x2c9a9d0e, 0x3c7a38f5, - 0x750bdabf, 0x3ff6c012, 0x67ff0b0d, 0xbc728956, 0xddd47645, - 0x3ff6cfdc, 0xb6f17309, 0x3c9c7aa9, 0x3c651a2f, 0x3ff6dfb2, - 0x683c88ab, 0xbc6bbe3a, 0x98593ae5, 0x3ff6ef92, 0x9e1ac8b2, - 0xbc90b974, 0xf9519484, 0x3ff6ff7d, 0x25860ef6, 0xbc883c0f, - 0x66f42e87, 0x3ff70f74, 0xd45aa65f, 0x3c59d644, 0xe8ec5f74, - 0x3ff71f75, 0x86887a99, 0xbc816e47, 0x86ead08a, 0x3ff72f82, - 0x2cd62c72, 0xbc920aa0, 0x48a58174, 0x3ff73f9a, 0x6c65d53c, - 0xbc90a8d9, 0x35d7cbfd, 0x3ff74fbd, 0x618a6e1c, 0x3c9047fd, - 0x564267c9, 0x3ff75feb, 0x57316dd3, 0xbc902459, 0xb1ab6e09, - 0x3ff77024, 0x169147f8, 0x3c9b7877, 0x4fde5d3f, 0x3ff78069, - 0x0a02162d, 0x3c9866b8, 0x38ac1cf6, 0x3ff790b9, 0x62aadd3e, - 0x3c9349a8, 0x73eb0187, 0x3ff7a114, 0xee04992f, 0xbc841577, - 0x0976cfdb, 0x3ff7b17b, 0x8468dc88, 0xbc9bebb5, 0x0130c132, - 0x3ff7c1ed, 0xd1164dd6, 0x3c9f124c, 0x62ff86f0, 0x3ff7d26a, - 0xfb72b8b4, 0x3c91bddb, 0x36cf4e62, 0x3ff7e2f3, 0xba15797e, - 0x3c705d02, 0x8491c491, 0x3ff7f387, 0xcf9311ae, 0xbc807f11, - 0x543e1a12, 0x3ff80427, 0x626d972b, 0xbc927c86, 0xadd106d9, - 0x3ff814d2, 0x0d151d4d, 0x3c946437, 0x994cce13, 0x3ff82589, - 0xd41532d8, 0xbc9d4c1d, 0x1eb941f7, 0x3ff8364c, 0x31df2bd5, - 0x3c999b9a, 0x4623c7ad, 0x3ff8471a, 0xa341cdfb, 0xbc88d684, - 0x179f5b21, 0x3ff857f4, 0xf8b216d0, 0xbc5ba748, 0x9b4492ed, - 0x3ff868d9, 0x9bd4f6ba, 0xbc9fc6f8, 0xd931a436, 0x3ff879ca, - 0xd2db47bd, 0x3c85d2d7, 0xd98a6699, 0x3ff88ac7, 0xf37cb53a, - 0x3c9994c2, 0xa478580f, 0x3ff89bd0, 0x4475202a, 0x3c9d5395, - 0x422aa0db, 0x3ff8ace5, 0x56864b27, 0x3c96e9f1, 0xbad61778, - 0x3ff8be05, 0xfc43446e, 0x3c9ecb5e, 0x16b5448c, 0x3ff8cf32, - 0x32e9e3aa, 0xbc70d55e, 0x5e0866d9, 0x3ff8e06a, 0x6fc9b2e6, - 0xbc97114a, 0x99157736, 0x3ff8f1ae, 0xa2e3976c, 0x3c85cc13, - 0xd0282c8a, 0x3ff902fe, 0x85fe3fd2, 0x3c9592ca, 0x0b91ffc6, - 0x3ff9145b, 0x2e582524, 0xbc9dd679, 0x53aa2fe2, 0x3ff925c3, - 0xa639db7f, 0xbc83455f, 0xb0cdc5e5, 0x3ff93737, 0x81b57ebc, - 0xbc675fc7, 0x2b5f98e5, 0x3ff948b8, 0x797d2d99, 0xbc8dc3d6, - 0xcbc8520f, 0x3ff95a44, 0x96a5f039, 0xbc764b7c, 0x9a7670b3, - 0x3ff96bdd, 0x7f19c896, 0xbc5ba596, 0x9fde4e50, 0x3ff97d82, - 0x7c1b85d1, 0xbc9d185b, 0xe47a22a2, 0x3ff98f33, 0xa24c78ec, - 0x3c7cabda, 0x70ca07ba, 0x3ff9a0f1, 0x91cee632, 0xbc9173bd, - 0x4d53fe0d, 0x3ff9b2bb, 0x4df6d518, 0xbc9dd84e, 0x82a3f090, - 0x3ff9c491, 0xb071f2be, 0x3c7c7c46, 0x194bb8d5, 0x3ff9d674, - 0xa3dd8233, 0xbc9516be, 0x19e32323, 0x3ff9e863, 0x78e64c6e, - 0x3c7824ca, 0x8d07f29e, 0x3ff9fa5e, 0xaaf1face, 0xbc84a9ce, - 0x7b5de565, 0x3ffa0c66, 0x5d1cd533, 0xbc935949, 0xed8eb8bb, - 0x3ffa1e7a, 0xee8be70e, 0x3c9c6618, 0xec4a2d33, 0x3ffa309b, - 0x7ddc36ab, 0x3c96305c, 0x80460ad8, 0x3ffa42c9, 0x589fb120, - 0xbc9aa780, 0xb23e255d, 0x3ffa5503, 0xdb8d41e1, 0xbc9d2f6e, - 0x8af46052, 0x3ffa674a, 0x30670366, 0x3c650f56, 0x1330b358, - 0x3ffa799e, 0xcac563c7, 0x3c9bcb7e, 0x53c12e59, 0x3ffa8bfe, - 0xb2ba15a9, 0xbc94f867, 0x5579fdbf, 0x3ffa9e6b, 0x0ef7fd31, - 0x3c90fac9, 0x21356eba, 0x3ffab0e5, 0xdae94545, 0x3c889c31, - 0xbfd3f37a, 0x3ffac36b, 0xcae76cd0, 0xbc8f9234, 0x3a3c2774, - 0x3ffad5ff, 0xb6b1b8e5, 0x3c97ef3b, 0x995ad3ad, 0x3ffae89f, - 0x345dcc81, 0x3c97a1cd, 0xe622f2ff, 0x3ffafb4c, 0x0f315ecd, - 0xbc94b2fc, 0x298db666, 0x3ffb0e07, 0x4c80e425, 0xbc9bdef5, - 0x6c9a8952, 0x3ffb20ce, 0x4a0756cc, 0x3c94dd02, 0xb84f15fb, - 0x3ffb33a2, 0x3084d708, 0xbc62805e, 0x15b749b1, 0x3ffb4684, - 0xe9df7c90, 0xbc7f763d, 0x8de5593a, 0x3ffb5972, 0xbbba6de3, - 0xbc9c71df, 0x29f1c52a, 0x3ffb6c6e, 0x52883f6e, 0x3c92a8f3, - 0xf2fb5e47, 0x3ffb7f76, 0x7e54ac3b, 0xbc75584f, 0xf22749e4, - 0x3ffb928c, 0x54cb65c6, 0xbc9b7216, 0x30a1064a, 0x3ffba5b0, - 0x0e54292e, 0xbc9efcd3, 0xb79a6f1f, 0x3ffbb8e0, 0xc9696205, - 0xbc3f52d1, 0x904bc1d2, 0x3ffbcc1e, 0x7a2d9e84, 0x3c823dd0, - 0xc3f3a207, 0x3ffbdf69, 0x60ea5b53, 0xbc3c2623, 0x5bd71e09, - 0x3ffbf2c2, 0x3f6b9c73, 0xbc9efdca, 0x6141b33d, 0x3ffc0628, - 0xa1fbca34, 0xbc8d8a5a, 0xdd85529c, 0x3ffc199b, 0x895048dd, - 0x3c811065, 0xd9fa652c, 0x3ffc2d1c, 0x17c8a5d7, 0xbc96e516, - 0x5fffd07a, 0x3ffc40ab, 0xe083c60a, 0x3c9b4537, 0x78fafb22, - 0x3ffc5447, 0x2493b5af, 0x3c912f07, 0x2e57d14b, 0x3ffc67f1, - 0xff483cad, 0x3c92884d, 0x8988c933, 0x3ffc7ba8, 0xbe255559, - 0xbc8e76bb, 0x9406e7b5, 0x3ffc8f6d, 0x48805c44, 0x3c71acbc, - 0x5751c4db, 0x3ffca340, 0xd10d08f5, 0xbc87f2be, 0xdcef9069, - 0x3ffcb720, 0xd1e949db, 0x3c7503cb, 0x2e6d1675, 0x3ffccb0f, - 0x86009092, 0xbc7d220f, 0x555dc3fa, 0x3ffcdf0b, 0x53829d72, - 0xbc8dd83b, 0x5b5bab74, 0x3ffcf315, 0xb86dff57, 0xbc9a08e9, - 0x4a07897c, 0x3ffd072d, 0x43797a9c, 0xbc9cbc37, 0x2b08c968, - 0x3ffd1b53, 0x219a36ee, 0x3c955636, 0x080d89f2, 0x3ffd2f87, - 0x719d8578, 0xbc9d487b, 0xeacaa1d6, 0x3ffd43c8, 0xbf5a1614, - 0x3c93db53, 0xdcfba487, 0x3ffd5818, 0xd75b3707, 0x3c82ed02, - 0xe862e6d3, 0x3ffd6c76, 0x4a8165a0, 0x3c5fe87a, 0x16c98398, - 0x3ffd80e3, 0x8beddfe8, 0xbc911ec1, 0x71ff6075, 0x3ffd955d, - 0xbb9af6be, 0x3c9a052d, 0x03db3285, 0x3ffda9e6, 0x696db532, - 0x3c9c2300, 0xd63a8315, 0x3ffdbe7c, 0x926b8be4, 0xbc9b76f1, - 0xf301b460, 0x3ffdd321, 0x78f018c3, 0x3c92da57, 0x641c0658, - 0x3ffde7d5, 0x8e79ba8f, 0xbc9ca552, 0x337b9b5f, 0x3ffdfc97, - 0x4f184b5c, 0xbc91a5cd, 0x6b197d17, 0x3ffe1167, 0xbd5c7f44, - 0xbc72b529, 0x14f5a129, 0x3ffe2646, 0x817a1496, 0xbc97b627, - 0x3b16ee12, 0x3ffe3b33, 0x31fdc68b, 0xbc99f4a4, 0xe78b3ff6, - 0x3ffe502e, 0x80a9cc8f, 0x3c839e89, 0x24676d76, 0x3ffe6539, - 0x7522b735, 0xbc863ff8, 0xfbc74c83, 0x3ffe7a51, 0xca0c8de2, - 0x3c92d522, 0x77cdb740, 0x3ffe8f79, 0x80b054b1, 0xbc910894, - 0xa2a490da, 0x3ffea4af, 0x179c2893, 0xbc9e9c23, 0x867cca6e, - 0x3ffeb9f4, 0x2293e4f2, 0x3c94832f, 0x2d8e67f1, 0x3ffecf48, - 0xb411ad8c, 0xbc9c93f3, 0xa2188510, 0x3ffee4aa, 0xa487568d, - 0x3c91c68d, 0xee615a27, 0x3ffefa1b, 0x86a4b6b0, 0x3c9dc7f4, - 0x1cb6412a, 0x3fff0f9c, 0x65181d45, 0xbc932200, 0x376bba97, - 0x3fff252b, 0xbf0d8e43, 0x3c93a1a5, 0x48dd7274, 0x3fff3ac9, - 0x3ed837de, 0xbc795a5a, 0x5b6e4540, 0x3fff5076, 0x2dd8a18b, - 0x3c99d3e1, 0x798844f8, 0x3fff6632, 0x3539343e, 0x3c9fa37b, - 0xad9cbe14, 0x3fff7bfd, 0xd006350a, 0xbc9dbb12, 0x02243c89, - 0x3fff91d8, 0xa779f689, 0xbc612ea8, 0x819e90d8, 0x3fffa7c1, - 0xf3a5931e, 0x3c874853, 0x3692d514, 0x3fffbdba, 0x15098eb6, - 0xbc796773, 0x2b8f71f1, 0x3fffd3c2, 0x966579e7, 0x3c62eb74, - 0x6b2a23d9, 0x3fffe9d9, 0x7442fde3, 0x3c74a603 - }; - - private static int[] eCoeff = { - 0xe78a6731, 0x3f55d87f, 0xd704a0c0, 0x3fac6b08, 0x6fba4e77, - 0x3f83b2ab, 0xff82c58f, 0x3fcebfbd, 0xfefa39ef, 0x3fe62e42, - 0x00000000, 0x00000000 - }; - - private static int[] coeffH = { - 0x00000000, 0xbfd61a00, 0x00000000, 0xbf5dabe1 - }; - - private static int[] highmaskLogX = { - 0xf8000000, 0xffffffff, 0x00000000, 0xfffff800 - }; - - private static int[] halfmask = { - 0xf8000000, 0xffffffff, 0xf8000000, 0xffffffff - }; - - private static int[] coeffPow = { - 0x6dc96112, 0xbf836578, 0xee241472, 0xbf9b0301, 0x9f95985a, - 0xbfb528db, 0xb3841d2a, 0xbfd619b6, 0x518775e3, 0x3f9004f2, - 0xac8349bb, 0x3fa76c9b, 0x486ececc, 0x3fc4635e, 0x161bb241, - 0xbf5dabe1, 0x9f95985a, 0xbfb528db, 0xf8b5787d, 0x3ef2531e, - 0x486ececb, 0x3fc4635e, 0x412055cc, 0xbdd61bb2 - }; - - private static int[] lTblPow = { - 0x00000000, 0x3ff00000, 0x00000000, 0x00000000, 0x20000000, - 0x3feff00a, 0x96621f95, 0x3e5b1856, 0xe0000000, 0x3fefe019, - 0xe5916f9e, 0xbe325278, 0x00000000, 0x3fefd02f, 0x859a1062, - 0x3e595fb7, 0xc0000000, 0x3fefc049, 0xb245f18f, 0xbe529c38, - 0xe0000000, 0x3fefb069, 0xad2880a7, 0xbe501230, 0x60000000, - 0x3fefa08f, 0xc8e72420, 0x3e597bd1, 0x80000000, 0x3fef90ba, - 0xc30c4500, 0xbe5d6c75, 0xe0000000, 0x3fef80ea, 0x02c63f43, - 0x3e2e1318, 0xc0000000, 0x3fef7120, 0xb3d4cccc, 0xbe44c52a, - 0x00000000, 0x3fef615c, 0xdbd91397, 0xbe4e7d6c, 0xa0000000, - 0x3fef519c, 0x65c5cd68, 0xbe522dc8, 0xa0000000, 0x3fef41e2, - 0x46d1306c, 0xbe5a840e, 0xe0000000, 0x3fef322d, 0xd2980e94, - 0x3e5071af, 0xa0000000, 0x3fef227e, 0x773abade, 0xbe5891e5, - 0xa0000000, 0x3fef12d4, 0xdc6bf46b, 0xbe5cccbe, 0xe0000000, - 0x3fef032f, 0xbc7247fa, 0xbe2bab83, 0x80000000, 0x3feef390, - 0xbcaa1e46, 0xbe53bb3b, 0x60000000, 0x3feee3f6, 0x5f6c682d, - 0xbe54c619, 0x80000000, 0x3feed461, 0x5141e368, 0xbe4b6d86, - 0xe0000000, 0x3feec4d1, 0xec678f76, 0xbe369af6, 0x80000000, - 0x3feeb547, 0x41301f55, 0xbe2d4312, 0x60000000, 0x3feea5c2, - 0x676da6bd, 0xbe4d8dd0, 0x60000000, 0x3fee9642, 0x57a891c4, - 0x3e51f991, 0xa0000000, 0x3fee86c7, 0xe4eb491e, 0x3e579bf9, - 0x20000000, 0x3fee7752, 0xfddc4a2c, 0xbe3356e6, 0xc0000000, - 0x3fee67e1, 0xd75b5bf1, 0xbe449531, 0x80000000, 0x3fee5876, - 0xbd423b8e, 0x3df54fe4, 0x60000000, 0x3fee4910, 0x330e51b9, - 0x3e54289c, 0x80000000, 0x3fee39af, 0x8651a95f, 0xbe55aad6, - 0xa0000000, 0x3fee2a53, 0x5e98c708, 0xbe2fc4a9, 0xe0000000, - 0x3fee1afc, 0x0989328d, 0x3e23958c, 0x40000000, 0x3fee0bab, - 0xee642abd, 0xbe425dd8, 0xa0000000, 0x3fedfc5e, 0xc394d236, - 0x3e526362, 0x20000000, 0x3feded17, 0xe104aa8e, 0x3e4ce247, - 0xc0000000, 0x3fedddd4, 0x265a9be4, 0xbe5bb77a, 0x40000000, - 0x3fedce97, 0x0ecac52f, 0x3e4a7cb1, 0xe0000000, 0x3fedbf5e, - 0x124cb3b8, 0x3e257024, 0x80000000, 0x3fedb02b, 0xe6d4febe, - 0xbe2033ee, 0x20000000, 0x3feda0fd, 0x39cca00e, 0xbe3ddabc, - 0xc0000000, 0x3fed91d3, 0xef8a552a, 0xbe543390, 0x40000000, - 0x3fed82af, 0xb8e85204, 0x3e513850, 0xe0000000, 0x3fed738f, - 0x3d59fe08, 0xbe5db728, 0x40000000, 0x3fed6475, 0x3aa7ead1, - 0x3e58804b, 0xc0000000, 0x3fed555f, 0xf8a35ba9, 0xbe5298b0, - 0x00000000, 0x3fed464f, 0x9a88dd15, 0x3e5a8cdb, 0x40000000, - 0x3fed3743, 0xb0b0a190, 0x3e598635, 0x80000000, 0x3fed283c, - 0xe2113295, 0xbe5c1119, 0x80000000, 0x3fed193a, 0xafbf1728, - 0xbe492e9c, 0x60000000, 0x3fed0a3d, 0xe4a4ccf3, 0x3e19b90e, - 0x20000000, 0x3fecfb45, 0xba3cbeb8, 0x3e406b50, 0xc0000000, - 0x3fecec51, 0x110f7ddd, 0x3e0d6806, 0x40000000, 0x3fecdd63, - 0x7dd7d508, 0xbe5a8943, 0x80000000, 0x3fecce79, 0x9b60f271, - 0xbe50676a, 0x80000000, 0x3fecbf94, 0x0b9ad660, 0x3e59174f, - 0x60000000, 0x3fecb0b4, 0x00823d9c, 0x3e5bbf72, 0x20000000, - 0x3feca1d9, 0x38a6ec89, 0xbe4d38f9, 0x80000000, 0x3fec9302, - 0x3a0b7d8e, 0x3e53dbfd, 0xc0000000, 0x3fec8430, 0xc6826b34, - 0xbe27c5c9, 0xc0000000, 0x3fec7563, 0x0c706381, 0xbe593653, - 0x60000000, 0x3fec669b, 0x7df34ec7, 0x3e461ab5, 0xe0000000, - 0x3fec57d7, 0x40e5e7e8, 0xbe5c3dae, 0x00000000, 0x3fec4919, - 0x5602770f, 0xbe55219d, 0xc0000000, 0x3fec3a5e, 0xec7911eb, - 0x3e5a5d25, 0x60000000, 0x3fec2ba9, 0xb39ea225, 0xbe53c00b, - 0x80000000, 0x3fec1cf8, 0x967a212e, 0x3e5a8ddf, 0x60000000, - 0x3fec0e4c, 0x580798bd, 0x3e5f53ab, 0x00000000, 0x3febffa5, - 0xb8282df6, 0xbe46b874, 0x20000000, 0x3febf102, 0xe33a6729, - 0x3e54963f, 0x00000000, 0x3febe264, 0x3b53e88a, 0xbe3adce1, - 0x60000000, 0x3febd3ca, 0xc2585084, 0x3e5cde9f, 0x80000000, - 0x3febc535, 0xa335c5ee, 0xbe39fd9c, 0x20000000, 0x3febb6a5, - 0x7325b04d, 0x3e42ba15, 0x60000000, 0x3feba819, 0x1564540f, - 0x3e3a9f35, 0x40000000, 0x3feb9992, 0x83fff592, 0xbe5465ce, - 0xa0000000, 0x3feb8b0f, 0xb9da63d3, 0xbe4b1a0a, 0x80000000, - 0x3feb7c91, 0x6d6f1ea4, 0x3e557657, 0x00000000, 0x3feb6e18, - 0x5e80a1bf, 0x3e4ddbb6, 0x00000000, 0x3feb5fa3, 0x1c9eacb5, - 0x3e592877, 0xa0000000, 0x3feb5132, 0x6d40beb3, 0xbe51858c, - 0xa0000000, 0x3feb42c6, 0xd740c67b, 0x3e427ad2, 0x40000000, - 0x3feb345f, 0xa3e0ccee, 0xbe5c2fc4, 0x40000000, 0x3feb25fc, - 0x8e752b50, 0xbe3da3c2, 0xc0000000, 0x3feb179d, 0xa892e7de, - 0x3e1fb481, 0xc0000000, 0x3feb0943, 0x21ed71e9, 0xbe365206, - 0x20000000, 0x3feafaee, 0x0e1380a3, 0x3e5c5b7b, 0x20000000, - 0x3feaec9d, 0x3c3d640e, 0xbe5dbbd0, 0x60000000, 0x3feade50, - 0x8f97a715, 0x3e3a8ec5, 0x20000000, 0x3fead008, 0x23ab2839, - 0x3e2fe98a, 0x40000000, 0x3feac1c4, 0xf4bbd50f, 0x3e54d8f6, - 0xe0000000, 0x3feab384, 0x14757c4d, 0xbe48774c, 0xc0000000, - 0x3feaa549, 0x7c7b0eea, 0x3e5b51bb, 0x20000000, 0x3fea9713, - 0xf56f7013, 0x3e386200, 0xe0000000, 0x3fea88e0, 0xbe428ebe, - 0xbe514af5, 0xe0000000, 0x3fea7ab2, 0x8d0e4496, 0x3e4f9165, - 0x60000000, 0x3fea6c89, 0xdbacc5d5, 0xbe5c063b, 0x20000000, - 0x3fea5e64, 0x3f19d970, 0xbe5a0c8c, 0x20000000, 0x3fea5043, - 0x09ea3e6b, 0x3e5065dc, 0x80000000, 0x3fea4226, 0x78df246c, - 0x3e5e05f6, 0x40000000, 0x3fea340e, 0x4057d4a0, 0x3e431b2b, - 0x40000000, 0x3fea25fa, 0x82867bb5, 0x3e4b76be, 0xa0000000, - 0x3fea17ea, 0x9436f40a, 0xbe5aad39, 0x20000000, 0x3fea09df, - 0x4b5253b3, 0x3e46380b, 0x00000000, 0x3fe9fbd8, 0x8fc52466, - 0xbe386f9b, 0x20000000, 0x3fe9edd5, 0x22d3f344, 0xbe538347, - 0x60000000, 0x3fe9dfd6, 0x1ac33522, 0x3e5dbc53, 0x00000000, - 0x3fe9d1dc, 0xeabdff1d, 0x3e40fc0c, 0xe0000000, 0x3fe9c3e5, - 0xafd30e73, 0xbe585e63, 0xe0000000, 0x3fe9b5f3, 0xa52f226a, - 0xbe43e8f9, 0x20000000, 0x3fe9a806, 0xecb8698d, 0xbe515b36, - 0x80000000, 0x3fe99a1c, 0xf2b4e89d, 0x3e48b62b, 0x20000000, - 0x3fe98c37, 0x7c9a88fb, 0x3e44414c, 0x00000000, 0x3fe97e56, - 0xda015741, 0xbe5d13ba, 0xe0000000, 0x3fe97078, 0x5fdace06, - 0x3e51b947, 0x00000000, 0x3fe962a0, 0x956ca094, 0x3e518785, - 0x40000000, 0x3fe954cb, 0x01164c1d, 0x3e5d5b57, 0xc0000000, - 0x3fe946fa, 0xe63b3767, 0xbe4f84e7, 0x40000000, 0x3fe9392e, - 0xe57cc2a9, 0x3e34eda3, 0xe0000000, 0x3fe92b65, 0x8c75b544, - 0x3e5766a0, 0xc0000000, 0x3fe91da1, 0x37d1d087, 0xbe5e2ab1, - 0x80000000, 0x3fe90fe1, 0xa953dc20, 0x3e5fa1f3, 0x80000000, - 0x3fe90225, 0xdbd3f369, 0x3e47d6db, 0xa0000000, 0x3fe8f46d, - 0x1c9be989, 0xbe5e2b0a, 0xa0000000, 0x3fe8e6b9, 0x3c93d76a, - 0x3e5c8618, 0xe0000000, 0x3fe8d909, 0x2182fc9a, 0xbe41aa9e, - 0x20000000, 0x3fe8cb5e, 0xe6b3539d, 0xbe530d19, 0x60000000, - 0x3fe8bdb6, 0x49e58cc3, 0xbe3bb374, 0xa0000000, 0x3fe8b012, - 0xa7cfeb8f, 0x3e56c412, 0x00000000, 0x3fe8a273, 0x8d52bc19, - 0x3e1429b8, 0x60000000, 0x3fe894d7, 0x4dc32c6c, 0xbe48604c, - 0xc0000000, 0x3fe8873f, 0x0c868e56, 0xbe564ee5, 0x00000000, - 0x3fe879ac, 0x56aee828, 0x3e5e2fd8, 0x60000000, 0x3fe86c1c, - 0x7ceab8ec, 0x3e493365, 0xc0000000, 0x3fe85e90, 0x78d4dadc, - 0xbe4f7f25, 0x00000000, 0x3fe85109, 0x0ccd8280, 0x3e31e7a2, - 0x40000000, 0x3fe84385, 0x34ba4e15, 0x3e328077, 0x80000000, - 0x3fe83605, 0xa670975a, 0xbe53eee5, 0xa0000000, 0x3fe82889, - 0xf61b77b2, 0xbe43a20a, 0xa0000000, 0x3fe81b11, 0x13e6643b, - 0x3e5e5fe5, 0xc0000000, 0x3fe80d9d, 0x82cc94e8, 0xbe5ff1f9, - 0xa0000000, 0x3fe8002d, 0x8a0c9c5d, 0xbe42b0e7, 0x60000000, - 0x3fe7f2c1, 0x22a16f01, 0x3e5d9ea0, 0x20000000, 0x3fe7e559, - 0xc38cd451, 0x3e506963, 0xc0000000, 0x3fe7d7f4, 0x9902bc71, - 0x3e4503d7, 0x40000000, 0x3fe7ca94, 0xdef2a3c0, 0x3e3d98ed, - 0xa0000000, 0x3fe7bd37, 0xed49abb0, 0x3e24c1ff, 0xe0000000, - 0x3fe7afde, 0xe3b0be70, 0xbe40c467, 0x00000000, 0x3fe7a28a, - 0xaf9f193c, 0xbe5dff6c, 0xe0000000, 0x3fe79538, 0xb74cf6b6, - 0xbe258ed0, 0xa0000000, 0x3fe787eb, 0x1d9127c7, 0x3e345fb0, - 0x40000000, 0x3fe77aa2, 0x1028c21d, 0xbe4619bd, 0xa0000000, - 0x3fe76d5c, 0x7cb0b5e4, 0x3e40f1a2, 0xe0000000, 0x3fe7601a, - 0x2b1bc4ad, 0xbe32e8bb, 0xe0000000, 0x3fe752dc, 0x6839f64e, - 0x3e41f57b, 0xc0000000, 0x3fe745a2, 0xc4121f7e, 0xbe52c40a, - 0x60000000, 0x3fe7386c, 0xd6852d72, 0xbe5c4e6b, 0xc0000000, - 0x3fe72b39, 0x91d690f7, 0xbe57f88f, 0xe0000000, 0x3fe71e0a, - 0x627a2159, 0xbe4425d5, 0xc0000000, 0x3fe710df, 0x50a54033, - 0x3e422b7e, 0x60000000, 0x3fe703b8, 0x3b0b5f91, 0x3e5d3857, - 0xe0000000, 0x3fe6f694, 0x84d628a2, 0xbe51f090, 0x00000000, - 0x3fe6e975, 0x306d8894, 0xbe414d83, 0xe0000000, 0x3fe6dc58, - 0x30bf24aa, 0xbe4650ca, 0x80000000, 0x3fe6cf40, 0xd4628d69, - 0xbe5db007, 0xc0000000, 0x3fe6c22b, 0xa2aae57b, 0xbe31d279, - 0xc0000000, 0x3fe6b51a, 0x860edf7e, 0xbe2d4c4a, 0x80000000, - 0x3fe6a80d, 0xf3559341, 0xbe5f7e98, 0xe0000000, 0x3fe69b03, - 0xa885899e, 0xbe5c2011, 0xe0000000, 0x3fe68dfd, 0x2bdc6d37, - 0x3e224a82, 0xa0000000, 0x3fe680fb, 0xc12ad1b9, 0xbe40cf56, - 0x00000000, 0x3fe673fd, 0x1bcdf659, 0xbdf52f2d, 0x00000000, - 0x3fe66702, 0x5df10408, 0x3e5663e0, 0xc0000000, 0x3fe65a0a, - 0xa4070568, 0xbe40b12f, 0x00000000, 0x3fe64d17, 0x71c54c47, - 0x3e5f5e8b, 0x00000000, 0x3fe64027, 0xbd4b7e83, 0x3e42ead6, - 0xa0000000, 0x3fe6333a, 0x61598bd2, 0xbe4c48d4, 0xc0000000, - 0x3fe62651, 0x6f538d61, 0x3e548401, 0xa0000000, 0x3fe6196c, - 0x14344120, 0xbe529af6, 0x00000000, 0x3fe60c8b, 0x5982c587, - 0xbe3e1e4f, 0x00000000, 0x3fe5ffad, 0xfe51d4ea, 0xbe4c897a, - 0x80000000, 0x3fe5f2d2, 0xfd46ebe1, 0x3e552e00, 0xa0000000, - 0x3fe5e5fb, 0xa4695699, 0x3e5ed471, 0x60000000, 0x3fe5d928, - 0x80d118ae, 0x3e456b61, 0xa0000000, 0x3fe5cc58, 0x304c330b, - 0x3e54dc29, 0x80000000, 0x3fe5bf8c, 0x0af2dedf, 0xbe3aa9bd, - 0xe0000000, 0x3fe5b2c3, 0x15fc9258, 0xbe479a37, 0xc0000000, - 0x3fe5a5fe, 0x9292c7ea, 0x3e188650, 0x20000000, 0x3fe5993d, - 0x33b4d380, 0x3e5d6d93, 0x20000000, 0x3fe58c7f, 0x02fd16c7, - 0x3e2fe961, 0xa0000000, 0x3fe57fc4, 0x4a05edb6, 0xbe4d55b4, - 0xa0000000, 0x3fe5730d, 0x3d443abb, 0xbe5e6954, 0x00000000, - 0x3fe5665a, 0x024acfea, 0x3e50e61b, 0x00000000, 0x3fe559aa, - 0xcc9edd09, 0xbe325403, 0x60000000, 0x3fe54cfd, 0x1fe26950, - 0x3e5d500e, 0x60000000, 0x3fe54054, 0x6c5ae164, 0xbe4a79b4, - 0xc0000000, 0x3fe533ae, 0x154b0287, 0xbe401571, 0xa0000000, - 0x3fe5270c, 0x0673f401, 0xbe56e56b, 0xe0000000, 0x3fe51a6d, - 0x751b639c, 0x3e235269, 0xa0000000, 0x3fe50dd2, 0x7c7b2bed, - 0x3ddec887, 0xc0000000, 0x3fe5013a, 0xafab4e17, 0x3e5e7575, - 0x60000000, 0x3fe4f4a6, 0x2e308668, 0x3e59aed6, 0x80000000, - 0x3fe4e815, 0xf33e2a76, 0xbe51f184, 0xe0000000, 0x3fe4db87, - 0x839f3e3e, 0x3e57db01, 0xc0000000, 0x3fe4cefd, 0xa9eda7bb, - 0x3e535e0f, 0x00000000, 0x3fe4c277, 0x2a8f66a5, 0x3e5ce451, - 0xc0000000, 0x3fe4b5f3, 0x05192456, 0xbe4e8518, 0xc0000000, - 0x3fe4a973, 0x4aa7cd1d, 0x3e46784a, 0x40000000, 0x3fe49cf7, - 0x8e23025e, 0xbe5749f2, 0x00000000, 0x3fe4907e, 0x18d30215, - 0x3e360f39, 0x20000000, 0x3fe48408, 0x63dcf2f3, 0x3e5e00fe, - 0xc0000000, 0x3fe47795, 0x46182d09, 0xbe5173d9, 0xa0000000, - 0x3fe46b26, 0x8f0e62aa, 0xbe48f281, 0xe0000000, 0x3fe45eba, - 0x5775c40c, 0xbe56aad4, 0x60000000, 0x3fe45252, 0x0fe25f69, - 0x3e48bd71, 0x40000000, 0x3fe445ed, 0xe9989ec5, 0x3e590d97, - 0x80000000, 0x3fe4398b, 0xb3d9ffe3, 0x3e479dbc, 0x20000000, - 0x3fe42d2d, 0x388e4d2e, 0xbe5eed80, 0xe0000000, 0x3fe420d1, - 0x6f797c18, 0x3e554b4c, 0x20000000, 0x3fe4147a, 0x31048bb4, - 0xbe5b1112, 0x80000000, 0x3fe40825, 0x2efba4f9, 0x3e48ebc7, - 0x40000000, 0x3fe3fbd4, 0x50201119, 0x3e40b701, 0x40000000, - 0x3fe3ef86, 0x0a4db32c, 0x3e551de8, 0xa0000000, 0x3fe3e33b, - 0x0c9c148b, 0xbe50c1f6, 0x20000000, 0x3fe3d6f4, 0xc9129447, - 0x3e533fa0, 0x00000000, 0x3fe3cab0, 0xaae5b5a0, 0xbe22b68e, - 0x20000000, 0x3fe3be6f, 0x02305e8a, 0xbe54fc08, 0x60000000, - 0x3fe3b231, 0x7f908258, 0x3e57dc05, 0x00000000, 0x3fe3a5f7, - 0x1a09af78, 0x3e08038b, 0xe0000000, 0x3fe399bf, 0x490643c1, - 0xbe5dbe42, 0xe0000000, 0x3fe38d8b, 0x5e8ad724, 0xbe3c2b72, - 0x20000000, 0x3fe3815b, 0xc67196b6, 0x3e1713cf, 0xa0000000, - 0x3fe3752d, 0x6182e429, 0xbe3ec14c, 0x40000000, 0x3fe36903, - 0xab6eb1ae, 0x3e5a2cc5, 0x40000000, 0x3fe35cdc, 0xfe5dc064, - 0xbe5c5878, 0x40000000, 0x3fe350b8, 0x0ba6b9e4, 0x3e51619b, - 0x80000000, 0x3fe34497, 0x857761aa, 0x3e5fff53, 0x00000000, - 0x3fe3387a, 0xf872d68c, 0x3e484f4d, 0xa0000000, 0x3fe32c5f, - 0x087e97c2, 0x3e52842e, 0x80000000, 0x3fe32048, 0x73d6d0c0, - 0xbe503edf, 0x80000000, 0x3fe31434, 0x0c1456a1, 0xbe5f72ad, - 0xa0000000, 0x3fe30823, 0x83a1a4d5, 0xbe5e65cc, 0xe0000000, - 0x3fe2fc15, 0x855a7390, 0xbe506438, 0x40000000, 0x3fe2f00b, - 0xa2898287, 0x3e3d22a2, 0xe0000000, 0x3fe2e403, 0x8b56f66f, - 0xbe5aa5fd, 0x80000000, 0x3fe2d7ff, 0x52db119a, 0x3e3a2e3d, - 0x60000000, 0x3fe2cbfe, 0xe2ddd4c0, 0xbe586469, 0x40000000, - 0x3fe2c000, 0x6b01bf10, 0x3e352b9d, 0x40000000, 0x3fe2b405, - 0xb07a1cdf, 0x3e5c5cda, 0x80000000, 0x3fe2a80d, 0xc7b5f868, - 0xbe5668b3, 0xc0000000, 0x3fe29c18, 0x185edf62, 0xbe563d66, - 0x00000000, 0x3fe29027, 0xf729e1cc, 0x3e59a9a0, 0x80000000, - 0x3fe28438, 0x6433c727, 0xbe43cc89, 0x00000000, 0x3fe2784d, - 0x41782631, 0xbe30750c, 0xa0000000, 0x3fe26c64, 0x914911b7, - 0xbe58290e, 0x40000000, 0x3fe2607f, 0x3dcc73e1, 0xbe4269cd, - 0x00000000, 0x3fe2549d, 0x2751bf70, 0xbe5a6998, 0xc0000000, - 0x3fe248bd, 0x4248b9fb, 0xbe4ddb00, 0x80000000, 0x3fe23ce1, - 0xf35cf82f, 0x3e561b71, 0x60000000, 0x3fe23108, 0x8e481a2d, - 0x3e518fb9, 0x60000000, 0x3fe22532, 0x5ab96edc, 0xbe5fafc5, - 0x40000000, 0x3fe2195f, 0x80943911, 0xbe07f819, 0x40000000, - 0x3fe20d8f, 0x386f2d6c, 0xbe54ba8b, 0x40000000, 0x3fe201c2, - 0xf29664ac, 0xbe5eb815, 0x20000000, 0x3fe1f5f8, 0x64f03390, - 0x3e5e320c, 0x20000000, 0x3fe1ea31, 0x747ff696, 0x3e5ef0a5, - 0x40000000, 0x3fe1de6d, 0x3e9ceb51, 0xbe5f8d27, 0x20000000, - 0x3fe1d2ac, 0x4ae0b55e, 0x3e5faa21, 0x20000000, 0x3fe1c6ee, - 0x28569a5e, 0x3e598a4f, 0x20000000, 0x3fe1bb33, 0x54b33e07, - 0x3e46130a, 0x20000000, 0x3fe1af7b, 0x024f1078, 0xbe4dbf93, - 0x00000000, 0x3fe1a3c6, 0xb0783bfa, 0x3e419248, 0xe0000000, - 0x3fe19813, 0x2f02b836, 0x3e4e02b7, 0xc0000000, 0x3fe18c64, - 0x28dec9d4, 0x3e09064f, 0x80000000, 0x3fe180b8, 0x45cbf406, - 0x3e5b1f46, 0x40000000, 0x3fe1750f, 0x03d9964c, 0x3e5b0a79, - 0x00000000, 0x3fe16969, 0x8b5b882b, 0xbe238086, 0xa0000000, - 0x3fe15dc5, 0x73bad6f8, 0xbdf1fca4, 0x20000000, 0x3fe15225, - 0x5385769c, 0x3e5e8d76, 0xa0000000, 0x3fe14687, 0x1676dc6b, - 0x3e571d08, 0x20000000, 0x3fe13aed, 0xa8c41c7f, 0xbe598a25, - 0x60000000, 0x3fe12f55, 0xc4e1aaf0, 0x3e435277, 0xa0000000, - 0x3fe123c0, 0x403638e1, 0xbe21aa7c, 0xc0000000, 0x3fe1182e, - 0x557a092b, 0xbdd0116b, 0xc0000000, 0x3fe10c9f, 0x7d779f66, - 0x3e4a61ba, 0xc0000000, 0x3fe10113, 0x2b09c645, 0xbe5d586e, - 0x20000000, 0x3fe0ea04, 0xea2cad46, 0x3e5aa97c, 0x20000000, - 0x3fe0d300, 0x23190e54, 0x3e50f1a7, 0xa0000000, 0x3fe0bc07, - 0x1379a5a6, 0xbe51619d, 0x60000000, 0x3fe0a51a, 0x926a3d4a, - 0x3e5cf019, 0xa0000000, 0x3fe08e38, 0xa8c24358, 0x3e35241e, - 0x20000000, 0x3fe07762, 0x24317e7a, 0x3e512cfa, 0x00000000, - 0x3fe06097, 0xfd9cf274, 0xbe55bef3, 0x00000000, 0x3fe049d7, - 0x3689b49d, 0xbe36d26d, 0x40000000, 0x3fe03322, 0xf72ef6c4, - 0xbe54cd08, 0xa0000000, 0x3fe01c78, 0x23702d2d, 0xbe5900bf, - 0x00000000, 0x3fe005da, 0x3f59c14c, 0x3e57d80b, 0x40000000, - 0x3fdfde8d, 0xad67766d, 0xbe57fad4, 0x40000000, 0x3fdfb17c, - 0x644f4ae7, 0x3e1ee43b, 0x40000000, 0x3fdf8481, 0x903234d2, - 0x3e501a86, 0x40000000, 0x3fdf579c, 0xafe9e509, 0xbe267c3e, - 0x00000000, 0x3fdf2acd, 0xb7dfda0b, 0xbe48149b, 0x40000000, - 0x3fdefe13, 0x3b94305e, 0x3e5f4ea7, 0x80000000, 0x3fded16f, - 0x5d95da61, 0xbe55c198, 0x00000000, 0x3fdea4e1, 0x406960c9, - 0xbdd99a19, 0x00000000, 0x3fde7868, 0xd22f3539, 0x3e470c78, - 0x80000000, 0x3fde4c04, 0x83eec535, 0xbe3e1232, 0x40000000, - 0x3fde1fb6, 0x3dfbffcb, 0xbe4b7d71, 0x40000000, 0x3fddf37d, - 0x7e1be4e0, 0xbe5b8f8f, 0x40000000, 0x3fddc759, 0x46dae887, - 0xbe350458, 0x80000000, 0x3fdd9b4a, 0xed6ecc49, 0xbe5f0045, - 0x80000000, 0x3fdd6f50, 0x2e9e883c, 0x3e2915da, 0x80000000, - 0x3fdd436b, 0xf0bccb32, 0x3e4a68c9, 0x80000000, 0x3fdd179b, - 0x9bbfc779, 0xbe54a26a, 0x00000000, 0x3fdcebe0, 0x7cea33ab, - 0x3e43c6b7, 0x40000000, 0x3fdcc039, 0xe740fd06, 0x3e5526c2, - 0x40000000, 0x3fdc94a7, 0x9eadeb1a, 0xbe396d8d, 0xc0000000, - 0x3fdc6929, 0xf0a8f95a, 0xbe5c0ab2, 0x80000000, 0x3fdc3dc0, - 0x6ee2693b, 0x3e0992e6, 0xc0000000, 0x3fdc126b, 0x5ac6b581, - 0xbe2834b6, 0x40000000, 0x3fdbe72b, 0x8cc226ff, 0x3e3596a6, - 0x00000000, 0x3fdbbbff, 0xf92a74bb, 0x3e3c5813, 0x00000000, - 0x3fdb90e7, 0x479664c0, 0xbe50d644, 0x00000000, 0x3fdb65e3, - 0x5004975b, 0xbe55258f, 0x00000000, 0x3fdb3af3, 0xe4b23194, - 0xbe588407, 0xc0000000, 0x3fdb1016, 0xe65d4d0a, 0x3e527c26, - 0x80000000, 0x3fdae54e, 0x814fddd6, 0x3e5962a2, 0x40000000, - 0x3fdaba9a, 0xe19d0913, 0xbe562f4e, 0x80000000, 0x3fda8ff9, - 0x43cfd006, 0xbe4cfdeb, 0x40000000, 0x3fda656c, 0x686f0a4e, - 0x3e5e47a8, 0xc0000000, 0x3fda3af2, 0x7200d410, 0x3e5e1199, - 0xc0000000, 0x3fda108c, 0xabd2266e, 0x3e5ee4d1, 0x40000000, - 0x3fd9e63a, 0x396f8f2c, 0x3e4dbffb, 0x00000000, 0x3fd9bbfb, - 0xe32b25dd, 0x3e5c3a54, 0x40000000, 0x3fd991cf, 0x431e4035, - 0xbe457925, 0x80000000, 0x3fd967b6, 0x7bed3dd3, 0x3e40c61d, - 0x00000000, 0x3fd93db1, 0xd7449365, 0x3e306419, 0x80000000, - 0x3fd913be, 0x1746e791, 0x3e56fcfc, 0x40000000, 0x3fd8e9df, - 0xf3a9028b, 0xbe5041b9, 0xc0000000, 0x3fd8c012, 0x56840c50, - 0xbe26e20a, 0x40000000, 0x3fd89659, 0x19763102, 0xbe51f466, - 0x80000000, 0x3fd86cb2, 0x7032de7c, 0xbe4d298a, 0x80000000, - 0x3fd8431e, 0xdeb39fab, 0xbe4361eb, 0x40000000, 0x3fd8199d, - 0x5d01cbe0, 0xbe5425b3, 0x80000000, 0x3fd7f02e, 0x3ce99aa9, - 0x3e146fa8, 0x80000000, 0x3fd7c6d2, 0xd1a262b9, 0xbe5a1a69, - 0xc0000000, 0x3fd79d88, 0x8606c236, 0x3e423a08, 0x80000000, - 0x3fd77451, 0x8fd1e1b7, 0x3e5a6a63, 0xc0000000, 0x3fd74b2c, - 0xe491456a, 0x3e42c1ca, 0x40000000, 0x3fd7221a, 0x4499a6d7, - 0x3e36a69a, 0x00000000, 0x3fd6f91a, 0x5237df94, 0xbe0f8f02, - 0x00000000, 0x3fd6d02c, 0xb6482c6e, 0xbe5abcf7, 0x00000000, - 0x3fd6a750, 0x1919fd61, 0xbe57ade2, 0x00000000, 0x3fd67e86, - 0xaa7a994d, 0xbe3f3fbd, 0x00000000, 0x3fd655ce, 0x67db014c, - 0x3e33c550, 0x00000000, 0x3fd62d28, 0xa82856b7, 0xbe1409d1, - 0xc0000000, 0x3fd60493, 0x1e6a300d, 0x3e55d899, 0x80000000, - 0x3fd5dc11, 0x1222bd5c, 0xbe35bfc0, 0xc0000000, 0x3fd5b3a0, - 0x6e8dc2d3, 0x3e5d4d79, 0x00000000, 0x3fd58b42, 0xe0e4ace6, - 0xbe517303, 0x80000000, 0x3fd562f4, 0xb306e0a8, 0x3e5edf0f, - 0xc0000000, 0x3fd53ab8, 0x6574bc54, 0x3e5ee859, 0x80000000, - 0x3fd5128e, 0xea902207, 0x3e5f6188, 0xc0000000, 0x3fd4ea75, - 0x9f911d79, 0x3e511735, 0x80000000, 0x3fd4c26e, 0xf9c77397, - 0xbe5b1643, 0x40000000, 0x3fd49a78, 0x15fc9258, 0x3e479a37, - 0x80000000, 0x3fd47293, 0xd5a04dd9, 0xbe426e56, 0xc0000000, - 0x3fd44abf, 0xe04042f5, 0x3e56f7c6, 0x40000000, 0x3fd422fd, - 0x1d8bf2c8, 0x3e5d8810, 0x00000000, 0x3fd3fb4c, 0x88a8ddee, - 0xbe311454, 0xc0000000, 0x3fd3d3ab, 0x3e3b5e47, 0xbe5d1b72, - 0x40000000, 0x3fd3ac1c, 0xc2ab5d59, 0x3e31b02b, 0xc0000000, - 0x3fd3849d, 0xd4e34b9e, 0x3e51cb2f, 0x40000000, 0x3fd35d30, - 0x177204fb, 0xbe2b8cd7, 0x80000000, 0x3fd335d3, 0xfcd38c82, - 0xbe4356e1, 0x80000000, 0x3fd30e87, 0x64f54acc, 0xbe4e6224, - 0x00000000, 0x3fd2e74c, 0xaa7975d9, 0x3e5dc0fe, 0x80000000, - 0x3fd2c021, 0x516dab3f, 0xbe50ffa3, 0x40000000, 0x3fd29907, - 0x2bfb7313, 0x3e5674a2, 0xc0000000, 0x3fd271fd, 0x0549fc99, - 0x3e385d29, 0xc0000000, 0x3fd24b04, 0x55b63073, 0xbe500c6d, - 0x00000000, 0x3fd2241c, 0x3f91953a, 0x3e389977, 0xc0000000, - 0x3fd1fd43, 0xa1543f71, 0xbe3487ab, 0xc0000000, 0x3fd1d67b, - 0x4ec8867c, 0x3df6a2dc, 0x00000000, 0x3fd1afc4, 0x4328e3bb, - 0x3e41d9c0, 0x80000000, 0x3fd1891c, 0x2e1cda84, 0x3e3bdd87, - 0x40000000, 0x3fd16285, 0x4b5331ae, 0xbe53128e, 0x00000000, - 0x3fd13bfe, 0xb9aec164, 0xbe52ac98, 0xc0000000, 0x3fd11586, - 0xd91e1316, 0xbe350630, 0x80000000, 0x3fd0ef1f, 0x7cacc12c, - 0x3e3f5219, 0x40000000, 0x3fd0c8c8, 0xbce277b7, 0x3e3d30c0, - 0x00000000, 0x3fd0a281, 0x2a63447d, 0xbe541377, 0x80000000, - 0x3fd07c49, 0xfac483b5, 0xbe5772ec, 0xc0000000, 0x3fd05621, - 0x36b8a570, 0xbe4fd4bd, 0xc0000000, 0x3fd03009, 0xbae505f7, - 0xbe450388, 0x80000000, 0x3fd00a01, 0x3e35aead, 0xbe5430fc, - 0x80000000, 0x3fcfc811, 0x707475ac, 0x3e38806e, 0x80000000, - 0x3fcf7c3f, 0xc91817fc, 0xbe40ccea, 0x80000000, 0x3fcf308c, - 0xae05d5e9, 0xbe4919b8, 0x80000000, 0x3fcee4f8, 0xae6cc9e6, - 0xbe530b94, 0x00000000, 0x3fce9983, 0x1efe3e8e, 0x3e57747e, - 0x00000000, 0x3fce4e2d, 0xda78d9bf, 0xbe59a608, 0x00000000, - 0x3fce02f5, 0x8abe2c2e, 0x3e4a35ad, 0x00000000, 0x3fcdb7dc, - 0x1495450d, 0xbe0872cc, 0x80000000, 0x3fcd6ce1, 0x86ee0ba0, - 0xbe4f59a0, 0x00000000, 0x3fcd2205, 0xe81ca888, 0x3e5402c3, - 0x00000000, 0x3fccd747, 0x3b4424b9, 0x3e5dfdc3, 0x80000000, - 0x3fcc8ca7, 0xd305b56c, 0x3e202da6, 0x00000000, 0x3fcc4226, - 0x399a6910, 0xbe482a1c, 0x80000000, 0x3fcbf7c2, 0x747f7938, - 0xbe587372, 0x80000000, 0x3fcbad7c, 0x6fc246a0, 0x3e50d83d, - 0x00000000, 0x3fcb6355, 0xee9e9be5, 0xbe5c35bd, 0x80000000, - 0x3fcb194a, 0x8416c0bc, 0x3e546d4f, 0x00000000, 0x3fcacf5e, - 0x49f7f08f, 0x3e56da76, 0x00000000, 0x3fca858f, 0x5dc30de2, - 0x3e5f390c, 0x00000000, 0x3fca3bde, 0x950583b6, 0xbe5e4169, - 0x80000000, 0x3fc9f249, 0x33631553, 0x3e52aeb1, 0x00000000, - 0x3fc9a8d3, 0xde8795a6, 0xbe59a504, 0x00000000, 0x3fc95f79, - 0x076bf41e, 0x3e5122fe, 0x80000000, 0x3fc9163c, 0x2914c8e7, - 0x3e3dd064, 0x00000000, 0x3fc8cd1d, 0x3a30eca3, 0xbe21b4aa, - 0x80000000, 0x3fc8841a, 0xb2a96650, 0xbe575444, 0x80000000, - 0x3fc83b34, 0x2376c0cb, 0xbe2a74c7, 0x80000000, 0x3fc7f26b, - 0xd8a0b653, 0xbe5181b6, 0x00000000, 0x3fc7a9bf, 0x32257882, - 0xbe4a78b4, 0x00000000, 0x3fc7612f, 0x1eee8bd9, 0xbe1bfe9d, - 0x80000000, 0x3fc718bb, 0x0c603cc4, 0x3e36fdc9, 0x80000000, - 0x3fc6d064, 0x3728b8cf, 0xbe1e542e, 0x80000000, 0x3fc68829, - 0xc79a4067, 0x3e5c380f, 0x00000000, 0x3fc6400b, 0xf69eac69, - 0x3e550a84, 0x80000000, 0x3fc5f808, 0xb7a780a4, 0x3e5d9224, - 0x80000000, 0x3fc5b022, 0xad9dfb1e, 0xbe55242f, 0x00000000, - 0x3fc56858, 0x659b18be, 0xbe4bfda3, 0x80000000, 0x3fc520a9, - 0x66ee3631, 0xbe57d769, 0x80000000, 0x3fc4d916, 0x1ec62819, - 0x3e2427f7, 0x80000000, 0x3fc4919f, 0xdec25369, 0xbe435431, - 0x00000000, 0x3fc44a44, 0xa8acfc4b, 0xbe3c62e8, 0x00000000, - 0x3fc40304, 0xcf1d3eab, 0xbdfba29f, 0x80000000, 0x3fc3bbdf, - 0x79aba3ea, 0xbdf1b7c8, 0x80000000, 0x3fc374d6, 0xb8d186da, - 0xbe5130cf, 0x80000000, 0x3fc32de8, 0x9d74f152, 0x3e2285b6, - 0x00000000, 0x3fc2e716, 0x50ae7ca9, 0xbe503920, 0x80000000, - 0x3fc2a05e, 0x6caed92e, 0xbe533924, 0x00000000, 0x3fc259c2, - 0x9cb5034e, 0xbe510e31, 0x80000000, 0x3fc21340, 0x12c4d378, - 0xbe540b43, 0x80000000, 0x3fc1ccd9, 0xcc418706, 0x3e59887a, - 0x00000000, 0x3fc1868e, 0x921f4106, 0xbe528e67, 0x80000000, - 0x3fc1405c, 0x3969441e, 0x3e5d8051, 0x00000000, 0x3fc0fa46, - 0xd941ef5b, 0x3e5f9079, 0x80000000, 0x3fc0b44a, 0x5a3e81b2, - 0xbe567691, 0x00000000, 0x3fc06e69, 0x9d66afe7, 0xbe4d43fb, - 0x00000000, 0x3fc028a2, 0x0a92a162, 0xbe52f394, 0x00000000, - 0x3fbfc5ea, 0x209897e5, 0x3e529e37, 0x00000000, 0x3fbf3ac5, - 0x8458bd7b, 0x3e582831, 0x00000000, 0x3fbeafd5, 0xb8d8b4b8, - 0xbe486b4a, 0x00000000, 0x3fbe2518, 0xe0a3b7b6, 0x3e5bafd2, - 0x00000000, 0x3fbd9a90, 0x2bf2710e, 0x3e383b2b, 0x00000000, - 0x3fbd103c, 0x73eb6ab7, 0xbe56d78d, 0x00000000, 0x3fbc861b, - 0x32ceaff5, 0xbe32dc5a, 0x00000000, 0x3fbbfc2e, 0xbee04cb7, - 0xbe4a71a4, 0x00000000, 0x3fbb7274, 0x35ae9577, 0x3e38142f, - 0x00000000, 0x3fbae8ee, 0xcbaddab4, 0xbe5490f0, 0x00000000, - 0x3fba5f9a, 0x95ce1114, 0x3e597c71, 0x00000000, 0x3fb9d67a, - 0x6d7c0f78, 0x3e3abc2d, 0x00000000, 0x3fb94d8d, 0x2841a782, - 0xbe566cbc, 0x00000000, 0x3fb8c4d2, 0x6ed429c6, 0xbe3cfff9, - 0x00000000, 0x3fb83c4a, 0xe4a49fbb, 0xbe552964, 0x00000000, - 0x3fb7b3f4, 0x2193d81e, 0xbe42fa72, 0x00000000, 0x3fb72bd0, - 0xdd70c122, 0x3e527a8c, 0x00000000, 0x3fb6a3df, 0x03108a54, - 0xbe450393, 0x00000000, 0x3fb61c1f, 0x30ff7954, 0x3e565840, - 0x00000000, 0x3fb59492, 0xdedd460c, 0xbe5422b5, 0x00000000, - 0x3fb50d36, 0x950f9f45, 0xbe5313f6, 0x00000000, 0x3fb4860b, - 0x582cdcb1, 0x3e506d39, 0x00000000, 0x3fb3ff12, 0x7216d3a6, - 0x3e4aa719, 0x00000000, 0x3fb3784a, 0x57a423fd, 0x3e5a9b9f, - 0x00000000, 0x3fb2f1b4, 0x7a138b41, 0xbe50b418, 0x00000000, - 0x3fb26b4e, 0x2fbfd7ea, 0x3e23a53e, 0x00000000, 0x3fb1e519, - 0x18913ccb, 0x3e465fc1, 0x00000000, 0x3fb15f15, 0x7ea24e21, - 0x3e042843, 0x00000000, 0x3fb0d941, 0x7c6d9c77, 0x3e59f61e, - 0x00000000, 0x3fb0539e, 0x114efd44, 0x3e4ccab7, 0x00000000, - 0x3faf9c56, 0x1777f657, 0x3e552f65, 0x00000000, 0x3fae91d2, - 0xc317b86a, 0xbe5a61e0, 0x00000000, 0x3fad87ac, 0xb7664efb, - 0xbe41f64e, 0x00000000, 0x3fac7de6, 0x5d3d03a9, 0x3e0807a0, - 0x00000000, 0x3fab7480, 0x743c38eb, 0xbe3726e1, 0x00000000, - 0x3faa6b78, 0x06a253f1, 0x3e5ad636, 0x00000000, 0x3fa962d0, - 0xa35f541b, 0x3e5a187a, 0x00000000, 0x3fa85a88, 0x4b86e446, - 0xbe508150, 0x00000000, 0x3fa7529c, 0x2589cacf, 0x3e52938a, - 0x00000000, 0x3fa64b10, 0xaf6b11f2, 0xbe3454cd, 0x00000000, - 0x3fa543e2, 0x97506fef, 0xbe5fdec5, 0x00000000, 0x3fa43d10, - 0xe75f7dd9, 0xbe388dd3, 0x00000000, 0x3fa3369c, 0xa4139632, - 0xbdea5177, 0x00000000, 0x3fa23086, 0x352d6f1e, 0xbe565ad6, - 0x00000000, 0x3fa12acc, 0x77449eb7, 0xbe50d5c7, 0x00000000, - 0x3fa0256e, 0x7478da78, 0x3e404724, 0x00000000, 0x3f9e40dc, - 0xf59cef7f, 0xbe539d0a, 0x00000000, 0x3f9c3790, 0x1511d43c, - 0x3e53c2c8, 0x00000000, 0x3f9a2f00, 0x9b8bff3c, 0xbe43b3e1, - 0x00000000, 0x3f982724, 0xad1e22a5, 0x3e46f0bd, 0x00000000, - 0x3f962000, 0x130d9356, 0x3e475ba0, 0x00000000, 0x3f941994, - 0x8f86f883, 0xbe513d0b, 0x00000000, 0x3f9213dc, 0x914d0dc8, - 0xbe534335, 0x00000000, 0x3f900ed8, 0x2d73e5e7, 0xbe22ba75, - 0x00000000, 0x3f8c1510, 0xc5b7d70e, 0x3e599c5d, 0x00000000, - 0x3f880de0, 0x8a27857e, 0xbe3d28c8, 0x00000000, 0x3f840810, - 0xda767328, 0x3e531b3d, 0x00000000, 0x3f8003b0, 0x77bacaf3, - 0xbe5f04e3, 0x00000000, 0x3f780150, 0xdf4b0720, 0x3e5a8bff, - 0x00000000, 0x3f6ffc40, 0x34c48e71, 0xbe3fcd99, 0x00000000, - 0x3f5ff6c0, 0x1ad218af, 0xbe4c78a7, 0x00000000, 0x00000000, - 0x00000000, 0x80000000 - }; - - private static int[] logTwoPow = { - 0xfefa39ef, 0x3fe62e42, 0xfefa39ef, 0xbfe62e42 - }; - - public void powIntrinsic(Register dest, Register value1, Register value2, CompilationResultBuilder crb, AMD64MacroAssembler masm) { - ArrayDataPointerConstant highSigMaskPtr = new ArrayDataPointerConstant(highSigMask, 16); - ArrayDataPointerConstant logTwoEPtr = new ArrayDataPointerConstant(logTwoE, 16); - ArrayDataPointerConstant highmaskYPtr = new ArrayDataPointerConstant(highmaskY, 16); - ArrayDataPointerConstant tExpPtr = new ArrayDataPointerConstant(tExp, 16); - ArrayDataPointerConstant eCoeffPtr = new ArrayDataPointerConstant(eCoeff, 16); - ArrayDataPointerConstant coeffHPtr = new ArrayDataPointerConstant(coeffH, 16); - ArrayDataPointerConstant highmaskLogXPtr = new ArrayDataPointerConstant(highmaskLogX, 16); - ArrayDataPointerConstant halfmaskPtr = new ArrayDataPointerConstant(halfmask, 8); - ArrayDataPointerConstant coeffPowPtr = new ArrayDataPointerConstant(coeffPow, 16); - ArrayDataPointerConstant lTblPowPtr = new ArrayDataPointerConstant(lTblPow, 16); - ArrayDataPointerConstant logTwoPowPtr = new ArrayDataPointerConstant(logTwoPow, 8); - - Label bb0 = new Label(); - Label bb1 = new Label(); - Label bb2 = new Label(); - Label bb3 = new Label(); - Label bb4 = new Label(); - Label bb5 = new Label(); - Label bb6 = new Label(); - Label bb7 = new Label(); - Label bb8 = new Label(); - Label bb9 = new Label(); - Label bb10 = new Label(); - Label bb11 = new Label(); - Label bb12 = new Label(); - Label bb13 = new Label(); - Label bb14 = new Label(); - Label bb15 = new Label(); - Label bb16 = new Label(); - Label bb18 = new Label(); - Label bb19 = new Label(); - Label bb20 = new Label(); - Label bb21 = new Label(); - Label bb22 = new Label(); - Label bb23 = new Label(); - Label bb24 = new Label(); - Label bb25 = new Label(); - Label bb26 = new Label(); - Label bb27 = new Label(); - Label bb28 = new Label(); - Label bb29 = new Label(); - Label bb30 = new Label(); - Label bb31 = new Label(); - Label bb32 = new Label(); - Label bb33 = new Label(); - Label bb34 = new Label(); - Label bb35 = new Label(); - Label bb36 = new Label(); - Label bb37 = new Label(); - Label bb38 = new Label(); - Label bb39 = new Label(); - Label bb40 = new Label(); - Label bb41 = new Label(); - Label bb42 = new Label(); - Label bb43 = new Label(); - Label bb44 = new Label(); - Label bb45 = new Label(); - Label bb46 = new Label(); - Label bb47 = new Label(); - Label bb48 = new Label(); - Label bb49 = new Label(); - Label bb50 = new Label(); - Label bb51 = new Label(); - Label bb53 = new Label(); - Label bb54 = new Label(); - Label bb55 = new Label(); - Label bb56 = new Label(); - - Register gpr1 = asRegister(gpr1Temp, AMD64Kind.QWORD); - Register gpr2 = asRegister(gpr2Temp, AMD64Kind.QWORD); - Register gpr3 = asRegister(rcxTemp, AMD64Kind.QWORD); - Register gpr4 = asRegister(gpr4Temp, AMD64Kind.QWORD); - Register gpr5 = asRegister(gpr5Temp, AMD64Kind.QWORD); - Register gpr6 = asRegister(gpr6Temp, AMD64Kind.QWORD); - Register gpr7 = asRegister(gpr7Temp, AMD64Kind.QWORD); - Register gpr8 = asRegister(gpr8Temp, AMD64Kind.QWORD); - - Register temp1 = asRegister(xmm1Temp, AMD64Kind.DOUBLE); - Register temp2 = asRegister(xmm2Temp, AMD64Kind.DOUBLE); - Register temp3 = asRegister(xmm3Temp, AMD64Kind.DOUBLE); - Register temp4 = asRegister(xmm4Temp, AMD64Kind.DOUBLE); - Register temp5 = asRegister(xmm5Temp, AMD64Kind.DOUBLE); - Register temp6 = asRegister(xmm6Temp, AMD64Kind.DOUBLE); - Register temp7 = asRegister(xmm7Temp, AMD64Kind.DOUBLE); - Register temp8 = asRegister(xmm8Temp, AMD64Kind.DOUBLE); - Register temp9 = asRegister(xmm9Temp, AMD64Kind.DOUBLE); - Register temp10 = asRegister(xmm10Temp, AMD64Kind.DOUBLE); - - setCrb(crb); - masm.movdqu(temp10, value1); - masm.movsd(temp8, value2); - if (dest.encoding != value1.encoding) { - masm.movdqu(dest, value1); - } - - masm.movq(temp9, externalAddress(logTwoEPtr)); // 0x00000000, - // 0x3ff72000 - masm.pextrw(gpr1, dest, 3); - masm.xorpd(temp2, temp2); - masm.movq(gpr2, 0x3ff0000000000000L); - masm.movdq(temp2, gpr2); - masm.movl(gpr5, 1069088768); - masm.movdq(temp7, gpr5); - masm.xorpd(temp1, temp1); - masm.movq(gpr6, 0x77f0000000000000L); - masm.movdq(temp1, gpr6); - masm.movdqu(temp3, dest); - masm.movl(gpr4, 32752); - masm.andl(gpr4, gpr1); - masm.subl(gpr4, 16368); - masm.movl(gpr3, gpr4); - masm.sarl(gpr4, 31); - masm.addl(gpr3, gpr4); - masm.xorl(gpr3, gpr4); - masm.por(dest, temp2); - masm.movdqu(temp6, externalAddress(highSigMaskPtr)); // 0x00000000, - // 0xfffff800, - // 0x00000000, - // 0xfffff800 - masm.psrlq(dest, 27); - masm.psrld(dest, 2); - masm.addl(gpr3, 16); - masm.bsrl(gpr3, gpr3); - masm.rcpps(dest, dest); - masm.psllq(temp3, 12); - masm.movl(gpr7, 8192); - masm.movdq(temp4, gpr7); - masm.psrlq(temp3, 12); - masm.subl(gpr1, 16); - masm.cmpl(gpr1, 32736); - masm.jcc(ConditionFlag.AboveEqual, bb0); - - masm.movl(gpr5, 0); - - masm.bind(bb1); - masm.mulss(dest, temp7); - masm.movl(gpr4, -1); - masm.subl(gpr3, 4); - masm.shll(gpr4); - masm.shlq(gpr4, 32); - masm.movdq(temp5, gpr4); - masm.por(temp3, temp1); - masm.subl(gpr1, 16351); - masm.cmpl(gpr1, 1); - masm.jcc(ConditionFlag.BelowEqual, bb2); - - masm.paddd(dest, temp4); - masm.pand(temp5, temp3); - masm.movdl(gpr4, dest); - masm.psllq(dest, 29); - - masm.bind(bb3); - masm.subsd(temp3, temp5); - masm.pand(dest, temp6); - masm.subl(gpr1, 1); - masm.sarl(gpr1, 4); - masm.cvtsi2sdl(temp7, gpr1); - masm.mulpd(temp5, dest); - - masm.bind(bb4); - masm.mulsd(temp3, dest); - masm.leaq(gpr8, externalAddress(coeffPowPtr)); - masm.movdqu(temp1, new AMD64Address(gpr8, 0)); // 0x6dc96112, - // 0xbf836578, - // 0xee241472, - // 0xbf9b0301 - masm.movdqu(temp4, new AMD64Address(gpr8, 16)); // 0x9f95985a, - // 0xbfb528db, - // 0xb3841d2a, - // 0xbfd619b6 - masm.movdqu(temp6, new AMD64Address(gpr8, 32)); // 0x518775e3, - // 0x3f9004f2, - // 0xac8349bb, - // 0x3fa76c9b - masm.movdqu(dest, new AMD64Address(gpr8, 48)); // 0x486ececc, - // 0x3fc4635e, - // 0x161bb241, - // 0xbf5dabe1 - masm.subsd(temp5, temp9); - masm.movl(gpr3, gpr1); - masm.sarl(gpr1, 31); - masm.addl(gpr3, gpr1); - masm.xorl(gpr1, gpr3); - masm.addl(gpr1, 1); - masm.bsrl(gpr1, gpr1); - masm.unpcklpd(temp5, temp3); - masm.addsd(temp3, temp5); - masm.leaq(gpr7, externalAddress(lTblPowPtr)); - masm.andl(gpr4, 16760832); - masm.shrl(gpr4, 10); - masm.addpd(temp5, new AMD64Address(gpr7, gpr4, Scale.Times1, -3648)); - masm.pshufd(temp2, temp3, 0x44); - masm.mulsd(temp3, temp3); - masm.mulpd(temp1, temp2); - masm.mulpd(temp4, temp2); - masm.addsd(temp5, temp7); - masm.mulsd(temp2, temp3); - masm.addpd(temp6, temp1); - masm.mulsd(temp3, temp3); - masm.addpd(dest, temp4); - masm.movdqu(temp1, temp8); - masm.pextrw(gpr3, temp8, 3); - masm.pshufd(temp7, temp5, 0xEE); - masm.movq(temp4, externalAddress(highmaskYPtr)); // 0x00000000, - // 0xfffffff8 - masm.mulpd(temp6, temp2); - masm.pshufd(temp3, temp3, 0x44); - masm.mulpd(dest, temp2); - masm.shll(gpr1, 4); - masm.subl(gpr1, 15872); - masm.andl(gpr3, 32752); - masm.addl(gpr1, gpr3); - masm.mulpd(temp3, temp6); - masm.cmpl(gpr1, 624); - masm.jcc(ConditionFlag.AboveEqual, bb5); - - masm.xorpd(temp6, temp6); - masm.movl(gpr4, 17080); - masm.pinsrw(temp6, gpr4, 3); - masm.movdqu(temp2, temp1); - masm.pand(temp4, temp1); - masm.subsd(temp1, temp4); - masm.mulsd(temp4, temp5); - masm.addsd(dest, temp7); - masm.mulsd(temp1, temp5); - masm.movdqu(temp7, temp6); - masm.addsd(temp6, temp4); - masm.leaq(gpr7, externalAddress(tExpPtr)); - masm.addpd(temp3, dest); - masm.movdl(gpr4, temp6); - masm.movl(gpr3, gpr4); - masm.andl(gpr4, 255); - masm.addl(gpr4, gpr4); - masm.movdqu(temp5, new AMD64Address(gpr7, gpr4, Scale.Times8, 0)); - masm.subsd(temp6, temp7); - masm.pshufd(dest, temp3, 0xEE); - masm.subsd(temp4, temp6); - masm.addsd(dest, temp3); - masm.addsd(temp4, temp1); - masm.mulsd(temp2, dest); - masm.leaq(gpr8, externalAddress(eCoeffPtr)); - masm.movdqu(temp7, new AMD64Address(gpr8, 0)); // 0xe78a6731, - // 0x3f55d87f, - // 0xd704a0c0, - // 0x3fac6b08 - masm.movdqu(temp3, new AMD64Address(gpr8, 16)); // 0x6fba4e77, - // 0x3f83b2ab, - // 0xff82c58f, - // 0x3fcebfbd - masm.shll(gpr3, 12); - masm.xorl(gpr3, gpr5); - masm.andl(gpr3, -1048576); - masm.movdq(temp6, gpr3); - masm.addsd(temp2, temp4); - masm.movq(gpr2, 0x3fe62e42fefa39efL); - masm.movdq(temp1, gpr2); - masm.pshufd(dest, temp2, 0x44); - masm.pshufd(temp4, temp2, 0x44); - masm.mulsd(temp1, temp2); - masm.pshufd(temp6, temp6, 0x11); - masm.mulpd(dest, dest); - masm.mulpd(temp7, temp4); - masm.paddd(temp5, temp6); - masm.mulsd(temp1, temp5); - masm.pshufd(temp6, temp5, 0xEE); - masm.mulsd(dest, dest); - masm.addpd(temp3, temp7); - masm.addsd(temp1, temp6); - masm.mulpd(dest, temp3); - masm.pshufd(temp3, dest, 0xEE); - masm.mulsd(dest, temp5); - masm.mulsd(temp3, temp5); - masm.addsd(dest, temp1); - masm.addsd(dest, temp3); - masm.addsd(dest, temp5); - masm.jmp(bb56); - - masm.bind(bb0); - masm.addl(gpr1, 16); - masm.movl(gpr4, 32752); - masm.andl(gpr4, gpr1); - masm.cmpl(gpr4, 32752); - masm.jcc(ConditionFlag.Equal, bb6); - - masm.testl(gpr1, 32768); - masm.jcc(ConditionFlag.NotEqual, bb7); - - masm.bind(bb8); - masm.movdqu(dest, temp10); - masm.movdqu(temp3, temp10); - masm.movdl(gpr4, temp3); - masm.psrlq(temp3, 32); - masm.movdl(gpr3, temp3); - masm.orl(gpr4, gpr3); - masm.cmpl(gpr4, 0); - masm.jcc(ConditionFlag.Equal, bb9); - - masm.xorpd(temp3, temp3); - masm.movl(gpr1, 18416); - masm.pinsrw(temp3, gpr1, 3); - masm.mulsd(dest, temp3); - masm.xorpd(temp2, temp2); - masm.movl(gpr1, 16368); - masm.pinsrw(temp2, gpr1, 3); - masm.movdqu(temp3, dest); - masm.pextrw(gpr1, dest, 3); - masm.por(dest, temp2); - masm.movl(gpr3, 18416); - masm.psrlq(dest, 27); - masm.psrld(dest, 2); - masm.rcpps(dest, dest); - masm.psllq(temp3, 12); - masm.movdqu(temp6, externalAddress(highSigMaskPtr)); // 0x00000000, - // 0xfffff800, - // 0x00000000, - // 0xfffff800 - masm.psrlq(temp3, 12); - masm.mulss(dest, temp7); - masm.movl(gpr4, -1024); - masm.movdl(temp5, gpr4); - masm.por(temp3, temp1); - masm.paddd(dest, temp4); - masm.psllq(temp5, 32); - masm.movdl(gpr4, dest); - masm.psllq(dest, 29); - masm.pand(temp5, temp3); - masm.movl(gpr5, 0); - masm.pand(dest, temp6); - masm.subsd(temp3, temp5); - masm.andl(gpr1, 32752); - masm.subl(gpr1, 18416); - masm.sarl(gpr1, 4); - masm.cvtsi2sdl(temp7, gpr1); - masm.mulpd(temp5, dest); - masm.jmp(bb4); - - masm.bind(bb10); - masm.movdqu(dest, temp10); - masm.movdqu(temp3, temp10); - masm.movdl(gpr4, temp3); - masm.psrlq(temp3, 32); - masm.movdl(gpr3, temp3); - masm.orl(gpr4, gpr3); - masm.cmpl(gpr4, 0); - masm.jcc(ConditionFlag.Equal, bb9); - - masm.xorpd(temp3, temp3); - masm.movl(gpr1, 18416); - masm.pinsrw(temp3, gpr1, 3); - masm.mulsd(dest, temp3); - masm.xorpd(temp2, temp2); - masm.movl(gpr1, 16368); - masm.pinsrw(temp2, gpr1, 3); - masm.movdqu(temp3, dest); - masm.pextrw(gpr1, dest, 3); - masm.por(dest, temp2); - masm.movl(gpr3, 18416); - masm.psrlq(dest, 27); - masm.psrld(dest, 2); - masm.rcpps(dest, dest); - masm.psllq(temp3, 12); - masm.movdqu(temp6, externalAddress(highSigMaskPtr)); // 0x00000000, - // 0xfffff800, - // 0x00000000, - // 0xfffff800 - masm.psrlq(temp3, 12); - masm.mulss(dest, temp7); - masm.movl(gpr4, -1024); - masm.movdl(temp5, gpr4); - masm.por(temp3, temp1); - masm.paddd(dest, temp4); - masm.psllq(temp5, 32); - masm.movdl(gpr4, dest); - masm.psllq(dest, 29); - masm.pand(temp5, temp3); - masm.movl(gpr5, Integer.MIN_VALUE); - masm.pand(dest, temp6); - masm.subsd(temp3, temp5); - masm.andl(gpr1, 32752); - masm.subl(gpr1, 18416); - masm.sarl(gpr1, 4); - masm.cvtsi2sdl(temp7, gpr1); - masm.mulpd(temp5, dest); - masm.jmp(bb4); - - masm.bind(bb5); - masm.cmpl(gpr1, 0); - masm.jcc(ConditionFlag.Less, bb11); - - masm.cmpl(gpr1, 752); - masm.jcc(ConditionFlag.AboveEqual, bb12); - - masm.addsd(dest, temp7); - masm.movq(temp4, externalAddress(halfmaskPtr)); // 0xf8000000, - // 0xffffffff - masm.addpd(temp3, dest); - masm.xorpd(temp6, temp6); - masm.movl(gpr1, 17080); - masm.pinsrw(temp6, gpr1, 3); - masm.pshufd(dest, temp3, 0xEE); - masm.addsd(dest, temp3); - masm.movdqu(temp3, temp5); - masm.addsd(temp5, dest); - masm.subsd(temp3, temp5); - masm.movdqu(temp7, temp5); - masm.pand(temp5, temp4); - masm.movdqu(temp2, temp1); - masm.pand(temp4, temp1); - masm.subsd(temp7, temp5); - masm.addsd(dest, temp3); - masm.subsd(temp1, temp4); - masm.mulsd(temp4, temp5); - masm.addsd(dest, temp7); - masm.mulsd(temp2, dest); - masm.movdqu(temp7, temp6); - masm.mulsd(temp1, temp5); - masm.addsd(temp6, temp4); - masm.movdl(gpr1, temp6); - masm.subsd(temp6, temp7); - masm.leaq(gpr7, externalAddress(tExpPtr)); - masm.movl(gpr3, gpr1); - masm.andl(gpr1, 255); - masm.addl(gpr1, gpr1); - masm.movdqu(temp5, new AMD64Address(gpr7, gpr1, Scale.Times8, 0)); - masm.addsd(temp2, temp1); - masm.leaq(gpr8, externalAddress(eCoeffPtr)); - masm.movdqu(temp7, new AMD64Address(gpr8, 0)); // 0xe78a6731, - // 0x3f55d87f, - // 0xd704a0c0, - // 0x3fac6b08 - masm.movdqu(temp3, new AMD64Address(gpr8, 16)); // 0x6fba4e77, - // 0x3f83b2ab, - // 0xff82c58f, - // 0x3fcebfbd - masm.subsd(temp4, temp6); - masm.pextrw(gpr4, temp6, 3); - masm.addsd(temp2, temp4); - masm.sarl(gpr3, 8); - masm.movl(gpr1, gpr3); - masm.sarl(gpr3, 1); - masm.subl(gpr1, gpr3); - masm.shll(gpr3, 20); - masm.xorl(gpr3, gpr5); - masm.movdl(temp6, gpr3); - masm.movq(temp1, new AMD64Address(gpr8, 32)); // 0xfefa39ef, - // 0x3fe62e42 - masm.andl(gpr4, 32767); - masm.cmpl(gpr4, 16529); - masm.jcc(ConditionFlag.Above, bb12); - - masm.pshufd(dest, temp2, 0x44); - masm.pshufd(temp4, temp2, 0x44); - masm.mulpd(dest, dest); - masm.mulpd(temp7, temp4); - masm.pshufd(temp6, temp6, 0x11); - masm.mulsd(temp1, temp2); - masm.mulsd(dest, dest); - masm.paddd(temp5, temp6); - masm.addpd(temp3, temp7); - masm.mulsd(temp1, temp5); - masm.pshufd(temp6, temp5, 0xEE); - masm.mulpd(dest, temp3); - masm.addsd(temp1, temp6); - masm.pshufd(temp3, dest, 0xEE); - masm.mulsd(dest, temp5); - masm.mulsd(temp3, temp5); - masm.shll(gpr1, 4); - masm.xorpd(temp4, temp4); - masm.addl(gpr1, 16368); - masm.pinsrw(temp4, gpr1, 3); - masm.addsd(dest, temp1); - masm.addsd(dest, temp3); - masm.movdqu(temp1, dest); - masm.addsd(dest, temp5); - masm.mulsd(dest, temp4); - masm.pextrw(gpr1, dest, 3); - masm.andl(gpr1, 32752); - masm.jcc(ConditionFlag.Equal, bb13); - - masm.cmpl(gpr1, 32752); - masm.jcc(ConditionFlag.Equal, bb14); - - masm.jmp(bb56); - - masm.bind(bb6); - masm.movdqu(temp1, temp8); - masm.movdqu(dest, temp10); - masm.movdqu(temp2, dest); - masm.movdl(gpr1, temp2); - masm.psrlq(temp2, 20); - masm.movdl(gpr4, temp2); - masm.orl(gpr1, gpr4); - masm.jcc(ConditionFlag.Equal, bb15); - - masm.movdl(gpr1, temp1); - masm.psrlq(temp1, 32); - masm.movdl(gpr4, temp1); - masm.movl(gpr3, gpr4); - masm.addl(gpr4, gpr4); - masm.orl(gpr1, gpr4); - masm.jcc(ConditionFlag.Equal, bb16); - - masm.addsd(dest, dest); - masm.jmp(bb56); - - masm.bind(bb16); - masm.xorpd(dest, dest); - masm.movl(gpr1, 16368); - masm.pinsrw(dest, gpr1, 3); - masm.jmp(bb56); - - masm.bind(bb18); - masm.addpd(dest, temp8); - masm.jmp(bb56); - - masm.bind(bb15); - masm.movdl(gpr1, temp1); - masm.movdqu(temp2, temp1); - masm.psrlq(temp1, 32); - masm.movdl(gpr4, temp1); - masm.movl(gpr3, gpr4); - masm.addl(gpr4, gpr4); - masm.orl(gpr1, gpr4); - masm.jcc(ConditionFlag.Equal, bb19); - - masm.pextrw(gpr1, temp2, 3); - masm.andl(gpr1, 32752); - masm.cmpl(gpr1, 32752); - masm.jcc(ConditionFlag.NotEqual, bb20); - - masm.movdl(gpr1, temp2); - masm.psrlq(temp2, 20); - masm.movdl(gpr4, temp2); - masm.orl(gpr1, gpr4); - masm.jcc(ConditionFlag.NotEqual, bb18); - - masm.bind(bb20); - masm.pextrw(gpr1, dest, 3); - masm.testl(gpr1, 32768); - masm.jcc(ConditionFlag.NotEqual, bb21); - - masm.testl(gpr3, Integer.MIN_VALUE); - masm.jcc(ConditionFlag.NotZero, bb22); - - masm.jmp(bb56); - - masm.bind(bb23); - masm.movdl(gpr1, temp8); - masm.testl(gpr1, 1); - masm.jcc(ConditionFlag.NotEqual, bb24); - - masm.testl(gpr1, 2); - masm.jcc(ConditionFlag.NotEqual, bb25); - - masm.jmp(bb24); - - masm.bind(bb21); - masm.shrl(gpr3, 20); - masm.andl(gpr3, 2047); - masm.cmpl(gpr3, 1075); - masm.jcc(ConditionFlag.Above, bb24); - - masm.jcc(ConditionFlag.Equal, bb26); - - masm.cmpl(gpr3, 1074); - masm.jcc(ConditionFlag.Above, bb23); - - masm.cmpl(gpr3, 1023); - masm.jcc(ConditionFlag.Below, bb24); - - masm.movdqu(temp1, temp8); - masm.movl(gpr1, 17208); - masm.xorpd(temp3, temp3); - masm.pinsrw(temp3, gpr1, 3); - masm.movdqu(temp4, temp3); - masm.addsd(temp3, temp1); - masm.subsd(temp4, temp3); - masm.addsd(temp1, temp4); - masm.pextrw(gpr1, temp1, 3); - masm.andl(gpr1, 32752); - masm.jcc(ConditionFlag.NotEqual, bb24); - - masm.movdl(gpr1, temp3); - masm.andl(gpr1, 1); - masm.jcc(ConditionFlag.Equal, bb24); - - masm.bind(bb25); - masm.pextrw(gpr1, temp8, 3); - masm.andl(gpr1, 32768); - masm.jcc(ConditionFlag.NotEqual, bb27); - - masm.jmp(bb56); - - masm.bind(bb27); - masm.xorpd(dest, dest); - masm.movl(gpr1, 32768); - masm.pinsrw(dest, gpr1, 3); - masm.jmp(bb56); - - masm.bind(bb24); - masm.pextrw(gpr1, temp8, 3); - masm.andl(gpr1, 32768); - masm.jcc(ConditionFlag.NotEqual, bb22); - - masm.xorpd(dest, dest); - masm.movl(gpr1, 32752); - masm.pinsrw(dest, gpr1, 3); - masm.jmp(bb56); - - masm.bind(bb26); - masm.movdl(gpr1, temp8); - masm.andl(gpr1, 1); - masm.jcc(ConditionFlag.Equal, bb24); - - masm.jmp(bb25); - - masm.bind(bb28); - masm.movdl(gpr1, temp1); - masm.psrlq(temp1, 20); - masm.movdl(gpr4, temp1); - masm.orl(gpr1, gpr4); - masm.jcc(ConditionFlag.Equal, bb29); - - masm.addsd(dest, temp8); - masm.jmp(bb56); - - masm.bind(bb29); - masm.movdqu(dest, temp10); - masm.pextrw(gpr1, dest, 3); - masm.cmpl(gpr1, 49136); - masm.jcc(ConditionFlag.NotEqual, bb30); - - masm.movdl(gpr3, dest); - masm.psrlq(dest, 20); - masm.movdl(gpr4, dest); - masm.orl(gpr3, gpr4); - masm.jcc(ConditionFlag.NotEqual, bb30); - - masm.xorpd(dest, dest); - masm.movl(gpr1, 32760); - masm.pinsrw(dest, gpr1, 3); - masm.jmp(bb56); - - masm.bind(bb30); - masm.andl(gpr1, 32752); - masm.subl(gpr1, 16368); - masm.pextrw(gpr4, temp8, 3); - masm.xorpd(dest, dest); - masm.xorl(gpr1, gpr4); - masm.andl(gpr1, 32768); - masm.jcc(ConditionFlag.Equal, bb31); - - masm.jmp(bb56); - - masm.bind(bb31); - masm.movl(gpr3, 32752); - masm.pinsrw(dest, gpr3, 3); - masm.jmp(bb56); - - masm.bind(bb32); - masm.movdl(gpr1, temp1); - masm.cmpl(gpr4, 17184); - masm.jcc(ConditionFlag.Above, bb33); - - masm.testl(gpr1, 1); - masm.jcc(ConditionFlag.NotEqual, bb34); - - masm.testl(gpr1, 2); - masm.jcc(ConditionFlag.Equal, bb35); - - masm.jmp(bb36); - - masm.bind(bb33); - masm.testl(gpr1, 1); - masm.jcc(ConditionFlag.Equal, bb35); - - masm.jmp(bb36); - - masm.bind(bb7); - masm.movdqu(temp2, temp10); - masm.movdl(gpr1, temp2); - masm.psrlq(temp2, 31); - masm.movdl(gpr3, temp2); - masm.orl(gpr1, gpr3); - masm.jcc(ConditionFlag.Equal, bb9); - - masm.pextrw(gpr4, temp8, 3); - masm.movdl(gpr1, temp8); - masm.movdqu(temp2, temp8); - masm.psrlq(temp2, 32); - masm.movdl(gpr3, temp2); - masm.addl(gpr3, gpr3); - masm.orl(gpr3, gpr1); - masm.jcc(ConditionFlag.Equal, bb37); - - masm.andl(gpr4, 32752); - masm.cmpl(gpr4, 32752); - masm.jcc(ConditionFlag.Equal, bb28); - - masm.cmpl(gpr4, 17200); - masm.jcc(ConditionFlag.Above, bb35); - - masm.cmpl(gpr4, 17184); - masm.jcc(ConditionFlag.AboveEqual, bb32); - - masm.cmpl(gpr4, 16368); - masm.jcc(ConditionFlag.Below, bb34); - - masm.movl(gpr1, 17208); - masm.xorpd(temp2, temp2); - masm.pinsrw(temp2, gpr1, 3); - masm.movdqu(temp4, temp2); - masm.addsd(temp2, temp1); - masm.subsd(temp4, temp2); - masm.addsd(temp1, temp4); - masm.pextrw(gpr1, temp1, 3); - masm.andl(gpr1, 32767); - masm.jcc(ConditionFlag.NotEqual, bb34); - - masm.movdl(gpr1, temp2); - masm.andl(gpr1, 1); - masm.jcc(ConditionFlag.Equal, bb35); - - masm.bind(bb36); - masm.xorpd(temp1, temp1); - masm.movl(gpr4, 30704); - masm.pinsrw(temp1, gpr4, 3); - masm.pextrw(gpr1, temp10, 3); - masm.movl(gpr4, 8192); - masm.movdl(temp4, gpr4); - masm.andl(gpr1, 32767); - masm.subl(gpr1, 16); - masm.jcc(ConditionFlag.Less, bb10); - - masm.movl(gpr4, gpr1); - masm.andl(gpr4, 32752); - masm.subl(gpr4, 16368); - masm.movl(gpr3, gpr4); - masm.sarl(gpr4, 31); - masm.addl(gpr3, gpr4); - masm.xorl(gpr3, gpr4); - masm.addl(gpr3, 16); - masm.bsrl(gpr3, gpr3); - masm.movl(gpr5, Integer.MIN_VALUE); - masm.jmp(bb1); - - masm.bind(bb34); - masm.xorpd(temp1, temp1); - masm.movl(gpr1, 32752); - masm.pinsrw(temp1, gpr1, 3); - masm.xorpd(dest, dest); - masm.mulsd(dest, temp1); - masm.jmp(bb56); - - masm.bind(bb35); - masm.xorpd(temp1, temp1); - masm.movl(gpr4, 30704); - masm.pinsrw(temp1, gpr4, 3); - masm.pextrw(gpr1, temp10, 3); - masm.movl(gpr4, 8192); - masm.movdl(temp4, gpr4); - masm.andl(gpr1, 32767); - masm.subl(gpr1, 16); - masm.jcc(ConditionFlag.Less, bb8); - - masm.movl(gpr4, gpr1); - masm.andl(gpr4, 32752); - masm.subl(gpr4, 16368); - masm.movl(gpr3, gpr4); - masm.sarl(gpr4, 31); - masm.addl(gpr3, gpr4); - masm.xorl(gpr3, gpr4); - masm.addl(gpr3, 16); - masm.bsrl(gpr3, gpr3); - masm.movl(gpr5, 0); - masm.jmp(bb1); - - masm.bind(bb19); - masm.xorpd(dest, dest); - masm.movl(gpr1, 16368); - masm.pinsrw(dest, gpr1, 3); - masm.jmp(bb56); - - masm.bind(bb22); - masm.xorpd(dest, dest); - masm.jmp(bb56); - - masm.bind(bb11); - masm.addl(gpr1, 384); - masm.cmpl(gpr1, 0); - masm.jcc(ConditionFlag.Less, bb38); - - masm.mulsd(temp5, temp1); - masm.addsd(dest, temp7); - masm.shrl(gpr5, 31); - masm.addpd(temp3, dest); - masm.pshufd(dest, temp3, 0xEE); - masm.addsd(temp3, dest); - masm.leaq(gpr7, externalAddress(logTwoPowPtr)); // 0xfefa39ef, - // 0x3fe62e42, - // 0xfefa39ef, - // 0xbfe62e42 - masm.movq(temp4, new AMD64Address(gpr7, gpr5, Scale.Times8, 0)); - masm.mulsd(temp1, temp3); - masm.xorpd(dest, dest); - masm.movl(gpr1, 16368); - masm.shll(gpr5, 15); - masm.orl(gpr1, gpr5); - masm.pinsrw(dest, gpr1, 3); - masm.addsd(temp5, temp1); - masm.mulsd(temp5, temp4); - masm.addsd(dest, temp5); - masm.jmp(bb56); - - masm.bind(bb38); - - masm.bind(bb37); - masm.xorpd(dest, dest); - masm.movl(gpr1, 16368); - masm.pinsrw(dest, gpr1, 3); - masm.jmp(bb56); - - masm.bind(bb39); - masm.xorpd(dest, dest); - masm.movl(gpr1, 16368); - masm.pinsrw(dest, gpr1, 3); - masm.jmp(bb56); - - masm.bind(bb9); - masm.movdqu(temp2, temp8); - masm.pextrw(gpr1, temp8, 3); - masm.andl(gpr1, 32752); - masm.cmpl(gpr1, 32752); - masm.jcc(ConditionFlag.NotEqual, bb40); - - masm.movdl(gpr1, temp2); - masm.psrlq(temp2, 20); - masm.movdl(gpr4, temp2); - masm.orl(gpr1, gpr4); - masm.jcc(ConditionFlag.NotEqual, bb18); - - masm.bind(bb40); - masm.movdl(gpr1, temp1); - masm.psrlq(temp1, 32); - masm.movdl(gpr4, temp1); - masm.movl(gpr3, gpr4); - masm.addl(gpr4, gpr4); - masm.orl(gpr1, gpr4); - masm.jcc(ConditionFlag.Equal, bb39); - - masm.shrl(gpr4, 21); - masm.cmpl(gpr4, 1075); - masm.jcc(ConditionFlag.Above, bb41); - - masm.jcc(ConditionFlag.Equal, bb42); - - masm.cmpl(gpr4, 1023); - masm.jcc(ConditionFlag.Below, bb41); - - masm.movdqu(temp1, temp8); - masm.movl(gpr1, 17208); - masm.xorpd(temp3, temp3); - masm.pinsrw(temp3, gpr1, 3); - masm.movdqu(temp4, temp3); - masm.addsd(temp3, temp1); - masm.subsd(temp4, temp3); - masm.addsd(temp1, temp4); - masm.pextrw(gpr1, temp1, 3); - masm.andl(gpr1, 32752); - masm.jcc(ConditionFlag.NotEqual, bb41); - - masm.movdl(gpr1, temp3); - masm.andl(gpr1, 1); - masm.jcc(ConditionFlag.Equal, bb41); - - masm.bind(bb43); - masm.movdqu(dest, temp10); - masm.testl(gpr3, Integer.MIN_VALUE); - masm.jcc(ConditionFlag.NotEqual, bb44); - - masm.jmp(bb56); - - masm.bind(bb42); - masm.movdl(gpr1, temp8); - masm.testl(gpr1, 1); - masm.jcc(ConditionFlag.NotEqual, bb43); - - masm.bind(bb41); - masm.testl(gpr3, Integer.MIN_VALUE); - masm.jcc(ConditionFlag.Equal, bb22); - - masm.xorpd(dest, dest); - - masm.bind(bb44); - masm.movl(gpr1, 16368); - masm.xorpd(temp1, temp1); - masm.pinsrw(temp1, gpr1, 3); - masm.divsd(temp1, dest); - masm.movdqu(dest, temp1); - masm.jmp(bb56); - - masm.bind(bb12); - masm.pextrw(gpr1, temp10, 3); - masm.pextrw(gpr4, temp8, 3); - masm.movl(gpr3, 32752); - masm.andl(gpr3, gpr4); - masm.cmpl(gpr3, 32752); - masm.jcc(ConditionFlag.Equal, bb45); - - masm.andl(gpr1, 32752); - masm.subl(gpr1, 16368); - masm.xorl(gpr4, gpr1); - masm.testl(gpr4, 32768); - masm.jcc(ConditionFlag.NotEqual, bb46); - - masm.bind(bb47); - masm.movl(gpr1, 32736); - masm.pinsrw(dest, gpr1, 3); - masm.shrl(gpr5, 16); - masm.orl(gpr1, gpr5); - masm.pinsrw(temp1, gpr1, 3); - masm.mulsd(dest, temp1); - - masm.bind(bb14); - masm.jmp(bb56); - - masm.bind(bb46); - masm.movl(gpr1, 16); - masm.pinsrw(dest, gpr1, 3); - masm.mulsd(dest, dest); - masm.testl(gpr3, Integer.MIN_VALUE); - masm.jcc(ConditionFlag.Equal, bb48); - - masm.movq(gpr2, 0x8000000000000000L); - masm.movdq(temp2, gpr2); - masm.xorpd(dest, temp2); - - masm.bind(bb48); - masm.jmp(bb56); - - masm.bind(bb13); - masm.pextrw(gpr3, temp5, 3); - masm.pextrw(gpr4, temp4, 3); - masm.movl(gpr1, -1); - masm.andl(gpr3, 32752); - masm.subl(gpr3, 16368); - masm.andl(gpr4, 32752); - masm.addl(gpr4, gpr3); - masm.movl(gpr3, -31); - masm.sarl(gpr4, 4); - masm.subl(gpr3, gpr4); - masm.jcc(ConditionFlag.LessEqual, bb49); - - masm.cmpl(gpr3, 20); - masm.jcc(ConditionFlag.Above, bb50); - - masm.shll(gpr1); - - masm.bind(bb49); - masm.movdl(dest, gpr1); - masm.psllq(dest, 32); - masm.pand(dest, temp5); - masm.subsd(temp5, dest); - masm.addsd(temp5, temp1); - masm.mulsd(dest, temp4); - masm.mulsd(temp5, temp4); - masm.addsd(dest, temp5); - - masm.bind(bb50); - masm.jmp(bb48); - - masm.bind(bb2); - masm.pextrw(gpr3, temp8, 3); - masm.movl(gpr4, Integer.MIN_VALUE); - masm.movdl(temp1, gpr4); - masm.xorpd(temp7, temp7); - masm.paddd(dest, temp4); - masm.movdl(gpr4, dest); - masm.psllq(dest, 29); - masm.paddq(temp1, temp3); - masm.pand(temp5, temp1); - masm.andl(gpr3, 32752); - masm.cmpl(gpr3, 16560); - masm.jcc(ConditionFlag.Less, bb3); - - masm.leaq(gpr7, externalAddress(lTblPowPtr)); - masm.leaq(gpr8, externalAddress(coeffHPtr)); - masm.movdqu(temp4, new AMD64Address(gpr8, 0)); // 0x00000000, - // 0xbfd61a00, - // 0x00000000, - // 0xbf5dabe1 - masm.pand(dest, temp6); - masm.subsd(temp3, temp5); - masm.addl(gpr1, 16351); - masm.shrl(gpr1, 4); - masm.subl(gpr1, 1022); - masm.cvtsi2sdl(temp7, gpr1); - masm.mulpd(temp5, dest); - masm.mulsd(temp3, dest); - masm.subsd(temp5, temp9); - masm.pshufd(temp1, temp4, 0xE); - masm.pshufd(temp2, temp3, 0x44); - masm.unpcklpd(temp5, temp3); - masm.addsd(temp3, temp5); - masm.andl(gpr4, 16760832); - masm.shrl(gpr4, 10); - masm.addpd(temp7, new AMD64Address(gpr7, gpr4, Scale.Times1, -3648)); - masm.movdqu(temp6, temp4); - masm.mulsd(temp4, temp5); - masm.movdqu(dest, temp1); - masm.mulsd(dest, temp5); - masm.mulsd(temp6, temp2); - masm.mulsd(temp1, temp2); - masm.movdqu(temp2, temp5); - masm.mulsd(temp4, temp5); - masm.addsd(temp5, dest); - masm.movdqu(dest, temp7); - masm.addsd(temp2, temp3); - masm.addsd(temp7, temp5); - masm.mulsd(temp6, temp2); - masm.subsd(dest, temp7); - masm.movdqu(temp2, temp7); - masm.addsd(temp7, temp4); - masm.addsd(dest, temp5); - masm.subsd(temp2, temp7); - masm.addsd(temp4, temp2); - masm.pshufd(temp2, temp5, 0xEE); - masm.movdqu(temp5, temp7); - masm.addsd(temp7, temp2); - masm.addsd(temp4, dest); - masm.leaq(gpr8, externalAddress(coeffPowPtr)); - masm.movdqu(dest, new AMD64Address(gpr8, 0)); // 0x6dc96112, - // 0xbf836578, - // 0xee241472, - // 0xbf9b0301 - masm.subsd(temp5, temp7); - masm.addsd(temp6, temp4); - masm.movdqu(temp4, temp7); - masm.addsd(temp5, temp2); - masm.addsd(temp7, temp1); - masm.movdqu(temp2, new AMD64Address(gpr8, 64)); // 0x486ececc, - // 0x3fc4635e, - // 0x161bb241, - // 0xbf5dabe1 - masm.subsd(temp4, temp7); - masm.addsd(temp6, temp5); - masm.addsd(temp4, temp1); - masm.pshufd(temp5, temp7, 0xEE); - masm.movapd(temp1, temp7); - masm.addsd(temp7, temp5); - masm.subsd(temp1, temp7); - masm.addsd(temp1, temp5); - masm.movdqu(temp5, new AMD64Address(gpr8, 80)); // 0x9f95985a, - // 0xbfb528db, - // 0xf8b5787d, - // 0x3ef2531e - masm.pshufd(temp3, temp3, 0x44); - masm.addsd(temp6, temp4); - masm.addsd(temp6, temp1); - masm.movdqu(temp1, new AMD64Address(gpr8, 32)); // 0x9f95985a, - // 0xbfb528db, - // 0xb3841d2a, - // 0xbfd619b6 - masm.mulpd(dest, temp3); - masm.mulpd(temp2, temp3); - masm.pshufd(temp4, temp3, 0x44); - masm.mulpd(temp3, temp3); - masm.addpd(dest, temp1); - masm.addpd(temp5, temp2); - masm.mulsd(temp4, temp3); - masm.movq(temp2, externalAddress(highmaskLogXPtr)); // 0xf8000000, - // 0xffffffff - masm.mulpd(temp3, temp3); - masm.movdqu(temp1, temp8); - masm.pextrw(gpr3, temp8, 3); - masm.mulpd(dest, temp4); - masm.pextrw(gpr1, temp7, 3); - masm.mulpd(temp5, temp4); - masm.mulpd(dest, temp3); - masm.leaq(gpr8, externalAddress(highmaskYPtr)); - masm.movq(temp4, new AMD64Address(gpr8, 8)); // 0x00000000, - // 0xffffffff - masm.pand(temp2, temp7); - masm.addsd(temp5, temp6); - masm.subsd(temp7, temp2); - masm.addpd(temp5, dest); - masm.andl(gpr1, 32752); - masm.subl(gpr1, 16368); - masm.andl(gpr3, 32752); - masm.cmpl(gpr3, 32752); - masm.jcc(ConditionFlag.Equal, bb45); - - masm.addl(gpr3, gpr1); - masm.cmpl(gpr3, 16576); - masm.jcc(ConditionFlag.AboveEqual, bb51); - - masm.pshufd(dest, temp5, 0xEE); - masm.pand(temp4, temp1); - masm.movdqu(temp3, temp1); - masm.addsd(temp5, dest); - masm.subsd(temp1, temp4); - masm.xorpd(temp6, temp6); - masm.movl(gpr4, 17080); - masm.pinsrw(temp6, gpr4, 3); - masm.addsd(temp7, temp5); - masm.mulsd(temp4, temp2); - masm.mulsd(temp1, temp2); - masm.movdqu(temp5, temp6); - masm.mulsd(temp3, temp7); - masm.addsd(temp6, temp4); - masm.addsd(temp1, temp3); - masm.leaq(gpr8, externalAddress(eCoeffPtr)); - masm.movdqu(temp7, new AMD64Address(gpr8, 0)); // 0xe78a6731, - // 0x3f55d87f, - // 0xd704a0c0, - // 0x3fac6b08 - masm.movdl(gpr4, temp6); - masm.subsd(temp6, temp5); - masm.leaq(gpr7, externalAddress(tExpPtr)); - masm.movl(gpr3, gpr4); - masm.andl(gpr4, 255); - masm.addl(gpr4, gpr4); - masm.movdqu(temp5, new AMD64Address(gpr7, gpr4, Scale.Times8, 0)); - masm.movdqu(temp3, new AMD64Address(gpr8, 16)); // 0x6fba4e77, - // 0x3f83b2ab, - // 0xff82c58f, - // 0x3fcebfbd - masm.movq(temp2, new AMD64Address(gpr8, 32)); // 0xfefa39ef, - // 0x3fe62e42 - masm.subsd(temp4, temp6); - masm.addsd(temp4, temp1); - masm.pextrw(gpr4, temp6, 3); - masm.shrl(gpr3, 8); - masm.movl(gpr1, gpr3); - masm.shrl(gpr3, 1); - masm.subl(gpr1, gpr3); - masm.shll(gpr3, 20); - masm.movdl(temp6, gpr3); - masm.pshufd(dest, temp4, 0x44); - masm.pshufd(temp1, temp4, 0x44); - masm.mulpd(dest, dest); - masm.mulpd(temp7, temp1); - masm.pshufd(temp6, temp6, 0x11); - masm.mulsd(temp2, temp4); - masm.andl(gpr4, 32767); - masm.cmpl(gpr4, 16529); - masm.jcc(ConditionFlag.Above, bb12); - - masm.mulsd(dest, dest); - masm.paddd(temp5, temp6); - masm.addpd(temp3, temp7); - masm.mulsd(temp2, temp5); - masm.pshufd(temp6, temp5, 0xEE); - masm.mulpd(dest, temp3); - masm.addsd(temp2, temp6); - masm.pshufd(temp3, dest, 0xEE); - masm.addl(gpr1, 1023); - masm.shll(gpr1, 20); - masm.orl(gpr1, gpr5); - masm.movdl(temp4, gpr1); - masm.mulsd(dest, temp5); - masm.mulsd(temp3, temp5); - masm.addsd(dest, temp2); - masm.psllq(temp4, 32); - masm.addsd(dest, temp3); - masm.movdqu(temp1, dest); - masm.addsd(dest, temp5); - masm.mulsd(dest, temp4); - masm.pextrw(gpr1, dest, 3); - masm.andl(gpr1, 32752); - masm.jcc(ConditionFlag.Equal, bb13); - - masm.cmpl(gpr1, 32752); - masm.jcc(ConditionFlag.Equal, bb14); - - masm.jmp(bb56); - - masm.bind(bb45); - masm.movdqu(dest, temp10); - masm.xorpd(temp2, temp2); - masm.movl(gpr1, 49136); - masm.pinsrw(temp2, gpr1, 3); - masm.addsd(temp2, dest); - masm.pextrw(gpr1, temp2, 3); - masm.cmpl(gpr1, 0); - masm.jcc(ConditionFlag.NotEqual, bb53); - - masm.xorpd(dest, dest); - masm.movl(gpr1, 32760); - masm.pinsrw(dest, gpr1, 3); - masm.jmp(bb56); - - masm.bind(bb53); - masm.movdqu(temp1, temp8); - masm.movdl(gpr4, temp1); - masm.movdqu(temp3, temp1); - masm.psrlq(temp3, 20); - masm.movdl(gpr3, temp3); - masm.orl(gpr3, gpr4); - masm.jcc(ConditionFlag.Equal, bb54); - - masm.addsd(temp1, temp1); - masm.movdqu(dest, temp1); - masm.jmp(bb56); - - masm.bind(bb51); - masm.pextrw(gpr1, temp1, 3); - masm.pextrw(gpr3, temp2, 3); - masm.xorl(gpr1, gpr3); - masm.testl(gpr1, 32768); - masm.jcc(ConditionFlag.Equal, bb47); - - masm.jmp(bb46); - - masm.bind(bb54); - masm.pextrw(gpr1, dest, 3); - masm.andl(gpr1, 32752); - masm.pextrw(gpr4, temp1, 3); - masm.xorpd(dest, dest); - masm.subl(gpr1, 16368); - masm.xorl(gpr1, gpr4); - masm.testl(gpr1, 32768); - masm.jcc(ConditionFlag.Equal, bb55); - - masm.jmp(bb56); - - masm.bind(bb55); - masm.movl(gpr4, 32752); - masm.pinsrw(dest, gpr4, 3); - masm.jmp(bb56); - - masm.bind(bb56); +/** + * AMD64MathIntrinsicBinaryOp assumes that the input values are stored in the xmm0 and xmm1 + * registers, and it will emit the output value into the xmm0 register. + * {@link #emitLIRWrapper(LIRGenerator, Value, Value)} is provided for emitting necessary mov LIRs + * before and after this LIR instruction. + */ +public abstract class AMD64MathIntrinsicBinaryOp extends AMD64LIRInstruction { + + @Def protected Value output; + @Use protected Value input0; + @Use protected Value input1; + @Temp protected Value[] temps; + + public AMD64MathIntrinsicBinaryOp(LIRInstructionClass type, Register... registers) { + super(type); + + input0 = xmm0.asValue(LIRKind.value(AMD64Kind.DOUBLE)); + input1 = xmm1.asValue(LIRKind.value(AMD64Kind.DOUBLE)); + output = xmm0.asValue(LIRKind.value(AMD64Kind.DOUBLE)); + + temps = registersToValues(registers); + } + + public final Variable emitLIRWrapper(LIRGenerator gen, Value x, Value y) { + LIRKind kind = LIRKind.combine(x, y); + RegisterValue xmm0Value = xmm0.asValue(kind); + gen.emitMove(xmm0Value, x); + RegisterValue xmm1Value = xmm1.asValue(kind); + gen.emitMove(xmm1Value, y); + gen.append(this); + Variable result = gen.newVariable(kind); + gen.emitMove(result, xmm0Value); + return result; } } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathIntrinsicUnaryOp.java 2019-03-12 08:09:28.483527123 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathIntrinsicUnaryOp.java 2019-03-12 08:09:28.107524683 +0100 @@ -24,3822 +24,48 @@ package org.graalvm.compiler.lir.amd64; -import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.ILLEGAL; -import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.REG; -import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.STACK; -import static jdk.vm.ci.code.ValueUtil.asRegister; +import static jdk.vm.ci.amd64.AMD64.xmm0; +import static org.graalvm.compiler.lir.amd64.AMD64HotSpotHelper.registersToValues; -import org.graalvm.compiler.asm.Label; -import org.graalvm.compiler.asm.amd64.AMD64Address; -import org.graalvm.compiler.asm.amd64.AMD64Address.Scale; -import org.graalvm.compiler.asm.amd64.AMD64Assembler.ConditionFlag; -import org.graalvm.compiler.asm.amd64.AMD64MacroAssembler; import org.graalvm.compiler.core.common.LIRKind; -import org.graalvm.compiler.debug.GraalError; import org.graalvm.compiler.lir.LIRInstructionClass; -import org.graalvm.compiler.lir.Opcode; -import org.graalvm.compiler.lir.asm.ArrayDataPointerConstant; -import org.graalvm.compiler.lir.asm.CompilationResultBuilder; +import org.graalvm.compiler.lir.Variable; import org.graalvm.compiler.lir.gen.LIRGeneratorTool; -import jdk.vm.ci.amd64.AMD64; -import jdk.vm.ci.amd64.AMD64.CPUFeature; import jdk.vm.ci.amd64.AMD64Kind; import jdk.vm.ci.code.Register; -import jdk.vm.ci.meta.AllocatableValue; +import jdk.vm.ci.code.RegisterValue; import jdk.vm.ci.meta.Value; -public final class AMD64MathIntrinsicUnaryOp extends AMD64LIRInstruction { - public static final LIRInstructionClass TYPE = LIRInstructionClass.create(AMD64MathIntrinsicUnaryOp.class); +/** + * AMD64MathIntrinsicUnaryOp assumes that the input value is stored at the xmm0 register, and will + * emit the output value into the xmm0 register as well. + * {@link #emitLIRWrapper(LIRGeneratorTool, Value)} is provided for emitting necessary mov LIRs + * before and after this LIR instruction. + */ +public abstract class AMD64MathIntrinsicUnaryOp extends AMD64LIRInstruction { - public enum UnaryIntrinsicOpcode { - LOG, - LOG10, - SIN, - COS, - TAN, - EXP - } - - @Opcode private final UnaryIntrinsicOpcode opcode; - @Def protected Value result; + @Def protected Value output; @Use protected Value input; - @Temp({REG, ILLEGAL}) protected Value xmm1Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value xmm2Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value xmm3Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value xmm4Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value xmm5Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value xmm6Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value xmm7Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value xmm8Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value xmm9Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value xmm10Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value gpr1Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value gpr2Temp = Value.ILLEGAL; - @Temp protected AllocatableValue rcxTemp; - @Temp({REG, ILLEGAL}) protected Value gpr4Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value gpr5Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value gpr6Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value gpr7Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value gpr8Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value gpr9Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value gpr10Temp = Value.ILLEGAL; - @Temp({STACK, ILLEGAL}) protected Value stackTemp = Value.ILLEGAL; - - CompilationResultBuilder internalCrb; - - public AMD64MathIntrinsicUnaryOp(LIRGeneratorTool tool, UnaryIntrinsicOpcode opcode, Value result, Value input, Value stackTemp) { - super(TYPE); - this.opcode = opcode; - this.result = result; - this.input = input; - if (opcode == UnaryIntrinsicOpcode.LOG || opcode == UnaryIntrinsicOpcode.LOG10 || - opcode == UnaryIntrinsicOpcode.SIN || opcode == UnaryIntrinsicOpcode.COS || - opcode == UnaryIntrinsicOpcode.TAN || opcode == UnaryIntrinsicOpcode.EXP) { - this.gpr1Temp = tool.newVariable(LIRKind.value(AMD64Kind.QWORD)); - this.gpr2Temp = tool.newVariable(LIRKind.value(AMD64Kind.QWORD)); - this.rcxTemp = AMD64.rcx.asValue(LIRKind.value(AMD64Kind.QWORD)); - this.gpr4Temp = tool.newVariable(LIRKind.value(AMD64Kind.QWORD)); - this.xmm1Temp = tool.newVariable(LIRKind.value(AMD64Kind.DOUBLE)); - this.xmm2Temp = tool.newVariable(LIRKind.value(AMD64Kind.DOUBLE)); - this.xmm3Temp = tool.newVariable(LIRKind.value(AMD64Kind.DOUBLE)); - this.xmm4Temp = tool.newVariable(LIRKind.value(AMD64Kind.DOUBLE)); - this.xmm5Temp = tool.newVariable(LIRKind.value(AMD64Kind.DOUBLE)); - this.xmm6Temp = tool.newVariable(LIRKind.value(AMD64Kind.DOUBLE)); - this.xmm7Temp = tool.newVariable(LIRKind.value(AMD64Kind.DOUBLE)); - - if (opcode == UnaryIntrinsicOpcode.EXP) { - this.gpr5Temp = tool.newVariable(LIRKind.value(AMD64Kind.QWORD)); - this.xmm8Temp = tool.newVariable(LIRKind.value(AMD64Kind.DOUBLE)); - this.xmm9Temp = tool.newVariable(LIRKind.value(AMD64Kind.DOUBLE)); - this.xmm10Temp = tool.newVariable(LIRKind.value(AMD64Kind.DOUBLE)); - } - - if (opcode == UnaryIntrinsicOpcode.TAN) { - this.gpr5Temp = tool.newVariable(LIRKind.value(AMD64Kind.QWORD)); - this.gpr6Temp = tool.newVariable(LIRKind.value(AMD64Kind.QWORD)); - this.gpr7Temp = tool.newVariable(LIRKind.value(AMD64Kind.QWORD)); - this.gpr8Temp = tool.newVariable(LIRKind.value(AMD64Kind.QWORD)); - this.gpr9Temp = tool.newVariable(LIRKind.value(AMD64Kind.QWORD)); - this.gpr10Temp = tool.newVariable(LIRKind.value(AMD64Kind.QWORD)); - } - - if (opcode == UnaryIntrinsicOpcode.SIN || opcode == UnaryIntrinsicOpcode.COS) { - this.gpr5Temp = tool.newVariable(LIRKind.value(AMD64Kind.QWORD)); - this.gpr6Temp = tool.newVariable(LIRKind.value(AMD64Kind.QWORD)); - this.gpr7Temp = tool.newVariable(LIRKind.value(AMD64Kind.QWORD)); - this.gpr8Temp = tool.newVariable(LIRKind.value(AMD64Kind.QWORD)); - this.gpr9Temp = tool.newVariable(LIRKind.value(AMD64Kind.QWORD)); - this.gpr10Temp = tool.newVariable(LIRKind.value(AMD64Kind.QWORD)); - this.xmm8Temp = tool.newVariable(LIRKind.value(AMD64Kind.DOUBLE)); - this.xmm9Temp = tool.newVariable(LIRKind.value(AMD64Kind.DOUBLE)); - } - - this.stackTemp = stackTemp; - } - } - - public AMD64MathIntrinsicUnaryOp(LIRGeneratorTool tool, UnaryIntrinsicOpcode opcode, Value result, Value input) { - this(tool, opcode, result, input, Value.ILLEGAL); - } - - private void setCrb(CompilationResultBuilder crb) { - internalCrb = crb; - } - - private AMD64Address externalAddress(ArrayDataPointerConstant curPtr) { - return (AMD64Address) internalCrb.recordDataReferenceInCode(curPtr); - } - - @Override - public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { - switch (opcode) { - case LOG: - logIntrinsic(asRegister(result, AMD64Kind.DOUBLE), asRegister(input, AMD64Kind.DOUBLE), crb, masm); - break; - case LOG10: - log10Intrinsic(asRegister(result, AMD64Kind.DOUBLE), asRegister(input, AMD64Kind.DOUBLE), crb, masm); - break; - case SIN: - sinIntrinsic(asRegister(result, AMD64Kind.DOUBLE), asRegister(input, AMD64Kind.DOUBLE), crb, masm); - break; - case COS: - cosIntrinsic(asRegister(result, AMD64Kind.DOUBLE), asRegister(input, AMD64Kind.DOUBLE), crb, masm); - break; - case TAN: - tanIntrinsic(asRegister(result, AMD64Kind.DOUBLE), asRegister(input, AMD64Kind.DOUBLE), crb, masm); - break; - case EXP: - expIntrinsic(asRegister(result, AMD64Kind.DOUBLE), asRegister(input, AMD64Kind.DOUBLE), crb, masm); - break; - default: - throw GraalError.shouldNotReachHere(); - } - } - - private static int[] logTwoTable = { - 0xfefa3800, 0x3fe62e42, 0x93c76730, 0x3d2ef357, 0xaa241800, - 0x3fe5ee82, 0x0cda46be, 0x3d220238, 0x5c364800, 0x3fe5af40, - 0xac10c9fb, 0x3d2dfa63, 0x26bb8c00, 0x3fe5707a, 0xff3303dd, - 0x3d09980b, 0x26867800, 0x3fe5322e, 0x5d257531, 0x3d05ccc4, - 0x835a5000, 0x3fe4f45a, 0x6d93b8fb, 0xbd2e6c51, 0x6f970c00, - 0x3fe4b6fd, 0xed4c541c, 0x3cef7115, 0x27e8a400, 0x3fe47a15, - 0xf94d60aa, 0xbd22cb6a, 0xf2f92400, 0x3fe43d9f, 0x481051f7, - 0xbcfd984f, 0x2125cc00, 0x3fe4019c, 0x30f0c74c, 0xbd26ce79, - 0x0c36c000, 0x3fe3c608, 0x7cfe13c2, 0xbd02b736, 0x17197800, - 0x3fe38ae2, 0xbb5569a4, 0xbd218b7a, 0xad9d8c00, 0x3fe35028, - 0x9527e6ac, 0x3d10b83f, 0x44340800, 0x3fe315da, 0xc5a0ed9c, - 0xbd274e93, 0x57b0e000, 0x3fe2dbf5, 0x07b9dc11, 0xbd17a6e5, - 0x6d0ec000, 0x3fe2a278, 0xe797882d, 0x3d206d2b, 0x1134dc00, - 0x3fe26962, 0x05226250, 0xbd0b61f1, 0xd8bebc00, 0x3fe230b0, - 0x6e48667b, 0x3d12fc06, 0x5fc61800, 0x3fe1f863, 0xc9fe81d3, - 0xbd2a7242, 0x49ae6000, 0x3fe1c078, 0xed70e667, 0x3cccacde, - 0x40f23c00, 0x3fe188ee, 0xf8ab4650, 0x3d14cc4e, 0xf6f29800, - 0x3fe151c3, 0xa293ae49, 0xbd2edd97, 0x23c75c00, 0x3fe11af8, - 0xbb9ddcb2, 0xbd258647, 0x8611cc00, 0x3fe0e489, 0x07801742, - 0x3d1c2998, 0xe2d05400, 0x3fe0ae76, 0x887e7e27, 0x3d1f486b, - 0x0533c400, 0x3fe078bf, 0x41edf5fd, 0x3d268122, 0xbe760400, - 0x3fe04360, 0xe79539e0, 0xbd04c45f, 0xe5b20800, 0x3fe00e5a, - 0xb1727b1c, 0xbd053ba3, 0xaf7a4800, 0x3fdfb358, 0x3c164935, - 0x3d0085fa, 0xee031800, 0x3fdf4aa7, 0x6f014a8b, 0x3d12cde5, - 0x56b41000, 0x3fdee2a1, 0x5a470251, 0x3d2f27f4, 0xc3ddb000, - 0x3fde7b42, 0x5372bd08, 0xbd246550, 0x1a272800, 0x3fde148a, - 0x07322938, 0xbd1326b2, 0x484c9800, 0x3fddae75, 0x60dc616a, - 0xbd1ea42d, 0x46def800, 0x3fdd4902, 0xe9a767a8, 0x3d235baf, - 0x18064800, 0x3fdce42f, 0x3ec7a6b0, 0xbd0797c3, 0xc7455800, - 0x3fdc7ff9, 0xc15249ae, 0xbd29b6dd, 0x693fa000, 0x3fdc1c60, - 0x7fe8e180, 0x3d2cec80, 0x1b80e000, 0x3fdbb961, 0xf40a666d, - 0x3d27d85b, 0x04462800, 0x3fdb56fa, 0x2d841995, 0x3d109525, - 0x5248d000, 0x3fdaf529, 0x52774458, 0xbd217cc5, 0x3c8ad800, - 0x3fda93ed, 0xbea77a5d, 0x3d1e36f2, 0x0224f800, 0x3fda3344, - 0x7f9d79f5, 0x3d23c645, 0xea15f000, 0x3fd9d32b, 0x10d0c0b0, - 0xbd26279e, 0x43135800, 0x3fd973a3, 0xa502d9f0, 0xbd152313, - 0x635bf800, 0x3fd914a8, 0x2ee6307d, 0xbd1766b5, 0xa88b3000, - 0x3fd8b639, 0xe5e70470, 0xbd205ae1, 0x776dc800, 0x3fd85855, - 0x3333778a, 0x3d2fd56f, 0x3bd81800, 0x3fd7fafa, 0xc812566a, - 0xbd272090, 0x687cf800, 0x3fd79e26, 0x2efd1778, 0x3d29ec7d, - 0x76c67800, 0x3fd741d8, 0x49dc60b3, 0x3d2d8b09, 0xe6af1800, - 0x3fd6e60e, 0x7c222d87, 0x3d172165, 0x3e9c6800, 0x3fd68ac8, - 0x2756eba0, 0x3d20a0d3, 0x0b3ab000, 0x3fd63003, 0xe731ae00, - 0xbd2db623, 0xdf596000, 0x3fd5d5bd, 0x08a465dc, 0xbd0a0b2a, - 0x53c8d000, 0x3fd57bf7, 0xee5d40ef, 0x3d1faded, 0x0738a000, - 0x3fd522ae, 0x8164c759, 0x3d2ebe70, 0x9e173000, 0x3fd4c9e0, - 0x1b0ad8a4, 0xbd2e2089, 0xc271c800, 0x3fd4718d, 0x0967d675, - 0xbd2f27ce, 0x23d5e800, 0x3fd419b4, 0xec90e09d, 0x3d08e436, - 0x77333000, 0x3fd3c252, 0xb606bd5c, 0x3d183b54, 0x76be1000, - 0x3fd36b67, 0xb0f177c8, 0x3d116ecd, 0xe1d36000, 0x3fd314f1, - 0xd3213cb8, 0xbd28e27a, 0x7cdc9000, 0x3fd2bef0, 0x4a5004f4, - 0x3d2a9cfa, 0x1134d800, 0x3fd26962, 0xdf5bb3b6, 0x3d2c93c1, - 0x6d0eb800, 0x3fd21445, 0xba46baea, 0x3d0a87de, 0x635a6800, - 0x3fd1bf99, 0x5147bdb7, 0x3d2ca6ed, 0xcbacf800, 0x3fd16b5c, - 0xf7a51681, 0x3d2b9acd, 0x8227e800, 0x3fd1178e, 0x63a5f01c, - 0xbd2c210e, 0x67616000, 0x3fd0c42d, 0x163ceae9, 0x3d27188b, - 0x604d5800, 0x3fd07138, 0x16ed4e91, 0x3cf89cdb, 0x5626c800, - 0x3fd01eae, 0x1485e94a, 0xbd16f08c, 0x6cb3b000, 0x3fcf991c, - 0xca0cdf30, 0x3d1bcbec, 0xe4dd0000, 0x3fcef5ad, 0x65bb8e11, - 0xbcca2115, 0xffe71000, 0x3fce530e, 0x6041f430, 0x3cc21227, - 0xb0d49000, 0x3fcdb13d, 0xf715b035, 0xbd2aff2a, 0xf2656000, - 0x3fcd1037, 0x75b6f6e4, 0xbd084a7e, 0xc6f01000, 0x3fcc6ffb, - 0xc5962bd2, 0xbcf1ec72, 0x383be000, 0x3fcbd087, 0x595412b6, - 0xbd2d4bc4, 0x575bd000, 0x3fcb31d8, 0x4eace1aa, 0xbd0c358d, - 0x3c8ae000, 0x3fca93ed, 0x50562169, 0xbd287243, 0x07089000, - 0x3fc9f6c4, 0x6865817a, 0x3d29904d, 0xdcf70000, 0x3fc95a5a, - 0x58a0ff6f, 0x3d07f228, 0xeb390000, 0x3fc8beaf, 0xaae92cd1, - 0xbd073d54, 0x6551a000, 0x3fc823c1, 0x9a631e83, 0x3d1e0ddb, - 0x85445000, 0x3fc7898d, 0x70914305, 0xbd1c6610, 0x8b757000, - 0x3fc6f012, 0xe59c21e1, 0xbd25118d, 0xbe8c1000, 0x3fc6574e, - 0x2c3c2e78, 0x3d19cf8b, 0x6b544000, 0x3fc5bf40, 0xeb68981c, - 0xbd127023, 0xe4a1b000, 0x3fc527e5, 0xe5697dc7, 0x3d2633e8, - 0x8333b000, 0x3fc4913d, 0x54fdb678, 0x3d258379, 0xa5993000, - 0x3fc3fb45, 0x7e6a354d, 0xbd2cd1d8, 0xb0159000, 0x3fc365fc, - 0x234b7289, 0x3cc62fa8, 0x0c868000, 0x3fc2d161, 0xcb81b4a1, - 0x3d039d6c, 0x2a49c000, 0x3fc23d71, 0x8fd3df5c, 0x3d100d23, - 0x7e23f000, 0x3fc1aa2b, 0x44389934, 0x3d2ca78e, 0x8227e000, - 0x3fc1178e, 0xce2d07f2, 0x3d21ef78, 0xb59e4000, 0x3fc08598, - 0x7009902c, 0xbd27e5dd, 0x39dbe000, 0x3fbfe891, 0x4fa10afd, - 0xbd2534d6, 0x830a2000, 0x3fbec739, 0xafe645e0, 0xbd2dc068, - 0x63844000, 0x3fbda727, 0x1fa71733, 0x3d1a8940, 0x01bc4000, - 0x3fbc8858, 0xc65aacd3, 0x3d2646d1, 0x8dad6000, 0x3fbb6ac8, - 0x2bf768e5, 0xbd139080, 0x40b1c000, 0x3fba4e76, 0xb94407c8, - 0xbd0e42b6, 0x5d594000, 0x3fb9335e, 0x3abd47da, 0x3d23115c, - 0x2f40e000, 0x3fb8197e, 0xf96ffdf7, 0x3d0f80dc, 0x0aeac000, - 0x3fb700d3, 0xa99ded32, 0x3cec1e8d, 0x4d97a000, 0x3fb5e95a, - 0x3c5d1d1e, 0xbd2c6906, 0x5d208000, 0x3fb4d311, 0x82f4e1ef, - 0xbcf53a25, 0xa7d1e000, 0x3fb3bdf5, 0xa5db4ed7, 0x3d2cc85e, - 0xa4472000, 0x3fb2aa04, 0xae9c697d, 0xbd20b6e8, 0xd1466000, - 0x3fb1973b, 0x560d9e9b, 0xbd25325d, 0xb59e4000, 0x3fb08598, - 0x7009902c, 0xbd17e5dd, 0xc006c000, 0x3faeea31, 0x4fc93b7b, - 0xbd0e113e, 0xcdddc000, 0x3faccb73, 0x47d82807, 0xbd1a68f2, - 0xd0fb0000, 0x3faaaef2, 0x353bb42e, 0x3d20fc1a, 0x149fc000, - 0x3fa894aa, 0xd05a267d, 0xbd197995, 0xf2d4c000, 0x3fa67c94, - 0xec19afa2, 0xbd029efb, 0xd42e0000, 0x3fa466ae, 0x75bdfd28, - 0xbd2c1673, 0x2f8d0000, 0x3fa252f3, 0xe021b67b, 0x3d283e9a, - 0x89e74000, 0x3fa0415d, 0x5cf1d753, 0x3d0111c0, 0xec148000, - 0x3f9c63d2, 0x3f9eb2f3, 0x3d2578c6, 0x28c90000, 0x3f984925, - 0x325a0c34, 0xbd2aa0ba, 0x25980000, 0x3f9432a9, 0x928637fe, - 0x3d098139, 0x58938000, 0x3f902056, 0x06e2f7d2, 0xbd23dc5b, - 0xa3890000, 0x3f882448, 0xda74f640, 0xbd275577, 0x75890000, - 0x3f801015, 0x999d2be8, 0xbd10c76b, 0x59580000, 0x3f700805, - 0xcb31c67b, 0x3d2166af, 0x00000000, 0x00000000, 0x00000000, - 0x80000000 - }; - - private static int[] logTwoData = { - 0xfefa3800, 0x3fa62e42, 0x93c76730, 0x3ceef357 - }; - - private static int[] coeffLogTwoData = { - 0x92492492, 0x3fc24924, 0x00000000, 0xbfd00000, 0x3d6fb175, - 0xbfc5555e, 0x55555555, 0x3fd55555, 0x9999999a, 0x3fc99999, - 0x00000000, 0xbfe00000 - }; - - /* - * Copyright (c) 2014, 2016, Intel Corporation. All rights reserved. Intel Math Library (LIBM) - * Source Code - * - * ALGORITHM DESCRIPTION - LOG() --------------------- - * - * x=2^k * mx, mx in [1,2) - * - * Get B~1/mx based on the output of rcpps instruction (B0) B = int((B0*2^7+0.5))/2^7 - * - * Reduced argument: r=B*mx-1.0 (computed accurately in high and low parts) - * - * Result: k*log(2) - log(B) + p(r) if |x-1| >= small value (2^-6) and p(r) is a degree 7 - * polynomial -log(B) read from data table (high, low parts) Result is formed from high and low - * parts. - * - * Special cases: log(NaN) = quiet NaN, and raise invalid exception log(+INF) = that INF log(0) - * = -INF with divide-by-zero exception raised log(1) = +0 log(x) = NaN with invalid exception - * raised if x < -0, including -INF - * - */ - - public void logIntrinsic(Register dest, Register value, CompilationResultBuilder crb, AMD64MacroAssembler masm) { - ArrayDataPointerConstant logTwoTablePtr = new ArrayDataPointerConstant(logTwoTable, 16); - ArrayDataPointerConstant logTwoDataPtr = new ArrayDataPointerConstant(logTwoData, 16); - ArrayDataPointerConstant coeffLogTwoDataPtr = new ArrayDataPointerConstant(coeffLogTwoData, 16); - - Label bb0 = new Label(); - Label bb1 = new Label(); - Label bb2 = new Label(); - Label bb3 = new Label(); - Label bb4 = new Label(); - Label bb5 = new Label(); - Label bb6 = new Label(); - Label bb7 = new Label(); - Label bb8 = new Label(); - - Register gpr1 = asRegister(gpr1Temp, AMD64Kind.QWORD); - Register gpr2 = asRegister(gpr2Temp, AMD64Kind.QWORD); - Register gpr3 = asRegister(rcxTemp, AMD64Kind.QWORD); - Register gpr4 = asRegister(gpr4Temp, AMD64Kind.QWORD); - - Register temp1 = asRegister(xmm1Temp, AMD64Kind.DOUBLE); - Register temp2 = asRegister(xmm2Temp, AMD64Kind.DOUBLE); - Register temp3 = asRegister(xmm3Temp, AMD64Kind.DOUBLE); - Register temp4 = asRegister(xmm4Temp, AMD64Kind.DOUBLE); - Register temp5 = asRegister(xmm5Temp, AMD64Kind.DOUBLE); - Register temp6 = asRegister(xmm6Temp, AMD64Kind.DOUBLE); - Register temp7 = asRegister(xmm7Temp, AMD64Kind.DOUBLE); - - AMD64Address stackSlot = (AMD64Address) crb.asAddress(stackTemp); - - setCrb(crb); - masm.movdq(stackSlot, value); - if (dest.encoding != value.encoding) { - masm.movdqu(dest, value); - } - masm.movq(gpr1, 0x3ff0000000000000L); - masm.movdq(temp2, gpr1); - masm.movq(gpr3, 0x77f0000000000000L); - masm.movdq(temp3, gpr3); - masm.movl(gpr2, 32768); - masm.movdl(temp4, gpr2); - masm.movq(gpr2, 0xffffe00000000000L); - masm.movdq(temp5, gpr2); - masm.movdqu(temp1, value); - masm.pextrw(gpr1, dest, 3); - masm.por(dest, temp2); - masm.movl(gpr2, 16352); - masm.psrlq(dest, 27); - masm.leaq(gpr4, externalAddress(logTwoTablePtr)); - masm.psrld(dest, 2); - masm.rcpps(dest, dest); - masm.psllq(temp1, 12); - masm.pshufd(temp6, temp5, 0xE4); - masm.psrlq(temp1, 12); - masm.subl(gpr1, 16); - masm.cmpl(gpr1, 32736); - masm.jcc(ConditionFlag.AboveEqual, bb0); - - masm.bind(bb1); - masm.paddd(dest, temp4); - masm.por(temp1, temp3); - masm.movdl(gpr3, dest); - masm.psllq(dest, 29); - masm.pand(temp5, temp1); - masm.pand(dest, temp6); - masm.subsd(temp1, temp5); - masm.mulpd(temp5, dest); - masm.andl(gpr1, 32752); - masm.subl(gpr1, gpr2); - masm.cvtsi2sdl(temp7, gpr1); - masm.mulsd(temp1, dest); - masm.movdq(temp6, externalAddress(logTwoDataPtr)); // 0xfefa3800, - // 0x3fa62e42 - masm.movdqu(temp3, externalAddress(coeffLogTwoDataPtr)); // 0x92492492, - // 0x3fc24924, - // 0x00000000, - // 0xbfd00000 - masm.subsd(temp5, temp2); - masm.andl(gpr3, 16711680); - masm.shrl(gpr3, 12); - masm.movdqu(dest, new AMD64Address(gpr4, gpr3, Scale.Times1, 0)); - masm.leaq(gpr4, externalAddress(coeffLogTwoDataPtr)); - masm.movdqu(temp4, new AMD64Address(gpr4, 16)); // 0x3d6fb175, - // 0xbfc5555e, - // 0x55555555, - // 0x3fd55555 - masm.addsd(temp1, temp5); - masm.movdqu(temp2, new AMD64Address(gpr4, 32)); // 0x9999999a, - // 0x3fc99999, - // 0x00000000, - // 0xbfe00000 - masm.mulsd(temp6, temp7); - if (masm.supports(CPUFeature.SSE3)) { - masm.movddup(temp5, temp1); - } else { - masm.movdqu(temp5, temp1); - masm.movlhps(temp5, temp5); - } - masm.leaq(gpr4, externalAddress(logTwoDataPtr)); - masm.mulsd(temp7, new AMD64Address(gpr4, 8)); // 0x93c76730, - // 0x3ceef357 - masm.mulsd(temp3, temp1); - masm.addsd(dest, temp6); - masm.mulpd(temp4, temp5); - masm.mulpd(temp5, temp5); - if (masm.supports(CPUFeature.SSE3)) { - masm.movddup(temp6, dest); - } else { - masm.movdqu(temp6, dest); - masm.movlhps(temp6, temp6); - } - masm.addsd(dest, temp1); - masm.addpd(temp4, temp2); - masm.mulpd(temp3, temp5); - masm.subsd(temp6, dest); - masm.mulsd(temp4, temp1); - masm.pshufd(temp2, dest, 0xEE); - masm.addsd(temp1, temp6); - masm.mulsd(temp5, temp5); - masm.addsd(temp7, temp2); - masm.addpd(temp4, temp3); - masm.addsd(temp1, temp7); - masm.mulpd(temp4, temp5); - masm.addsd(temp1, temp4); - masm.pshufd(temp5, temp4, 0xEE); - masm.addsd(temp1, temp5); - masm.addsd(dest, temp1); - masm.jmp(bb8); - - masm.bind(bb0); - masm.movdq(dest, stackSlot); - masm.movdq(temp1, stackSlot); - masm.addl(gpr1, 16); - masm.cmpl(gpr1, 32768); - masm.jcc(ConditionFlag.AboveEqual, bb2); - - masm.cmpl(gpr1, 16); - masm.jcc(ConditionFlag.Below, bb3); - - masm.bind(bb4); - masm.addsd(dest, dest); - masm.jmp(bb8); - - masm.bind(bb5); - masm.jcc(ConditionFlag.Above, bb4); - - masm.cmpl(gpr3, 0); - masm.jcc(ConditionFlag.Above, bb4); - - masm.jmp(bb6); - - masm.bind(bb3); - masm.xorpd(temp1, temp1); - masm.addsd(temp1, dest); - masm.movdl(gpr3, temp1); - masm.psrlq(temp1, 32); - masm.movdl(gpr2, temp1); - masm.orl(gpr3, gpr2); - masm.cmpl(gpr3, 0); - masm.jcc(ConditionFlag.Equal, bb7); - - masm.xorpd(temp1, temp1); - masm.movl(gpr1, 18416); - masm.pinsrw(temp1, gpr1, 3); - masm.mulsd(dest, temp1); - masm.movdqu(temp1, dest); - masm.pextrw(gpr1, dest, 3); - masm.por(dest, temp2); - masm.psrlq(dest, 27); - masm.movl(gpr2, 18416); - masm.psrld(dest, 2); - masm.rcpps(dest, dest); - masm.psllq(temp1, 12); - masm.pshufd(temp6, temp5, 0xE4); - masm.psrlq(temp1, 12); - masm.jmp(bb1); - - masm.bind(bb2); - masm.movdl(gpr3, temp1); - masm.psrlq(temp1, 32); - masm.movdl(gpr2, temp1); - masm.addl(gpr2, gpr2); - masm.cmpl(gpr2, -2097152); - masm.jcc(ConditionFlag.AboveEqual, bb5); - - masm.orl(gpr3, gpr2); - masm.cmpl(gpr3, 0); - masm.jcc(ConditionFlag.Equal, bb7); - - masm.bind(bb6); - masm.xorpd(temp1, temp1); - masm.xorpd(dest, dest); - masm.movl(gpr1, 32752); - masm.pinsrw(temp1, gpr1, 3); - masm.mulsd(dest, temp1); - masm.jmp(bb8); - - masm.bind(bb7); - masm.xorpd(temp1, temp1); - masm.xorpd(dest, dest); - masm.movl(gpr1, 49136); - masm.pinsrw(dest, gpr1, 3); - masm.divsd(dest, temp1); - - masm.bind(bb8); - } - - private static int[] highmaskLogTen = { - 0xf8000000, 0xffffffff, 0x00000000, 0xffffe000 - }; - - private static int[] logTenE = { - 0x00000000, 0x3fdbc000, 0xbf2e4108, 0x3f5a7a6c - }; - - private static int[] logTenTable = { - 0x509f7800, 0x3fd34413, 0x1f12b358, 0x3d1fef31, 0x80333400, - 0x3fd32418, 0xc671d9d0, 0xbcf542bf, 0x51195000, 0x3fd30442, - 0x78a4b0c3, 0x3d18216a, 0x6fc79400, 0x3fd2e490, 0x80fa389d, - 0xbc902869, 0x89d04000, 0x3fd2c502, 0x75c2f564, 0x3d040754, - 0x4ddd1c00, 0x3fd2a598, 0xd219b2c3, 0xbcfa1d84, 0x6baa7c00, - 0x3fd28651, 0xfd9abec1, 0x3d1be6d3, 0x94028800, 0x3fd2672d, - 0xe289a455, 0xbd1ede5e, 0x78b86400, 0x3fd2482c, 0x6734d179, - 0x3d1fe79b, 0xcca3c800, 0x3fd2294d, 0x981a40b8, 0xbced34ea, - 0x439c5000, 0x3fd20a91, 0xcc392737, 0xbd1a9cc3, 0x92752c00, - 0x3fd1ebf6, 0x03c9afe7, 0x3d1e98f8, 0x6ef8dc00, 0x3fd1cd7d, - 0x71dae7f4, 0x3d08a86c, 0x8fe4dc00, 0x3fd1af25, 0xee9185a1, - 0xbcff3412, 0xace59400, 0x3fd190ee, 0xc2cab353, 0x3cf17ed9, - 0x7e925000, 0x3fd172d8, 0x6952c1b2, 0x3cf1521c, 0xbe694400, - 0x3fd154e2, 0xcacb79ca, 0xbd0bdc78, 0x26cbac00, 0x3fd1370d, - 0xf71f4de1, 0xbd01f8be, 0x72fa0800, 0x3fd11957, 0x55bf910b, - 0x3c946e2b, 0x5f106000, 0x3fd0fbc1, 0x39e639c1, 0x3d14a84b, - 0xa802a800, 0x3fd0de4a, 0xd3f31d5d, 0xbd178385, 0x0b992000, - 0x3fd0c0f3, 0x3843106f, 0xbd1f602f, 0x486ce800, 0x3fd0a3ba, - 0x8819497c, 0x3cef987a, 0x1de49400, 0x3fd086a0, 0x1caa0467, - 0x3d0faec7, 0x4c30cc00, 0x3fd069a4, 0xa4424372, 0xbd1618fc, - 0x94490000, 0x3fd04cc6, 0x946517d2, 0xbd18384b, 0xb7e84000, - 0x3fd03006, 0xe0109c37, 0xbd19a6ac, 0x798a0c00, 0x3fd01364, - 0x5121e864, 0xbd164cf7, 0x38ce8000, 0x3fcfedbf, 0x46214d1a, - 0xbcbbc402, 0xc8e62000, 0x3fcfb4ef, 0xdab93203, 0x3d1e0176, - 0x2cb02800, 0x3fcf7c5a, 0x2a2ea8e4, 0xbcfec86a, 0xeeeaa000, - 0x3fcf43fd, 0xc18e49a4, 0x3cf110a8, 0x9bb6e800, 0x3fcf0bda, - 0x923cc9c0, 0xbd15ce99, 0xc093f000, 0x3fced3ef, 0x4d4b51e9, - 0x3d1a04c7, 0xec58f800, 0x3fce9c3c, 0x163cad59, 0x3cac8260, - 0x9a907000, 0x3fce2d7d, 0x3fa93646, 0x3ce4a1c0, 0x37311000, - 0x3fcdbf99, 0x32abd1fd, 0x3d07ea9d, 0x6744b800, 0x3fcd528c, - 0x4dcbdfd4, 0xbd1b08e2, 0xe36de800, 0x3fcce653, 0x0b7b7f7f, - 0xbd1b8f03, 0x77506800, 0x3fcc7aec, 0xa821c9fb, 0x3d13c163, - 0x00ff8800, 0x3fcc1053, 0x536bca76, 0xbd074ee5, 0x70719800, - 0x3fcba684, 0xd7da9b6b, 0xbd1fbf16, 0xc6f8d800, 0x3fcb3d7d, - 0xe2220bb3, 0x3d1a295d, 0x16c15800, 0x3fcad53c, 0xe724911e, - 0xbcf55822, 0x82533800, 0x3fca6dbc, 0x6d982371, 0x3cac567c, - 0x3c19e800, 0x3fca06fc, 0x84d17d80, 0x3d1da204, 0x85ef8000, - 0x3fc9a0f8, 0x54466a6a, 0xbd002204, 0xb0ac2000, 0x3fc93bae, - 0xd601fd65, 0x3d18840c, 0x1bb9b000, 0x3fc8d71c, 0x7bf58766, - 0xbd14f897, 0x34aae800, 0x3fc8733e, 0x3af6ac24, 0xbd0f5c45, - 0x76d68000, 0x3fc81012, 0x4303e1a1, 0xbd1f9a80, 0x6af57800, - 0x3fc7ad96, 0x43fbcb46, 0x3cf4c33e, 0xa6c51000, 0x3fc74bc7, - 0x70f0eac5, 0xbd192e3b, 0xccab9800, 0x3fc6eaa3, 0xc0093dfe, - 0xbd0faf15, 0x8b60b800, 0x3fc68a28, 0xde78d5fd, 0xbc9ea4ee, - 0x9d987000, 0x3fc62a53, 0x962bea6e, 0xbd194084, 0xc9b0e800, - 0x3fc5cb22, 0x888dd999, 0x3d1fe201, 0xe1634800, 0x3fc56c93, - 0x16ada7ad, 0x3d1b1188, 0xc176c000, 0x3fc50ea4, 0x4159b5b5, - 0xbcf09c08, 0x51766000, 0x3fc4b153, 0x84393d23, 0xbcf6a89c, - 0x83695000, 0x3fc4549d, 0x9f0b8bbb, 0x3d1c4b8c, 0x538d5800, - 0x3fc3f881, 0xf49df747, 0x3cf89b99, 0xc8138000, 0x3fc39cfc, - 0xd503b834, 0xbd13b99f, 0xf0df0800, 0x3fc3420d, 0xf011b386, - 0xbd05d8be, 0xe7466800, 0x3fc2e7b2, 0xf39c7bc2, 0xbd1bb94e, - 0xcdd62800, 0x3fc28de9, 0x05e6d69b, 0xbd10ed05, 0xd015d800, - 0x3fc234b0, 0xe29b6c9d, 0xbd1ff967, 0x224ea800, 0x3fc1dc06, - 0x727711fc, 0xbcffb30d, 0x01540000, 0x3fc183e8, 0x39786c5a, - 0x3cc23f57, 0xb24d9800, 0x3fc12c54, 0xc905a342, 0x3d003a1d, - 0x82835800, 0x3fc0d54a, 0x9b9920c0, 0x3d03b25a, 0xc72ac000, - 0x3fc07ec7, 0x46f26a24, 0x3cf0fa41, 0xdd35d800, 0x3fc028ca, - 0x41d9d6dc, 0x3d034a65, 0x52474000, 0x3fbfa6a4, 0x44f66449, - 0x3d19cad3, 0x2da3d000, 0x3fbefcb8, 0x67832999, 0x3d18400f, - 0x32a10000, 0x3fbe53ce, 0x9c0e3b1a, 0xbcff62fd, 0x556b7000, - 0x3fbdabe3, 0x02976913, 0xbcf8243b, 0x97e88000, 0x3fbd04f4, - 0xec793797, 0x3d1c0578, 0x09647000, 0x3fbc5eff, 0x05fc0565, - 0xbd1d799e, 0xc6426000, 0x3fbbb9ff, 0x4625f5ed, 0x3d1f5723, - 0xf7afd000, 0x3fbb15f3, 0xdd5aae61, 0xbd1a7e1e, 0xd358b000, - 0x3fba72d8, 0x3314e4d3, 0x3d17bc91, 0x9b1f5000, 0x3fb9d0ab, - 0x9a4d514b, 0x3cf18c9b, 0x9cd4e000, 0x3fb92f69, 0x7e4496ab, - 0x3cf1f96d, 0x31f4f000, 0x3fb88f10, 0xf56479e7, 0x3d165818, - 0xbf628000, 0x3fb7ef9c, 0x26bf486d, 0xbd1113a6, 0xb526b000, - 0x3fb7510c, 0x1a1c3384, 0x3ca9898d, 0x8e31e000, 0x3fb6b35d, - 0xb3875361, 0xbd0661ac, 0xd01de000, 0x3fb6168c, 0x2a7cacfa, - 0xbd1bdf10, 0x0af23000, 0x3fb57a98, 0xff868816, 0x3cf046d0, - 0xd8ea0000, 0x3fb4df7c, 0x1515fbe7, 0xbd1fd529, 0xde3b2000, - 0x3fb44538, 0x6e59a132, 0x3d1faeee, 0xc8df9000, 0x3fb3abc9, - 0xf1322361, 0xbd198807, 0x505f1000, 0x3fb3132d, 0x0888e6ab, - 0x3d1e5380, 0x359bd000, 0x3fb27b61, 0xdfbcbb22, 0xbcfe2724, - 0x429ee000, 0x3fb1e463, 0x6eb4c58c, 0xbcfe4dd6, 0x4a673000, - 0x3fb14e31, 0x4ce1ac9b, 0x3d1ba691, 0x28b96000, 0x3fb0b8c9, - 0x8c7813b8, 0xbd0b3872, 0xc1f08000, 0x3fb02428, 0xc2bc8c2c, - 0x3cb5ea6b, 0x05a1a000, 0x3faf209c, 0x72e8f18e, 0xbce8df84, - 0xc0b5e000, 0x3fadfa6d, 0x9fdef436, 0x3d087364, 0xaf416000, - 0x3facd5c2, 0x1068c3a9, 0x3d0827e7, 0xdb356000, 0x3fabb296, - 0x120a34d3, 0x3d101a9f, 0x5dfea000, 0x3faa90e6, 0xdaded264, - 0xbd14c392, 0x6034c000, 0x3fa970ad, 0x1c9d06a9, 0xbd1b705e, - 0x194c6000, 0x3fa851e8, 0x83996ad9, 0xbd0117bc, 0xcf4ac000, - 0x3fa73492, 0xb1a94a62, 0xbca5ea42, 0xd67b4000, 0x3fa618a9, - 0x75aed8ca, 0xbd07119b, 0x9126c000, 0x3fa4fe29, 0x5291d533, - 0x3d12658f, 0x6f4d4000, 0x3fa3e50e, 0xcd2c5cd9, 0x3d1d5c70, - 0xee608000, 0x3fa2cd54, 0xd1008489, 0x3d1a4802, 0x9900e000, - 0x3fa1b6f9, 0x54fb5598, 0xbd16593f, 0x06bb6000, 0x3fa0a1f9, - 0x64ef57b4, 0xbd17636b, 0xb7940000, 0x3f9f1c9f, 0xee6a4737, - 0x3cb5d479, 0x91aa0000, 0x3f9cf7f5, 0x3a16373c, 0x3d087114, - 0x156b8000, 0x3f9ad5ed, 0x836c554a, 0x3c6900b0, 0xd4764000, - 0x3f98b67f, 0xed12f17b, 0xbcffc974, 0x77dec000, 0x3f9699a7, - 0x232ce7ea, 0x3d1e35bb, 0xbfbf4000, 0x3f947f5d, 0xd84ffa6e, - 0x3d0e0a49, 0x82c7c000, 0x3f92679c, 0x8d170e90, 0xbd14d9f2, - 0xadd20000, 0x3f90525d, 0x86d9f88e, 0x3cdeb986, 0x86f10000, - 0x3f8c7f36, 0xb9e0a517, 0x3ce29faa, 0xb75c8000, 0x3f885e9e, - 0x542568cb, 0xbd1f7bdb, 0x46b30000, 0x3f8442e8, 0xb954e7d9, - 0x3d1e5287, 0xb7e60000, 0x3f802c07, 0x22da0b17, 0xbd19fb27, - 0x6c8b0000, 0x3f7833e3, 0x821271ef, 0xbd190f96, 0x29910000, - 0x3f701936, 0xbc3491a5, 0xbd1bcf45, 0x354a0000, 0x3f600fe3, - 0xc0ff520a, 0xbd19d71c, 0x00000000, 0x00000000, 0x00000000, - 0x00000000 - }; - - private static int[] logTwoLogTenData = { - 0x509f7800, 0x3f934413, 0x1f12b358, 0x3cdfef31 - }; - - private static int[] coeffLogTenData = { - 0xc1a5f12e, 0x40358874, 0x64d4ef0d, 0xc0089309, 0x385593b1, - 0xc025c917, 0xdc963467, 0x3ffc6a02, 0x7f9d3aa1, 0x4016ab9f, - 0xdc77b115, 0xbff27af2 - }; - - /* - * Copyright (c) 2014, 2016, Intel Corporation. All rights reserved. Intel Math Library (LIBM) - * Source Code - * - * ALGORITHM DESCRIPTION - LOG10() --------------------- - * - * Let x=2^k * mx, mx in [1,2) - * - * Get B~1/mx based on the output of rcpss instruction (B0) B = int((B0*LH*2^7+0.5))/2^7 LH is a - * short approximation for log10(e) - * - * Reduced argument: r=B*mx-LH (computed accurately in high and low parts) - * - * Result: k*log10(2) - log(B) + p(r) p(r) is a degree 7 polynomial -log(B) read from data table - * (high, low parts) Result is formed from high and low parts - * - * Special cases: log10(0) = -INF with divide-by-zero exception raised log10(1) = +0 log10(x) = - * NaN with invalid exception raised if x < -0, including -INF log10(+INF) = +INF - * - */ - - public void log10Intrinsic(Register dest, Register value, CompilationResultBuilder crb, AMD64MacroAssembler masm) { - ArrayDataPointerConstant highmaskLogTenPtr = new ArrayDataPointerConstant(highmaskLogTen, 16); - ArrayDataPointerConstant logTenEPtr = new ArrayDataPointerConstant(logTenE, 16); - ArrayDataPointerConstant logTenTablePtr = new ArrayDataPointerConstant(logTenTable, 16); - ArrayDataPointerConstant logTwoLogTenDataPtr = new ArrayDataPointerConstant(logTwoLogTenData, 16); - ArrayDataPointerConstant coeffLogTenDataPtr = new ArrayDataPointerConstant(coeffLogTenData, 16); - - Label bb0 = new Label(); - Label bb1 = new Label(); - Label bb2 = new Label(); - Label bb3 = new Label(); - Label bb4 = new Label(); - Label bb5 = new Label(); - Label bb6 = new Label(); - Label bb7 = new Label(); - Label bb8 = new Label(); - - Register gpr1 = asRegister(gpr1Temp, AMD64Kind.QWORD); - Register gpr2 = asRegister(gpr2Temp, AMD64Kind.QWORD); - Register gpr3 = asRegister(rcxTemp, AMD64Kind.QWORD); - Register gpr4 = asRegister(gpr4Temp, AMD64Kind.QWORD); - - Register temp1 = asRegister(xmm1Temp, AMD64Kind.DOUBLE); - Register temp2 = asRegister(xmm2Temp, AMD64Kind.DOUBLE); - Register temp3 = asRegister(xmm3Temp, AMD64Kind.DOUBLE); - Register temp4 = asRegister(xmm4Temp, AMD64Kind.DOUBLE); - Register temp5 = asRegister(xmm5Temp, AMD64Kind.DOUBLE); - Register temp6 = asRegister(xmm6Temp, AMD64Kind.DOUBLE); - Register temp7 = asRegister(xmm7Temp, AMD64Kind.DOUBLE); - - AMD64Address stackSlot = (AMD64Address) crb.asAddress(stackTemp); - - setCrb(crb); - masm.movdq(stackSlot, value); - if (dest.encoding != value.encoding) { - masm.movdqu(dest, value); - } - masm.movdqu(temp5, externalAddress(highmaskLogTenPtr)); // 0xf8000000, - // 0xffffffff, - // 0x00000000, - // 0xffffe000 - masm.xorpd(temp2, temp2); - masm.movl(gpr1, 16368); - masm.pinsrw(temp2, gpr1, 3); - masm.movl(gpr2, 1054736384); - masm.movdl(temp7, gpr2); - masm.xorpd(temp3, temp3); - masm.movl(gpr3, 30704); - masm.pinsrw(temp3, gpr3, 3); - masm.movl(gpr3, 32768); - masm.movdl(temp4, gpr3); - masm.movdqu(temp1, value); - masm.pextrw(gpr1, dest, 3); - masm.por(dest, temp2); - masm.movl(gpr2, 16352); - masm.psrlq(dest, 27); - masm.movdqu(temp2, externalAddress(logTenEPtr)); // 0x00000000, - // 0x3fdbc000, - // 0xbf2e4108, - // 0x3f5a7a6c - masm.psrld(dest, 2); - masm.rcpps(dest, dest); - masm.psllq(temp1, 12); - masm.pshufd(temp6, temp5, 0x4E); - masm.psrlq(temp1, 12); - masm.subl(gpr1, 16); - masm.cmpl(gpr1, 32736); - masm.jcc(ConditionFlag.AboveEqual, bb0); - - masm.bind(bb1); - masm.mulss(dest, temp7); - masm.por(temp1, temp3); - masm.andpd(temp5, temp1); - masm.paddd(dest, temp4); - masm.movdqu(temp3, externalAddress(coeffLogTenDataPtr)); // 0xc1a5f12e, - // 0x40358874, - // 0x64d4ef0d, - // 0xc0089309 - masm.leaq(gpr4, externalAddress(coeffLogTenDataPtr)); - masm.movdqu(temp4, new AMD64Address(gpr4, 16)); // 0x385593b1, - // 0xc025c917, - // 0xdc963467, - // 0x3ffc6a02 - masm.subsd(temp1, temp5); - masm.movdl(gpr3, dest); - masm.psllq(dest, 29); - masm.andpd(dest, temp6); - masm.movdq(temp6, externalAddress(logTwoLogTenDataPtr)); // 0x509f7800, - // 0x3f934413 - masm.andl(gpr1, 32752); - masm.subl(gpr1, gpr2); - masm.cvtsi2sdl(temp7, gpr1); - masm.mulpd(temp5, dest); - masm.mulsd(temp1, dest); - masm.subsd(temp5, temp2); - masm.movdqu(temp2, new AMD64Address(gpr4, 32)); // 0x7f9d3aa1, - // 0x4016ab9f, - // 0xdc77b115, - // 0xbff27af2 - masm.leaq(gpr4, externalAddress(logTenTablePtr)); - masm.andl(gpr3, 16711680); - masm.shrl(gpr3, 12); - masm.movdqu(dest, new AMD64Address(gpr4, gpr3, Scale.Times1, -1504)); - masm.addsd(temp1, temp5); - masm.mulsd(temp6, temp7); - masm.pshufd(temp5, temp1, 0x44); - masm.leaq(gpr4, externalAddress(logTwoLogTenDataPtr)); - masm.mulsd(temp7, new AMD64Address(gpr4, 8)); // 0x1f12b358, - // 0x3cdfef31 - masm.mulsd(temp3, temp1); - masm.addsd(dest, temp6); - masm.mulpd(temp4, temp5); - masm.leaq(gpr4, externalAddress(logTenEPtr)); - masm.movdq(temp6, new AMD64Address(gpr4, 8)); // 0xbf2e4108, - // 0x3f5a7a6c - masm.mulpd(temp5, temp5); - masm.addpd(temp4, temp2); - masm.mulpd(temp3, temp5); - masm.pshufd(temp2, dest, 0xE4); - masm.addsd(dest, temp1); - masm.mulsd(temp4, temp1); - masm.subsd(temp2, dest); - masm.mulsd(temp6, temp1); - masm.addsd(temp1, temp2); - masm.pshufd(temp2, dest, 0xEE); - masm.mulsd(temp5, temp5); - masm.addsd(temp7, temp2); - masm.addsd(temp1, temp6); - masm.addpd(temp4, temp3); - masm.addsd(temp1, temp7); - masm.mulpd(temp4, temp5); - masm.addsd(temp1, temp4); - masm.pshufd(temp5, temp4, 0xEE); - masm.addsd(temp1, temp5); - masm.addsd(dest, temp1); - masm.jmp(bb8); - - masm.bind(bb0); - masm.movdq(dest, stackSlot); - masm.movdq(temp1, stackSlot); - masm.addl(gpr1, 16); - masm.cmpl(gpr1, 32768); - masm.jcc(ConditionFlag.AboveEqual, bb2); - - masm.cmpl(gpr1, 16); - masm.jcc(ConditionFlag.Below, bb3); - - masm.bind(bb4); - masm.addsd(dest, dest); - masm.jmp(bb8); - - masm.bind(bb5); - masm.jcc(ConditionFlag.Above, bb4); - - masm.cmpl(gpr3, 0); - masm.jcc(ConditionFlag.Above, bb4); - - masm.jmp(bb6); - - masm.bind(bb3); - masm.xorpd(temp1, temp1); - masm.addsd(temp1, dest); - masm.movdl(gpr3, temp1); - masm.psrlq(temp1, 32); - masm.movdl(gpr2, temp1); - masm.orl(gpr3, gpr2); - masm.cmpl(gpr3, 0); - masm.jcc(ConditionFlag.Equal, bb7); - - masm.xorpd(temp1, temp1); - masm.xorpd(temp2, temp2); - masm.movl(gpr1, 18416); - masm.pinsrw(temp1, gpr1, 3); - masm.mulsd(dest, temp1); - masm.movl(gpr1, 16368); - masm.pinsrw(temp2, gpr1, 3); - masm.movdqu(temp1, dest); - masm.pextrw(gpr1, dest, 3); - masm.por(dest, temp2); - masm.movl(gpr2, 18416); - masm.psrlq(dest, 27); - masm.movdqu(temp2, externalAddress(logTenEPtr)); // 0x00000000, - // 0x3fdbc000, - // 0xbf2e4108, - // 0x3f5a7a6c - masm.psrld(dest, 2); - masm.rcpps(dest, dest); - masm.psllq(temp1, 12); - masm.pshufd(temp6, temp5, 0x4E); - masm.psrlq(temp1, 12); - masm.jmp(bb1); - - masm.bind(bb2); - masm.movdl(gpr3, temp1); - masm.psrlq(temp1, 32); - masm.movdl(gpr2, temp1); - masm.addl(gpr2, gpr2); - masm.cmpl(gpr2, -2097152); - masm.jcc(ConditionFlag.AboveEqual, bb5); - - masm.orl(gpr3, gpr2); - masm.cmpl(gpr3, 0); - masm.jcc(ConditionFlag.Equal, bb7); - - masm.bind(bb6); - masm.xorpd(temp1, temp1); - masm.xorpd(dest, dest); - masm.movl(gpr1, 32752); - masm.pinsrw(temp1, gpr1, 3); - masm.mulsd(dest, temp1); - masm.jmp(bb8); - - masm.bind(bb7); - masm.xorpd(temp1, temp1); - masm.xorpd(dest, dest); - masm.movl(gpr1, 49136); - masm.pinsrw(dest, gpr1, 3); - masm.divsd(dest, temp1); - - masm.bind(bb8); - } - - /* - * Copyright (c) 2014, 2016, Intel Corporation. All rights reserved. Intel Math Library (LIBM) - * Source Code - * - * ALGORITHM DESCRIPTION - SIN() --------------------- - * - * 1. RANGE REDUCTION - * - * We perform an initial range reduction from X to r with - * - * X =~= N * pi/32 + r - * - * so that |r| <= pi/64 + epsilon. We restrict inputs to those where |N| <= 932560. Beyond this, - * the range reduction is insufficiently accurate. For extremely small inputs, denormalization - * can occur internally, impacting performance. This means that the main path is actually only - * taken for 2^-252 <= |X| < 90112. - * - * To avoid branches, we perform the range reduction to full accuracy each time. - * - * X - N * (P_1 + P_2 + P_3) - * - * where P_1 and P_2 are 32-bit numbers (so multiplication by N is exact) and P_3 is a 53-bit - * number. Together, these approximate pi well enough for all cases in the restricted range. - * - * The main reduction sequence is: - * - * y = 32/pi * x N = integer(y) (computed by adding and subtracting off SHIFTER) - * - * m_1 = N * P_1 m_2 = N * P_2 r_1 = x - m_1 r = r_1 - m_2 (this r can be used for most of the - * calculation) - * - * c_1 = r_1 - r m_3 = N * P_3 c_2 = c_1 - m_2 c = c_2 - m_3 - * - * 2. MAIN ALGORITHM - * - * The algorithm uses a table lookup based on B = M * pi / 32 where M = N mod 64. The stored - * values are: sigma closest power of 2 to cos(B) C_hl 53-bit cos(B) - sigma S_hi + S_lo 2 * - * 53-bit sin(B) - * - * The computation is organized as follows: - * - * sin(B + r + c) = [sin(B) + sigma * r] + r * (cos(B) - sigma) + sin(B) * [cos(r + c) - 1] + - * cos(B) * [sin(r + c) - r] - * - * which is approximately: - * - * [S_hi + sigma * r] + C_hl * r + S_lo + S_hi * [(cos(r) - 1) - r * c] + (C_hl + sigma) * - * [(sin(r) - r) + c] - * - * and this is what is actually computed. We separate this sum into four parts: - * - * hi + med + pols + corr - * - * where - * - * hi = S_hi + sigma r med = C_hl * r pols = S_hi * (cos(r) - 1) + (C_hl + sigma) * (sin(r) - r) - * corr = S_lo + c * ((C_hl + sigma) - S_hi * r) - * - * 3. POLYNOMIAL - * - * The polynomial S_hi * (cos(r) - 1) + (C_hl + sigma) * (sin(r) - r) can be rearranged freely, - * since it is quite small, so we exploit parallelism to the fullest. - * - * psc4 = SC_4 * r_1 msc4 = psc4 * r r2 = r * r msc2 = SC_2 * r2 r4 = r2 * r2 psc3 = SC_3 + msc4 - * psc1 = SC_1 + msc2 msc3 = r4 * psc3 sincospols = psc1 + msc3 pols = sincospols * - * - * 4. CORRECTION TERM - * - * This is where the "c" component of the range reduction is taken into account; recall that - * just "r" is used for most of the calculation. - * - * -c = m_3 - c_2 -d = S_hi * r - (C_hl + sigma) corr = -c * -d + S_lo - * - * 5. COMPENSATED SUMMATIONS - * - * The two successive compensated summations add up the high and medium parts, leaving just the - * low parts to add up at the end. - * - * rs = sigma * r res_int = S_hi + rs k_0 = S_hi - res_int k_2 = k_0 + rs med = C_hl * r res_hi - * = res_int + med k_1 = res_int - res_hi k_3 = k_1 + med - * - * 6. FINAL SUMMATION - * - * We now add up all the small parts: - * - * res_lo = pols(hi) + pols(lo) + corr + k_1 + k_3 - * - * Now the overall result is just: - * - * res_hi + res_lo - * - * 7. SMALL ARGUMENTS - * - * If |x| < SNN (SNN meaning the smallest normal number), we simply perform 0.1111111 cdots 1111 - * * x. For SNN <= |x|, we do 2^-55 * (2^55 * x - x). - * - * Special cases: sin(NaN) = quiet NaN, and raise invalid exception sin(INF) = NaN and raise - * invalid exception sin(+/-0) = +/-0 - * - */ - - public int[] oneHalf = { - 0x00000000, 0x3fe00000, 0x00000000, 0x3fe00000 - }; - - public int[] pTwo = { - 0x1a600000, 0x3d90b461, 0x1a600000, 0x3d90b461 - }; - - public int[] scFour = { - 0xa556c734, 0x3ec71de3, 0x1a01a01a, 0x3efa01a0 - }; - - public int[] cTable = { - 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, - 0x00000000, 0x00000000, 0x3ff00000, 0x176d6d31, 0xbf73b92e, - 0xbc29b42c, 0x3fb917a6, 0xe0000000, 0xbc3e2718, 0x00000000, - 0x3ff00000, 0x011469fb, 0xbf93ad06, 0x3c69a60b, 0x3fc8f8b8, - 0xc0000000, 0xbc626d19, 0x00000000, 0x3ff00000, 0x939d225a, - 0xbfa60bea, 0x2ed59f06, 0x3fd29406, 0xa0000000, 0xbc75d28d, - 0x00000000, 0x3ff00000, 0x866b95cf, 0xbfb37ca1, 0xa6aea963, - 0x3fd87de2, 0xe0000000, 0xbc672ced, 0x00000000, 0x3ff00000, - 0x73fa1279, 0xbfbe3a68, 0x3806f63b, 0x3fde2b5d, 0x20000000, - 0x3c5e0d89, 0x00000000, 0x3ff00000, 0x5bc57974, 0xbfc59267, - 0x39ae68c8, 0x3fe1c73b, 0x20000000, 0x3c8b25dd, 0x00000000, - 0x3ff00000, 0x53aba2fd, 0xbfcd0dfe, 0x25091dd6, 0x3fe44cf3, - 0x20000000, 0x3c68076a, 0x00000000, 0x3ff00000, 0x99fcef32, - 0x3fca8279, 0x667f3bcd, 0x3fe6a09e, 0x20000000, 0xbc8bdd34, - 0x00000000, 0x3fe00000, 0x94247758, 0x3fc133cc, 0x6b151741, - 0x3fe8bc80, 0x20000000, 0xbc82c5e1, 0x00000000, 0x3fe00000, - 0x9ae68c87, 0x3fac73b3, 0x290ea1a3, 0x3fea9b66, 0xe0000000, - 0x3c39f630, 0x00000000, 0x3fe00000, 0x7f909c4e, 0xbf9d4a2c, - 0xf180bdb1, 0x3fec38b2, 0x80000000, 0xbc76e0b1, 0x00000000, - 0x3fe00000, 0x65455a75, 0xbfbe0875, 0xcf328d46, 0x3fed906b, - 0x20000000, 0x3c7457e6, 0x00000000, 0x3fe00000, 0x76acf82d, - 0x3fa4a031, 0x56c62dda, 0x3fee9f41, 0xe0000000, 0x3c8760b1, - 0x00000000, 0x3fd00000, 0x0e5967d5, 0xbfac1d1f, 0xcff75cb0, - 0x3fef6297, 0x20000000, 0x3c756217, 0x00000000, 0x3fd00000, - 0x0f592f50, 0xbf9ba165, 0xa3d12526, 0x3fefd88d, 0x40000000, - 0xbc887df6, 0x00000000, 0x3fc00000, 0x00000000, 0x00000000, - 0x00000000, 0x3ff00000, 0x00000000, 0x00000000, 0x00000000, - 0x00000000, 0x0f592f50, 0x3f9ba165, 0xa3d12526, 0x3fefd88d, - 0x40000000, 0xbc887df6, 0x00000000, 0xbfc00000, 0x0e5967d5, - 0x3fac1d1f, 0xcff75cb0, 0x3fef6297, 0x20000000, 0x3c756217, - 0x00000000, 0xbfd00000, 0x76acf82d, 0xbfa4a031, 0x56c62dda, - 0x3fee9f41, 0xe0000000, 0x3c8760b1, 0x00000000, 0xbfd00000, - 0x65455a75, 0x3fbe0875, 0xcf328d46, 0x3fed906b, 0x20000000, - 0x3c7457e6, 0x00000000, 0xbfe00000, 0x7f909c4e, 0x3f9d4a2c, - 0xf180bdb1, 0x3fec38b2, 0x80000000, 0xbc76e0b1, 0x00000000, - 0xbfe00000, 0x9ae68c87, 0xbfac73b3, 0x290ea1a3, 0x3fea9b66, - 0xe0000000, 0x3c39f630, 0x00000000, 0xbfe00000, 0x94247758, - 0xbfc133cc, 0x6b151741, 0x3fe8bc80, 0x20000000, 0xbc82c5e1, - 0x00000000, 0xbfe00000, 0x99fcef32, 0xbfca8279, 0x667f3bcd, - 0x3fe6a09e, 0x20000000, 0xbc8bdd34, 0x00000000, 0xbfe00000, - 0x53aba2fd, 0x3fcd0dfe, 0x25091dd6, 0x3fe44cf3, 0x20000000, - 0x3c68076a, 0x00000000, 0xbff00000, 0x5bc57974, 0x3fc59267, - 0x39ae68c8, 0x3fe1c73b, 0x20000000, 0x3c8b25dd, 0x00000000, - 0xbff00000, 0x73fa1279, 0x3fbe3a68, 0x3806f63b, 0x3fde2b5d, - 0x20000000, 0x3c5e0d89, 0x00000000, 0xbff00000, 0x866b95cf, - 0x3fb37ca1, 0xa6aea963, 0x3fd87de2, 0xe0000000, 0xbc672ced, - 0x00000000, 0xbff00000, 0x939d225a, 0x3fa60bea, 0x2ed59f06, - 0x3fd29406, 0xa0000000, 0xbc75d28d, 0x00000000, 0xbff00000, - 0x011469fb, 0x3f93ad06, 0x3c69a60b, 0x3fc8f8b8, 0xc0000000, - 0xbc626d19, 0x00000000, 0xbff00000, 0x176d6d31, 0x3f73b92e, - 0xbc29b42c, 0x3fb917a6, 0xe0000000, 0xbc3e2718, 0x00000000, - 0xbff00000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, - 0x00000000, 0x00000000, 0x00000000, 0xbff00000, 0x176d6d31, - 0x3f73b92e, 0xbc29b42c, 0xbfb917a6, 0xe0000000, 0x3c3e2718, - 0x00000000, 0xbff00000, 0x011469fb, 0x3f93ad06, 0x3c69a60b, - 0xbfc8f8b8, 0xc0000000, 0x3c626d19, 0x00000000, 0xbff00000, - 0x939d225a, 0x3fa60bea, 0x2ed59f06, 0xbfd29406, 0xa0000000, - 0x3c75d28d, 0x00000000, 0xbff00000, 0x866b95cf, 0x3fb37ca1, - 0xa6aea963, 0xbfd87de2, 0xe0000000, 0x3c672ced, 0x00000000, - 0xbff00000, 0x73fa1279, 0x3fbe3a68, 0x3806f63b, 0xbfde2b5d, - 0x20000000, 0xbc5e0d89, 0x00000000, 0xbff00000, 0x5bc57974, - 0x3fc59267, 0x39ae68c8, 0xbfe1c73b, 0x20000000, 0xbc8b25dd, - 0x00000000, 0xbff00000, 0x53aba2fd, 0x3fcd0dfe, 0x25091dd6, - 0xbfe44cf3, 0x20000000, 0xbc68076a, 0x00000000, 0xbff00000, - 0x99fcef32, 0xbfca8279, 0x667f3bcd, 0xbfe6a09e, 0x20000000, - 0x3c8bdd34, 0x00000000, 0xbfe00000, 0x94247758, 0xbfc133cc, - 0x6b151741, 0xbfe8bc80, 0x20000000, 0x3c82c5e1, 0x00000000, - 0xbfe00000, 0x9ae68c87, 0xbfac73b3, 0x290ea1a3, 0xbfea9b66, - 0xe0000000, 0xbc39f630, 0x00000000, 0xbfe00000, 0x7f909c4e, - 0x3f9d4a2c, 0xf180bdb1, 0xbfec38b2, 0x80000000, 0x3c76e0b1, - 0x00000000, 0xbfe00000, 0x65455a75, 0x3fbe0875, 0xcf328d46, - 0xbfed906b, 0x20000000, 0xbc7457e6, 0x00000000, 0xbfe00000, - 0x76acf82d, 0xbfa4a031, 0x56c62dda, 0xbfee9f41, 0xe0000000, - 0xbc8760b1, 0x00000000, 0xbfd00000, 0x0e5967d5, 0x3fac1d1f, - 0xcff75cb0, 0xbfef6297, 0x20000000, 0xbc756217, 0x00000000, - 0xbfd00000, 0x0f592f50, 0x3f9ba165, 0xa3d12526, 0xbfefd88d, - 0x40000000, 0x3c887df6, 0x00000000, 0xbfc00000, 0x00000000, - 0x00000000, 0x00000000, 0xbff00000, 0x00000000, 0x00000000, - 0x00000000, 0x00000000, 0x0f592f50, 0xbf9ba165, 0xa3d12526, - 0xbfefd88d, 0x40000000, 0x3c887df6, 0x00000000, 0x3fc00000, - 0x0e5967d5, 0xbfac1d1f, 0xcff75cb0, 0xbfef6297, 0x20000000, - 0xbc756217, 0x00000000, 0x3fd00000, 0x76acf82d, 0x3fa4a031, - 0x56c62dda, 0xbfee9f41, 0xe0000000, 0xbc8760b1, 0x00000000, - 0x3fd00000, 0x65455a75, 0xbfbe0875, 0xcf328d46, 0xbfed906b, - 0x20000000, 0xbc7457e6, 0x00000000, 0x3fe00000, 0x7f909c4e, - 0xbf9d4a2c, 0xf180bdb1, 0xbfec38b2, 0x80000000, 0x3c76e0b1, - 0x00000000, 0x3fe00000, 0x9ae68c87, 0x3fac73b3, 0x290ea1a3, - 0xbfea9b66, 0xe0000000, 0xbc39f630, 0x00000000, 0x3fe00000, - 0x94247758, 0x3fc133cc, 0x6b151741, 0xbfe8bc80, 0x20000000, - 0x3c82c5e1, 0x00000000, 0x3fe00000, 0x99fcef32, 0x3fca8279, - 0x667f3bcd, 0xbfe6a09e, 0x20000000, 0x3c8bdd34, 0x00000000, - 0x3fe00000, 0x53aba2fd, 0xbfcd0dfe, 0x25091dd6, 0xbfe44cf3, - 0x20000000, 0xbc68076a, 0x00000000, 0x3ff00000, 0x5bc57974, - 0xbfc59267, 0x39ae68c8, 0xbfe1c73b, 0x20000000, 0xbc8b25dd, - 0x00000000, 0x3ff00000, 0x73fa1279, 0xbfbe3a68, 0x3806f63b, - 0xbfde2b5d, 0x20000000, 0xbc5e0d89, 0x00000000, 0x3ff00000, - 0x866b95cf, 0xbfb37ca1, 0xa6aea963, 0xbfd87de2, 0xe0000000, - 0x3c672ced, 0x00000000, 0x3ff00000, 0x939d225a, 0xbfa60bea, - 0x2ed59f06, 0xbfd29406, 0xa0000000, 0x3c75d28d, 0x00000000, - 0x3ff00000, 0x011469fb, 0xbf93ad06, 0x3c69a60b, 0xbfc8f8b8, - 0xc0000000, 0x3c626d19, 0x00000000, 0x3ff00000, 0x176d6d31, - 0xbf73b92e, 0xbc29b42c, 0xbfb917a6, 0xe0000000, 0x3c3e2718, - 0x00000000, 0x3ff00000 - }; - - public int[] scTwo = { - 0x11111111, 0x3f811111, 0x55555555, 0x3fa55555 - }; - - public int[] scThree = { - 0x1a01a01a, 0xbf2a01a0, 0x16c16c17, 0xbf56c16c - }; - - public int[] scOne = { - 0x55555555, 0xbfc55555, 0x00000000, 0xbfe00000 - }; - - public int[] piInvTable = { - 0x00000000, 0x00000000, 0xa2f9836e, 0x4e441529, 0xfc2757d1, - 0xf534ddc0, 0xdb629599, 0x3c439041, 0xfe5163ab, 0xdebbc561, - 0xb7246e3a, 0x424dd2e0, 0x06492eea, 0x09d1921c, 0xfe1deb1c, - 0xb129a73e, 0xe88235f5, 0x2ebb4484, 0xe99c7026, 0xb45f7e41, - 0x3991d639, 0x835339f4, 0x9c845f8b, 0xbdf9283b, 0x1ff897ff, - 0xde05980f, 0xef2f118b, 0x5a0a6d1f, 0x6d367ecf, 0x27cb09b7, - 0x4f463f66, 0x9e5fea2d, 0x7527bac7, 0xebe5f17b, 0x3d0739f7, - 0x8a5292ea, 0x6bfb5fb1, 0x1f8d5d08, 0x56033046, 0xfc7b6bab, - 0xf0cfbc21 - }; - - public int[] piFour = { - 0x40000000, 0x3fe921fb, 0x18469899, 0x3e64442d - }; - - public int[] piThirtyTwoInv = { - 0x6dc9c883, 0x40245f30 - }; - - public int[] shifter = { - 0x00000000, 0x43380000 - }; - - public int[] signMask = { - 0x00000000, 0x80000000 - }; - - public int[] pThree = { - 0x2e037073, 0x3b63198a - }; - - public int[] allOnes = { - 0xffffffff, 0x3fefffff - }; - - public int[] twoPowFiftyFive = { - 0x00000000, 0x43600000 - }; - - public int[] twoPowFiftyFiveM = { - 0x00000000, 0x3c800000 - }; - - public int[] pOne = { - 0x54400000, 0x3fb921fb - }; - - public void sinIntrinsic(Register dest, Register value, CompilationResultBuilder crb, AMD64MacroAssembler masm) { - ArrayDataPointerConstant oneHalfPtr = new ArrayDataPointerConstant(oneHalf, 16); - ArrayDataPointerConstant pTwoPtr = new ArrayDataPointerConstant(pTwo, 16); - ArrayDataPointerConstant scFourPtr = new ArrayDataPointerConstant(scFour, 16); - ArrayDataPointerConstant cTablePtr = new ArrayDataPointerConstant(cTable, 16); - ArrayDataPointerConstant scTwoPtr = new ArrayDataPointerConstant(scTwo, 16); - ArrayDataPointerConstant scThreePtr = new ArrayDataPointerConstant(scThree, 16); - ArrayDataPointerConstant scOnePtr = new ArrayDataPointerConstant(scOne, 16); - ArrayDataPointerConstant piInvTablePtr = new ArrayDataPointerConstant(piInvTable, 16); - ArrayDataPointerConstant piFourPtr = new ArrayDataPointerConstant(piFour, 16); - ArrayDataPointerConstant piThirtyTwoInvPtr = new ArrayDataPointerConstant(piThirtyTwoInv, 8); - ArrayDataPointerConstant shifterPtr = new ArrayDataPointerConstant(shifter, 8); - ArrayDataPointerConstant signMaskPtr = new ArrayDataPointerConstant(signMask, 8); - ArrayDataPointerConstant pThreePtr = new ArrayDataPointerConstant(pThree, 8); - ArrayDataPointerConstant allOnesPtr = new ArrayDataPointerConstant(allOnes, 8); - ArrayDataPointerConstant twoPowFiftyFivePtr = new ArrayDataPointerConstant(twoPowFiftyFive, 8); - ArrayDataPointerConstant twoPowFiftyFiveMPtr = new ArrayDataPointerConstant(twoPowFiftyFiveM, 8); - ArrayDataPointerConstant pOnePtr = new ArrayDataPointerConstant(pOne, 8); - - Label bb0 = new Label(); - Label bb1 = new Label(); - Label bb2 = new Label(); - Label bb4 = new Label(); - Label bb5 = new Label(); - Label bb6 = new Label(); - Label bb8 = new Label(); - Label bb9 = new Label(); - Label bb10 = new Label(); - Label bb11 = new Label(); - Label bb12 = new Label(); - Label bb13 = new Label(); - Label bb14 = new Label(); - Label bb15 = new Label(); - - Register gpr1 = asRegister(gpr1Temp, AMD64Kind.QWORD); - Register gpr2 = asRegister(gpr2Temp, AMD64Kind.QWORD); - Register gpr3 = asRegister(rcxTemp, AMD64Kind.QWORD); - Register gpr4 = asRegister(gpr4Temp, AMD64Kind.QWORD); - Register gpr5 = asRegister(gpr5Temp, AMD64Kind.QWORD); - Register gpr6 = asRegister(gpr6Temp, AMD64Kind.QWORD); - Register gpr7 = asRegister(gpr7Temp, AMD64Kind.QWORD); - Register gpr8 = asRegister(gpr8Temp, AMD64Kind.QWORD); - Register gpr9 = asRegister(gpr9Temp, AMD64Kind.QWORD); - Register gpr10 = asRegister(gpr10Temp, AMD64Kind.QWORD); - - Register temp1 = asRegister(xmm1Temp, AMD64Kind.DOUBLE); - Register temp2 = asRegister(xmm2Temp, AMD64Kind.DOUBLE); - Register temp3 = asRegister(xmm3Temp, AMD64Kind.DOUBLE); - Register temp4 = asRegister(xmm4Temp, AMD64Kind.DOUBLE); - Register temp5 = asRegister(xmm5Temp, AMD64Kind.DOUBLE); - Register temp6 = asRegister(xmm6Temp, AMD64Kind.DOUBLE); - Register temp7 = asRegister(xmm7Temp, AMD64Kind.DOUBLE); - Register temp8 = asRegister(xmm8Temp, AMD64Kind.DOUBLE); - Register temp9 = asRegister(xmm9Temp, AMD64Kind.DOUBLE); - - AMD64Address stackSlot = (AMD64Address) crb.asAddress(stackTemp); - - setCrb(crb); - masm.movsd(stackSlot, value); - if (dest.encoding != value.encoding) { - masm.movdqu(dest, value); - } - - masm.leaq(gpr1, stackSlot); - masm.movl(gpr1, new AMD64Address(gpr1, 4)); - masm.movdq(temp1, externalAddress(piThirtyTwoInvPtr)); // 0x6dc9c883, - // 0x40245f30 - masm.movdq(temp2, externalAddress(shifterPtr)); // 0x00000000, - // 0x43380000 - - masm.andl(gpr1, 2147418112); - masm.subl(gpr1, 808452096); - masm.cmpl(gpr1, 281346048); - masm.jcc(ConditionFlag.Above, bb0); - - masm.mulsd(temp1, dest); - masm.movdqu(temp5, externalAddress(oneHalfPtr)); // 0x00000000, - // 0x3fe00000, - // 0x00000000, - // 0x3fe00000 - masm.movdq(temp4, externalAddress(signMaskPtr)); // 0x00000000, - // 0x80000000 - masm.pand(temp4, dest); - masm.por(temp5, temp4); - masm.addpd(temp1, temp5); - masm.cvttsd2sil(gpr4, temp1); - masm.cvtsi2sdl(temp1, gpr4); - masm.movdqu(temp6, externalAddress(pTwoPtr)); // 0x1a600000, - // 0x3d90b461, - // 0x1a600000, - // 0x3d90b461 - masm.movq(gpr7, 0x3fb921fb54400000L); - masm.movdq(temp3, gpr7); - masm.movdqu(temp5, externalAddress(scFourPtr)); // 0xa556c734, - // 0x3ec71de3, - // 0x1a01a01a, - // 0x3efa01a0 - masm.pshufd(temp4, dest, 0x44); - masm.mulsd(temp3, temp1); - if (masm.supports(CPUFeature.SSE3)) { - masm.movddup(temp1, temp1); - } else { - masm.movlhps(temp1, temp1); - } - masm.andl(gpr4, 63); - masm.shll(gpr4, 5); - masm.leaq(gpr1, externalAddress(cTablePtr)); - masm.addq(gpr1, gpr4); - masm.movdqu(temp8, new AMD64Address(gpr1, 0)); - masm.mulpd(temp6, temp1); - masm.mulsd(temp1, externalAddress(pThreePtr)); // 0x2e037073, - // 0x3b63198a - masm.subsd(temp4, temp3); - masm.subsd(dest, temp3); - if (masm.supports(CPUFeature.SSE3)) { - masm.movddup(temp3, temp4); - } else { - masm.movdqu(temp3, temp4); - masm.movlhps(temp3, temp3); - } - masm.subsd(temp4, temp6); - masm.pshufd(dest, dest, 0x44); - masm.pshufd(temp7, temp8, 0xE); - masm.movdqu(temp2, temp8); - masm.movdqu(temp9, temp7); - masm.mulpd(temp5, dest); - masm.subpd(dest, temp6); - masm.mulsd(temp7, temp4); - masm.subsd(temp3, temp4); - masm.mulpd(temp5, dest); - masm.mulpd(dest, dest); - masm.subsd(temp3, temp6); - masm.movdqu(temp6, externalAddress(scTwoPtr)); // 0x11111111, - // 0x3f811111, - // 0x55555555, - // 0x3fa55555 - masm.subsd(temp1, temp3); - masm.movdq(temp3, new AMD64Address(gpr1, 24)); - masm.addsd(temp2, temp3); - masm.subsd(temp7, temp2); - masm.mulsd(temp2, temp4); - masm.mulpd(temp6, dest); - masm.mulsd(temp3, temp4); - masm.mulpd(temp2, dest); - masm.mulpd(dest, dest); - masm.addpd(temp5, externalAddress(scThreePtr)); // 0x1a01a01a, - // 0xbf2a01a0, - // 0x16c16c17, - // 0xbf56c16c - masm.mulsd(temp4, temp8); - masm.addpd(temp6, externalAddress(scOnePtr)); // 0x55555555, - // 0xbfc55555, - // 0x00000000, - // 0xbfe00000 - masm.mulpd(temp5, dest); - masm.movdqu(dest, temp3); - masm.addsd(temp3, temp9); - masm.mulpd(temp1, temp7); - masm.movdqu(temp7, temp4); - masm.addsd(temp4, temp3); - masm.addpd(temp6, temp5); - masm.subsd(temp9, temp3); - masm.subsd(temp3, temp4); - masm.addsd(temp1, new AMD64Address(gpr1, 16)); - masm.mulpd(temp6, temp2); - masm.addsd(temp9, dest); - masm.addsd(temp3, temp7); - masm.addsd(temp1, temp9); - masm.addsd(temp1, temp3); - masm.addsd(temp1, temp6); - masm.unpckhpd(temp6, temp6); - masm.movdqu(dest, temp4); - masm.addsd(temp1, temp6); - masm.addsd(dest, temp1); - masm.jmp(bb15); - - masm.bind(bb14); - masm.xorpd(temp1, temp1); - masm.xorpd(dest, dest); - masm.divsd(dest, temp1); - masm.jmp(bb15); - - masm.bind(bb0); - masm.jcc(ConditionFlag.Greater, bb1); - - masm.shrl(gpr1, 20); - masm.cmpl(gpr1, 3325); - masm.jcc(ConditionFlag.NotEqual, bb2); - - masm.mulsd(dest, externalAddress(allOnesPtr)); // 0xffffffff, - // 0x3fefffff - masm.jmp(bb15); - - masm.bind(bb2); - masm.movdq(temp3, externalAddress(twoPowFiftyFivePtr)); // 0x00000000, - // 0x43600000 - masm.mulsd(temp3, dest); - masm.subsd(temp3, dest); - masm.mulsd(temp3, externalAddress(twoPowFiftyFiveMPtr)); // 0x00000000, - // 0x3c800000 - masm.jmp(bb15); - - masm.bind(bb1); - masm.pextrw(gpr3, dest, 3); - masm.andl(gpr3, 32752); - masm.cmpl(gpr3, 32752); - masm.jcc(ConditionFlag.Equal, bb14); - - masm.subl(gpr3, 16224); - masm.shrl(gpr3, 7); - masm.andl(gpr3, 65532); - masm.leaq(gpr10, externalAddress(piInvTablePtr)); - masm.addq(gpr3, gpr10); - masm.movdq(gpr1, dest); - masm.movl(gpr9, new AMD64Address(gpr3, 20)); - masm.movl(gpr7, new AMD64Address(gpr3, 24)); - masm.movl(gpr4, gpr1); - masm.shrq(gpr1, 21); - masm.orl(gpr1, Integer.MIN_VALUE); - masm.shrl(gpr1, 11); - masm.movl(gpr8, gpr9); - masm.imulq(gpr9, gpr4); - masm.imulq(gpr8, gpr1); - masm.imulq(gpr7, gpr1); - masm.movl(gpr5, new AMD64Address(gpr3, 16)); - masm.movl(gpr6, new AMD64Address(gpr3, 12)); - masm.movl(gpr10, gpr9); - masm.shrq(gpr9, 32); - masm.addq(gpr8, gpr9); - masm.addq(gpr10, gpr7); - masm.movl(gpr7, gpr10); - masm.shrq(gpr10, 32); - masm.addq(gpr8, gpr10); - masm.movl(gpr9, gpr5); - masm.imulq(gpr5, gpr4); - masm.imulq(gpr9, gpr1); - masm.movl(gpr10, gpr6); - masm.imulq(gpr6, gpr4); - masm.movl(gpr2, gpr5); - masm.shrq(gpr5, 32); - masm.addq(gpr8, gpr2); - masm.movl(gpr2, gpr8); - masm.shrq(gpr8, 32); - masm.addq(gpr9, gpr5); - masm.addq(gpr9, gpr8); - masm.shlq(gpr2, 32); - masm.orq(gpr7, gpr2); - masm.imulq(gpr10, gpr1); - masm.movl(gpr8, new AMD64Address(gpr3, 8)); - masm.movl(gpr5, new AMD64Address(gpr3, 4)); - masm.movl(gpr2, gpr6); - masm.shrq(gpr6, 32); - masm.addq(gpr9, gpr2); - masm.movl(gpr2, gpr9); - masm.shrq(gpr9, 32); - masm.addq(gpr10, gpr6); - masm.addq(gpr10, gpr9); - masm.movq(gpr6, gpr8); - masm.imulq(gpr8, gpr4); - masm.imulq(gpr6, gpr1); - masm.movl(gpr9, gpr8); - masm.shrq(gpr8, 32); - masm.addq(gpr10, gpr9); - masm.movl(gpr9, gpr10); - masm.shrq(gpr10, 32); - masm.addq(gpr6, gpr8); - masm.addq(gpr6, gpr10); - masm.movq(gpr8, gpr5); - masm.imulq(gpr5, gpr4); - masm.imulq(gpr8, gpr1); - masm.shlq(gpr9, 32); - masm.orq(gpr9, gpr2); - masm.movl(gpr1, new AMD64Address(gpr3, 0)); - masm.movl(gpr10, gpr5); - masm.shrq(gpr5, 32); - masm.addq(gpr6, gpr10); - masm.movl(gpr10, gpr6); - masm.shrq(gpr6, 32); - masm.addq(gpr8, gpr5); - masm.addq(gpr8, gpr6); - masm.imulq(gpr4, gpr1); - masm.pextrw(gpr2, dest, 3); - masm.leaq(gpr6, externalAddress(piInvTablePtr)); - masm.subq(gpr3, gpr6); - masm.addl(gpr3, gpr3); - masm.addl(gpr3, gpr3); - masm.addl(gpr3, gpr3); - masm.addl(gpr3, 19); - masm.movl(gpr5, 32768); - masm.andl(gpr5, gpr2); - masm.shrl(gpr2, 4); - masm.andl(gpr2, 2047); - masm.subl(gpr2, 1023); - masm.subl(gpr3, gpr2); - masm.addq(gpr8, gpr4); - masm.movl(gpr4, gpr3); - masm.addl(gpr4, 32); - masm.cmpl(gpr3, 1); - masm.jcc(ConditionFlag.Less, bb4); - - masm.negl(gpr3); - masm.addl(gpr3, 29); - masm.shll(gpr8); - masm.movl(gpr6, gpr8); - masm.andl(gpr8, 536870911); - masm.testl(gpr8, 268435456); - masm.jcc(ConditionFlag.NotEqual, bb5); - - masm.shrl(gpr8); - masm.movl(gpr2, 0); - masm.shlq(gpr8, 32); - masm.orq(gpr8, gpr10); - - masm.bind(bb6); - - masm.cmpq(gpr8, 0); - masm.jcc(ConditionFlag.Equal, bb8); - - masm.bind(bb9); - masm.bsrq(gpr10, gpr8); - masm.movl(gpr3, 29); - masm.subl(gpr3, gpr10); - masm.jcc(ConditionFlag.LessEqual, bb10); - - masm.shlq(gpr8); - masm.movq(gpr1, gpr9); - masm.shlq(gpr9); - masm.addl(gpr4, gpr3); - masm.negl(gpr3); - masm.addl(gpr3, 64); - masm.shrq(gpr1); - masm.shrq(gpr7); - masm.orq(gpr8, gpr1); - masm.orq(gpr9, gpr7); - - masm.bind(bb11); - masm.cvtsi2sdq(dest, gpr8); - masm.shrq(gpr9, 1); - masm.cvtsi2sdq(temp3, gpr9); - masm.xorpd(temp4, temp4); - masm.shll(gpr4, 4); - masm.negl(gpr4); - masm.addl(gpr4, 16368); - masm.orl(gpr4, gpr5); - masm.xorl(gpr4, gpr2); - masm.pinsrw(temp4, gpr4, 3); - masm.leaq(gpr1, externalAddress(piFourPtr)); - masm.movdqu(temp2, new AMD64Address(gpr1, 0)); // 0x40000000, - // 0x3fe921fb, - // 0x18469899, - // 0x3e64442d - masm.xorpd(temp5, temp5); - masm.subl(gpr4, 1008); - masm.pinsrw(temp5, gpr4, 3); - masm.mulsd(dest, temp4); - masm.shll(gpr5, 16); - masm.sarl(gpr5, 31); - masm.mulsd(temp3, temp5); - masm.movdqu(temp1, dest); - masm.pshufd(temp6, temp2, 0xE); - masm.mulsd(dest, temp2); - masm.shrl(gpr6, 29); - masm.addsd(temp1, temp3); - masm.mulsd(temp3, temp2); - masm.addl(gpr6, gpr5); - masm.xorl(gpr6, gpr5); - masm.mulsd(temp6, temp1); - masm.movl(gpr1, gpr6); - masm.addsd(temp6, temp3); - masm.movdqu(temp2, dest); - masm.addsd(dest, temp6); - masm.subsd(temp2, dest); - masm.addsd(temp6, temp2); - - masm.bind(bb12); - masm.movdq(temp1, externalAddress(piThirtyTwoInvPtr)); // 0x6dc9c883, - // 0x40245f30 - masm.mulsd(temp1, dest); - masm.movdq(temp5, externalAddress(oneHalfPtr)); // 0x00000000, - // 0x3fe00000, - // 0x00000000, - // 0x3fe00000 - masm.movdq(temp4, externalAddress(signMaskPtr)); // 0x00000000, - // 0x80000000 - masm.pand(temp4, dest); - masm.por(temp5, temp4); - masm.addpd(temp1, temp5); - masm.cvttsd2sil(gpr4, temp1); - masm.cvtsi2sdl(temp1, gpr4); - masm.movdq(temp3, externalAddress(pOnePtr)); // 0x54400000, - // 0x3fb921fb - masm.movdqu(temp2, externalAddress(pTwoPtr)); // 0x1a600000, - // 0x3d90b461, - // 0x1a600000, - // 0x3d90b461 - masm.mulsd(temp3, temp1); - masm.unpcklpd(temp1, temp1); - masm.shll(gpr1, 3); - masm.addl(gpr4, 1865216); - masm.movdqu(temp4, dest); - masm.addl(gpr4, gpr1); - masm.andl(gpr4, 63); - masm.movdqu(temp5, externalAddress(scFourPtr)); // 0x54400000, - // 0x3fb921fb - masm.leaq(gpr1, externalAddress(cTablePtr)); - masm.shll(gpr4, 5); - masm.addq(gpr1, gpr4); - masm.movdqu(temp8, new AMD64Address(gpr1, 0)); - masm.mulpd(temp2, temp1); - masm.subsd(dest, temp3); - masm.mulsd(temp1, externalAddress(pThreePtr)); // 0x2e037073, - // 0x3b63198a - masm.subsd(temp4, temp3); - masm.unpcklpd(dest, dest); - masm.movdqu(temp3, temp4); - masm.subsd(temp4, temp2); - masm.mulpd(temp5, dest); - masm.subpd(dest, temp2); - masm.pshufd(temp7, temp8, 0xE); - masm.movdqu(temp9, temp7); - masm.mulsd(temp7, temp4); - masm.subsd(temp3, temp4); - masm.mulpd(temp5, dest); - masm.mulpd(dest, dest); - masm.subsd(temp3, temp2); - masm.movdqu(temp2, temp8); - masm.subsd(temp1, temp3); - masm.movdq(temp3, new AMD64Address(gpr1, 24)); - masm.addsd(temp2, temp3); - masm.subsd(temp7, temp2); - masm.subsd(temp1, temp6); - masm.movdqu(temp6, externalAddress(scTwoPtr)); // 0x11111111, - // 0x3f811111, - // 0x55555555, - // 0x3fa55555 - masm.mulsd(temp2, temp4); - masm.mulpd(temp6, dest); - masm.mulsd(temp3, temp4); - masm.mulpd(temp2, dest); - masm.mulpd(dest, dest); - masm.addpd(temp5, externalAddress(scThreePtr)); // 0x1a01a01a, - // 0xbf2a01a0, - // 0x16c16c17, - // 0xbf56c16c - masm.mulsd(temp4, temp8); - masm.addpd(temp6, externalAddress(scOnePtr)); // 0x55555555, - // 0xbfc55555, - // 0x00000000, - // 0xbfe00000 - masm.mulpd(temp5, dest); - masm.movdqu(dest, temp3); - masm.addsd(temp3, temp9); - masm.mulpd(temp1, temp7); - masm.movdqu(temp7, temp4); - masm.addsd(temp4, temp3); - masm.addpd(temp6, temp5); - masm.subsd(temp9, temp3); - masm.subsd(temp3, temp4); - masm.addsd(temp1, new AMD64Address(gpr1, 16)); - masm.mulpd(temp6, temp2); - masm.addsd(temp9, dest); - masm.addsd(temp3, temp7); - masm.addsd(temp1, temp9); - masm.addsd(temp1, temp3); - masm.addsd(temp1, temp6); - masm.unpckhpd(temp6, temp6); - masm.movdqu(dest, temp4); - masm.addsd(temp1, temp6); - masm.addsd(dest, temp1); - masm.jmp(bb15); - - masm.bind(bb8); - masm.addl(gpr4, 64); - masm.movq(gpr8, gpr9); - masm.movq(gpr9, gpr7); - masm.movl(gpr7, 0); - masm.cmpq(gpr8, 0); - masm.jcc(ConditionFlag.NotEqual, bb9); - - masm.addl(gpr4, 64); - masm.movq(gpr8, gpr9); - masm.movq(gpr9, gpr7); - masm.cmpq(gpr8, 0); - masm.jcc(ConditionFlag.NotEqual, bb9); - - masm.xorpd(dest, dest); - masm.xorpd(temp6, temp6); - masm.jmp(bb12); - - masm.bind(bb10); - masm.jcc(ConditionFlag.Equal, bb11); - - masm.negl(gpr3); - masm.shrq(gpr9); - masm.movq(gpr1, gpr8); - masm.shrq(gpr8); - masm.subl(gpr4, gpr3); - masm.negl(gpr3); - masm.addl(gpr3, 64); - masm.shlq(gpr1); - masm.orq(gpr9, gpr1); - masm.jmp(bb11); - - masm.bind(bb4); - masm.negl(gpr3); - masm.shlq(gpr8, 32); - masm.orq(gpr8, gpr10); - masm.shlq(gpr8); - masm.movq(gpr6, gpr8); - masm.testl(gpr8, Integer.MIN_VALUE); - masm.jcc(ConditionFlag.NotEqual, bb13); - - masm.shrl(gpr8); - masm.movl(gpr2, 0); - masm.shrq(gpr6, 3); - masm.jmp(bb6); - - masm.bind(bb5); - masm.shrl(gpr8); - masm.movl(gpr2, 536870912); - masm.shrl(gpr2); - masm.shlq(gpr8, 32); - masm.orq(gpr8, gpr10); - masm.shlq(gpr2, 32); - masm.addl(gpr6, 536870912); - masm.movl(gpr3, 0); - masm.movl(gpr10, 0); - masm.subq(gpr3, gpr7); - masm.sbbq(gpr10, gpr9); - masm.sbbq(gpr2, gpr8); - masm.movq(gpr7, gpr3); - masm.movq(gpr9, gpr10); - masm.movq(gpr8, gpr2); - masm.movl(gpr2, 32768); - masm.jmp(bb6); - - masm.bind(bb13); - masm.shrl(gpr8); - masm.movq(gpr2, 0x100000000L); - masm.shrq(gpr2); - masm.movl(gpr3, 0); - masm.movl(gpr10, 0); - masm.subq(gpr3, gpr7); - masm.sbbq(gpr10, gpr9); - masm.sbbq(gpr2, gpr8); - masm.movq(gpr7, gpr3); - masm.movq(gpr9, gpr10); - masm.movq(gpr8, gpr2); - masm.movl(gpr2, 32768); - masm.shrq(gpr6, 3); - masm.addl(gpr6, 536870912); - masm.jmp(bb6); - - masm.bind(bb15); - } - - /* - * Copyright (c) 2014, 2016, Intel Corporation. All rights reserved. Intel Math Library (LIBM) - * Source Code - * - * ALGORITHM DESCRIPTION - COS() --------------------- - * - * 1. RANGE REDUCTION - * - * We perform an initial range reduction from X to r with - * - * X =~= N * pi/32 + r - * - * so that |r| <= pi/64 + epsilon. We restrict inputs to those where |N| <= 932560. Beyond this, - * the range reduction is insufficiently accurate. For extremely small inputs, denormalization - * can occur internally, impacting performance. This means that the main path is actually only - * taken for 2^-252 <= |X| < 90112. - * - * To avoid branches, we perform the range reduction to full accuracy each time. - * - * X - N * (P_1 + P_2 + P_3) - * - * where P_1 and P_2 are 32-bit numbers (so multiplication by N is exact) and P_3 is a 53-bit - * number. Together, these approximate pi well enough for all cases in the restricted range. - * - * The main reduction sequence is: - * - * y = 32/pi * x N = integer(y) (computed by adding and subtracting off SHIFTER) - * - * m_1 = N * P_1 m_2 = N * P_2 r_1 = x - m_1 r = r_1 - m_2 (this r can be used for most of the - * calculation) - * - * c_1 = r_1 - r m_3 = N * P_3 c_2 = c_1 - m_2 c = c_2 - m_3 - * - * 2. MAIN ALGORITHM - * - * The algorithm uses a table lookup based on B = M * pi / 32 where M = N mod 64. The stored - * values are: sigma closest power of 2 to cos(B) C_hl 53-bit cos(B) - sigma S_hi + S_lo 2 * - * 53-bit sin(B) - * - * The computation is organized as follows: - * - * sin(B + r + c) = [sin(B) + sigma * r] + r * (cos(B) - sigma) + sin(B) * [cos(r + c) - 1] + - * cos(B) * [sin(r + c) - r] - * - * which is approximately: - * - * [S_hi + sigma * r] + C_hl * r + S_lo + S_hi * [(cos(r) - 1) - r * c] + (C_hl + sigma) * - * [(sin(r) - r) + c] - * - * and this is what is actually computed. We separate this sum into four parts: - * - * hi + med + pols + corr - * - * where - * - * hi = S_hi + sigma r med = C_hl * r pols = S_hi * (cos(r) - 1) + (C_hl + sigma) * (sin(r) - r) - * corr = S_lo + c * ((C_hl + sigma) - S_hi * r) - * - * 3. POLYNOMIAL - * - * The polynomial S_hi * (cos(r) - 1) + (C_hl + sigma) * (sin(r) - r) can be rearranged freely, - * since it is quite small, so we exploit parallelism to the fullest. - * - * psc4 = SC_4 * r_1 msc4 = psc4 * r r2 = r * r msc2 = SC_2 * r2 r4 = r2 * r2 psc3 = SC_3 + msc4 - * psc1 = SC_1 + msc2 msc3 = r4 * psc3 sincospols = psc1 + msc3 pols = sincospols * - * - * 4. CORRECTION TERM - * - * This is where the "c" component of the range reduction is taken into account; recall that - * just "r" is used for most of the calculation. - * - * -c = m_3 - c_2 -d = S_hi * r - (C_hl + sigma) corr = -c * -d + S_lo - * - * 5. COMPENSATED SUMMATIONS - * - * The two successive compensated summations add up the high and medium parts, leaving just the - * low parts to add up at the end. - * - * rs = sigma * r res_int = S_hi + rs k_0 = S_hi - res_int k_2 = k_0 + rs med = C_hl * r res_hi - * = res_int + med k_1 = res_int - res_hi k_3 = k_1 + med - * - * 6. FINAL SUMMATION - * - * We now add up all the small parts: - * - * res_lo = pols(hi) + pols(lo) + corr + k_1 + k_3 - * - * Now the overall result is just: - * - * res_hi + res_lo - * - * 7. SMALL ARGUMENTS - * - * Inputs with |X| < 2^-252 are treated specially as 1 - |x|. - * - * Special cases: cos(NaN) = quiet NaN, and raise invalid exception cos(INF) = NaN and raise - * invalid exception cos(0) = 1 - * - */ - - public int[] one = { - 0x00000000, 0x3ff00000 - }; + @Temp protected Value[] temps; - public void cosIntrinsic(Register dest, Register value, CompilationResultBuilder crb, AMD64MacroAssembler masm) { - ArrayDataPointerConstant oneHalfPtr = new ArrayDataPointerConstant(oneHalf, 16); - ArrayDataPointerConstant pTwoPtr = new ArrayDataPointerConstant(pTwo, 16); - ArrayDataPointerConstant scFourPtr = new ArrayDataPointerConstant(scFour, 16); - ArrayDataPointerConstant cTablePtr = new ArrayDataPointerConstant(cTable, 16); - ArrayDataPointerConstant scTwoPtr = new ArrayDataPointerConstant(scTwo, 16); - ArrayDataPointerConstant scThreePtr = new ArrayDataPointerConstant(scThree, 16); - ArrayDataPointerConstant scOnePtr = new ArrayDataPointerConstant(scOne, 16); - ArrayDataPointerConstant piInvTablePtr = new ArrayDataPointerConstant(piInvTable, 16); - ArrayDataPointerConstant piFourPtr = new ArrayDataPointerConstant(piFour, 16); - ArrayDataPointerConstant piThirtyTwoInvPtr = new ArrayDataPointerConstant(piThirtyTwoInv, 8); - ArrayDataPointerConstant signMaskPtr = new ArrayDataPointerConstant(signMask, 8); - ArrayDataPointerConstant pThreePtr = new ArrayDataPointerConstant(pThree, 8); - ArrayDataPointerConstant pOnePtr = new ArrayDataPointerConstant(pOne, 8); - ArrayDataPointerConstant onePtr = new ArrayDataPointerConstant(one, 8); + public AMD64MathIntrinsicUnaryOp(LIRInstructionClass type, Register... registers) { + super(type); - Label bb0 = new Label(); - Label bb1 = new Label(); - Label bb3 = new Label(); - Label bb4 = new Label(); - Label bb5 = new Label(); - Label bb6 = new Label(); - Label bb7 = new Label(); - Label bb8 = new Label(); - Label bb9 = new Label(); - Label bb10 = new Label(); - Label bb11 = new Label(); - Label bb12 = new Label(); - Label bb13 = new Label(); - Label bb14 = new Label(); + input = xmm0.asValue(LIRKind.value(AMD64Kind.DOUBLE)); + output = xmm0.asValue(LIRKind.value(AMD64Kind.DOUBLE)); - Register gpr1 = asRegister(gpr1Temp, AMD64Kind.QWORD); - Register gpr2 = asRegister(gpr2Temp, AMD64Kind.QWORD); - Register gpr3 = asRegister(rcxTemp, AMD64Kind.QWORD); - Register gpr4 = asRegister(gpr4Temp, AMD64Kind.QWORD); - Register gpr5 = asRegister(gpr5Temp, AMD64Kind.QWORD); - Register gpr6 = asRegister(gpr6Temp, AMD64Kind.QWORD); - Register gpr7 = asRegister(gpr7Temp, AMD64Kind.QWORD); - Register gpr8 = asRegister(gpr8Temp, AMD64Kind.QWORD); - Register gpr9 = asRegister(gpr9Temp, AMD64Kind.QWORD); - Register gpr10 = asRegister(gpr10Temp, AMD64Kind.QWORD); - - Register temp1 = asRegister(xmm1Temp, AMD64Kind.DOUBLE); - Register temp2 = asRegister(xmm2Temp, AMD64Kind.DOUBLE); - Register temp3 = asRegister(xmm3Temp, AMD64Kind.DOUBLE); - Register temp4 = asRegister(xmm4Temp, AMD64Kind.DOUBLE); - Register temp5 = asRegister(xmm5Temp, AMD64Kind.DOUBLE); - Register temp6 = asRegister(xmm6Temp, AMD64Kind.DOUBLE); - Register temp7 = asRegister(xmm7Temp, AMD64Kind.DOUBLE); - Register temp8 = asRegister(xmm8Temp, AMD64Kind.DOUBLE); - Register temp9 = asRegister(xmm9Temp, AMD64Kind.DOUBLE); - - AMD64Address stackSlot = (AMD64Address) crb.asAddress(stackTemp); - - setCrb(crb); - masm.movdq(stackSlot, value); - if (dest.encoding != value.encoding) { - masm.movdqu(dest, value); - } - - masm.leaq(gpr1, stackSlot); - masm.movl(gpr1, new AMD64Address(gpr1, 4)); - masm.movdq(temp1, externalAddress(piThirtyTwoInvPtr)); // 0x6dc9c883, - // 0x40245f30 - - masm.andl(gpr1, 2147418112); - masm.subl(gpr1, 808452096); - masm.cmpl(gpr1, 281346048); - masm.jcc(ConditionFlag.Above, bb0); - - masm.mulsd(temp1, dest); - masm.movdqu(temp5, externalAddress(oneHalfPtr)); // 0x00000000, - // 0x3fe00000, - // 0x00000000, - // 0x3fe00000 - masm.movdq(temp4, externalAddress(signMaskPtr)); // 0x00000000, - // 0x80000000 - masm.pand(temp4, dest); - masm.por(temp5, temp4); - masm.addpd(temp1, temp5); - masm.cvttsd2sil(gpr4, temp1); - masm.cvtsi2sdl(temp1, gpr4); - masm.movdqu(temp2, externalAddress(pTwoPtr)); // 0x1a600000, - // 0x3d90b461, - // 0x1a600000, - // 0x3d90b461 - masm.movdq(temp3, externalAddress(pOnePtr)); // 0x54400000, - // 0x3fb921fb - masm.mulsd(temp3, temp1); - masm.unpcklpd(temp1, temp1); - masm.addq(gpr4, 1865232); - masm.movdqu(temp4, dest); - masm.andq(gpr4, 63); - masm.movdqu(temp5, externalAddress(scFourPtr)); // 0xa556c734, - // 0x3ec71de3, - // 0x1a01a01a, - // 0x3efa01a0 - masm.leaq(gpr1, externalAddress(cTablePtr)); - masm.shlq(gpr4, 5); - masm.addq(gpr1, gpr4); - masm.movdqu(temp8, new AMD64Address(gpr1, 0)); - masm.mulpd(temp2, temp1); - masm.subsd(dest, temp3); - masm.mulsd(temp1, externalAddress(pThreePtr)); // 0x2e037073, - // 0x3b63198a - masm.subsd(temp4, temp3); - masm.unpcklpd(dest, dest); - masm.movdqu(temp3, temp4); - masm.subsd(temp4, temp2); - masm.mulpd(temp5, dest); - masm.subpd(dest, temp2); - masm.pshufd(temp7, temp8, 0xE); - masm.movdqu(temp6, externalAddress(scTwoPtr)); // 0x11111111, - // 0x3f811111, - // 0x55555555, - // 0x3fa55555 - masm.mulsd(temp7, temp4); - masm.subsd(temp3, temp4); - masm.mulpd(temp5, dest); - masm.mulpd(dest, dest); - masm.subsd(temp3, temp2); - masm.movdqu(temp2, temp8); - masm.subsd(temp1, temp3); - masm.movdq(temp3, new AMD64Address(gpr1, 24)); - masm.addsd(temp2, temp3); - masm.subsd(temp7, temp2); - masm.mulsd(temp2, temp4); - masm.mulpd(temp6, dest); - masm.mulsd(temp3, temp4); - masm.mulpd(temp2, dest); - masm.mulpd(dest, dest); - masm.addpd(temp5, externalAddress(scThreePtr)); // 0x1a01a01a, - // 0xbf2a01a0, - // 0x16c16c17, - // 0xbf56c16c - masm.mulsd(temp4, temp8); - masm.pshufd(temp9, temp8, 0xE); - masm.addpd(temp6, externalAddress(scOnePtr)); // 0x55555555, - // 0xbfc55555, - // 0x00000000, - // 0xbfe00000 - masm.mulpd(temp5, dest); - masm.movdqu(dest, temp3); - masm.addsd(temp3, temp9); - masm.mulpd(temp1, temp7); - masm.movdqu(temp7, temp4); - masm.addsd(temp4, temp3); - masm.addpd(temp6, temp5); - masm.subsd(temp9, temp3); - masm.subsd(temp3, temp4); - masm.addsd(temp1, new AMD64Address(gpr1, 16)); - masm.mulpd(temp6, temp2); - masm.addsd(dest, temp9); - masm.addsd(temp3, temp7); - masm.addsd(dest, temp1); - masm.addsd(dest, temp3); - masm.addsd(dest, temp6); - masm.unpckhpd(temp6, temp6); - masm.addsd(dest, temp6); - masm.addsd(dest, temp4); - masm.jmp(bb13); - - masm.bind(bb14); - masm.xorpd(temp1, temp1); - masm.xorpd(dest, dest); - masm.divsd(dest, temp1); - masm.jmp(bb13); - - masm.bind(bb0); - masm.jcc(ConditionFlag.Greater, bb1); - - masm.pextrw(gpr1, dest, 3); - masm.andl(gpr1, 32767); - masm.pinsrw(dest, gpr1, 3); - masm.movdq(temp1, externalAddress(onePtr)); // 0x00000000, - // 0x3ff00000 - masm.subsd(temp1, dest); - masm.movdqu(dest, temp1); - masm.jmp(bb13); - - masm.bind(bb1); - masm.pextrw(gpr3, dest, 3); - masm.andl(gpr3, 32752); - masm.cmpl(gpr3, 32752); - masm.jcc(ConditionFlag.Equal, bb14); - - masm.subl(gpr3, 16224); - masm.shrl(gpr3, 7); - masm.andl(gpr3, 65532); - masm.leaq(gpr10, externalAddress(piInvTablePtr)); - masm.addq(gpr3, gpr10); - masm.movdq(gpr1, dest); - masm.movl(gpr9, new AMD64Address(gpr3, 20)); - masm.movl(gpr7, new AMD64Address(gpr3, 24)); - masm.movl(gpr4, gpr1); - masm.shrq(gpr1, 21); - masm.orl(gpr1, Integer.MIN_VALUE); - masm.shrl(gpr1, 11); - masm.movl(gpr8, gpr9); - masm.imulq(gpr9, gpr4); - masm.imulq(gpr8, gpr1); - masm.imulq(gpr7, gpr1); - masm.movl(gpr5, new AMD64Address(gpr3, 16)); - masm.movl(gpr6, new AMD64Address(gpr3, 12)); - masm.movl(gpr10, gpr9); - masm.shrq(gpr9, 32); - masm.addq(gpr8, gpr9); - masm.addq(gpr10, gpr7); - masm.movl(gpr7, gpr10); - masm.shrq(gpr10, 32); - masm.addq(gpr8, gpr10); - masm.movl(gpr9, gpr5); - masm.imulq(gpr5, gpr4); - masm.imulq(gpr9, gpr1); - masm.movl(gpr10, gpr6); - masm.imulq(gpr6, gpr4); - masm.movl(gpr2, gpr5); - masm.shrq(gpr5, 32); - masm.addq(gpr8, gpr2); - masm.movl(gpr2, gpr8); - masm.shrq(gpr8, 32); - masm.addq(gpr9, gpr5); - masm.addq(gpr9, gpr8); - masm.shlq(gpr2, 32); - masm.orq(gpr7, gpr2); - masm.imulq(gpr10, gpr1); - masm.movl(gpr8, new AMD64Address(gpr3, 8)); - masm.movl(gpr5, new AMD64Address(gpr3, 4)); - masm.movl(gpr2, gpr6); - masm.shrq(gpr6, 32); - masm.addq(gpr9, gpr2); - masm.movl(gpr2, gpr9); - masm.shrq(gpr9, 32); - masm.addq(gpr10, gpr6); - masm.addq(gpr10, gpr9); - masm.movq(gpr6, gpr8); - masm.imulq(gpr8, gpr4); - masm.imulq(gpr6, gpr1); - masm.movl(gpr9, gpr8); - masm.shrq(gpr8, 32); - masm.addq(gpr10, gpr9); - masm.movl(gpr9, gpr10); - masm.shrq(gpr10, 32); - masm.addq(gpr6, gpr8); - masm.addq(gpr6, gpr10); - masm.movq(gpr8, gpr5); - masm.imulq(gpr5, gpr4); - masm.imulq(gpr8, gpr1); - masm.shlq(gpr9, 32); - masm.orq(gpr9, gpr2); - masm.movl(gpr1, new AMD64Address(gpr3, 0)); - masm.movl(gpr10, gpr5); - masm.shrq(gpr5, 32); - masm.addq(gpr6, gpr10); - masm.movl(gpr10, gpr6); - masm.shrq(gpr6, 32); - masm.addq(gpr8, gpr5); - masm.addq(gpr8, gpr6); - masm.imulq(gpr4, gpr1); - masm.pextrw(gpr2, dest, 3); - masm.leaq(gpr6, externalAddress(piInvTablePtr)); - masm.subq(gpr3, gpr6); - masm.addl(gpr3, gpr3); - masm.addl(gpr3, gpr3); - masm.addl(gpr3, gpr3); - masm.addl(gpr3, 19); - masm.movl(gpr5, 32768); - masm.andl(gpr5, gpr2); - masm.shrl(gpr2, 4); - masm.andl(gpr2, 2047); - masm.subl(gpr2, 1023); - masm.subl(gpr3, gpr2); - masm.addq(gpr8, gpr4); - masm.movl(gpr4, gpr3); - masm.addl(gpr4, 32); - masm.cmpl(gpr3, 1); - masm.jcc(ConditionFlag.Less, bb3); - - masm.negl(gpr3); - masm.addl(gpr3, 29); - masm.shll(gpr8); - masm.movl(gpr6, gpr8); - masm.andl(gpr8, 536870911); - masm.testl(gpr8, 268435456); - masm.jcc(ConditionFlag.NotEqual, bb4); - - masm.shrl(gpr8); - masm.movl(gpr2, 0); - masm.shlq(gpr8, 32); - masm.orq(gpr8, gpr10); - - masm.bind(bb5); - - masm.bind(bb6); - masm.cmpq(gpr8, 0); - masm.jcc(ConditionFlag.Equal, bb7); - - masm.bind(bb8); - masm.bsrq(gpr10, gpr8); - masm.movl(gpr3, 29); - masm.subl(gpr3, gpr10); - masm.jcc(ConditionFlag.LessEqual, bb9); - - masm.shlq(gpr8); - masm.movq(gpr1, gpr9); - masm.shlq(gpr9); - masm.addl(gpr4, gpr3); - masm.negl(gpr3); - masm.addl(gpr3, 64); - masm.shrq(gpr1); - masm.shrq(gpr7); - masm.orq(gpr8, gpr1); - masm.orq(gpr9, gpr7); - - masm.bind(bb10); - masm.cvtsi2sdq(dest, gpr8); - masm.shrq(gpr9, 1); - masm.cvtsi2sdq(temp3, gpr9); - masm.xorpd(temp4, temp4); - masm.shll(gpr4, 4); - masm.negl(gpr4); - masm.addl(gpr4, 16368); - masm.orl(gpr4, gpr5); - masm.xorl(gpr4, gpr2); - masm.pinsrw(temp4, gpr4, 3); - masm.leaq(gpr2, externalAddress(piFourPtr)); - masm.movdqu(temp2, new AMD64Address(gpr2, 0)); // 0x40000000, - // 0x3fe921fb, - // 0x18469899, - // 0x3e64442d - masm.xorpd(temp5, temp5); - masm.subl(gpr4, 1008); - masm.pinsrw(temp5, gpr4, 3); - masm.mulsd(dest, temp4); - masm.shll(gpr5, 16); - masm.sarl(gpr5, 31); - masm.mulsd(temp3, temp5); - masm.movdqu(temp1, dest); - masm.mulsd(dest, temp2); - masm.pshufd(temp6, temp2, 0xE); - masm.shrl(gpr6, 29); - masm.addsd(temp1, temp3); - masm.mulsd(temp3, temp2); - masm.addl(gpr6, gpr5); - masm.xorl(gpr6, gpr5); - masm.mulsd(temp6, temp1); - masm.movl(gpr1, gpr6); - masm.addsd(temp6, temp3); - masm.movdqu(temp2, dest); - masm.addsd(dest, temp6); - masm.subsd(temp2, dest); - masm.addsd(temp6, temp2); - - masm.bind(bb11); - masm.movq(temp1, externalAddress(piThirtyTwoInvPtr)); // 0x6dc9c883, - // 0x40245f30 - masm.mulsd(temp1, dest); - masm.movdq(temp5, externalAddress(oneHalfPtr)); // 0x00000000, - // 0x3fe00000, - // 0x00000000, - // 0x3fe00000 - masm.movdq(temp4, externalAddress(signMaskPtr)); // 0x00000000, - // 0x80000000 - masm.pand(temp4, dest); - masm.por(temp5, temp4); - masm.addpd(temp1, temp5); - masm.cvttsd2siq(gpr4, temp1); - masm.cvtsi2sdq(temp1, gpr4); - masm.movdq(temp3, externalAddress(pOnePtr)); // 0x54400000, - // 0x3fb921fb - masm.movdqu(temp2, externalAddress(pTwoPtr)); // 0x1a600000, - // 0x3d90b461, - // 0x1a600000, - // 0x3d90b461 - masm.mulsd(temp3, temp1); - masm.unpcklpd(temp1, temp1); - masm.shll(gpr1, 3); - masm.addl(gpr4, 1865232); - masm.movdqu(temp4, dest); - masm.addl(gpr4, gpr1); - masm.andl(gpr4, 63); - masm.movdqu(temp5, externalAddress(scFourPtr)); // 0xa556c734, - // 0x3ec71de3, - // 0x1a01a01a, - // 0x3efa01a0 - masm.leaq(gpr1, externalAddress(cTablePtr)); - masm.shll(gpr4, 5); - masm.addq(gpr1, gpr4); - masm.movdqu(temp8, new AMD64Address(gpr1, 0)); - masm.mulpd(temp2, temp1); - masm.subsd(dest, temp3); - masm.mulsd(temp1, externalAddress(pThreePtr)); // 0x2e037073, - // 0x3b63198a - masm.subsd(temp4, temp3); - masm.unpcklpd(dest, dest); - masm.movdqu(temp3, temp4); - masm.subsd(temp4, temp2); - masm.mulpd(temp5, dest); - masm.pshufd(temp7, temp8, 0xE); - masm.movdqu(temp9, temp7); - masm.subpd(dest, temp2); - masm.mulsd(temp7, temp4); - masm.subsd(temp3, temp4); - masm.mulpd(temp5, dest); - masm.mulpd(dest, dest); - masm.subsd(temp3, temp2); - masm.movdqu(temp2, temp8); - masm.subsd(temp1, temp3); - masm.movdq(temp3, new AMD64Address(gpr1, 24)); - masm.addsd(temp2, temp3); - masm.subsd(temp7, temp2); - masm.subsd(temp1, temp6); - masm.movdqu(temp6, externalAddress(scTwoPtr)); // 0x11111111, - // 0x3f811111, - // 0x55555555, - // 0x3fa55555 - masm.mulsd(temp2, temp4); - masm.mulpd(temp6, dest); - masm.mulsd(temp3, temp4); - masm.mulpd(temp2, dest); - masm.mulpd(dest, dest); - masm.addpd(temp5, externalAddress(scThreePtr)); // 0x1a01a01a, - // 0xbf2a01a0, - // 0x16c16c17, - // 0xbf56c16c - masm.mulsd(temp4, temp8); - masm.addpd(temp6, externalAddress(scOnePtr)); // 0x55555555, - // 0xbfc55555, - // 0x00000000, - // 0xbfe00000 - masm.mulpd(temp5, dest); - masm.movdqu(dest, temp3); - masm.addsd(temp3, temp9); - masm.mulpd(temp1, temp7); - masm.movdqu(temp7, temp4); - masm.addsd(temp4, temp3); - masm.addpd(temp6, temp5); - masm.subsd(temp9, temp3); - masm.subsd(temp3, temp4); - masm.addsd(temp1, new AMD64Address(gpr1, 16)); - masm.mulpd(temp6, temp2); - masm.addsd(temp9, dest); - masm.addsd(temp3, temp7); - masm.addsd(temp1, temp9); - masm.addsd(temp1, temp3); - masm.addsd(temp1, temp6); - masm.unpckhpd(temp6, temp6); - masm.movdqu(dest, temp4); - masm.addsd(temp1, temp6); - masm.addsd(dest, temp1); - masm.jmp(bb13); - - masm.bind(bb7); - masm.addl(gpr4, 64); - masm.movq(gpr8, gpr9); - masm.movq(gpr9, gpr7); - masm.movl(gpr7, 0); - masm.cmpq(gpr8, 0); - masm.jcc(ConditionFlag.NotEqual, bb8); - - masm.addl(gpr4, 64); - masm.movq(gpr8, gpr9); - masm.movq(gpr9, gpr7); - masm.cmpq(gpr8, 0); - masm.jcc(ConditionFlag.NotEqual, bb8); - - masm.xorpd(dest, dest); - masm.xorpd(temp6, temp6); - masm.jmp(bb11); - - masm.bind(bb9); - masm.jcc(ConditionFlag.Equal, bb10); - - masm.negl(gpr3); - masm.shrq(gpr9); - masm.movq(gpr1, gpr8); - masm.shrq(gpr8); - masm.subl(gpr4, gpr3); - masm.negl(gpr3); - masm.addl(gpr3, 64); - masm.shlq(gpr1); - masm.orq(gpr9, gpr1); - masm.jmp(bb10); - - masm.bind(bb3); - masm.negl(gpr3); - masm.shlq(gpr8, 32); - masm.orq(gpr8, gpr10); - masm.shlq(gpr8); - masm.movq(gpr6, gpr8); - masm.testl(gpr8, Integer.MIN_VALUE); - masm.jcc(ConditionFlag.NotEqual, bb12); - - masm.shrl(gpr8); - masm.movl(gpr2, 0); - masm.shrq(gpr6, 3); - masm.jmp(bb6); - - masm.bind(bb4); - masm.shrl(gpr8); - masm.movl(gpr2, 536870912); - masm.shrl(gpr2); - masm.shlq(gpr8, 32); - masm.orq(gpr8, gpr10); - masm.shlq(gpr2, 32); - masm.addl(gpr6, 536870912); - masm.movl(gpr3, 0); - masm.movl(gpr10, 0); - masm.subq(gpr3, gpr7); - masm.sbbq(gpr10, gpr9); - masm.sbbq(gpr2, gpr8); - masm.movq(gpr7, gpr3); - masm.movq(gpr9, gpr10); - masm.movq(gpr8, gpr2); - masm.movl(gpr2, 32768); - masm.jmp(bb5); - - masm.bind(bb12); - masm.shrl(gpr8); - masm.movq(gpr2, 0x100000000L); - masm.shrq(gpr2); - masm.movl(gpr3, 0); - masm.movl(gpr10, 0); - masm.subq(gpr3, gpr7); - masm.sbbq(gpr10, gpr9); - masm.sbbq(gpr2, gpr8); - masm.movq(gpr7, gpr3); - masm.movq(gpr9, gpr10); - masm.movq(gpr8, gpr2); - masm.movl(gpr2, 32768); - masm.shrq(gpr6, 3); - masm.addl(gpr6, 536870912); - masm.jmp(bb6); - - masm.bind(bb13); + temps = registersToValues(registers); } - /* - * Copyright (c) 2014, 2016, Intel Corporation. All rights reserved. Intel Math Library (LIBM) - * Source Code - * - * ALGORITHM DESCRIPTION - TAN() --------------------- - * - * Polynomials coefficients and other constants. - * - * Note that in this algorithm, there is a different polynomial for each breakpoint, so there - * are 32 sets of polynomial coefficients as well as 32 instances of the other constants. - * - * The polynomial coefficients and constants are offset from the start of the main block as - * follows: - * - * 0: c8 | c0 16: c9 | c1 32: c10 | c2 48: c11 | c3 64: c12 | c4 80: c13 | c5 96: c14 | c6 112: - * c15 | c7 128: T_hi 136: T_lo 144: Sigma 152: T_hl 160: Tau 168: Mask 176: (end of block) - * - * The total table size is therefore 5632 bytes. - * - * Note that c0 and c1 are always zero. We could try storing other constants here, and just - * loading the low part of the SIMD register in these cases, after ensuring the high part is - * zero. - * - * The higher terms of the polynomial are computed in the *low* part of the SIMD register. This - * is so we can overlap the multiplication by r^8 and the unpacking of the other part. - * - * The constants are: T_hi + T_lo = accurate constant term in power series Sigma + T_hl = - * accurate coefficient of r in power series (Sigma=1 bit) Tau = multiplier for the reciprocal, - * always -1 or 0 - * - * The basic reconstruction formula using these constants is: - * - * High = tau * recip_hi + t_hi Med = (sgn * r + t_hl * r)_hi Low = (sgn * r + t_hl * r)_lo + - * tau * recip_lo + T_lo + (T_hl + sigma) * c + pol - * - * where pol = c0 + c1 * r + c2 * r^2 + ... + c15 * r^15 - * - * (c0 = c1 = 0, but using them keeps SIMD regularity) - * - * We then do a compensated sum High + Med, add the low parts together and then do the final - * sum. - * - * Here recip_hi + recip_lo is an accurate reciprocal of the remainder modulo pi/2 - * - * Special cases: tan(NaN) = quiet NaN, and raise invalid exception tan(INF) = NaN and raise - * invalid exception tan(+/-0) = +/-0 - * - */ - - private static int[] oneHalfTan = { - 0x00000000, 0x3fe00000, 0x00000000, 0x3fe00000 - }; - - private static int[] mulSixteen = { - 0x00000000, 0x40300000, 0x00000000, 0x3ff00000 - }; - - private static int[] signMaskTan = { - 0x00000000, 0x80000000, 0x00000000, 0x80000000 - }; - - private static int[] piThirtyTwoInvTan = { - 0x6dc9c883, 0x3fe45f30, 0x6dc9c883, 0x40245f30 - }; - - private static int[] pOneTan = { - 0x54444000, 0x3fb921fb, 0x54440000, 0x3fb921fb - }; - - private static int[] pTwoTan = { - 0x67674000, 0xbd32e7b9, 0x4c4c0000, 0x3d468c23 - }; - - private static int[] pThreeTan = { - 0x3707344a, 0x3aa8a2e0, 0x03707345, 0x3ae98a2e - }; - - private static int[] cTableTan = { - 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x882c10fa, - 0x3f9664f4, 0x00000000, 0x00000000, 0x00000000, 0x00000000, - 0x00000000, 0x00000000, 0x55e6c23d, 0x3f8226e3, 0x55555555, - 0x3fd55555, 0x00000000, 0x00000000, 0x00000000, 0x00000000, - 0x0e157de0, 0x3f6d6d3d, 0x11111111, 0x3fc11111, 0x00000000, - 0x00000000, 0x00000000, 0x00000000, 0x452b75e3, 0x3f57da36, - 0x1ba1ba1c, 0x3faba1ba, 0x00000000, 0x00000000, 0x00000000, - 0x00000000, 0x00000000, 0x3ff00000, 0x00000000, 0x00000000, - 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x4e435f9b, - 0x3f953f83, 0x00000000, 0x00000000, 0x3c6e8e46, 0x3f9b74ea, - 0x00000000, 0x00000000, 0xda5b7511, 0x3f85ad63, 0xdc230b9b, - 0x3fb97558, 0x26cb3788, 0x3f881308, 0x76fc4985, 0x3fd62ac9, - 0x77bb08ba, 0x3f757c85, 0xb6247521, 0x3fb1381e, 0x5922170c, - 0x3f754e95, 0x8746482d, 0x3fc27f83, 0x11055b30, 0x3f64e391, - 0x3e666320, 0x3fa3e609, 0x0de9dae3, 0x3f6301df, 0x1f1dca06, - 0x3fafa8ae, 0x8c5b2da2, 0x3fb936bb, 0x4e88f7a5, 0x3c587d05, - 0x00000000, 0x3ff00000, 0xa8935dd9, 0x3f83dde2, 0x00000000, - 0x00000000, 0x00000000, 0x00000000, 0x5a279ea3, 0x3faa3407, - 0x00000000, 0x00000000, 0x432d65fa, 0x3fa70153, 0x00000000, - 0x00000000, 0x891a4602, 0x3f9d03ef, 0xd62ca5f8, 0x3fca77d9, - 0xb35f4628, 0x3f97a265, 0x433258fa, 0x3fd8cf51, 0xb58fd909, - 0x3f8f88e3, 0x01771cea, 0x3fc2b154, 0xf3562f8e, 0x3f888f57, - 0xc028a723, 0x3fc7370f, 0x20b7f9f0, 0x3f80f44c, 0x214368e9, - 0x3fb6dfaa, 0x28891863, 0x3f79b4b6, 0x172dbbf0, 0x3fb6cb8e, - 0xe0553158, 0x3fc975f5, 0x593fe814, 0x3c2ef5d3, 0x00000000, - 0x3ff00000, 0x03dec550, 0x3fa44203, 0x00000000, 0x00000000, - 0x00000000, 0x00000000, 0x9314533e, 0x3fbb8ec5, 0x00000000, - 0x00000000, 0x09aa36d0, 0x3fb6d3f4, 0x00000000, 0x00000000, - 0xdcb427fd, 0x3fb13950, 0xd87ab0bb, 0x3fd5335e, 0xce0ae8a5, - 0x3fabb382, 0x79143126, 0x3fddba41, 0x5f2b28d4, 0x3fa552f1, - 0x59f21a6d, 0x3fd015ab, 0x22c27d95, 0x3fa0e984, 0xe19fc6aa, - 0x3fd0576c, 0x8f2c2950, 0x3f9a4898, 0xc0b3f22c, 0x3fc59462, - 0x1883a4b8, 0x3f94b61c, 0x3f838640, 0x3fc30eb8, 0x355c63dc, - 0x3fd36a08, 0x1dce993d, 0xbc6d704d, 0x00000000, 0x3ff00000, - 0x2b82ab63, 0x3fb78e92, 0x00000000, 0x00000000, 0x00000000, - 0x00000000, 0x56f37042, 0x3fccfc56, 0x00000000, 0x00000000, - 0xaa563951, 0x3fc90125, 0x00000000, 0x00000000, 0x3d0e7c5d, - 0x3fc50533, 0x9bed9b2e, 0x3fdf0ed9, 0x5fe7c47c, 0x3fc1f250, - 0x96c125e5, 0x3fe2edd9, 0x5a02bbd8, 0x3fbe5c71, 0x86362c20, - 0x3fda08b7, 0x4b4435ed, 0x3fb9d342, 0x4b494091, 0x3fd911bd, - 0xb56658be, 0x3fb5e4c7, 0x93a2fd76, 0x3fd3c092, 0xda271794, - 0x3fb29910, 0x3303df2b, 0x3fd189be, 0x99fcef32, 0x3fda8279, - 0xb68c1467, 0x3c708b2f, 0x00000000, 0x3ff00000, 0x980c4337, - 0x3fc5f619, 0x00000000, 0x00000000, 0x00000000, 0x00000000, - 0xcc03e501, 0x3fdff10f, 0x00000000, 0x00000000, 0x44a4e845, - 0x3fddb63b, 0x00000000, 0x00000000, 0x3768ad9f, 0x3fdb72a4, - 0x3dd01cca, 0x3fe5fdb9, 0xa61d2811, 0x3fd972b2, 0x5645ad0b, - 0x3fe977f9, 0xd013b3ab, 0x3fd78ca3, 0xbf0bf914, 0x3fe4f192, - 0x4d53e730, 0x3fd5d060, 0x3f8b9000, 0x3fe49933, 0xe2b82f08, - 0x3fd4322a, 0x5936a835, 0x3fe27ae1, 0xb1c61c9b, 0x3fd2b3fb, - 0xef478605, 0x3fe1659e, 0x190834ec, 0x3fe11ab7, 0xcdb625ea, - 0xbc8e564b, 0x00000000, 0x3ff00000, 0xb07217e3, 0x3fd248f1, - 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x2b2c49d0, - 0x3ff2de9c, 0x00000000, 0x00000000, 0x2655bc98, 0x3ff33e58, - 0x00000000, 0x00000000, 0xff691fa2, 0x3ff3972e, 0xe93463bd, - 0x3feeed87, 0x070e10a0, 0x3ff3f5b2, 0xf4d790a4, 0x3ff20c10, - 0xa04e8ea3, 0x3ff4541a, 0x386accd3, 0x3ff1369e, 0x222a66dd, - 0x3ff4b521, 0x22a9777e, 0x3ff20817, 0x52a04a6e, 0x3ff5178f, - 0xddaa0031, 0x3ff22137, 0x4447d47c, 0x3ff57c01, 0x1e9c7f1d, - 0x3ff29311, 0x2ab7f990, 0x3fe561b8, 0x209c7df1, 0x3c87a8c5, - 0x00000000, 0x3ff00000, 0x4170bcc6, 0x3fdc92d8, 0x00000000, - 0x00000000, 0x00000000, 0x00000000, 0xc7ab4d5a, 0x40085e24, - 0x00000000, 0x00000000, 0xe93ea75d, 0x400b963d, 0x00000000, - 0x00000000, 0x94a7f25a, 0x400f37e2, 0x4b6261cb, 0x3ff5f984, - 0x5a9dd812, 0x4011aab0, 0x74c30018, 0x3ffaf5a5, 0x7f2ce8e3, - 0x4013fe8b, 0xfe8e54fa, 0x3ffd7334, 0x670d618d, 0x4016a10c, - 0x4db97058, 0x4000e012, 0x24df44dd, 0x40199c5f, 0x697d6ece, - 0x4003006e, 0x83298b82, 0x401cfc4d, 0x19d490d6, 0x40058c19, - 0x2ae42850, 0x3fea4300, 0x118e20e6, 0xbc7a6db8, 0x00000000, - 0x40000000, 0xe33345b8, 0xbfd4e526, 0x00000000, 0x00000000, - 0x00000000, 0x00000000, 0x65965966, 0x40219659, 0x00000000, - 0x00000000, 0x882c10fa, 0x402664f4, 0x00000000, 0x00000000, - 0x83cd3723, 0x402c8342, 0x00000000, 0x40000000, 0x55e6c23d, - 0x403226e3, 0x55555555, 0x40055555, 0x34451939, 0x40371c96, - 0xaaaaaaab, 0x400aaaaa, 0x0e157de0, 0x403d6d3d, 0x11111111, - 0x40111111, 0xa738201f, 0x4042bbce, 0x05b05b06, 0x4015b05b, - 0x452b75e3, 0x4047da36, 0x1ba1ba1c, 0x401ba1ba, 0x00000000, - 0x3ff00000, 0x00000000, 0x00000000, 0x00000000, 0x40000000, - 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, - 0x00000000, 0x4f48b8d3, 0xbf33eaf9, 0x00000000, 0x00000000, - 0x0cf7586f, 0x3f20b8ea, 0x00000000, 0x00000000, 0xd0258911, - 0xbf0abaf3, 0x23e49fe9, 0xbfab5a8c, 0x2d53222e, 0x3ef60d15, - 0x21169451, 0x3fa172b2, 0xbb254dbc, 0xbee1d3b5, 0xdbf93b8e, - 0xbf84c7db, 0x05b4630b, 0x3ecd3364, 0xee9aada7, 0x3f743924, - 0x794a8297, 0xbeb7b7b9, 0xe015f797, 0xbf5d41f5, 0xe41a4a56, - 0x3ea35dfb, 0xe4c2a251, 0x3f49a2ab, 0x5af9e000, 0xbfce49ce, - 0x8c743719, 0x3d1eb860, 0x00000000, 0x00000000, 0x1b4863cf, - 0x3fd78294, 0x00000000, 0x3ff00000, 0x00000000, 0xfffffff8, - 0x535ad890, 0xbf2b9320, 0x00000000, 0x00000000, 0x018fdf1f, - 0x3f16d61d, 0x00000000, 0x00000000, 0x0359f1be, 0xbf0139e4, - 0xa4317c6d, 0xbfa67e17, 0x82672d0f, 0x3eebb405, 0x2f1b621e, - 0x3f9f455b, 0x51ccf238, 0xbed55317, 0xf437b9ac, 0xbf804bee, - 0xc791a2b5, 0x3ec0e993, 0x919a1db2, 0x3f7080c2, 0x336a5b0e, - 0xbeaa48a2, 0x0a268358, 0xbf55a443, 0xdfd978e4, 0x3e94b61f, - 0xd7767a58, 0x3f431806, 0x2aea0000, 0xbfc9bbe8, 0x7723ea61, - 0xbd3a2369, 0x00000000, 0x00000000, 0xdf7796ff, 0x3fd6e642, - 0x00000000, 0x3ff00000, 0x00000000, 0xfffffff8, 0xb9ff07ce, - 0xbf231c78, 0x00000000, 0x00000000, 0xa5517182, 0x3f0ff0e0, - 0x00000000, 0x00000000, 0x790b4cbc, 0xbef66191, 0x848a46c6, - 0xbfa21ac0, 0xb16435fa, 0x3ee1d3ec, 0x2a1aa832, 0x3f9c71ea, - 0xfdd299ef, 0xbec9dd1a, 0x3f8dbaaf, 0xbf793363, 0x309fc6ea, - 0x3eb415d6, 0xbee60471, 0x3f6b83ba, 0x94a0a697, 0xbe9dae11, - 0x3e5c67b3, 0xbf4fd07b, 0x9a8f3e3e, 0x3e86bd75, 0xa4beb7a4, - 0x3f3d1eb1, 0x29cfc000, 0xbfc549ce, 0xbf159358, 0xbd397b33, - 0x00000000, 0x00000000, 0x871fee6c, 0x3fd666f0, 0x00000000, - 0x3ff00000, 0x00000000, 0xfffffff8, 0x7d98a556, 0xbf1a3958, - 0x00000000, 0x00000000, 0x9d88dc01, 0x3f0704c2, 0x00000000, - 0x00000000, 0x73742a2b, 0xbeed054a, 0x58844587, 0xbf9c2a13, - 0x55688a79, 0x3ed7a326, 0xee33f1d6, 0x3f9a48f4, 0xa8dc9888, - 0xbebf8939, 0xaad4b5b8, 0xbf72f746, 0x9102efa1, 0x3ea88f82, - 0xdabc29cf, 0x3f678228, 0x9289afb8, 0xbe90f456, 0x741fb4ed, - 0xbf46f3a3, 0xa97f6663, 0x3e79b4bf, 0xca89ff3f, 0x3f36db70, - 0xa8a2a000, 0xbfc0ee13, 0x3da24be1, 0xbd338b9f, 0x00000000, - 0x00000000, 0x11cd6c69, 0x3fd601fd, 0x00000000, 0x3ff00000, - 0x00000000, 0xfffffff8, 0x1a154b97, 0xbf116b01, 0x00000000, - 0x00000000, 0x2d427630, 0x3f0147bf, 0x00000000, 0x00000000, - 0xb93820c8, 0xbee264d4, 0xbb6cbb18, 0xbf94ab8c, 0x888d4d92, - 0x3ed0568b, 0x60730f7c, 0x3f98b19b, 0xe4b1fb11, 0xbeb2f950, - 0x22cf9f74, 0xbf6b21cd, 0x4a3ff0a6, 0x3e9f499e, 0xfd2b83ce, - 0x3f64aad7, 0x637b73af, 0xbe83487c, 0xe522591a, 0xbf3fc092, - 0xa158e8bc, 0x3e6e3aae, 0xe5e82ffa, 0x3f329d2f, 0xd636a000, - 0xbfb9477f, 0xc2c2d2bc, 0xbd135ef9, 0x00000000, 0x00000000, - 0xf2fdb123, 0x3fd5b566, 0x00000000, 0x3ff00000, 0x00000000, - 0xfffffff8, 0xc41acb64, 0xbf05448d, 0x00000000, 0x00000000, - 0xdbb03d6f, 0x3efb7ad2, 0x00000000, 0x00000000, 0x9e42962d, - 0xbed5aea5, 0x2579f8ef, 0xbf8b2398, 0x288a1ed9, 0x3ec81441, - 0xb0198dc5, 0x3f979a3a, 0x2fdfe253, 0xbea57cd3, 0x5766336f, - 0xbf617caa, 0x600944c3, 0x3e954ed6, 0xa4e0aaf8, 0x3f62c646, - 0x6b8fb29c, 0xbe74e3a3, 0xdc4c0409, 0xbf33f952, 0x9bffe365, - 0x3e6301ec, 0xb8869e44, 0x3f2fc566, 0xe1e04000, 0xbfb0cc62, - 0x016b907f, 0xbd119cbc, 0x00000000, 0x00000000, 0xe6b9d8fa, - 0x3fd57fb3, 0x00000000, 0x3ff00000, 0x00000000, 0xfffffff8, - 0x5daf22a6, 0xbef429d7, 0x00000000, 0x00000000, 0x06bca545, - 0x3ef7a27d, 0x00000000, 0x00000000, 0x7211c19a, 0xbec41c3e, - 0x956ed53e, 0xbf7ae3f4, 0xee750e72, 0x3ec3901b, 0x91d443f5, - 0x3f96f713, 0x36661e6c, 0xbe936e09, 0x506f9381, 0xbf5122e8, - 0xcb6dd43f, 0x3e9041b9, 0x6698b2ff, 0x3f61b0c7, 0x576bf12b, - 0xbe625a8a, 0xe5a0e9dc, 0xbf23499d, 0x110384dd, 0x3e5b1c2c, - 0x68d43db6, 0x3f2cb899, 0x6ecac000, 0xbfa0c414, 0xcd7dd58c, - 0x3d13500f, 0x00000000, 0x00000000, 0x85a2c8fb, 0x3fd55fe0, - 0x00000000, 0x3ff00000, 0x00000000, 0xfffffff8, 0x00000000, - 0x00000000, 0x00000000, 0x00000000, 0x2bf70ebe, 0x3ef66a8f, - 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, - 0x00000000, 0xd644267f, 0x3ec22805, 0x16c16c17, 0x3f96c16c, - 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0xc4e09162, - 0x3e8d6db2, 0xbc011567, 0x3f61566a, 0x00000000, 0x00000000, - 0x00000000, 0x00000000, 0x1f79955c, 0x3e57da4e, 0x9334ef0b, - 0x3f2bbd77, 0x00000000, 0x00000000, 0x00000000, 0x00000000, - 0x00000000, 0x00000000, 0x55555555, 0x3fd55555, 0x00000000, - 0x3ff00000, 0x00000000, 0xfffffff8, 0x5daf22a6, 0x3ef429d7, - 0x00000000, 0x00000000, 0x06bca545, 0x3ef7a27d, 0x00000000, - 0x00000000, 0x7211c19a, 0x3ec41c3e, 0x956ed53e, 0x3f7ae3f4, - 0xee750e72, 0x3ec3901b, 0x91d443f5, 0x3f96f713, 0x36661e6c, - 0x3e936e09, 0x506f9381, 0x3f5122e8, 0xcb6dd43f, 0x3e9041b9, - 0x6698b2ff, 0x3f61b0c7, 0x576bf12b, 0x3e625a8a, 0xe5a0e9dc, - 0x3f23499d, 0x110384dd, 0x3e5b1c2c, 0x68d43db6, 0x3f2cb899, - 0x6ecac000, 0x3fa0c414, 0xcd7dd58c, 0xbd13500f, 0x00000000, - 0x00000000, 0x85a2c8fb, 0x3fd55fe0, 0x00000000, 0x3ff00000, - 0x00000000, 0xfffffff8, 0xc41acb64, 0x3f05448d, 0x00000000, - 0x00000000, 0xdbb03d6f, 0x3efb7ad2, 0x00000000, 0x00000000, - 0x9e42962d, 0x3ed5aea5, 0x2579f8ef, 0x3f8b2398, 0x288a1ed9, - 0x3ec81441, 0xb0198dc5, 0x3f979a3a, 0x2fdfe253, 0x3ea57cd3, - 0x5766336f, 0x3f617caa, 0x600944c3, 0x3e954ed6, 0xa4e0aaf8, - 0x3f62c646, 0x6b8fb29c, 0x3e74e3a3, 0xdc4c0409, 0x3f33f952, - 0x9bffe365, 0x3e6301ec, 0xb8869e44, 0x3f2fc566, 0xe1e04000, - 0x3fb0cc62, 0x016b907f, 0x3d119cbc, 0x00000000, 0x00000000, - 0xe6b9d8fa, 0x3fd57fb3, 0x00000000, 0x3ff00000, 0x00000000, - 0xfffffff8, 0x1a154b97, 0x3f116b01, 0x00000000, 0x00000000, - 0x2d427630, 0x3f0147bf, 0x00000000, 0x00000000, 0xb93820c8, - 0x3ee264d4, 0xbb6cbb18, 0x3f94ab8c, 0x888d4d92, 0x3ed0568b, - 0x60730f7c, 0x3f98b19b, 0xe4b1fb11, 0x3eb2f950, 0x22cf9f74, - 0x3f6b21cd, 0x4a3ff0a6, 0x3e9f499e, 0xfd2b83ce, 0x3f64aad7, - 0x637b73af, 0x3e83487c, 0xe522591a, 0x3f3fc092, 0xa158e8bc, - 0x3e6e3aae, 0xe5e82ffa, 0x3f329d2f, 0xd636a000, 0x3fb9477f, - 0xc2c2d2bc, 0x3d135ef9, 0x00000000, 0x00000000, 0xf2fdb123, - 0x3fd5b566, 0x00000000, 0x3ff00000, 0x00000000, 0xfffffff8, - 0x7d98a556, 0x3f1a3958, 0x00000000, 0x00000000, 0x9d88dc01, - 0x3f0704c2, 0x00000000, 0x00000000, 0x73742a2b, 0x3eed054a, - 0x58844587, 0x3f9c2a13, 0x55688a79, 0x3ed7a326, 0xee33f1d6, - 0x3f9a48f4, 0xa8dc9888, 0x3ebf8939, 0xaad4b5b8, 0x3f72f746, - 0x9102efa1, 0x3ea88f82, 0xdabc29cf, 0x3f678228, 0x9289afb8, - 0x3e90f456, 0x741fb4ed, 0x3f46f3a3, 0xa97f6663, 0x3e79b4bf, - 0xca89ff3f, 0x3f36db70, 0xa8a2a000, 0x3fc0ee13, 0x3da24be1, - 0x3d338b9f, 0x00000000, 0x00000000, 0x11cd6c69, 0x3fd601fd, - 0x00000000, 0x3ff00000, 0x00000000, 0xfffffff8, 0xb9ff07ce, - 0x3f231c78, 0x00000000, 0x00000000, 0xa5517182, 0x3f0ff0e0, - 0x00000000, 0x00000000, 0x790b4cbc, 0x3ef66191, 0x848a46c6, - 0x3fa21ac0, 0xb16435fa, 0x3ee1d3ec, 0x2a1aa832, 0x3f9c71ea, - 0xfdd299ef, 0x3ec9dd1a, 0x3f8dbaaf, 0x3f793363, 0x309fc6ea, - 0x3eb415d6, 0xbee60471, 0x3f6b83ba, 0x94a0a697, 0x3e9dae11, - 0x3e5c67b3, 0x3f4fd07b, 0x9a8f3e3e, 0x3e86bd75, 0xa4beb7a4, - 0x3f3d1eb1, 0x29cfc000, 0x3fc549ce, 0xbf159358, 0x3d397b33, - 0x00000000, 0x00000000, 0x871fee6c, 0x3fd666f0, 0x00000000, - 0x3ff00000, 0x00000000, 0xfffffff8, 0x535ad890, 0x3f2b9320, - 0x00000000, 0x00000000, 0x018fdf1f, 0x3f16d61d, 0x00000000, - 0x00000000, 0x0359f1be, 0x3f0139e4, 0xa4317c6d, 0x3fa67e17, - 0x82672d0f, 0x3eebb405, 0x2f1b621e, 0x3f9f455b, 0x51ccf238, - 0x3ed55317, 0xf437b9ac, 0x3f804bee, 0xc791a2b5, 0x3ec0e993, - 0x919a1db2, 0x3f7080c2, 0x336a5b0e, 0x3eaa48a2, 0x0a268358, - 0x3f55a443, 0xdfd978e4, 0x3e94b61f, 0xd7767a58, 0x3f431806, - 0x2aea0000, 0x3fc9bbe8, 0x7723ea61, 0x3d3a2369, 0x00000000, - 0x00000000, 0xdf7796ff, 0x3fd6e642, 0x00000000, 0x3ff00000, - 0x00000000, 0xfffffff8, 0x4f48b8d3, 0x3f33eaf9, 0x00000000, - 0x00000000, 0x0cf7586f, 0x3f20b8ea, 0x00000000, 0x00000000, - 0xd0258911, 0x3f0abaf3, 0x23e49fe9, 0x3fab5a8c, 0x2d53222e, - 0x3ef60d15, 0x21169451, 0x3fa172b2, 0xbb254dbc, 0x3ee1d3b5, - 0xdbf93b8e, 0x3f84c7db, 0x05b4630b, 0x3ecd3364, 0xee9aada7, - 0x3f743924, 0x794a8297, 0x3eb7b7b9, 0xe015f797, 0x3f5d41f5, - 0xe41a4a56, 0x3ea35dfb, 0xe4c2a251, 0x3f49a2ab, 0x5af9e000, - 0x3fce49ce, 0x8c743719, 0xbd1eb860, 0x00000000, 0x00000000, - 0x1b4863cf, 0x3fd78294, 0x00000000, 0x3ff00000, 0x00000000, - 0xfffffff8, 0x65965966, 0xc0219659, 0x00000000, 0x00000000, - 0x882c10fa, 0x402664f4, 0x00000000, 0x00000000, 0x83cd3723, - 0xc02c8342, 0x00000000, 0xc0000000, 0x55e6c23d, 0x403226e3, - 0x55555555, 0x40055555, 0x34451939, 0xc0371c96, 0xaaaaaaab, - 0xc00aaaaa, 0x0e157de0, 0x403d6d3d, 0x11111111, 0x40111111, - 0xa738201f, 0xc042bbce, 0x05b05b06, 0xc015b05b, 0x452b75e3, - 0x4047da36, 0x1ba1ba1c, 0x401ba1ba, 0x00000000, 0xbff00000, - 0x00000000, 0x00000000, 0x00000000, 0x40000000, 0x00000000, - 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, - 0xc7ab4d5a, 0xc0085e24, 0x00000000, 0x00000000, 0xe93ea75d, - 0x400b963d, 0x00000000, 0x00000000, 0x94a7f25a, 0xc00f37e2, - 0x4b6261cb, 0xbff5f984, 0x5a9dd812, 0x4011aab0, 0x74c30018, - 0x3ffaf5a5, 0x7f2ce8e3, 0xc013fe8b, 0xfe8e54fa, 0xbffd7334, - 0x670d618d, 0x4016a10c, 0x4db97058, 0x4000e012, 0x24df44dd, - 0xc0199c5f, 0x697d6ece, 0xc003006e, 0x83298b82, 0x401cfc4d, - 0x19d490d6, 0x40058c19, 0x2ae42850, 0xbfea4300, 0x118e20e6, - 0x3c7a6db8, 0x00000000, 0x40000000, 0xe33345b8, 0xbfd4e526, - 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x2b2c49d0, - 0xbff2de9c, 0x00000000, 0x00000000, 0x2655bc98, 0x3ff33e58, - 0x00000000, 0x00000000, 0xff691fa2, 0xbff3972e, 0xe93463bd, - 0xbfeeed87, 0x070e10a0, 0x3ff3f5b2, 0xf4d790a4, 0x3ff20c10, - 0xa04e8ea3, 0xbff4541a, 0x386accd3, 0xbff1369e, 0x222a66dd, - 0x3ff4b521, 0x22a9777e, 0x3ff20817, 0x52a04a6e, 0xbff5178f, - 0xddaa0031, 0xbff22137, 0x4447d47c, 0x3ff57c01, 0x1e9c7f1d, - 0x3ff29311, 0x2ab7f990, 0xbfe561b8, 0x209c7df1, 0xbc87a8c5, - 0x00000000, 0x3ff00000, 0x4170bcc6, 0x3fdc92d8, 0x00000000, - 0x00000000, 0x00000000, 0x00000000, 0xcc03e501, 0xbfdff10f, - 0x00000000, 0x00000000, 0x44a4e845, 0x3fddb63b, 0x00000000, - 0x00000000, 0x3768ad9f, 0xbfdb72a4, 0x3dd01cca, 0xbfe5fdb9, - 0xa61d2811, 0x3fd972b2, 0x5645ad0b, 0x3fe977f9, 0xd013b3ab, - 0xbfd78ca3, 0xbf0bf914, 0xbfe4f192, 0x4d53e730, 0x3fd5d060, - 0x3f8b9000, 0x3fe49933, 0xe2b82f08, 0xbfd4322a, 0x5936a835, - 0xbfe27ae1, 0xb1c61c9b, 0x3fd2b3fb, 0xef478605, 0x3fe1659e, - 0x190834ec, 0xbfe11ab7, 0xcdb625ea, 0x3c8e564b, 0x00000000, - 0x3ff00000, 0xb07217e3, 0x3fd248f1, 0x00000000, 0x00000000, - 0x00000000, 0x00000000, 0x56f37042, 0xbfccfc56, 0x00000000, - 0x00000000, 0xaa563951, 0x3fc90125, 0x00000000, 0x00000000, - 0x3d0e7c5d, 0xbfc50533, 0x9bed9b2e, 0xbfdf0ed9, 0x5fe7c47c, - 0x3fc1f250, 0x96c125e5, 0x3fe2edd9, 0x5a02bbd8, 0xbfbe5c71, - 0x86362c20, 0xbfda08b7, 0x4b4435ed, 0x3fb9d342, 0x4b494091, - 0x3fd911bd, 0xb56658be, 0xbfb5e4c7, 0x93a2fd76, 0xbfd3c092, - 0xda271794, 0x3fb29910, 0x3303df2b, 0x3fd189be, 0x99fcef32, - 0xbfda8279, 0xb68c1467, 0xbc708b2f, 0x00000000, 0x3ff00000, - 0x980c4337, 0x3fc5f619, 0x00000000, 0x00000000, 0x00000000, - 0x00000000, 0x9314533e, 0xbfbb8ec5, 0x00000000, 0x00000000, - 0x09aa36d0, 0x3fb6d3f4, 0x00000000, 0x00000000, 0xdcb427fd, - 0xbfb13950, 0xd87ab0bb, 0xbfd5335e, 0xce0ae8a5, 0x3fabb382, - 0x79143126, 0x3fddba41, 0x5f2b28d4, 0xbfa552f1, 0x59f21a6d, - 0xbfd015ab, 0x22c27d95, 0x3fa0e984, 0xe19fc6aa, 0x3fd0576c, - 0x8f2c2950, 0xbf9a4898, 0xc0b3f22c, 0xbfc59462, 0x1883a4b8, - 0x3f94b61c, 0x3f838640, 0x3fc30eb8, 0x355c63dc, 0xbfd36a08, - 0x1dce993d, 0x3c6d704d, 0x00000000, 0x3ff00000, 0x2b82ab63, - 0x3fb78e92, 0x00000000, 0x00000000, 0x00000000, 0x00000000, - 0x5a279ea3, 0xbfaa3407, 0x00000000, 0x00000000, 0x432d65fa, - 0x3fa70153, 0x00000000, 0x00000000, 0x891a4602, 0xbf9d03ef, - 0xd62ca5f8, 0xbfca77d9, 0xb35f4628, 0x3f97a265, 0x433258fa, - 0x3fd8cf51, 0xb58fd909, 0xbf8f88e3, 0x01771cea, 0xbfc2b154, - 0xf3562f8e, 0x3f888f57, 0xc028a723, 0x3fc7370f, 0x20b7f9f0, - 0xbf80f44c, 0x214368e9, 0xbfb6dfaa, 0x28891863, 0x3f79b4b6, - 0x172dbbf0, 0x3fb6cb8e, 0xe0553158, 0xbfc975f5, 0x593fe814, - 0xbc2ef5d3, 0x00000000, 0x3ff00000, 0x03dec550, 0x3fa44203, - 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x4e435f9b, - 0xbf953f83, 0x00000000, 0x00000000, 0x3c6e8e46, 0x3f9b74ea, - 0x00000000, 0x00000000, 0xda5b7511, 0xbf85ad63, 0xdc230b9b, - 0xbfb97558, 0x26cb3788, 0x3f881308, 0x76fc4985, 0x3fd62ac9, - 0x77bb08ba, 0xbf757c85, 0xb6247521, 0xbfb1381e, 0x5922170c, - 0x3f754e95, 0x8746482d, 0x3fc27f83, 0x11055b30, 0xbf64e391, - 0x3e666320, 0xbfa3e609, 0x0de9dae3, 0x3f6301df, 0x1f1dca06, - 0x3fafa8ae, 0x8c5b2da2, 0xbfb936bb, 0x4e88f7a5, 0xbc587d05, - 0x00000000, 0x3ff00000, 0xa8935dd9, 0x3f83dde2, 0x00000000, - 0x00000000, 0x00000000, 0x00000000 - }; - - private static int[] maskThirtyFiveTan = { - 0xfffc0000, 0xffffffff, 0x00000000, 0x00000000 - }; - - private static int[] qElevenTan = { - 0xb8fe4d77, 0x3f82609a - }; - - private static int[] qNineTan = { - 0xbf847a43, 0x3f9664a0 - }; - - private static int[] qSevenTan = { - 0x52c4c8ab, 0x3faba1ba - }; - - private static int[] qFiveTan = { - 0x11092746, 0x3fc11111 - }; - - private static int[] qThreeTan = { - 0x55555612, 0x3fd55555 - }; - - private static int[] piInvTableTan = { - 0x00000000, 0x00000000, 0xa2f9836e, 0x4e441529, 0xfc2757d1, - 0xf534ddc0, 0xdb629599, 0x3c439041, 0xfe5163ab, 0xdebbc561, - 0xb7246e3a, 0x424dd2e0, 0x06492eea, 0x09d1921c, 0xfe1deb1c, - 0xb129a73e, 0xe88235f5, 0x2ebb4484, 0xe99c7026, 0xb45f7e41, - 0x3991d639, 0x835339f4, 0x9c845f8b, 0xbdf9283b, 0x1ff897ff, - 0xde05980f, 0xef2f118b, 0x5a0a6d1f, 0x6d367ecf, 0x27cb09b7, - 0x4f463f66, 0x9e5fea2d, 0x7527bac7, 0xebe5f17b, 0x3d0739f7, - 0x8a5292ea, 0x6bfb5fb1, 0x1f8d5d08, 0x56033046, 0xfc7b6bab, - 0xf0cfbc21 - }; - - private static int[] piFourTan = { - 0x00000000, 0x3fe921fb, 0x4611a626, 0x3e85110b - }; - - private static int[] qqTwoTan = { - 0x676733af, 0x3d32e7b9 - }; - - private static int[] twoPowFiftyFiveTan = { - 0x00000000, 0x43600000 - }; - - private static int[] twoPowMFiftyFiveTan = { - 0x00000000, 0x3c800000 - }; - - public void tanIntrinsic(Register dest, Register value, CompilationResultBuilder crb, AMD64MacroAssembler masm) { - ArrayDataPointerConstant oneHalfTanPtr = new ArrayDataPointerConstant(oneHalfTan, 16); - ArrayDataPointerConstant mulSixteenPtr = new ArrayDataPointerConstant(mulSixteen, 16); - ArrayDataPointerConstant signMaskTanPtr = new ArrayDataPointerConstant(signMaskTan, 16); - ArrayDataPointerConstant piThirtyTwoInvTanPtr = new ArrayDataPointerConstant(piThirtyTwoInvTan, 16); - ArrayDataPointerConstant pOneTanPtr = new ArrayDataPointerConstant(pOneTan, 16); - ArrayDataPointerConstant pTwoTanPtr = new ArrayDataPointerConstant(pTwoTan, 16); - ArrayDataPointerConstant pThreeTanPtr = new ArrayDataPointerConstant(pThreeTan, 16); - ArrayDataPointerConstant cTableTanPtr = new ArrayDataPointerConstant(cTableTan, 16); - ArrayDataPointerConstant maskThirtyFiveTanPtr = new ArrayDataPointerConstant(maskThirtyFiveTan, 16); - ArrayDataPointerConstant qElevenTanPtr = new ArrayDataPointerConstant(qElevenTan, 16); - ArrayDataPointerConstant qNineTanPtr = new ArrayDataPointerConstant(qNineTan, 16); - ArrayDataPointerConstant qSevenTanPtr = new ArrayDataPointerConstant(qSevenTan, 8); - ArrayDataPointerConstant qFiveTanPtr = new ArrayDataPointerConstant(qFiveTan, 16); - ArrayDataPointerConstant qThreeTanPtr = new ArrayDataPointerConstant(qThreeTan, 16); - ArrayDataPointerConstant piInvTableTanPtr = new ArrayDataPointerConstant(piInvTableTan, 16); - ArrayDataPointerConstant piFourTanPtr = new ArrayDataPointerConstant(piFourTan, 8); - ArrayDataPointerConstant qqTwoTanPtr = new ArrayDataPointerConstant(qqTwoTan, 8); - ArrayDataPointerConstant onePtr = new ArrayDataPointerConstant(one, 8); - ArrayDataPointerConstant twoPowFiftyFiveTanPtr = new ArrayDataPointerConstant(twoPowFiftyFiveTan, 8); - ArrayDataPointerConstant twoPowMFiftyFiveTanPtr = new ArrayDataPointerConstant(twoPowMFiftyFiveTan, 8); - - Label bb0 = new Label(); - Label bb1 = new Label(); - Label bb2 = new Label(); - Label bb3 = new Label(); - Label bb5 = new Label(); - Label bb6 = new Label(); - Label bb8 = new Label(); - Label bb9 = new Label(); - Label bb10 = new Label(); - Label bb11 = new Label(); - Label bb12 = new Label(); - Label bb13 = new Label(); - Label bb14 = new Label(); - Label bb15 = new Label(); - - Register gpr1 = asRegister(gpr1Temp, AMD64Kind.QWORD); - Register gpr2 = asRegister(gpr2Temp, AMD64Kind.QWORD); - Register gpr3 = asRegister(rcxTemp, AMD64Kind.QWORD); - Register gpr4 = asRegister(gpr4Temp, AMD64Kind.QWORD); - Register gpr5 = asRegister(gpr5Temp, AMD64Kind.QWORD); - Register gpr6 = asRegister(gpr6Temp, AMD64Kind.QWORD); - Register gpr7 = asRegister(gpr7Temp, AMD64Kind.QWORD); - Register gpr8 = asRegister(gpr8Temp, AMD64Kind.QWORD); - Register gpr9 = asRegister(gpr9Temp, AMD64Kind.QWORD); - Register gpr10 = asRegister(gpr10Temp, AMD64Kind.QWORD); - - Register temp1 = asRegister(xmm1Temp, AMD64Kind.DOUBLE); - Register temp2 = asRegister(xmm2Temp, AMD64Kind.DOUBLE); - Register temp3 = asRegister(xmm3Temp, AMD64Kind.DOUBLE); - Register temp4 = asRegister(xmm4Temp, AMD64Kind.DOUBLE); - Register temp5 = asRegister(xmm5Temp, AMD64Kind.DOUBLE); - Register temp6 = asRegister(xmm6Temp, AMD64Kind.DOUBLE); - Register temp7 = asRegister(xmm7Temp, AMD64Kind.DOUBLE); - - setCrb(crb); - if (dest.encoding != value.encoding) { - masm.movdqu(dest, value); - } - - masm.pextrw(gpr1, dest, 3); - masm.andl(gpr1, 32767); - masm.subl(gpr1, 16314); - masm.cmpl(gpr1, 270); - masm.jcc(ConditionFlag.Above, bb0); - - masm.movdqu(temp5, externalAddress(oneHalfTanPtr)); // 0x00000000, - // 0x3fe00000, - // 0x00000000, - // 0x3fe00000 - masm.movdqu(temp6, externalAddress(mulSixteenPtr)); // 0x00000000, - // 0x40300000, - // 0x00000000, - // 0x3ff00000 - masm.unpcklpd(dest, dest); - masm.movdqu(temp4, externalAddress(signMaskTanPtr)); // 0x00000000, - // 0x80000000, - // 0x00000000, - // 0x80000000 - masm.andpd(temp4, dest); - masm.movdqu(temp1, externalAddress(piThirtyTwoInvTanPtr)); // 0x6dc9c883, - // 0x3fe45f30, - // 0x6dc9c883, - // 0x40245f30 - masm.mulpd(temp1, dest); - masm.por(temp5, temp4); - masm.addpd(temp1, temp5); - masm.movdqu(temp7, temp1); - masm.unpckhpd(temp7, temp7); - masm.cvttsd2sil(gpr4, temp7); - masm.cvttpd2dq(temp1, temp1); - masm.cvtdq2pd(temp1, temp1); - masm.mulpd(temp1, temp6); - masm.movdqu(temp3, externalAddress(pOneTanPtr)); // 0x54444000, - // 0x3fb921fb, - // 0x54440000, - // 0x3fb921fb - masm.movdq(temp5, externalAddress(qqTwoTanPtr)); // 0x676733af, - // 0x3d32e7b9 - masm.addq(gpr4, 469248); - masm.movdqu(temp4, externalAddress(pTwoTanPtr)); // 0x67674000, - // 0xbd32e7b9, - // 0x4c4c0000, - // 0x3d468c23 - masm.mulpd(temp3, temp1); - masm.andq(gpr4, 31); - masm.mulsd(temp5, temp1); - masm.movq(gpr3, gpr4); - masm.mulpd(temp4, temp1); - masm.shlq(gpr3, 1); - masm.subpd(dest, temp3); - masm.mulpd(temp1, externalAddress(pThreeTanPtr)); // 0x3707344a, - // 0x3aa8a2e0, - // 0x03707345, - // 0x3ae98a2e - masm.addq(gpr4, gpr3); - masm.shlq(gpr3, 2); - masm.addq(gpr4, gpr3); - masm.addsd(temp5, dest); - masm.movdqu(temp2, dest); - masm.subpd(dest, temp4); - masm.movdq(temp6, externalAddress(onePtr)); // 0x00000000, - // 0x3ff00000 - masm.shlq(gpr4, 4); - masm.leaq(gpr1, externalAddress(cTableTanPtr)); - masm.andpd(temp5, externalAddress(maskThirtyFiveTanPtr)); // 0xfffc0000, - // 0xffffffff, - // 0x00000000, - // 0x00000000 - masm.movdqu(temp3, dest); - masm.addq(gpr1, gpr4); - masm.subpd(temp2, dest); - masm.unpckhpd(dest, dest); - masm.divsd(temp6, temp5); - masm.subpd(temp2, temp4); - masm.movdqu(temp7, new AMD64Address(gpr1, 16)); - masm.subsd(temp3, temp5); - masm.mulpd(temp7, dest); - masm.subpd(temp2, temp1); - masm.movdqu(temp1, new AMD64Address(gpr1, 48)); - masm.mulpd(temp1, dest); - masm.movdqu(temp4, new AMD64Address(gpr1, 96)); - masm.mulpd(temp4, dest); - masm.addsd(temp2, temp3); - masm.movdqu(temp3, dest); - masm.mulpd(dest, dest); - masm.addpd(temp7, new AMD64Address(gpr1, 0)); - masm.addpd(temp1, new AMD64Address(gpr1, 32)); - masm.mulpd(temp1, dest); - masm.addpd(temp4, new AMD64Address(gpr1, 80)); - masm.addpd(temp7, temp1); - masm.movdqu(temp1, new AMD64Address(gpr1, 112)); - masm.mulpd(temp1, dest); - masm.mulpd(dest, dest); - masm.addpd(temp4, temp1); - masm.movdqu(temp1, new AMD64Address(gpr1, 64)); - masm.mulpd(temp1, dest); - masm.addpd(temp7, temp1); - masm.movdqu(temp1, temp3); - masm.mulpd(temp3, dest); - masm.mulsd(dest, dest); - masm.mulpd(temp1, new AMD64Address(gpr1, 144)); - masm.mulpd(temp4, temp3); - masm.movdqu(temp3, temp1); - masm.addpd(temp7, temp4); - masm.movdqu(temp4, temp1); - masm.mulsd(dest, temp7); - masm.unpckhpd(temp7, temp7); - masm.addsd(dest, temp7); - masm.unpckhpd(temp1, temp1); - masm.addsd(temp3, temp1); - masm.subsd(temp4, temp3); - masm.addsd(temp1, temp4); - masm.movdqu(temp4, temp2); - masm.movdq(temp7, new AMD64Address(gpr1, 144)); - masm.unpckhpd(temp2, temp2); - masm.addsd(temp7, new AMD64Address(gpr1, 152)); - masm.mulsd(temp7, temp2); - masm.addsd(temp7, new AMD64Address(gpr1, 136)); - masm.addsd(temp7, temp1); - masm.addsd(dest, temp7); - masm.movdq(temp7, externalAddress(onePtr)); // 0x00000000, - // 0x3ff00000 - masm.mulsd(temp4, temp6); - masm.movdq(temp2, new AMD64Address(gpr1, 168)); - masm.andpd(temp2, temp6); - masm.mulsd(temp5, temp2); - masm.mulsd(temp6, new AMD64Address(gpr1, 160)); - masm.subsd(temp7, temp5); - masm.subsd(temp2, new AMD64Address(gpr1, 128)); - masm.subsd(temp7, temp4); - masm.mulsd(temp7, temp6); - masm.movdqu(temp4, temp3); - masm.subsd(temp3, temp2); - masm.addsd(temp2, temp3); - masm.subsd(temp4, temp2); - masm.addsd(dest, temp4); - masm.subsd(dest, temp7); - masm.addsd(dest, temp3); - masm.jmp(bb15); - - masm.bind(bb0); - masm.jcc(ConditionFlag.Greater, bb1); - - masm.pextrw(gpr1, dest, 3); - masm.movl(gpr4, gpr1); - masm.andl(gpr1, 32752); - masm.jcc(ConditionFlag.Equal, bb2); - - masm.andl(gpr4, 32767); - masm.cmpl(gpr4, 15904); - masm.jcc(ConditionFlag.Below, bb3); - - masm.movdqu(temp2, dest); - masm.movdqu(temp3, dest); - masm.movdq(temp1, externalAddress(qElevenTanPtr)); // 0xb8fe4d77, - // 0x3f82609a - masm.mulsd(temp2, dest); - masm.mulsd(temp3, temp2); - masm.mulsd(temp1, temp2); - masm.addsd(temp1, externalAddress(qNineTanPtr)); // 0xbf847a43, - // 0x3f9664a0 - masm.mulsd(temp1, temp2); - masm.addsd(temp1, externalAddress(qSevenTanPtr)); // 0x52c4c8ab, - // 0x3faba1ba - masm.mulsd(temp1, temp2); - masm.addsd(temp1, externalAddress(qFiveTanPtr)); // 0x11092746, - // 0x3fc11111 - masm.mulsd(temp1, temp2); - masm.addsd(temp1, externalAddress(qThreeTanPtr)); // 0x55555612, - // 0x3fd55555 - masm.mulsd(temp1, temp3); - masm.addsd(dest, temp1); - masm.jmp(bb15); - - masm.bind(bb3); - masm.movdq(temp3, externalAddress(twoPowFiftyFiveTanPtr)); // 0x00000000, - // 0x43600000 - masm.mulsd(temp3, dest); - masm.addsd(dest, temp3); - masm.mulsd(dest, externalAddress(twoPowMFiftyFiveTanPtr)); // 0x00000000, - // 0x3c800000 - masm.jmp(bb15); - - masm.bind(bb14); - masm.xorpd(temp1, temp1); - masm.xorpd(dest, dest); - masm.divsd(dest, temp1); - masm.jmp(bb15); - - masm.bind(bb2); - masm.movdqu(temp1, dest); - masm.mulsd(temp1, temp1); - masm.jmp(bb15); - - masm.bind(bb1); - masm.pextrw(gpr3, dest, 3); - masm.andl(gpr3, 32752); - masm.cmpl(gpr3, 32752); - masm.jcc(ConditionFlag.Equal, bb14); - - masm.subl(gpr3, 16224); - masm.shrl(gpr3, 7); - masm.andl(gpr3, 65532); - masm.leaq(gpr10, externalAddress(piInvTableTanPtr)); - masm.addq(gpr3, gpr10); - masm.movdq(gpr1, dest); - masm.movl(gpr9, new AMD64Address(gpr3, 20)); - masm.movl(gpr7, new AMD64Address(gpr3, 24)); - masm.movl(gpr4, gpr1); - masm.shrq(gpr1, 21); - masm.orl(gpr1, Integer.MIN_VALUE); - masm.shrl(gpr1, 11); - masm.movl(gpr8, gpr9); - masm.imulq(gpr9, gpr4); - masm.imulq(gpr8, gpr1); - masm.imulq(gpr7, gpr1); - masm.movl(gpr5, new AMD64Address(gpr3, 16)); - masm.movl(gpr6, new AMD64Address(gpr3, 12)); - masm.movl(gpr10, gpr9); - masm.shrq(gpr9, 32); - masm.addq(gpr8, gpr9); - masm.addq(gpr10, gpr7); - masm.movl(gpr7, gpr10); - masm.shrq(gpr10, 32); - masm.addq(gpr8, gpr10); - masm.movl(gpr9, gpr5); - masm.imulq(gpr5, gpr4); - masm.imulq(gpr9, gpr1); - masm.movl(gpr10, gpr6); - masm.imulq(gpr6, gpr4); - masm.movl(gpr2, gpr5); - masm.shrq(gpr5, 32); - masm.addq(gpr8, gpr2); - masm.movl(gpr2, gpr8); - masm.shrq(gpr8, 32); - masm.addq(gpr9, gpr5); - masm.addq(gpr9, gpr8); - masm.shlq(gpr2, 32); - masm.orq(gpr7, gpr2); - masm.imulq(gpr10, gpr1); - masm.movl(gpr8, new AMD64Address(gpr3, 8)); - masm.movl(gpr5, new AMD64Address(gpr3, 4)); - masm.movl(gpr2, gpr6); - masm.shrq(gpr6, 32); - masm.addq(gpr9, gpr2); - masm.movl(gpr2, gpr9); - masm.shrq(gpr9, 32); - masm.addq(gpr10, gpr6); - masm.addq(gpr10, gpr9); - masm.movq(gpr6, gpr8); - masm.imulq(gpr8, gpr4); - masm.imulq(gpr6, gpr1); - masm.movl(gpr9, gpr8); - masm.shrq(gpr8, 32); - masm.addq(gpr10, gpr9); - masm.movl(gpr9, gpr10); - masm.shrq(gpr10, 32); - masm.addq(gpr6, gpr8); - masm.addq(gpr6, gpr10); - masm.movq(gpr8, gpr5); - masm.imulq(gpr5, gpr4); - masm.imulq(gpr8, gpr1); - masm.shlq(gpr9, 32); - masm.orq(gpr9, gpr2); - masm.movl(gpr1, new AMD64Address(gpr3, 0)); - masm.movl(gpr10, gpr5); - masm.shrq(gpr5, 32); - masm.addq(gpr6, gpr10); - masm.movl(gpr10, gpr6); - masm.shrq(gpr6, 32); - masm.addq(gpr8, gpr5); - masm.addq(gpr8, gpr6); - masm.imulq(gpr4, gpr1); - masm.pextrw(gpr2, dest, 3); - masm.leaq(gpr6, externalAddress(piInvTableTanPtr)); - masm.subq(gpr3, gpr6); - masm.addl(gpr3, gpr3); - masm.addl(gpr3, gpr3); - masm.addl(gpr3, gpr3); - masm.addl(gpr3, 19); - masm.movl(gpr5, 32768); - masm.andl(gpr5, gpr2); - masm.shrl(gpr2, 4); - masm.andl(gpr2, 2047); - masm.subl(gpr2, 1023); - masm.subl(gpr3, gpr2); - masm.addq(gpr8, gpr4); - masm.movl(gpr4, gpr3); - masm.addl(gpr4, 32); - masm.cmpl(gpr3, 0); - masm.jcc(ConditionFlag.Less, bb5); - - masm.negl(gpr3); - masm.addl(gpr3, 29); - masm.shll(gpr8); - masm.movl(gpr6, gpr8); - masm.andl(gpr8, 1073741823); - masm.testl(gpr8, 536870912); - masm.jcc(ConditionFlag.NotEqual, bb6); - - masm.shrl(gpr8); - masm.movl(gpr2, 0); - masm.shlq(gpr8, 32); - masm.orq(gpr8, gpr10); - - masm.bind(bb8); - masm.cmpq(gpr8, 0); - masm.jcc(ConditionFlag.Equal, bb9); - - masm.bind(bb10); - masm.bsrq(gpr10, gpr8); - masm.movl(gpr3, 29); - masm.subl(gpr3, gpr10); - masm.jcc(ConditionFlag.LessEqual, bb11); - - masm.shlq(gpr8); - masm.movq(gpr1, gpr9); - masm.shlq(gpr9); - masm.addl(gpr4, gpr3); - masm.negl(gpr3); - masm.addl(gpr3, 64); - masm.shrq(gpr1); - masm.shrq(gpr7); - masm.orq(gpr8, gpr1); - masm.orq(gpr9, gpr7); - - masm.bind(bb12); - masm.cvtsi2sdq(dest, gpr8); - masm.shrq(gpr9, 1); - masm.cvtsi2sdq(temp3, gpr9); - masm.xorpd(temp4, temp4); - masm.shll(gpr4, 4); - masm.negl(gpr4); - masm.addl(gpr4, 16368); - masm.orl(gpr4, gpr5); - masm.xorl(gpr4, gpr2); - masm.pinsrw(temp4, gpr4, 3); - masm.leaq(gpr1, externalAddress(piFourTanPtr)); - masm.movdq(temp2, new AMD64Address(gpr1, 0)); // 0x00000000, - // 0x3fe921fb, - masm.movdq(temp7, new AMD64Address(gpr1, 8)); // 0x4611a626, - // 0x3e85110b - masm.xorpd(temp5, temp5); - masm.subl(gpr4, 1008); - masm.pinsrw(temp5, gpr4, 3); - masm.mulsd(dest, temp4); - masm.shll(gpr5, 16); - masm.sarl(gpr5, 31); - masm.mulsd(temp3, temp5); - masm.movdqu(temp1, dest); - masm.mulsd(dest, temp2); - masm.shrl(gpr6, 30); - masm.addsd(temp1, temp3); - masm.mulsd(temp3, temp2); - masm.addl(gpr6, gpr5); - masm.xorl(gpr6, gpr5); - masm.mulsd(temp7, temp1); - masm.movl(gpr1, gpr6); - masm.addsd(temp7, temp3); - masm.movdqu(temp2, dest); - masm.addsd(dest, temp7); - masm.subsd(temp2, dest); - masm.addsd(temp7, temp2); - masm.movdqu(temp1, externalAddress(piThirtyTwoInvTanPtr)); // 0x6dc9c883, - // 0x3fe45f30, - // 0x6dc9c883, - // 0x40245f30 - if (masm.supports(CPUFeature.SSE3)) { - masm.movddup(dest, dest); - } else { - masm.movlhps(dest, dest); - } - masm.movdqu(temp4, externalAddress(signMaskTanPtr)); // 0x00000000, - // 0x80000000, - // 0x00000000, - // 0x80000000 - masm.andpd(temp4, dest); - masm.mulpd(temp1, dest); - if (masm.supports(CPUFeature.SSE3)) { - masm.movddup(temp7, temp7); - } else { - masm.movlhps(temp7, temp7); - } - masm.movdqu(temp5, externalAddress(oneHalfTanPtr)); // 0x00000000, - // 0x3fe00000, - // 0x00000000, - // 0x3fe00000 - masm.movdqu(temp6, externalAddress(mulSixteenPtr)); // 0x00000000, - // 0x40300000, - // 0x00000000, - // 0x3ff00000 - masm.por(temp5, temp4); - masm.addpd(temp1, temp5); - masm.movdqu(temp5, temp1); - masm.unpckhpd(temp5, temp5); - masm.cvttsd2sil(gpr4, temp5); - masm.cvttpd2dq(temp1, temp1); - masm.cvtdq2pd(temp1, temp1); - masm.mulpd(temp1, temp6); - masm.movdqu(temp3, externalAddress(pOneTanPtr)); // 0x54444000, - // 0x3fb921fb, - // 0x54440000, - // 0x3fb921fb - masm.movdq(temp5, externalAddress(qqTwoTanPtr)); // 0x676733af, - // 0x3d32e7b9 - masm.shll(gpr1, 4); - masm.addl(gpr4, 469248); - masm.movdqu(temp4, externalAddress(pTwoTanPtr)); // 0x67674000, - // 0xbd32e7b9, - // 0x4c4c0000, - // 0x3d468c23 - masm.mulpd(temp3, temp1); - masm.addl(gpr4, gpr1); - masm.andl(gpr4, 31); - masm.mulsd(temp5, temp1); - masm.movl(gpr3, gpr4); - masm.mulpd(temp4, temp1); - masm.shll(gpr3, 1); - masm.subpd(dest, temp3); - masm.mulpd(temp1, externalAddress(pThreeTanPtr)); // 0x3707344a, - // 0x3aa8a2e0, - // 0x03707345, - // 0x3ae98a2e - masm.addl(gpr4, gpr3); - masm.shll(gpr3, 2); - masm.addl(gpr4, gpr3); - masm.addsd(temp5, dest); - masm.movdqu(temp2, dest); - masm.subpd(dest, temp4); - masm.movdq(temp6, externalAddress(onePtr)); // 0x00000000, - // 0x3ff00000 - masm.shll(gpr4, 4); - masm.leaq(gpr1, externalAddress(cTableTanPtr)); - masm.andpd(temp5, externalAddress(maskThirtyFiveTanPtr)); // 0xfffc0000, - // 0xffffffff, - // 0x00000000, - // 0x00000000 - masm.movdqu(temp3, dest); - masm.addq(gpr1, gpr4); - masm.subpd(temp2, dest); - masm.unpckhpd(dest, dest); - masm.divsd(temp6, temp5); - masm.subpd(temp2, temp4); - masm.subsd(temp3, temp5); - masm.subpd(temp2, temp1); - masm.movdqu(temp1, new AMD64Address(gpr1, 48)); - masm.addpd(temp2, temp7); - masm.movdqu(temp7, new AMD64Address(gpr1, 16)); - masm.mulpd(temp7, dest); - masm.movdqu(temp4, new AMD64Address(gpr1, 96)); - masm.mulpd(temp1, dest); - masm.mulpd(temp4, dest); - masm.addsd(temp2, temp3); - masm.movdqu(temp3, dest); - masm.mulpd(dest, dest); - masm.addpd(temp7, new AMD64Address(gpr1, 0)); - masm.addpd(temp1, new AMD64Address(gpr1, 32)); - masm.mulpd(temp1, dest); - masm.addpd(temp4, new AMD64Address(gpr1, 80)); - masm.addpd(temp7, temp1); - masm.movdqu(temp1, new AMD64Address(gpr1, 112)); - masm.mulpd(temp1, dest); - masm.mulpd(dest, dest); - masm.addpd(temp4, temp1); - masm.movdqu(temp1, new AMD64Address(gpr1, 64)); - masm.mulpd(temp1, dest); - masm.addpd(temp7, temp1); - masm.movdqu(temp1, temp3); - masm.mulpd(temp3, dest); - masm.mulsd(dest, dest); - masm.mulpd(temp1, new AMD64Address(gpr1, 144)); - masm.mulpd(temp4, temp3); - masm.movdqu(temp3, temp1); - masm.addpd(temp7, temp4); - masm.movdqu(temp4, temp1); - masm.mulsd(dest, temp7); - masm.unpckhpd(temp7, temp7); - masm.addsd(dest, temp7); - masm.unpckhpd(temp1, temp1); - masm.addsd(temp3, temp1); - masm.subsd(temp4, temp3); - masm.addsd(temp1, temp4); - masm.movdqu(temp4, temp2); - masm.movdq(temp7, new AMD64Address(gpr1, 144)); - masm.unpckhpd(temp2, temp2); - masm.addsd(temp7, new AMD64Address(gpr1, 152)); - masm.mulsd(temp7, temp2); - masm.addsd(temp7, new AMD64Address(gpr1, 136)); - masm.addsd(temp7, temp1); - masm.addsd(dest, temp7); - masm.movdq(temp7, externalAddress(onePtr)); // 0x00000000, - // 0x3ff00000 - masm.mulsd(temp4, temp6); - masm.movdq(temp2, new AMD64Address(gpr1, 168)); - masm.andpd(temp2, temp6); - masm.mulsd(temp5, temp2); - masm.mulsd(temp6, new AMD64Address(gpr1, 160)); - masm.subsd(temp7, temp5); - masm.subsd(temp2, new AMD64Address(gpr1, 128)); - masm.subsd(temp7, temp4); - masm.mulsd(temp7, temp6); - masm.movdqu(temp4, temp3); - masm.subsd(temp3, temp2); - masm.addsd(temp2, temp3); - masm.subsd(temp4, temp2); - masm.addsd(dest, temp4); - masm.subsd(dest, temp7); - masm.addsd(dest, temp3); - masm.jmp(bb15); - - masm.bind(bb9); - masm.addl(gpr4, 64); - masm.movq(gpr8, gpr9); - masm.movq(gpr9, gpr7); - masm.movl(gpr7, 0); - masm.cmpq(gpr8, 0); - masm.jcc(ConditionFlag.NotEqual, bb10); - - masm.addl(gpr4, 64); - masm.movq(gpr8, gpr9); - masm.movq(gpr9, gpr7); - masm.cmpq(gpr8, 0); - masm.jcc(ConditionFlag.NotEqual, bb10); - - masm.jmp(bb12); - - masm.bind(bb11); - masm.jcc(ConditionFlag.Equal, bb12); - - masm.negl(gpr3); - masm.shrq(gpr9); - masm.movq(gpr1, gpr8); - masm.shrq(gpr8); - masm.subl(gpr4, gpr3); - masm.negl(gpr3); - masm.addl(gpr3, 64); - masm.shlq(gpr1); - masm.orq(gpr9, gpr1); - masm.jmp(bb12); - - masm.bind(bb5); - masm.notl(gpr3); - masm.shlq(gpr8, 32); - masm.orq(gpr8, gpr10); - masm.shlq(gpr8); - masm.movq(gpr6, gpr8); - masm.testl(gpr8, Integer.MIN_VALUE); - masm.jcc(ConditionFlag.NotEqual, bb13); - - masm.shrl(gpr8); - masm.movl(gpr2, 0); - masm.shrq(gpr6, 2); - masm.jmp(bb8); - - masm.bind(bb6); - masm.shrl(gpr8); - masm.movl(gpr2, 1073741824); - masm.shrl(gpr2); - masm.shlq(gpr8, 32); - masm.orq(gpr8, gpr10); - masm.shlq(gpr2, 32); - masm.addl(gpr6, 1073741824); - masm.movl(gpr3, 0); - masm.movl(gpr10, 0); - masm.subq(gpr3, gpr7); - masm.sbbq(gpr10, gpr9); - masm.sbbq(gpr2, gpr8); - masm.movq(gpr7, gpr3); - masm.movq(gpr9, gpr10); - masm.movq(gpr8, gpr2); - masm.movl(gpr2, 32768); - masm.jmp(bb8); - - masm.bind(bb13); - masm.shrl(gpr8); - masm.movq(gpr2, 0x100000000L); - masm.shrq(gpr2); - masm.movl(gpr3, 0); - masm.movl(gpr10, 0); - masm.subq(gpr3, gpr7); - masm.sbbq(gpr10, gpr9); - masm.sbbq(gpr2, gpr8); - masm.movq(gpr7, gpr3); - masm.movq(gpr9, gpr10); - masm.movq(gpr8, gpr2); - masm.movl(gpr2, 32768); - masm.shrq(gpr6, 2); - masm.addl(gpr6, 1073741824); - masm.jmp(bb8); - - masm.bind(bb15); + public final Variable emitLIRWrapper(LIRGeneratorTool gen, Value value) { + LIRKind kind = LIRKind.combine(value); + RegisterValue xmm0Value = xmm0.asValue(kind); + gen.emitMove(xmm0Value, value); + gen.append(this); + Variable result = gen.newVariable(kind); + gen.emitMove(result, xmm0Value); + return result; } - /* - * Copyright (c) 2014, 2016, Intel Corporation. All rights reserved. Intel Math Library (LIBM) - * Source Code - * - * ALGORITHM DESCRIPTION - EXP() --------------------- - * - * Description: Let K = 64 (table size). x x/log(2) n e = 2 = 2 * T[j] * (1 + P(y)) where x = - * m*log(2)/K + y, y in [-log(2)/K..log(2)/K] m = n*K + j, m,n,j - signed integer, j in - * [-K/2..K/2] j/K values of 2 are tabulated as T[j] = T_hi[j] ( 1 + T_lo[j]). - * - * P(y) is a minimax polynomial approximation of exp(x)-1 on small interval - * [-log(2)/K..log(2)/K] (were calculated by Maple V). - * - * To avoid problems with arithmetic overflow and underflow, n n1 n2 value of 2 is safely - * computed as 2 * 2 where n1 in [-BIAS/2..BIAS/2] where BIAS is a value of exponent bias. - * - * Special cases: exp(NaN) = NaN exp(+INF) = +INF exp(-INF) = 0 exp(x) = 1 for subnormals for - * finite argument, only exp(0)=1 is exact For IEEE double if x > 709.782712893383973096 then - * exp(x) overflow if x < -745.133219101941108420 then exp(x) underflow - * - */ - - private static int[] cvExp = { - 0x652b82fe, 0x40571547, 0x652b82fe, 0x40571547, 0xfefa0000, - 0x3f862e42, 0xfefa0000, 0x3f862e42, 0xbc9e3b3a, 0x3d1cf79a, - 0xbc9e3b3a, 0x3d1cf79a, 0xfffffffe, 0x3fdfffff, 0xfffffffe, - 0x3fdfffff, 0xe3289860, 0x3f56c15c, 0x555b9e25, 0x3fa55555, - 0xc090cf0f, 0x3f811115, 0x55548ba1, 0x3fc55555 - }; - - private static int[] shifterExp = { - 0x00000000, 0x43380000, 0x00000000, 0x43380000 - }; - - private static int[] mMaskExp = { - 0xffffffc0, 0x00000000, 0xffffffc0, 0x00000000 - }; - - private static int[] biasExp = { - 0x0000ffc0, 0x00000000, 0x0000ffc0, 0x00000000 - }; - - private static int[] tblAddrExp = { - 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x0e03754d, - 0x3cad7bbf, 0x3e778060, 0x00002c9a, 0x3567f613, 0x3c8cd252, - 0xd3158574, 0x000059b0, 0x61e6c861, 0x3c60f74e, 0x18759bc8, - 0x00008745, 0x5d837b6c, 0x3c979aa6, 0x6cf9890f, 0x0000b558, - 0x702f9cd1, 0x3c3ebe3d, 0x32d3d1a2, 0x0000e3ec, 0x1e63bcd8, - 0x3ca3516e, 0xd0125b50, 0x00011301, 0x26f0387b, 0x3ca4c554, - 0xaea92ddf, 0x0001429a, 0x62523fb6, 0x3ca95153, 0x3c7d517a, - 0x000172b8, 0x3f1353bf, 0x3c8b898c, 0xeb6fcb75, 0x0001a35b, - 0x3e3a2f5f, 0x3c9aecf7, 0x3168b9aa, 0x0001d487, 0x44a6c38d, - 0x3c8a6f41, 0x88628cd6, 0x0002063b, 0xe3a8a894, 0x3c968efd, - 0x6e756238, 0x0002387a, 0x981fe7f2, 0x3c80472b, 0x65e27cdd, - 0x00026b45, 0x6d09ab31, 0x3c82f7e1, 0xf51fdee1, 0x00029e9d, - 0x720c0ab3, 0x3c8b3782, 0xa6e4030b, 0x0002d285, 0x4db0abb6, - 0x3c834d75, 0x0a31b715, 0x000306fe, 0x5dd3f84a, 0x3c8fdd39, - 0xb26416ff, 0x00033c08, 0xcc187d29, 0x3ca12f8c, 0x373aa9ca, - 0x000371a7, 0x738b5e8b, 0x3ca7d229, 0x34e59ff6, 0x0003a7db, - 0xa72a4c6d, 0x3c859f48, 0x4c123422, 0x0003dea6, 0x259d9205, - 0x3ca8b846, 0x21f72e29, 0x0004160a, 0x60c2ac12, 0x3c4363ed, - 0x6061892d, 0x00044e08, 0xdaa10379, 0x3c6ecce1, 0xb5c13cd0, - 0x000486a2, 0xbb7aafb0, 0x3c7690ce, 0xd5362a27, 0x0004bfda, - 0x9b282a09, 0x3ca083cc, 0x769d2ca6, 0x0004f9b2, 0xc1aae707, - 0x3ca509b0, 0x569d4f81, 0x0005342b, 0x18fdd78e, 0x3c933505, - 0x36b527da, 0x00056f47, 0xe21c5409, 0x3c9063e1, 0xdd485429, - 0x0005ab07, 0x2b64c035, 0x3c9432e6, 0x15ad2148, 0x0005e76f, - 0x99f08c0a, 0x3ca01284, 0xb03a5584, 0x0006247e, 0x0073dc06, - 0x3c99f087, 0x82552224, 0x00066238, 0x0da05571, 0x3c998d4d, - 0x667f3bcc, 0x0006a09e, 0x86ce4786, 0x3ca52bb9, 0x3c651a2e, - 0x0006dfb2, 0x206f0dab, 0x3ca32092, 0xe8ec5f73, 0x00071f75, - 0x8e17a7a6, 0x3ca06122, 0x564267c8, 0x00075feb, 0x461e9f86, - 0x3ca244ac, 0x73eb0186, 0x0007a114, 0xabd66c55, 0x3c65ebe1, - 0x36cf4e62, 0x0007e2f3, 0xbbff67d0, 0x3c96fe9f, 0x994cce12, - 0x00082589, 0x14c801df, 0x3c951f14, 0x9b4492ec, 0x000868d9, - 0xc1f0eab4, 0x3c8db72f, 0x422aa0db, 0x0008ace5, 0x59f35f44, - 0x3c7bf683, 0x99157736, 0x0008f1ae, 0x9c06283c, 0x3ca360ba, - 0xb0cdc5e4, 0x00093737, 0x20f962aa, 0x3c95e8d1, 0x9fde4e4f, - 0x00097d82, 0x2b91ce27, 0x3c71affc, 0x82a3f090, 0x0009c491, - 0x589a2ebd, 0x3c9b6d34, 0x7b5de564, 0x000a0c66, 0x9ab89880, - 0x3c95277c, 0xb23e255c, 0x000a5503, 0x6e735ab3, 0x3c846984, - 0x5579fdbf, 0x000a9e6b, 0x92cb3387, 0x3c8c1a77, 0x995ad3ad, - 0x000ae89f, 0xdc2d1d96, 0x3ca22466, 0xb84f15fa, 0x000b33a2, - 0xb19505ae, 0x3ca1112e, 0xf2fb5e46, 0x000b7f76, 0x0a5fddcd, - 0x3c74ffd7, 0x904bc1d2, 0x000bcc1e, 0x30af0cb3, 0x3c736eae, - 0xdd85529c, 0x000c199b, 0xd10959ac, 0x3c84e08f, 0x2e57d14b, - 0x000c67f1, 0x6c921968, 0x3c676b2c, 0xdcef9069, 0x000cb720, - 0x36df99b3, 0x3c937009, 0x4a07897b, 0x000d072d, 0xa63d07a7, - 0x3c74a385, 0xdcfba487, 0x000d5818, 0xd5c192ac, 0x3c8e5a50, - 0x03db3285, 0x000da9e6, 0x1c4a9792, 0x3c98bb73, 0x337b9b5e, - 0x000dfc97, 0x603a88d3, 0x3c74b604, 0xe78b3ff6, 0x000e502e, - 0x92094926, 0x3c916f27, 0xa2a490d9, 0x000ea4af, 0x41aa2008, - 0x3c8ec3bc, 0xee615a27, 0x000efa1b, 0x31d185ee, 0x3c8a64a9, - 0x5b6e4540, 0x000f5076, 0x4d91cd9d, 0x3c77893b, 0x819e90d8, - 0x000fa7c1 - }; - - private static int[] allOnesExp = { - 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff - }; - - private static int[] expBias = { - 0x00000000, 0x3ff00000, 0x00000000, 0x3ff00000 - }; - - private static int[] xMaxExp = { - 0xffffffff, 0x7fefffff - }; - - private static int[] xMinExp = { - 0x00000000, 0x00100000 - }; - - private static int[] infExp = { - 0x00000000, 0x7ff00000 - }; - - private static int[] zeroExp = { - 0x00000000, 0x00000000 - }; - - public void expIntrinsic(Register dest, Register value, CompilationResultBuilder crb, AMD64MacroAssembler masm) { - ArrayDataPointerConstant onePtr = new ArrayDataPointerConstant(one, 16); - ArrayDataPointerConstant cvExpPtr = new ArrayDataPointerConstant(cvExp, 16); - ArrayDataPointerConstant shifterExpPtr = new ArrayDataPointerConstant(shifterExp, 8); - ArrayDataPointerConstant mMaskExpPtr = new ArrayDataPointerConstant(mMaskExp, 16); - ArrayDataPointerConstant biasExpPtr = new ArrayDataPointerConstant(biasExp, 16); - ArrayDataPointerConstant tblAddrExpPtr = new ArrayDataPointerConstant(tblAddrExp, 16); - ArrayDataPointerConstant expBiasPtr = new ArrayDataPointerConstant(expBias, 8); - ArrayDataPointerConstant xMaxExpPtr = new ArrayDataPointerConstant(xMaxExp, 8); - ArrayDataPointerConstant xMinExpPtr = new ArrayDataPointerConstant(xMinExp, 8); - ArrayDataPointerConstant infExpPtr = new ArrayDataPointerConstant(infExp, 8); - ArrayDataPointerConstant zeroExpPtr = new ArrayDataPointerConstant(zeroExp, 8); - ArrayDataPointerConstant allOnesExpPtr = new ArrayDataPointerConstant(allOnesExp, 8); - - Label bb0 = new Label(); - Label bb1 = new Label(); - Label bb2 = new Label(); - Label bb3 = new Label(); - Label bb4 = new Label(); - Label bb5 = new Label(); - Label bb7 = new Label(); - Label bb8 = new Label(); - Label bb9 = new Label(); - Label bb10 = new Label(); - Label bb11 = new Label(); - Label bb12 = new Label(); - Label bb14 = new Label(); - - Register gpr1 = asRegister(gpr1Temp, AMD64Kind.QWORD); - Register gpr2 = asRegister(gpr2Temp, AMD64Kind.QWORD); - Register gpr3 = asRegister(rcxTemp, AMD64Kind.QWORD); - Register gpr4 = asRegister(gpr4Temp, AMD64Kind.QWORD); - Register gpr5 = asRegister(gpr5Temp, AMD64Kind.QWORD); - - Register temp1 = asRegister(xmm1Temp, AMD64Kind.DOUBLE); - Register temp2 = asRegister(xmm2Temp, AMD64Kind.DOUBLE); - Register temp3 = asRegister(xmm3Temp, AMD64Kind.DOUBLE); - Register temp4 = asRegister(xmm4Temp, AMD64Kind.DOUBLE); - Register temp5 = asRegister(xmm5Temp, AMD64Kind.DOUBLE); - Register temp6 = asRegister(xmm6Temp, AMD64Kind.DOUBLE); - Register temp7 = asRegister(xmm7Temp, AMD64Kind.DOUBLE); - Register temp8 = asRegister(xmm8Temp, AMD64Kind.DOUBLE); - Register temp9 = asRegister(xmm9Temp, AMD64Kind.DOUBLE); - Register temp10 = asRegister(xmm10Temp, AMD64Kind.DOUBLE); - - AMD64Address stackSlot = (AMD64Address) crb.asAddress(stackTemp); - - setCrb(crb); - masm.movsd(stackSlot, value); - if (dest.encoding != value.encoding) { - masm.movdqu(dest, value); - } - - masm.movdqu(temp9, externalAddress(mMaskExpPtr)); // 0xffffffc0, - // 0x00000000, - // 0xffffffc0, - // 0x00000000 - masm.movdqu(temp10, externalAddress(biasExpPtr)); // 0x0000ffc0, - // 0x00000000, - // 0x0000ffc0, - // 0x00000000 - masm.unpcklpd(dest, dest); - masm.leaq(gpr5, stackSlot); - masm.leaq(gpr2, externalAddress(cvExpPtr)); - masm.movdqu(temp1, new AMD64Address(gpr2, 0)); // 0x652b82fe, - // 0x40571547, - // 0x652b82fe, - // 0x40571547 - masm.movdqu(temp6, externalAddress(shifterExpPtr)); // 0x00000000, - // 0x43380000, - // 0x00000000, - // 0x43380000 - masm.movdqu(temp2, new AMD64Address(gpr2, 16)); // 0xfefa0000, - // 0x3f862e42, - // 0xfefa0000, - // 0x3f862e42 - masm.movdqu(temp3, new AMD64Address(gpr2, 32)); // 0xbc9e3b3a, - // 0x3d1cf79a, - // 0xbc9e3b3a, - // 0x3d1cf79a - masm.pextrw(gpr1, dest, 3); - masm.andl(gpr1, 32767); - masm.movl(gpr4, 16527); - masm.subl(gpr4, gpr1); - masm.subl(gpr1, 15504); - masm.orl(gpr4, gpr1); - masm.cmpl(gpr4, Integer.MIN_VALUE); - masm.jcc(ConditionFlag.AboveEqual, bb0); - - masm.leaq(gpr4, externalAddress(tblAddrExpPtr)); - masm.movdqu(temp8, new AMD64Address(gpr2, 48)); // 0xfffffffe, - // 0x3fdfffff, - // 0xfffffffe, - // 0x3fdfffff - masm.movdqu(temp4, new AMD64Address(gpr2, 64)); // 0xe3289860, - // 0x3f56c15c, - // 0x555b9e25, - // 0x3fa55555 - masm.movdqu(temp5, new AMD64Address(gpr2, 80)); // 0xc090cf0f, - // 0x3f811115, - // 0x55548ba1, - // 0x3fc55555 - masm.mulpd(temp1, dest); - masm.addpd(temp1, temp6); - masm.movapd(temp7, temp1); - masm.movdl(gpr1, temp1); - masm.pand(temp7, temp9); - masm.subpd(temp1, temp6); - masm.mulpd(temp2, temp1); - masm.mulpd(temp3, temp1); - masm.paddq(temp7, temp10); - masm.subpd(dest, temp2); - masm.movl(gpr3, gpr1); - masm.andl(gpr3, 63); - masm.shll(gpr3, 4); - masm.movdqu(temp2, new AMD64Address(gpr3, gpr4, Scale.Times1, 0)); - masm.sarl(gpr1, 6); - masm.psllq(temp7, 46); - masm.subpd(dest, temp3); - masm.mulpd(temp4, dest); - masm.movl(gpr4, gpr1); - masm.movapd(temp6, dest); - masm.movapd(temp1, dest); - masm.mulpd(temp6, temp6); - masm.mulpd(dest, temp6); - masm.addpd(temp5, temp4); - masm.mulsd(dest, temp6); - masm.mulpd(temp6, temp8); - masm.addsd(temp1, temp2); - masm.unpckhpd(temp2, temp2); - masm.mulpd(dest, temp5); - masm.addsd(temp1, dest); - masm.por(temp2, temp7); - masm.unpckhpd(dest, dest); - masm.addsd(dest, temp1); - masm.addsd(dest, temp6); - masm.addl(gpr4, 894); - masm.cmpl(gpr4, 1916); - masm.jcc(ConditionFlag.Above, bb1); - - masm.mulsd(dest, temp2); - masm.addsd(dest, temp2); - masm.jmp(bb14); - - masm.bind(bb1); - masm.movdqu(temp6, externalAddress(expBiasPtr)); // 0x00000000, - // 0x3ff00000, - // 0x00000000, - // 0x3ff00000 - masm.xorpd(temp3, temp3); - masm.movdqu(temp4, externalAddress(allOnesExpPtr)); // 0xffffffff, - // 0xffffffff, - // 0xffffffff, - // 0xffffffff - masm.movl(gpr4, -1022); - masm.subl(gpr4, gpr1); - masm.movdl(temp5, gpr4); - masm.psllq(temp4, temp5); - masm.movl(gpr3, gpr1); - masm.sarl(gpr1, 1); - masm.pinsrw(temp3, gpr1, 3); - masm.psllq(temp3, 4); - masm.psubd(temp2, temp3); - masm.mulsd(dest, temp2); - masm.cmpl(gpr4, 52); - masm.jcc(ConditionFlag.Greater, bb2); - - masm.pand(temp4, temp2); - masm.paddd(temp3, temp6); - masm.subsd(temp2, temp4); - masm.addsd(dest, temp2); - masm.cmpl(gpr3, 1023); - masm.jcc(ConditionFlag.GreaterEqual, bb3); - - masm.pextrw(gpr3, dest, 3); - masm.andl(gpr3, 32768); - masm.orl(gpr4, gpr3); - masm.cmpl(gpr4, 0); - masm.jcc(ConditionFlag.Equal, bb4); - - masm.movapd(temp6, dest); - masm.addsd(dest, temp4); - masm.mulsd(dest, temp3); - masm.pextrw(gpr3, dest, 3); - masm.andl(gpr3, 32752); - masm.cmpl(gpr3, 0); - masm.jcc(ConditionFlag.Equal, bb5); - - masm.jmp(bb14); - - masm.bind(bb5); - masm.mulsd(temp6, temp3); - masm.mulsd(temp4, temp3); - masm.movdqu(dest, temp6); - masm.pxor(temp6, temp4); - masm.psrad(temp6, 31); - masm.pshufd(temp6, temp6, 85); - masm.psllq(dest, 1); - masm.psrlq(dest, 1); - masm.pxor(dest, temp6); - masm.psrlq(temp6, 63); - masm.paddq(dest, temp6); - masm.paddq(dest, temp4); - masm.jmp(bb14); - - masm.bind(bb4); - masm.addsd(dest, temp4); - masm.mulsd(dest, temp3); - masm.jmp(bb14); - - masm.bind(bb3); - masm.addsd(dest, temp4); - masm.mulsd(dest, temp3); - masm.pextrw(gpr3, dest, 3); - masm.andl(gpr3, 32752); - masm.cmpl(gpr3, 32752); - masm.jcc(ConditionFlag.AboveEqual, bb7); - - masm.jmp(bb14); - - masm.bind(bb2); - masm.paddd(temp3, temp6); - masm.addpd(dest, temp2); - masm.mulsd(dest, temp3); - masm.jmp(bb14); - - masm.bind(bb8); - masm.movsd(dest, externalAddress(xMaxExpPtr)); // 0xffffffff, - // 0x7fefffff - masm.movsd(temp8, externalAddress(xMinExpPtr)); // 0x00000000, - // 0x00100000 - masm.cmpl(gpr1, 2146435072); - masm.jcc(ConditionFlag.AboveEqual, bb9); - - masm.movl(gpr1, new AMD64Address(gpr5, 4)); - masm.cmpl(gpr1, Integer.MIN_VALUE); - masm.jcc(ConditionFlag.AboveEqual, bb10); - - masm.mulsd(dest, dest); - - masm.bind(bb7); - masm.jmp(bb14); - - masm.bind(bb10); - masm.mulsd(dest, temp8); - masm.jmp(bb14); - - masm.bind(bb9); - masm.movl(gpr4, stackSlot); - masm.cmpl(gpr1, 2146435072); - masm.jcc(ConditionFlag.Above, bb11); - - masm.cmpl(gpr4, 0); - masm.jcc(ConditionFlag.NotEqual, bb11); - - masm.movl(gpr1, new AMD64Address(gpr5, 4)); - masm.cmpl(gpr1, 2146435072); - masm.jcc(ConditionFlag.NotEqual, bb12); - - masm.movsd(dest, externalAddress(infExpPtr)); // 0x00000000, - // 0x7ff00000 - masm.jmp(bb14); - - masm.bind(bb12); - masm.movsd(dest, externalAddress(zeroExpPtr)); // 0x00000000, - // 0x00000000 - masm.jmp(bb14); - - masm.bind(bb11); - masm.movsd(dest, stackSlot); - masm.addsd(dest, dest); - masm.jmp(bb14); - - masm.bind(bb0); - masm.movl(gpr1, new AMD64Address(gpr5, 4)); - masm.andl(gpr1, 2147483647); - masm.cmpl(gpr1, 1083179008); - masm.jcc(ConditionFlag.AboveEqual, bb8); - - masm.addsd(dest, externalAddress(onePtr)); // 0x00000000, - // 0x3ff00000 - masm.bind(bb14); - } } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64Move.java 2019-03-12 08:09:29.047530783 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64Move.java 2019-03-12 08:09:28.675528369 +0100 @@ -698,7 +698,7 @@ if (crb.mustReplaceWithNullRegister(input)) { masm.movq(result, crb.nullRegister); } else { - masm.movq(result, 0x0L); + masm.movslq(result, 0); } } else if (crb.target.inlineObjects) { crb.recordInlineDataInCode(input); --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.jtt/src/org/graalvm/compiler/lir/jtt/SPARCBranchBailoutTest.java 2019-03-12 08:09:29.551534054 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.jtt/src/org/graalvm/compiler/lir/jtt/SPARCBranchBailoutTest.java 2019-03-12 08:09:29.179531639 +0100 @@ -25,17 +25,15 @@ package org.graalvm.compiler.lir.jtt; import org.graalvm.compiler.api.directives.GraalDirectives; -import org.graalvm.compiler.core.common.PermanentBailoutException; +import org.graalvm.compiler.asm.BranchTargetOutOfBoundsException; import org.graalvm.compiler.debug.DebugContext; import org.graalvm.compiler.debug.DebugContext.Scope; -import org.graalvm.compiler.debug.GraalError; import org.graalvm.compiler.lir.LIRInstruction; import org.graalvm.compiler.lir.LIRInstructionClass; import org.graalvm.compiler.lir.asm.CompilationResultBuilder; import org.graalvm.compiler.lir.gen.LIRGeneratorTool; import org.graalvm.compiler.nodes.StructuredGraph; import org.graalvm.compiler.nodes.StructuredGraph.AllowAssumptions; -import org.junit.Assert; import org.junit.Assume; import org.junit.Test; @@ -82,18 +80,14 @@ } @SuppressWarnings("try") - @Test + @Test(expected = BranchTargetOutOfBoundsException.class) public void testBailoutOnBranchOverflow() throws Throwable { Assume.assumeTrue(getBackend().getTarget().arch instanceof SPARC); ResolvedJavaMethod m = getResolvedJavaMethod("testBranch"); DebugContext debug = getDebugContext(); - try { - try (Scope s = debug.disable()) { - StructuredGraph graph = parseEager(m, AllowAssumptions.YES, debug); - compile(m, graph); - } - } catch (GraalError e) { - Assert.assertEquals(PermanentBailoutException.class, e.getCause().getClass()); + try (Scope s = debug.disable()) { + StructuredGraph graph = parseEager(m, AllowAssumptions.YES, debug); + compile(m, graph); } } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/LIR.java 2019-03-12 08:09:30.031537169 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/LIR.java 2019-03-12 08:09:29.663534780 +0100 @@ -28,12 +28,14 @@ import java.util.Arrays; import java.util.List; +import org.graalvm.compiler.asm.Label; import org.graalvm.compiler.core.common.cfg.AbstractBlockBase; import org.graalvm.compiler.core.common.cfg.AbstractControlFlowGraph; import org.graalvm.compiler.core.common.cfg.BlockMap; import org.graalvm.compiler.debug.DebugContext; import org.graalvm.compiler.lir.StandardOp.BlockEndOp; import org.graalvm.compiler.lir.StandardOp.LabelOp; +import org.graalvm.compiler.lir.StandardOp.LabelHoldingOp; import org.graalvm.compiler.lir.gen.LIRGenerator; import org.graalvm.compiler.options.OptionValues; @@ -233,8 +235,11 @@ continue; } for (LIRInstruction inst : lirInstructions.get(block)) { - if (inst instanceof LabelOp) { - ((LabelOp) inst).getLabel().reset(); + if (inst instanceof LabelHoldingOp) { + Label label = ((LabelHoldingOp) inst).getLabel(); + if (label != null) { + label.reset(); + } } } } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/LIRValueUtil.java 2019-03-12 08:09:30.519540336 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/LIRValueUtil.java 2019-03-12 08:09:30.147537922 +0100 @@ -71,6 +71,11 @@ return asConstantValue(value).getJavaConstant(); } + public static boolean isNullConstant(Value value) { + assert value != null; + return isJavaConstant(value) && asJavaConstant(value).isNull(); + } + public static boolean isIntConstant(Value value, long expected) { if (isJavaConstant(value)) { JavaConstant javaConstant = asJavaConstant(value); --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/StandardOp.java 2019-03-12 08:09:31.007543502 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/StandardOp.java 2019-03-12 08:09:30.643541140 +0100 @@ -70,10 +70,14 @@ boolean makeNullCheckFor(Value value, LIRFrameState nullCheckState, int implicitNullCheckLimit); } + public interface LabelHoldingOp { + Label getLabel(); + } + /** * LIR operation that defines the position of a label. */ - public static final class LabelOp extends LIRInstruction { + public static final class LabelOp extends LIRInstruction implements LabelHoldingOp { public static final LIRInstructionClass TYPE = LIRInstructionClass.create(LabelOp.class); public static final EnumSet incomingFlags = EnumSet.of(REG, STACK); @@ -155,6 +159,7 @@ crb.asm.bind(label); } + @Override public Label getLabel() { return label; } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/Variable.java 2019-03-12 08:09:31.499546695 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/Variable.java 2019-03-12 08:09:31.131544307 +0100 @@ -33,7 +33,7 @@ * Represents a value that is yet to be bound to a machine location (such as a {@link RegisterValue} * or {@link StackSlot}) by a register allocator. */ -public final class Variable extends AllocatableValue { +public class Variable extends AllocatableValue { /** * The identifier of the variable. This is a non-zero index in a contiguous 0-based name space. --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/alloc/lsra/LinearScanLifetimeAnalysisPhase.java 2019-03-12 08:09:31.987549862 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/alloc/lsra/LinearScanLifetimeAnalysisPhase.java 2019-03-12 08:09:31.623547500 +0100 @@ -176,7 +176,7 @@ ValueConsumer useConsumer = (operand, mode, flags) -> { if (isVariable(operand)) { - int operandNum = allocator.operandNumber(operand); + int operandNum = getOperandNumber(operand); if (!liveKillScratch.get(operandNum)) { liveGenScratch.set(operandNum); if (debug.isLogEnabled()) { @@ -194,7 +194,7 @@ }; ValueConsumer stateConsumer = (operand, mode, flags) -> { if (LinearScan.isVariableOrRegister(operand)) { - int operandNum = allocator.operandNumber(operand); + int operandNum = getOperandNumber(operand); if (!liveKillScratch.get(operandNum)) { liveGenScratch.set(operandNum); if (debug.isLogEnabled()) { @@ -205,7 +205,7 @@ }; ValueConsumer defConsumer = (operand, mode, flags) -> { if (isVariable(operand)) { - int varNum = allocator.operandNumber(operand); + int varNum = getOperandNumber(operand); liveKillScratch.set(varNum); if (debug.isLogEnabled()) { debug.log("liveKill for operand %d(%s)", varNum, operand); @@ -268,7 +268,7 @@ */ if (isRegister(operand)) { if (allocator.isProcessed(operand)) { - liveKill.set(allocator.operandNumber(operand)); + liveKill.set(getOperandNumber(operand)); } } } @@ -281,11 +281,15 @@ */ if (isRegister(operand) && block != allocator.getLIR().getControlFlowGraph().getStartBlock()) { if (allocator.isProcessed(operand)) { - assert liveKill.get(allocator.operandNumber(operand)) : "using fixed register " + asRegister(operand) + " that is not defined in this block " + block; + assert liveKill.get(getOperandNumber(operand)) : "using fixed register " + asRegister(operand) + " that is not defined in this block " + block; } } } + protected int getOperandNumber(Value operand) { + return allocator.operandNumber(operand); + } + /** * Performs a backward dataflow analysis to compute global live sets (i.e. * {@link BlockData#liveIn} and {@link BlockData#liveOut}) for each block. --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/alloc/lsra/ssa/SSALinearScanLifetimeAnalysisPhase.java 2019-03-12 08:09:32.475553028 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/alloc/lsra/ssa/SSALinearScanLifetimeAnalysisPhase.java 2019-03-12 08:09:32.107550641 +0100 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -24,14 +24,17 @@ package org.graalvm.compiler.lir.alloc.lsra.ssa; +import java.util.BitSet; import java.util.EnumSet; +import org.graalvm.compiler.core.common.cfg.AbstractBlockBase; import org.graalvm.compiler.debug.DebugContext; +import org.graalvm.compiler.lir.LIR; import org.graalvm.compiler.lir.LIRInstruction; import org.graalvm.compiler.lir.LIRInstruction.OperandFlag; import org.graalvm.compiler.lir.LIRInstruction.OperandMode; +import org.graalvm.compiler.lir.StandardOp; import org.graalvm.compiler.lir.StandardOp.LabelOp; -import org.graalvm.compiler.lir.ValueConsumer; import org.graalvm.compiler.lir.alloc.lsra.Interval; import org.graalvm.compiler.lir.alloc.lsra.Interval.RegisterPriority; import org.graalvm.compiler.lir.alloc.lsra.LinearScan; @@ -53,17 +56,39 @@ if (hintAtDef && op instanceof LabelOp) { LabelOp label = (LabelOp) op; + if (!label.isPhiIn()) { + return; + } Interval to = allocator.getOrCreateInterval((AllocatableValue) targetValue); - SSAUtil.forEachPhiRegisterHint(allocator.getLIR(), allocator.blockForId(label.id()), label, targetValue, mode, (ValueConsumer) (registerHint, valueMode, valueFlags) -> { - if (LinearScan.isVariableOrRegister(registerHint)) { - Interval from = allocator.getOrCreateInterval((AllocatableValue) registerHint); - - setHint(debug, op, to, from); - setHint(debug, op, from, to); + LIR lir = allocator.getLIR(); + AbstractBlockBase block = allocator.blockForId(label.id()); + assert mode == OperandMode.DEF : "Wrong operand mode: " + mode; + assert lir.getLIRforBlock(block).get(0).equals(label) : String.format("Block %s and Label %s do not match!", block, label); + + int idx = SSAUtil.indexOfValue(label, targetValue); + assert idx >= 0 : String.format("Value %s not in label %s", targetValue, label); + + BitSet blockLiveIn = allocator.getBlockData(block).liveIn; + + AbstractBlockBase selectedPredecessor = null; + AllocatableValue selectedSource = null; + for (AbstractBlockBase pred : block.getPredecessors()) { + if (selectedPredecessor == null || pred.getRelativeFrequency() > selectedPredecessor.getRelativeFrequency()) { + StandardOp.JumpOp jump = SSAUtil.phiOut(lir, pred); + Value sourceValue = jump.getOutgoingValue(idx); + if (LinearScan.isVariableOrRegister(sourceValue) && !blockLiveIn.get(getOperandNumber(sourceValue))) { + selectedSource = (AllocatableValue) sourceValue; + selectedPredecessor = pred; + } } - }); + } + if (selectedSource != null) { + Interval from = allocator.getOrCreateInterval(selectedSource); + setHint(debug, op, to, from); + setHint(debug, op, from, to); + } } } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/asm/CompilationResultBuilder.java 2019-03-12 08:09:32.963556195 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/asm/CompilationResultBuilder.java 2019-03-12 08:09:32.599553833 +0100 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2013, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -38,6 +38,7 @@ import jdk.internal.vm.compiler.collections.Equivalence; import org.graalvm.compiler.asm.AbstractAddress; import org.graalvm.compiler.asm.Assembler; +import org.graalvm.compiler.asm.Label; import org.graalvm.compiler.code.CompilationResult; import org.graalvm.compiler.code.CompilationResult.CodeAnnotation; import org.graalvm.compiler.code.DataSection.Data; @@ -46,7 +47,6 @@ import org.graalvm.compiler.core.common.cfg.AbstractBlockBase; import org.graalvm.compiler.core.common.spi.ForeignCallsProvider; import org.graalvm.compiler.core.common.type.DataPointerConstant; -import org.graalvm.compiler.debug.Assertions; import org.graalvm.compiler.debug.DebugContext; import org.graalvm.compiler.debug.GraalError; import org.graalvm.compiler.graph.NodeSourcePosition; @@ -54,12 +54,14 @@ import org.graalvm.compiler.lir.LIRFrameState; import org.graalvm.compiler.lir.LIRInstruction; import org.graalvm.compiler.lir.LabelRef; +import org.graalvm.compiler.lir.StandardOp.LabelHoldingOp; import org.graalvm.compiler.lir.framemap.FrameMap; import org.graalvm.compiler.options.Option; import org.graalvm.compiler.options.OptionKey; import org.graalvm.compiler.options.OptionType; import org.graalvm.compiler.options.OptionValues; +import jdk.vm.ci.code.BailoutException; import jdk.vm.ci.code.CodeCacheProvider; import jdk.vm.ci.code.DebugInfo; import jdk.vm.ci.code.Register; @@ -155,6 +157,20 @@ private Consumer beforeOp; private Consumer afterOp; + /** + * These position maps are used for estimating offsets of forward branches. Used for + * architectures where certain branch instructions have limited displacement such as ARM tbz or + * SPARC cbcond. + */ + private EconomicMap labelBindLirPositions; + private EconomicMap lirPositions; + /** + * This flag is for setting the + * {@link CompilationResultBuilder#labelWithinRange(LIRInstruction, Label, int)} into a + * conservative mode and always answering false. + */ + private boolean conservativeLabelOffsets = false; + public final boolean mustReplaceWithNullRegister(JavaConstant nullConstant) { return !nullRegister.equals(Register.None) && JavaConstant.NULL_POINTER.equals(nullConstant); } @@ -179,14 +195,6 @@ this.debug = debug; assert frameContext != null; this.dataCache = dataCache; - - if (dataBuilder.needDetailedPatchingInformation() || Assertions.assertionsEnabled()) { - /* - * Always enabled in debug mode, even when the VM does not request detailed information, - * to increase test coverage. - */ - asm.setCodePatchingAnnotationConsumer(assemblerCodeAnnotation -> compilationResult.addAnnotation(new AssemblerAnnotation(assemblerCodeAnnotation))); - } } public void setTotalFrameSize(int frameSize) { @@ -543,6 +551,8 @@ if (op.getPosition() != null) { crb.recordSourceMapping(start, crb.asm.position(), op.getPosition()); } + } catch (BailoutException e) { + throw e; } catch (AssertionError t) { throw new GraalError(t); } catch (RuntimeException t) { @@ -559,6 +569,8 @@ if (dataCache != null) { dataCache.clear(); } + lir = null; + currentBlockIndex = 0; } public void setOpCallback(Consumer beforeOp, Consumer afterOp) { @@ -570,4 +582,57 @@ return options; } + /** + * Builds up a map for label and LIR instruction positions where labels are or labels pointing + * to. + */ + public void buildLabelOffsets(LIR generatedLIR) { + labelBindLirPositions = EconomicMap.create(Equivalence.IDENTITY); + lirPositions = EconomicMap.create(Equivalence.IDENTITY); + int instructionPosition = 0; + for (AbstractBlockBase block : generatedLIR.codeEmittingOrder()) { + if (block != null) { + for (LIRInstruction op : generatedLIR.getLIRforBlock(block)) { + if (op instanceof LabelHoldingOp) { + Label label = ((LabelHoldingOp) op).getLabel(); + if (label != null) { + labelBindLirPositions.put(label, instructionPosition); + } + lirPositions.put(op, instructionPosition); + } + instructionPosition++; + } + } + } + } + + /** + * Answers the code generator whether the jump from instruction to label is within disp LIR + * instructions. + * + * @param disp Maximum number of LIR instructions between label and instruction + */ + public boolean labelWithinRange(LIRInstruction instruction, Label label, int disp) { + if (conservativeLabelOffsets) { + return false; + } + Integer labelPosition = labelBindLirPositions.get(label); + Integer instructionPosition = lirPositions.get(instruction); + boolean result; + if (labelPosition != null && instructionPosition != null) { + result = Math.abs(labelPosition - instructionPosition) < disp; + } else { + result = false; + } + return result; + } + + /** + * Sets this CompilationResultBuilder into conservative mode. If set, + * {@link CompilationResultBuilder#labelWithinRange(LIRInstruction, Label, int)} always returns + * false. + */ + public void setConservativeLabelRanges() { + this.conservativeLabelOffsets = true; + } } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/asm/DataBuilder.java 2019-03-12 08:09:33.431559231 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/asm/DataBuilder.java 2019-03-12 08:09:33.071556895 +0100 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -29,13 +29,5 @@ import jdk.vm.ci.meta.Constant; public abstract class DataBuilder { - - /** - * When the method returns true, then Graal must produce detailed information that allows code - * patching without decoding instructions, i.e., Graal must produce annotations for the machine - * code that describe the exact locations of operands within instructions. - */ - public abstract boolean needDetailedPatchingInformation(); - public abstract Data createDataItem(Constant c); } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/gen/ArithmeticLIRGenerator.java 2019-03-12 08:09:33.923562423 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/gen/ArithmeticLIRGenerator.java 2019-03-12 08:09:33.555560036 +0100 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2009, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2009, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -110,4 +110,9 @@ return isAdd ? emitAdd(resultKind, a, b, setFlags) : emitSub(resultKind, a, b, setFlags); } + public Value emitRor(Value value, Value distance) { + // (value >>> distance) | (value << -distance) + return emitOr(emitUShr(value, distance), emitShl(value, emitNegate(distance))); + } + } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/gen/LIRGenerator.java 2019-03-12 08:09:34.403565538 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/gen/LIRGenerator.java 2019-03-12 08:09:34.035563150 +0100 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2009, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2009, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -35,8 +35,8 @@ import java.util.ArrayList; import java.util.List; +import java.util.Optional; -import jdk.vm.ci.code.RegisterConfig; import org.graalvm.compiler.asm.Label; import org.graalvm.compiler.core.common.LIRKind; import org.graalvm.compiler.core.common.calc.Condition; @@ -59,6 +59,7 @@ import org.graalvm.compiler.lir.StandardOp.BlockEndOp; import org.graalvm.compiler.lir.StandardOp.LabelOp; import org.graalvm.compiler.lir.StandardOp.SaveRegistersOp; +import org.graalvm.compiler.lir.hashing.Hasher; import org.graalvm.compiler.lir.SwitchStrategy; import org.graalvm.compiler.lir.Variable; import org.graalvm.compiler.options.Option; @@ -70,6 +71,7 @@ import jdk.vm.ci.code.CodeCacheProvider; import jdk.vm.ci.code.Register; import jdk.vm.ci.code.RegisterAttributes; +import jdk.vm.ci.code.RegisterConfig; import jdk.vm.ci.code.StackSlot; import jdk.vm.ci.code.TargetDescription; import jdk.vm.ci.meta.AllocatableValue; @@ -449,29 +451,51 @@ @Override public void emitStrategySwitch(JavaConstant[] keyConstants, double[] keyProbabilities, LabelRef[] keyTargets, LabelRef defaultTarget, Variable value) { - int keyCount = keyConstants.length; SwitchStrategy strategy = SwitchStrategy.getBestStrategy(keyProbabilities, keyConstants, keyTargets); + + int keyCount = keyConstants.length; + double minDensity = 1 / Math.sqrt(strategy.getAverageEffort()); + Optional hasher = hasherFor(keyConstants, minDensity); + double hashTableSwitchDensity = hasher.map(h -> keyCount / (double) h.cardinality()).orElse(0d); long valueRange = keyConstants[keyCount - 1].asLong() - keyConstants[0].asLong() + 1; double tableSwitchDensity = keyCount / (double) valueRange; + /* * This heuristic tries to find a compromise between the effort for the best switch strategy * and the density of a tableswitch. If the effort for the strategy is at least 4, then a * tableswitch is preferred if better than a certain value that starts at 0.5 and lowers * gradually with additional effort. */ - if (strategy.getAverageEffort() < 4 || tableSwitchDensity < (1 / Math.sqrt(strategy.getAverageEffort()))) { + if (strategy.getAverageEffort() < 4d || (tableSwitchDensity < minDensity && hashTableSwitchDensity < minDensity)) { emitStrategySwitch(strategy, value, keyTargets, defaultTarget); } else { - int minValue = keyConstants[0].asInt(); - assert valueRange < Integer.MAX_VALUE; - LabelRef[] targets = new LabelRef[(int) valueRange]; - for (int i = 0; i < valueRange; i++) { - targets[i] = defaultTarget; - } - for (int i = 0; i < keyCount; i++) { - targets[keyConstants[i].asInt() - minValue] = keyTargets[i]; + if (hashTableSwitchDensity > tableSwitchDensity) { + Hasher h = hasher.get(); + int cardinality = h.cardinality(); + LabelRef[] targets = new LabelRef[cardinality]; + JavaConstant[] keys = new JavaConstant[cardinality]; + for (int i = 0; i < cardinality; i++) { + keys[i] = JavaConstant.INT_0; + targets[i] = defaultTarget; + } + for (int i = 0; i < keyCount; i++) { + int idx = h.hash(keyConstants[i].asLong()); + keys[idx] = keyConstants[i]; + targets[idx] = keyTargets[i]; + } + emitHashTableSwitch(h, keys, defaultTarget, targets, value); + } else { + int minValue = keyConstants[0].asInt(); + assert valueRange < Integer.MAX_VALUE; + LabelRef[] targets = new LabelRef[(int) valueRange]; + for (int i = 0; i < valueRange; i++) { + targets[i] = defaultTarget; + } + for (int i = 0; i < keyCount; i++) { + targets[keyConstants[i].asInt() - minValue] = keyTargets[i]; + } + emitTableSwitch(minValue, defaultTarget, targets, value); } - emitTableSwitch(minValue, defaultTarget, targets, value); } } @@ -480,6 +504,16 @@ protected abstract void emitTableSwitch(int lowKey, LabelRef defaultTarget, LabelRef[] targets, Value key); + @SuppressWarnings("unused") + protected Optional hasherFor(JavaConstant[] keyConstants, double minDensity) { + return Optional.empty(); + } + + @SuppressWarnings("unused") + protected void emitHashTableSwitch(Hasher hasher, JavaConstant[] keys, LabelRef defaultTarget, LabelRef[] targets, Value value) { + throw new UnsupportedOperationException(getClass().getSimpleName() + " doesn't support hash table switches"); + } + @Override public void beforeRegisterAllocation() { } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/gen/LIRGeneratorTool.java 2019-03-12 08:09:34.891568704 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/gen/LIRGeneratorTool.java 2019-03-12 08:09:34.527566342 +0100 @@ -24,6 +24,9 @@ package org.graalvm.compiler.lir.gen; +import java.util.BitSet; +import java.util.List; + import org.graalvm.compiler.core.common.CompressEncoding; import org.graalvm.compiler.core.common.LIRKind; import org.graalvm.compiler.core.common.calc.Condition; @@ -39,11 +42,13 @@ import org.graalvm.compiler.lir.LabelRef; import org.graalvm.compiler.lir.SwitchStrategy; import org.graalvm.compiler.lir.Variable; +import org.graalvm.compiler.lir.VirtualStackSlot; import jdk.vm.ci.code.CodeCacheProvider; import jdk.vm.ci.code.Register; import jdk.vm.ci.code.RegisterAttributes; import jdk.vm.ci.code.RegisterConfig; +import jdk.vm.ci.code.StackSlot; import jdk.vm.ci.code.TargetDescription; import jdk.vm.ci.code.ValueKindFactory; import jdk.vm.ci.meta.AllocatableValue; @@ -264,6 +269,11 @@ Variable emitArrayEquals(JavaKind kind, Value array1, Value array2, Value length, int constantLength, boolean directPointers); @SuppressWarnings("unused") + default Variable emitArrayEquals(JavaKind kind1, JavaKind kind2, Value array1, Value array2, Value length, int constantLength, boolean directPointers) { + throw GraalError.unimplemented("Array.equals with different types substitution is not implemented on this architecture"); + } + + @SuppressWarnings("unused") default Variable emitArrayIndexOf(JavaKind kind, boolean findTwoConsecutive, Value sourcePointer, Value sourceCount, Value... searchValues) { throw GraalError.unimplemented("String.indexOf substitution is not implemented on this architecture"); } @@ -313,4 +323,21 @@ * after this fence will execute until all previous instructions have retired. */ void emitSpeculationFence(); + + default VirtualStackSlot allocateStackSlots(int slots, BitSet objects, List outObjectStackSlots) { + return getResult().getFrameMapBuilder().allocateStackSlots(slots, objects, outObjectStackSlots); + } + + default Value emitReadCallerStackPointer(Stamp wordStamp) { + /* + * We do not know the frame size yet. So we load the address of the first spill slot + * relative to the beginning of the frame, which is equivalent to the stack pointer of the + * caller. + */ + return emitAddress(StackSlot.get(getLIRKind(wordStamp), 0, true)); + } + + default Value emitReadReturnAddress(Stamp wordStamp, int returnAddressSize) { + return emitMove(StackSlot.get(getLIRKind(wordStamp), -returnAddressSize, true)); + } } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/ssa/SSAUtil.java 2019-03-12 08:09:35.367571792 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/ssa/SSAUtil.java 2019-03-12 08:09:35.003569430 +0100 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -31,11 +31,9 @@ import org.graalvm.compiler.core.common.cfg.AbstractBlockBase; import org.graalvm.compiler.lir.LIR; import org.graalvm.compiler.lir.LIRInstruction; -import org.graalvm.compiler.lir.LIRInstruction.OperandMode; import org.graalvm.compiler.lir.StandardOp.BlockEndOp; import org.graalvm.compiler.lir.StandardOp.JumpOp; import org.graalvm.compiler.lir.StandardOp.LabelOp; -import org.graalvm.compiler.lir.ValueConsumer; import jdk.vm.ci.meta.Value; @@ -114,13 +112,6 @@ return (JumpOp) op; } - public static JumpOp phiOutOrNull(LIR lir, AbstractBlockBase block) { - if (block.getSuccessorCount() != 1) { - return null; - } - return phiOut(lir, block); - } - public static int phiOutIndex(LIR lir, AbstractBlockBase block) { assert block.getSuccessorCount() == 1; ArrayList instructions = lir.getLIRforBlock(block); @@ -149,10 +140,6 @@ return new SSAVerifier(lir).verify(); } - public static boolean isMerge(AbstractBlockBase block) { - return block.getPredecessorCount() > 1; - } - public static void verifyPhi(LIR lir, AbstractBlockBase merge) { assert merge.getPredecessorCount() > 1; for (AbstractBlockBase pred : merge.getPredecessors()) { @@ -163,25 +150,7 @@ } } - public static void forEachPhiRegisterHint(LIR lir, AbstractBlockBase block, LabelOp label, Value targetValue, OperandMode mode, ValueConsumer valueConsumer) { - assert mode == OperandMode.DEF : "Wrong operand mode: " + mode; - assert lir.getLIRforBlock(block).get(0).equals(label) : String.format("Block %s and Label %s do not match!", block, label); - - if (!label.isPhiIn()) { - return; - } - int idx = indexOfValue(label, targetValue); - assert idx >= 0 : String.format("Value %s not in label %s", targetValue, label); - - for (AbstractBlockBase pred : block.getPredecessors()) { - JumpOp jump = phiOut(lir, pred); - Value sourceValue = jump.getOutgoingValue(idx); - valueConsumer.visitValue(jump, sourceValue, null, null); - } - - } - - private static int indexOfValue(LabelOp label, Value value) { + public static int indexOfValue(LabelOp label, Value value) { for (int i = 0; i < label.getIncomingSize(); i++) { if (label.getIncomingValue(i).equals(value)) { return i; @@ -189,20 +158,4 @@ } return -1; } - - public static int numPhiOut(LIR lir, AbstractBlockBase block) { - if (block.getSuccessorCount() != 1) { - // cannot be a phi_out block - return 0; - } - return numPhiIn(lir, block.getSuccessors()[0]); - } - - private static int numPhiIn(LIR lir, AbstractBlockBase block) { - if (!isMerge(block)) { - return 0; - } - return phiIn(lir, block).getPhiSize(); - } - } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop.phases/src/org/graalvm/compiler/loop/phases/LoopTransformations.java 2019-03-12 08:09:35.859574984 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop.phases/src/org/graalvm/compiler/loop/phases/LoopTransformations.java 2019-03-12 08:09:35.491572597 +0100 @@ -54,7 +54,6 @@ import org.graalvm.compiler.nodes.IfNode; import org.graalvm.compiler.nodes.LogicNode; import org.graalvm.compiler.nodes.LoopBeginNode; -import org.graalvm.compiler.nodes.LoopExitNode; import org.graalvm.compiler.nodes.NodeView; import org.graalvm.compiler.nodes.PhiNode; import org.graalvm.compiler.nodes.SafepointNode; @@ -231,11 +230,11 @@ graph.getDebug().log("LoopTransformations.insertPrePostLoops %s", loop); LoopFragmentWhole preLoop = loop.whole(); CountedLoopInfo preCounted = loop.counted(); - IfNode preLimit = preCounted.getLimitTest(); - assert preLimit != null; LoopBeginNode preLoopBegin = loop.loopBegin(); - LoopExitNode preLoopExitNode = preLoopBegin.getSingleLoopExit(); - FixedNode continuationNode = preLoopExitNode.next(); + AbstractBeginNode preLoopExitNode = preCounted.getCountedExit(); + + assert preLoop.nodes().contains(preLoopBegin); + assert preLoop.nodes().contains(preLoopExitNode); // Each duplication is inserted after the original, ergo create the post loop first LoopFragmentWhole mainLoop = preLoop.duplicate(); @@ -249,9 +248,9 @@ LoopBeginNode postLoopBegin = postLoop.getDuplicatedNode(preLoopBegin); postLoopBegin.setPostLoop(); - EndNode postEndNode = getBlockEndAfterLoopExit(postLoopBegin); + AbstractBeginNode postLoopExitNode = postLoop.getDuplicatedNode(preLoopExitNode); + EndNode postEndNode = getBlockEndAfterLoopExit(postLoopExitNode); AbstractMergeNode postMergeNode = postEndNode.merge(); - LoopExitNode postLoopExitNode = postLoopBegin.getSingleLoopExit(); // Update the main loop phi initialization to carry from the pre loop for (PhiNode prePhiNode : preLoopBegin.phis()) { @@ -259,13 +258,16 @@ mainPhiNode.setValueAt(0, prePhiNode); } - EndNode mainEndNode = getBlockEndAfterLoopExit(mainLoopBegin); + AbstractBeginNode mainLoopExitNode = mainLoop.getDuplicatedNode(preLoopExitNode); + EndNode mainEndNode = getBlockEndAfterLoopExit(mainLoopExitNode); AbstractMergeNode mainMergeNode = mainEndNode.merge(); AbstractEndNode postEntryNode = postLoopBegin.forwardEnd(); + // Exits have been merged, find the continuation below the merge + FixedNode continuationNode = mainMergeNode.next(); + // In the case of no Bounds tests, we just flow right into the main loop AbstractBeginNode mainLandingNode = BeginNode.begin(postEntryNode); - LoopExitNode mainLoopExitNode = mainLoopBegin.getSingleLoopExit(); mainLoopExitNode.setNext(mainLandingNode); preLoopExitNode.setNext(mainLoopBegin.forwardEnd()); @@ -337,8 +339,8 @@ /** * Find the end of the block following the LoopExit. */ - private static EndNode getBlockEndAfterLoopExit(LoopBeginNode curLoopBegin) { - FixedNode node = curLoopBegin.getSingleLoopExit().next(); + private static EndNode getBlockEndAfterLoopExit(AbstractBeginNode exit) { + FixedNode node = exit.next(); // Find the last node after the exit blocks starts return getBlockEnd(node); } @@ -423,6 +425,9 @@ condition.getDebug().log(DebugContext.VERBOSE_LEVEL, "isUnrollableLoop %s doubling the stride overflows %d", loopBegin, stride); return false; } + if (!loop.canDuplicateLoop()) { + return false; + } if (loopBegin.isMainLoop() || loopBegin.isSimpleLoop()) { // Flow-less loops to partial unroll for now. 3 blocks corresponds to an if that either // exits or continues the loop. There might be fixed and floating work within the loop --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/BasicInductionVariable.java 2019-03-12 08:09:36.323577994 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/BasicInductionVariable.java 2019-03-12 08:09:35.955575607 +0100 @@ -143,7 +143,7 @@ } @Override - public ValueNode extremumNode(boolean assumePositiveTripCount, Stamp stamp) { + public ValueNode extremumNode(boolean assumeLoopEntered, Stamp stamp) { Stamp fromStamp = phi.stamp(NodeView.DEFAULT); StructuredGraph graph = graph(); ValueNode stride = strideNode(); @@ -152,7 +152,7 @@ stride = IntegerConvertNode.convert(stride, stamp, graph(), NodeView.DEFAULT); initNode = IntegerConvertNode.convert(initNode, stamp, graph(), NodeView.DEFAULT); } - ValueNode maxTripCount = loop.counted().maxTripCountNode(assumePositiveTripCount); + ValueNode maxTripCount = loop.counted().maxTripCountNode(assumeLoopEntered); if (!maxTripCount.stamp(NodeView.DEFAULT).isCompatible(stamp)) { maxTripCount = IntegerConvertNode.convert(maxTripCount, stamp, graph(), NodeView.DEFAULT); } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/CountedLoopInfo.java 2019-03-12 08:09:36.787581004 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/CountedLoopInfo.java 2019-03-12 08:09:36.419578617 +0100 @@ -24,10 +24,12 @@ package org.graalvm.compiler.loop; -import static org.graalvm.compiler.loop.MathUtil.add; -import static org.graalvm.compiler.loop.MathUtil.sub; +import static java.lang.Math.abs; import static org.graalvm.compiler.loop.MathUtil.unsignedDivBefore; +import static org.graalvm.compiler.nodes.calc.BinaryArithmeticNode.add; +import static org.graalvm.compiler.nodes.calc.BinaryArithmeticNode.sub; +import org.graalvm.compiler.core.common.NumUtil; import org.graalvm.compiler.core.common.type.IntegerStamp; import org.graalvm.compiler.core.common.type.Stamp; import org.graalvm.compiler.core.common.util.UnsignedLong; @@ -37,16 +39,16 @@ import org.graalvm.compiler.nodes.ConstantNode; import org.graalvm.compiler.nodes.GuardNode; import org.graalvm.compiler.nodes.IfNode; +import org.graalvm.compiler.nodes.LogicNode; import org.graalvm.compiler.nodes.NodeView; import org.graalvm.compiler.nodes.StructuredGraph; import org.graalvm.compiler.nodes.ValueNode; -import org.graalvm.compiler.nodes.calc.CompareNode; import org.graalvm.compiler.nodes.calc.ConditionalNode; import org.graalvm.compiler.nodes.calc.IntegerLessThanNode; import org.graalvm.compiler.nodes.calc.NegateNode; import org.graalvm.compiler.nodes.extended.GuardingNode; +import org.graalvm.compiler.nodes.util.GraphUtil; -import jdk.vm.ci.code.CodeUtil; import jdk.vm.ci.meta.DeoptimizationAction; import jdk.vm.ci.meta.DeoptimizationReason; import jdk.vm.ci.meta.SpeculationLog; @@ -92,42 +94,41 @@ * * THIS VALUE SHOULD BE TREATED AS UNSIGNED. * - * @param assumePositive if true the check that the loop is entered at all will be omitted. + * @param assumeLoopEntered if true the check that the loop is entered at all will be omitted. */ - public ValueNode maxTripCountNode(boolean assumePositive) { + public ValueNode maxTripCountNode(boolean assumeLoopEntered) { StructuredGraph graph = iv.valueNode().graph(); Stamp stamp = iv.valueNode().stamp(NodeView.DEFAULT); ValueNode max; ValueNode min; - ValueNode range; ValueNode absStride; if (iv.direction() == Direction.Up) { absStride = iv.strideNode(); - range = sub(graph, end, iv.initNode()); max = end; min = iv.initNode(); } else { assert iv.direction() == Direction.Down; - absStride = graph.maybeAddOrUnique(NegateNode.create(iv.strideNode(), NodeView.DEFAULT)); - range = sub(graph, iv.initNode(), end); + absStride = NegateNode.create(iv.strideNode(), NodeView.DEFAULT); max = iv.initNode(); min = end; } + ValueNode range = sub(max, min); - ConstantNode one = ConstantNode.forIntegerStamp(stamp, 1, graph); + ConstantNode one = ConstantNode.forIntegerStamp(stamp, 1); if (oneOff) { - range = add(graph, range, one); + range = add(range, one); } // round-away-from-zero divison: (range + stride -/+ 1) / stride - ValueNode denominator = add(graph, range, sub(graph, absStride, one)); + ValueNode denominator = add(range, sub(absStride, one)); ValueNode div = unsignedDivBefore(graph, loop.entryPoint(), denominator, absStride, null); - if (assumePositive) { - return div; + if (assumeLoopEntered) { + return graph.addOrUniqueWithInputs(div); } - ConstantNode zero = ConstantNode.forIntegerStamp(stamp, 0, graph); - return graph.unique(new ConditionalNode(graph.unique(new IntegerLessThanNode(max, min)), zero, div)); + ConstantNode zero = ConstantNode.forIntegerStamp(stamp, 0); + LogicNode noEntryCheck = IntegerLessThanNode.create(max, min, NodeView.DEFAULT); + return graph.addOrUniqueWithInputs(ConditionalNode.create(noEntryCheck, zero, div, NodeView.DEFAULT)); } /** @@ -173,7 +174,7 @@ } public boolean isExactTripCount() { - return loop.loopBegin().loopExits().count() == 1; + return loop.loop().getNaturalExits().size() == 1; } public ValueNode exactTripCountNode() { @@ -216,6 +217,15 @@ return body; } + public AbstractBeginNode getCountedExit() { + if (getLimitTest().trueSuccessor() == getBody()) { + return getLimitTest().falseSuccessor(); + } else { + assert getLimitTest().falseSuccessor() == getBody(); + return getLimitTest().trueSuccessor(); + } + } + public Direction getDirection() { return iv.direction(); } @@ -228,30 +238,48 @@ return loop.loopBegin().getOverflowGuard(); } + public boolean counterNeverOverflows() { + if (iv.isConstantStride() && abs(iv.constantStride()) == 1) { + return true; + } + IntegerStamp endStamp = (IntegerStamp) end.stamp(NodeView.DEFAULT); + ValueNode strideNode = iv.strideNode(); + IntegerStamp strideStamp = (IntegerStamp) strideNode.stamp(NodeView.DEFAULT); + GraphUtil.tryKillUnused(strideNode); + if (getDirection() == Direction.Up) { + long max = NumUtil.maxValue(endStamp.getBits()); + return endStamp.upperBound() <= max - (strideStamp.upperBound() - 1) - (oneOff ? 1 : 0); + } else if (getDirection() == Direction.Down) { + long min = NumUtil.minValue(endStamp.getBits()); + return min + (1 - strideStamp.lowerBound()) + (oneOff ? 1 : 0) <= endStamp.lowerBound(); + } + return false; + } + @SuppressWarnings("try") public GuardingNode createOverFlowGuard() { GuardingNode overflowGuard = getOverFlowGuard(); - if (overflowGuard != null) { + if (overflowGuard != null || counterNeverOverflows()) { return overflowGuard; } try (DebugCloseable position = loop.loopBegin().withNodeSourcePosition()) { IntegerStamp stamp = (IntegerStamp) iv.valueNode().stamp(NodeView.DEFAULT); StructuredGraph graph = iv.valueNode().graph(); - CompareNode cond; // we use a negated guard with a < condition to achieve a >= + LogicNode cond; // we use a negated guard with a < condition to achieve a >= ConstantNode one = ConstantNode.forIntegerStamp(stamp, 1, graph); if (iv.direction() == Direction.Up) { - ValueNode v1 = sub(graph, ConstantNode.forIntegerStamp(stamp, CodeUtil.maxValue(stamp.getBits()), graph), sub(graph, iv.strideNode(), one)); + ValueNode v1 = sub(ConstantNode.forIntegerStamp(stamp, NumUtil.maxValue(stamp.getBits())), sub(iv.strideNode(), one)); if (oneOff) { - v1 = sub(graph, v1, one); + v1 = sub(v1, one); } - cond = graph.unique(new IntegerLessThanNode(v1, end)); + cond = graph.addOrUniqueWithInputs(IntegerLessThanNode.create(v1, end, NodeView.DEFAULT)); } else { assert iv.direction() == Direction.Down; - ValueNode v1 = add(graph, ConstantNode.forIntegerStamp(stamp, CodeUtil.minValue(stamp.getBits()), graph), sub(graph, one, iv.strideNode())); + ValueNode v1 = add(ConstantNode.forIntegerStamp(stamp, NumUtil.minValue(stamp.getBits())), sub(one, iv.strideNode())); if (oneOff) { - v1 = add(graph, v1, one); + v1 = add(v1, one); } - cond = graph.unique(new IntegerLessThanNode(end, v1)); + cond = graph.addOrUniqueWithInputs(IntegerLessThanNode.create(end, v1, NodeView.DEFAULT)); } assert graph.getGuardsStage().allowsFloatingGuards(); overflowGuard = graph.unique(new GuardNode(cond, AbstractBeginNode.prevBegin(loop.entryPoint()), DeoptimizationReason.LoopLimitCheck, DeoptimizationAction.InvalidateRecompile, true, --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/DefaultLoopPolicies.java 2019-03-12 08:09:37.251584014 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/DefaultLoopPolicies.java 2019-03-12 08:09:36.887581653 +0100 @@ -33,6 +33,7 @@ import org.graalvm.compiler.core.common.util.UnsignedLong; import org.graalvm.compiler.debug.CounterKey; import org.graalvm.compiler.debug.DebugContext; +import org.graalvm.compiler.debug.GraalError; import org.graalvm.compiler.graph.Node; import org.graalvm.compiler.graph.NodeBitMap; import org.graalvm.compiler.nodes.AbstractBeginNode; @@ -46,6 +47,7 @@ import org.graalvm.compiler.nodes.StructuredGraph; import org.graalvm.compiler.nodes.VirtualState; import org.graalvm.compiler.nodes.VirtualState.VirtualClosure; +import org.graalvm.compiler.nodes.calc.CompareNode; import org.graalvm.compiler.nodes.cfg.Block; import org.graalvm.compiler.nodes.cfg.ControlFlowGraph; import org.graalvm.compiler.nodes.debug.ControlFlowAnchorNode; @@ -64,9 +66,10 @@ @Option(help = "", type = OptionType.Expert) public static final OptionKey LoopUnswitchTrivial = new OptionKey<>(10); @Option(help = "", type = OptionType.Expert) public static final OptionKey LoopUnswitchFrequencyBoost = new OptionKey<>(10.0); - @Option(help = "", type = OptionType.Expert) public static final OptionKey FullUnrollMaxNodes = new OptionKey<>(300); + @Option(help = "", type = OptionType.Expert) public static final OptionKey FullUnrollMaxNodes = new OptionKey<>(400); + @Option(help = "", type = OptionType.Expert) public static final OptionKey FullUnrollConstantCompareBoost = new OptionKey<>(15); @Option(help = "", type = OptionType.Expert) public static final OptionKey FullUnrollMaxIterations = new OptionKey<>(600); - @Option(help = "", type = OptionType.Expert) public static final OptionKey ExactFullUnrollMaxNodes = new OptionKey<>(1200); + @Option(help = "", type = OptionType.Expert) public static final OptionKey ExactFullUnrollMaxNodes = new OptionKey<>(800); @Option(help = "", type = OptionType.Expert) public static final OptionKey ExactPartialUnrollMaxNodes = new OptionKey<>(200); @Option(help = "", type = OptionType.Expert) public static final OptionKey UnrollMaxIterations = new OptionKey<>(16); @@ -87,7 +90,7 @@ @Override public boolean shouldFullUnroll(LoopEx loop) { - if (!loop.isCounted() || !loop.counted().isConstantMaxTripCount()) { + if (!loop.isCounted() || !loop.counted().isConstantMaxTripCount() || !loop.counted().counterNeverOverflows()) { return false; } OptionValues options = loop.entryPoint().getOptions(); @@ -96,9 +99,27 @@ if (maxTrips.equals(0)) { return loop.canDuplicateLoop(); } - int maxNodes = (counted.isExactTripCount() && counted.isConstantExactTripCount()) ? Options.ExactFullUnrollMaxNodes.getValue(options) : Options.FullUnrollMaxNodes.getValue(options); - maxNodes = Math.min(maxNodes, Math.max(0, MaximumDesiredSize.getValue(options) - loop.loopBegin().graph().getNodeCount())); - int size = Math.max(1, loop.size() - 1 - loop.loopBegin().phis().count()); + if (maxTrips.isGreaterThan(Options.FullUnrollMaxIterations.getValue(options))) { + return false; + } + int globalMax = MaximumDesiredSize.getValue(options) - loop.loopBegin().graph().getNodeCount(); + if (globalMax <= 0) { + return false; + } + int maxNodes = counted.isExactTripCount() ? Options.ExactFullUnrollMaxNodes.getValue(options) : Options.FullUnrollMaxNodes.getValue(options); + for (Node usage : counted.getCounter().valueNode().usages()) { + if (usage instanceof CompareNode) { + CompareNode compare = (CompareNode) usage; + if (compare.getY().isConstant()) { + maxNodes += Options.FullUnrollConstantCompareBoost.getValue(options); + } + } + } + maxNodes = Math.min(maxNodes, globalMax); + int size = loop.inside().nodes().count(); + size -= 2; // remove the counted if and its non-exit begin + size -= loop.loopBegin().loopEnds().count(); + GraalError.guarantee(size >= 0, "Wrong size"); /* @formatter:off * The check below should not throw ArithmeticException because: * maxTrips is guaranteed to be >= 1 by the check above @@ -107,7 +128,7 @@ * - 1 <= size <= Integer.MAX_VALUE * @formatter:on */ - if (maxTrips.isLessOrEqualTo(Options.FullUnrollMaxIterations.getValue(options)) && maxTrips.minus(1).times(size).isLessOrEqualTo(maxNodes)) { + if (maxTrips.minus(1).times(size).isLessOrEqualTo(maxNodes)) { // check whether we're allowed to unroll this loop return loop.canDuplicateLoop(); } else { @@ -240,5 +261,4 @@ return false; } } - } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/DerivedConvertedInductionVariable.java 2019-03-12 08:09:37.735587154 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/DerivedConvertedInductionVariable.java 2019-03-12 08:09:37.367584767 +0100 @@ -81,8 +81,8 @@ } @Override - public ValueNode extremumNode(boolean assumePositiveTripCount, Stamp s) { - return base.extremumNode(assumePositiveTripCount, s); + public ValueNode extremumNode(boolean assumeLoopEntered, Stamp s) { + return base.extremumNode(assumeLoopEntered, s); } @Override --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/DerivedOffsetInductionVariable.java 2019-03-12 08:09:38.215590268 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/DerivedOffsetInductionVariable.java 2019-03-12 08:09:37.851587906 +0100 @@ -99,8 +99,8 @@ } @Override - public ValueNode extremumNode(boolean assumePositiveTripCount, Stamp stamp) { - return op(base.extremumNode(assumePositiveTripCount, stamp), IntegerConvertNode.convert(offset, stamp, graph(), NodeView.DEFAULT)); + public ValueNode extremumNode(boolean assumeLoopEntered, Stamp stamp) { + return op(base.extremumNode(assumeLoopEntered, stamp), IntegerConvertNode.convert(offset, stamp, graph(), NodeView.DEFAULT)); } @Override --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/DerivedScaledInductionVariable.java 2019-03-12 08:09:38.707593459 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/DerivedScaledInductionVariable.java 2019-03-12 08:09:38.335591046 +0100 @@ -110,8 +110,8 @@ } @Override - public ValueNode extremumNode(boolean assumePositiveTripCount, Stamp stamp) { - return mul(graph(), base.extremumNode(assumePositiveTripCount, stamp), IntegerConvertNode.convert(scale, stamp, graph(), NodeView.DEFAULT)); + public ValueNode extremumNode(boolean assumeLoopEntered, Stamp stamp) { + return mul(graph(), base.extremumNode(assumeLoopEntered, stamp), IntegerConvertNode.convert(scale, stamp, graph(), NodeView.DEFAULT)); } @Override --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/InductionVariable.java 2019-03-12 08:09:39.207596702 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/InductionVariable.java 2019-03-12 08:09:38.835594289 +0100 @@ -99,7 +99,7 @@ return extremumNode(false, valueNode().stamp(NodeView.DEFAULT)); } - public abstract ValueNode extremumNode(boolean assumePositiveTripCount, Stamp stamp); + public abstract ValueNode extremumNode(boolean assumeLoopEntered, Stamp stamp); public abstract boolean isConstantExtremum(); --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/LoopEx.java 2019-03-12 08:09:39.703599920 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/LoopEx.java 2019-03-12 08:09:39.339597558 +0100 @@ -76,8 +76,6 @@ import org.graalvm.compiler.nodes.extended.ValueAnchorNode; import org.graalvm.compiler.nodes.util.GraphUtil; -import jdk.vm.ci.code.BytecodeFrame; - public class LoopEx { private final Loop loop; private LoopFragmentInside inside; @@ -85,6 +83,7 @@ private CountedLoopInfo counted; private LoopsData data; private EconomicMap ivs; + private boolean countedLoopChecked; LoopEx(Loop loop, LoopsData data) { this.loop = loop; @@ -143,10 +142,12 @@ } public boolean isCounted() { + assert countedLoopChecked; return counted != null; } public CountedLoopInfo counted() { + assert countedLoopChecked; return counted; } @@ -211,6 +212,10 @@ } public boolean detectCounted() { + if (countedLoopChecked) { + return isCounted(); + } + countedLoopChecked = true; LoopBeginNode loopBegin = loopBegin(); FixedNode next = loopBegin.next(); while (next instanceof FixedGuardNode || next instanceof ValueAnchorNode || next instanceof FullInfopointNode) { @@ -219,8 +224,8 @@ if (next instanceof IfNode) { IfNode ifNode = (IfNode) next; boolean negated = false; - if (!loopBegin.isLoopExit(ifNode.falseSuccessor())) { - if (!loopBegin.isLoopExit(ifNode.trueSuccessor())) { + if (!isCfgLoopExit(ifNode.falseSuccessor())) { + if (!isCfgLoopExit(ifNode.trueSuccessor())) { return false; } negated = true; @@ -301,7 +306,7 @@ } break; default: - throw GraalError.shouldNotReachHere(); + throw GraalError.shouldNotReachHere(condition.toString()); } counted = new CountedLoopInfo(this, iv, ifNode, limit, oneOff, negated ? ifNode.falseSuccessor() : ifNode.trueSuccessor()); return true; @@ -309,6 +314,11 @@ return false; } + private boolean isCfgLoopExit(AbstractBeginNode begin) { + Block block = data.getCFG().blockFor(begin); + return loop.getDepth() > block.getLoopDepth() || loop.isNaturalExit(block); + } + public LoopsData loopsData() { return data; } @@ -321,7 +331,7 @@ work.add(cfg.blockFor(branch)); while (!work.isEmpty()) { Block b = work.remove(); - if (loop().getExits().contains(b)) { + if (loop().isLoopExit(b)) { assert !exits.contains(b.getBeginNode()); exits.add(b.getBeginNode()); } else if (blocks.add(b.getBeginNode())) { @@ -465,7 +475,7 @@ } if (node instanceof FrameState) { FrameState frameState = (FrameState) node; - if (frameState.bci == BytecodeFrame.AFTER_EXCEPTION_BCI || frameState.bci == BytecodeFrame.UNWIND_BCI) { + if (frameState.isExceptionHandlingBCI()) { return false; } } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/LoopFragment.java 2019-03-12 08:09:40.199603137 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/LoopFragment.java 2019-03-12 08:09:39.827600723 +0100 @@ -56,6 +56,7 @@ import org.graalvm.compiler.nodes.ValueProxyNode; import org.graalvm.compiler.nodes.VirtualState; import org.graalvm.compiler.nodes.cfg.Block; +import org.graalvm.compiler.nodes.cfg.ControlFlowGraph; import org.graalvm.compiler.nodes.java.MonitorEnterNode; import org.graalvm.compiler.nodes.spi.NodeWithState; import org.graalvm.compiler.nodes.virtual.CommitAllocationNode; @@ -144,7 +145,18 @@ protected abstract void beforeDuplication(); - protected abstract void finishDuplication(); + protected void finishDuplication() { + LoopEx originalLoopEx = original().loop(); + ControlFlowGraph cfg = originalLoopEx.loopsData().getCFG(); + for (LoopExitNode exit : originalLoopEx.loopBegin().loopExits().snapshot()) { + if (!originalLoopEx.loop().isLoopExit(cfg.blockFor(exit))) { + // this LoopExitNode is too low, we need to remove it otherwise it will be below + // merged exits + exit.removeExit(); + } + } + + } protected void patchNodes(final DuplicationReplacement dataFix) { if (isDuplicate() && !nodesReady) { @@ -388,42 +400,6 @@ }; } - public static NodeIterable toHirExits(final Iterable blocks) { - return new NodeIterable() { - - @Override - public Iterator iterator() { - final Iterator it = blocks.iterator(); - return new Iterator() { - - @Override - public void remove() { - throw new UnsupportedOperationException(); - } - - /** - * Return the true LoopExitNode for this loop or the BeginNode for the block. - */ - @Override - public AbstractBeginNode next() { - Block next = it.next(); - LoopExitNode exit = next.getLoopExit(); - if (exit != null) { - return exit; - } - return next.getBeginNode(); - } - - @Override - public boolean hasNext() { - return it.hasNext(); - } - }; - } - - }; - } - /** * Merges the early exits (i.e. loop exits) that were duplicated as part of this fragment, with * the original fragment's exits. @@ -431,13 +407,12 @@ protected void mergeEarlyExits() { assert isDuplicate(); StructuredGraph graph = graph(); - for (AbstractBeginNode earlyExit : LoopFragment.toHirBlocks(original().loop().loop().getExits())) { - LoopExitNode loopEarlyExit = (LoopExitNode) earlyExit; - FixedNode next = loopEarlyExit.next(); - if (loopEarlyExit.isDeleted() || !this.original().contains(loopEarlyExit)) { + for (AbstractBeginNode earlyExit : LoopFragment.toHirBlocks(original().loop().loop().getLoopExits())) { + FixedNode next = earlyExit.next(); + if (earlyExit.isDeleted() || !this.original().contains(earlyExit)) { continue; } - AbstractBeginNode newEarlyExit = getDuplicatedNode(loopEarlyExit); + AbstractBeginNode newEarlyExit = getDuplicatedNode(earlyExit); if (newEarlyExit == null) { continue; } @@ -446,72 +421,79 @@ EndNode newEnd = graph.add(new EndNode()); merge.addForwardEnd(originalEnd); merge.addForwardEnd(newEnd); - loopEarlyExit.setNext(originalEnd); + earlyExit.setNext(originalEnd); newEarlyExit.setNext(newEnd); merge.setNext(next); - FrameState exitState = loopEarlyExit.stateAfter(); - if (exitState != null) { - FrameState originalExitState = exitState; - exitState = exitState.duplicateWithVirtualState(); - loopEarlyExit.setStateAfter(exitState); - merge.setStateAfter(originalExitState); - /* - * Using the old exit's state as the merge's state is necessary because some of the - * VirtualState nodes contained in the old exit's state may be shared by other - * dominated VirtualStates. Those dominated virtual states need to see the - * proxy->phi update that are applied below. - * - * We now update the original fragment's nodes accordingly: - */ - originalExitState.applyToVirtual(node -> original.nodes.clearAndGrow(node)); - exitState.applyToVirtual(node -> original.nodes.markAndGrow(node)); - } - FrameState finalExitState = exitState; - - for (Node anchored : loopEarlyExit.anchored().snapshot()) { - anchored.replaceFirstInput(loopEarlyExit, merge); - } - - boolean newEarlyExitIsLoopExit = newEarlyExit instanceof LoopExitNode; - for (ProxyNode vpn : loopEarlyExit.proxies().snapshot()) { - if (vpn.hasNoUsages()) { - continue; - } - if (vpn.value() == null) { - assert vpn instanceof GuardProxyNode; - vpn.replaceAtUsages(null); - continue; - } - final ValueNode replaceWith; - ValueNode newVpn = prim(newEarlyExitIsLoopExit ? vpn : vpn.value()); - if (newVpn != null) { - PhiNode phi; - if (vpn instanceof ValueProxyNode) { - phi = graph.addWithoutUnique(new ValuePhiNode(vpn.stamp(NodeView.DEFAULT), merge)); - } else if (vpn instanceof GuardProxyNode) { - phi = graph.addWithoutUnique(new GuardPhiNode(merge)); + FrameState exitState = null; + if (earlyExit instanceof LoopExitNode) { + LoopExitNode earlyLoopExit = (LoopExitNode) earlyExit; + exitState = earlyLoopExit.stateAfter(); + if (exitState != null) { + FrameState originalExitState = exitState; + exitState = exitState.duplicateWithVirtualState(); + earlyLoopExit.setStateAfter(exitState); + merge.setStateAfter(originalExitState); + /* + * Using the old exit's state as the merge's state is necessary because some of + * the VirtualState nodes contained in the old exit's state may be shared by + * other dominated VirtualStates. Those dominated virtual states need to see the + * proxy->phi update that are applied below. + * + * We now update the original fragment's nodes accordingly: + */ + originalExitState.applyToVirtual(node -> original.nodes.clearAndGrow(node)); + exitState.applyToVirtual(node -> original.nodes.markAndGrow(node)); + } + } + + for (Node anchored : earlyExit.anchored().snapshot()) { + anchored.replaceFirstInput(earlyExit, merge); + } + + if (earlyExit instanceof LoopExitNode) { + LoopExitNode earlyLoopExit = (LoopExitNode) earlyExit; + FrameState finalExitState = exitState; + boolean newEarlyExitIsLoopExit = newEarlyExit instanceof LoopExitNode; + for (ProxyNode vpn : earlyLoopExit.proxies().snapshot()) { + if (vpn.hasNoUsages()) { + continue; + } + if (vpn.value() == null) { + assert vpn instanceof GuardProxyNode; + vpn.replaceAtUsages(null); + continue; + } + final ValueNode replaceWith; + ValueNode newVpn = prim(newEarlyExitIsLoopExit ? vpn : vpn.value()); + if (newVpn != null) { + PhiNode phi; + if (vpn instanceof ValueProxyNode) { + phi = graph.addWithoutUnique(new ValuePhiNode(vpn.stamp(NodeView.DEFAULT), merge)); + } else if (vpn instanceof GuardProxyNode) { + phi = graph.addWithoutUnique(new GuardPhiNode(merge)); + } else { + throw GraalError.shouldNotReachHere(); + } + phi.addInput(vpn); + phi.addInput(newVpn); + replaceWith = phi; } else { - throw GraalError.shouldNotReachHere(); + replaceWith = vpn.value(); } - phi.addInput(vpn); - phi.addInput(newVpn); - replaceWith = phi; - } else { - replaceWith = vpn.value(); - } - vpn.replaceAtMatchingUsages(replaceWith, usage -> { - if (merge.isPhiAtMerge(usage)) { - return false; - } - if (usage instanceof VirtualState) { - VirtualState stateUsage = (VirtualState) usage; - if (finalExitState != null && finalExitState.isPartOfThisState(stateUsage)) { + vpn.replaceAtMatchingUsages(replaceWith, usage -> { + if (merge.isPhiAtMerge(usage)) { return false; } - } - return true; - }); + if (usage instanceof VirtualState) { + VirtualState stateUsage = (VirtualState) usage; + if (finalExitState != null && finalExitState.isPartOfThisState(stateUsage)) { + return false; + } + } + return true; + }); + } } } } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/LoopFragmentInside.java 2019-03-12 08:09:40.703606406 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/LoopFragmentInside.java 2019-03-12 08:09:40.339604044 +0100 @@ -391,11 +391,6 @@ } @Override - protected void finishDuplication() { - // TODO (gd) ? - } - - @Override protected void beforeDuplication() { // Nothing to do } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/LoopFragmentWhole.java 2019-03-12 08:09:41.191609571 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/LoopFragmentWhole.java 2019-03-12 08:09:40.819607158 +0100 @@ -24,9 +24,7 @@ package org.graalvm.compiler.loop; -import jdk.internal.vm.compiler.collections.EconomicSet; import org.graalvm.compiler.core.common.cfg.Loop; -import org.graalvm.compiler.debug.DebugContext; import org.graalvm.compiler.graph.Graph; import org.graalvm.compiler.graph.Graph.DuplicationReplacement; import org.graalvm.compiler.graph.Node; @@ -34,8 +32,6 @@ import org.graalvm.compiler.nodes.EndNode; import org.graalvm.compiler.nodes.FixedNode; import org.graalvm.compiler.nodes.LoopBeginNode; -import org.graalvm.compiler.nodes.LoopExitNode; -import org.graalvm.compiler.nodes.StructuredGraph; import org.graalvm.compiler.nodes.ValueNode; import org.graalvm.compiler.nodes.cfg.Block; @@ -68,7 +64,7 @@ public NodeBitMap nodes() { if (nodes == null) { Loop loop = loop().loop(); - nodes = LoopFragment.computeNodes(graph(), LoopFragment.toHirBlocks(loop.getBlocks()), LoopFragment.toHirExits(loop.getExits())); + nodes = LoopFragment.computeNodes(graph(), LoopFragment.toHirBlocks(loop.getBlocks()), LoopFragment.toHirBlocks(loop.getLoopExits())); } return nodes; } @@ -108,46 +104,8 @@ } @Override - protected void finishDuplication() { - // TODO (gd) ? - } - - void cleanupLoopExits() { - LoopBeginNode loopBegin = original().loop().loopBegin(); - assert nodes == null || nodes.contains(loopBegin); - StructuredGraph graph = loopBegin.graph(); - if (graph.getGuardsStage() == StructuredGraph.GuardsStage.AFTER_FSA) { - // After FrameStateAssignment ControlFlowGraph treats loop exits differently which means - // that the LoopExitNodes can be in a block which post dominates the true loop exit. For - // cloning to work right they must agree. - EconomicSet exits = EconomicSet.create(); - for (Block exitBlock : original().loop().loop().getExits()) { - LoopExitNode exitNode = exitBlock.getLoopExit(); - if (exitNode == null) { - exitNode = graph.add(new LoopExitNode(loopBegin)); - graph.addAfterFixed(exitBlock.getBeginNode(), exitNode); - if (nodes != null) { - nodes.mark(exitNode); - } - graph.getDebug().dump(DebugContext.VERBOSE_LEVEL, graph, "Adjusting loop exit node for %s", loopBegin); - } - exits.add(exitNode); - } - for (LoopExitNode exitNode : loopBegin.loopExits()) { - if (!exits.contains(exitNode)) { - if (nodes != null) { - nodes.clear(exitNode); - } - graph.removeFixed(exitNode); - } - } - } - - } - - @Override protected void beforeDuplication() { - cleanupLoopExits(); + // nothing to do } @Override --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/MathUtil.java 2019-03-12 08:09:41.675612710 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/MathUtil.java 2019-03-12 08:09:41.307610323 +0100 @@ -88,7 +88,7 @@ if (isConstantOne(divisor)) { return dividend; } - ValueNode div = graph.addOrUniqueWithInputs(createDiv.apply(dividend, divisor)); + ValueNode div = createDiv.apply(dividend, divisor); if (div instanceof FixedBinaryNode) { FixedBinaryNode fixedDiv = (FixedBinaryNode) div; if (before.predecessor() instanceof FixedBinaryNode) { @@ -98,7 +98,7 @@ return binaryPredecessor; } } - graph.addBeforeFixed(before, fixedDiv); + graph.addBeforeFixed(before, graph.addOrUniqueWithInputs(fixedDiv)); } return div; } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.microbenchmarks/src/org/graalvm/compiler/microbenchmarks/lir/GraalCompilerState.java 2019-03-12 08:09:42.163615875 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.microbenchmarks/src/org/graalvm/compiler/microbenchmarks/lir/GraalCompilerState.java 2019-03-12 08:09:41.799613514 +0100 @@ -46,12 +46,13 @@ import org.graalvm.compiler.core.common.CompilationIdentifier; import org.graalvm.compiler.core.common.alloc.ComputeBlockOrder; import org.graalvm.compiler.core.common.cfg.AbstractBlockBase; +import org.graalvm.compiler.core.gen.LIRCompilerBackend; +import org.graalvm.compiler.core.gen.LIRGenerationProvider; import org.graalvm.compiler.core.target.Backend; -import org.graalvm.compiler.debug.DebugHandlersFactory; import org.graalvm.compiler.debug.DebugContext; +import org.graalvm.compiler.debug.DebugHandlersFactory; import org.graalvm.compiler.lir.LIR; import org.graalvm.compiler.lir.asm.CompilationResultBuilderFactory; -import org.graalvm.compiler.lir.framemap.FrameMapBuilder; import org.graalvm.compiler.lir.gen.LIRGenerationResult; import org.graalvm.compiler.lir.gen.LIRGeneratorTool; import org.graalvm.compiler.lir.phases.AllocationPhase.AllocationContext; @@ -88,6 +89,7 @@ import jdk.vm.ci.meta.ConstantReflectionProvider; import jdk.vm.ci.meta.MetaAccessProvider; import jdk.vm.ci.meta.ResolvedJavaMethod; +import jdk.vm.ci.meta.SpeculationLog; /** * State providing a new copy of a graph for each invocation of a benchmark. Subclasses of this @@ -374,10 +376,10 @@ linearScanOrder = ComputeBlockOrder.computeLinearScanOrder(blocks.length, startBlock); LIR lir = new LIR(cfg, linearScanOrder, codeEmittingOrder, getGraphOptions(), getGraphDebug()); - FrameMapBuilder frameMapBuilder = request.backend.newFrameMapBuilder(registerConfig); - lirGenRes = request.backend.newLIRGenerationResult(graph.compilationId(), lir, frameMapBuilder, request.graph, stub); - lirGenTool = request.backend.newLIRGenerator(lirGenRes); - nodeLirGen = request.backend.newNodeLIRBuilder(request.graph, lirGenTool); + LIRGenerationProvider lirBackend = (LIRGenerationProvider) request.backend; + lirGenRes = lirBackend.newLIRGenerationResult(graph.compilationId(), lir, registerConfig, request.graph, stub); + lirGenTool = lirBackend.newLIRGenerator(lirGenRes); + nodeLirGen = lirBackend.newNodeLIRBuilder(request.graph, lirGenTool); } protected OptionValues getGraphOptions() { @@ -460,8 +462,10 @@ */ protected final void emitCode() { int bytecodeSize = request.graph.method() == null ? 0 : request.graph.getBytecodeSize(); + SpeculationLog speculationLog = null; request.compilationResult.setHasUnsafeAccess(request.graph.hasUnsafeAccess()); - GraalCompiler.emitCode(request.backend, request.graph.getAssumptions(), request.graph.method(), request.graph.getMethods(), request.graph.getFields(), bytecodeSize, lirGenRes, + LIRCompilerBackend.emitCode(request.backend, request.graph.getAssumptions(), request.graph.method(), request.graph.getMethods(), request.graph.getFields(), + speculationLog, bytecodeSize, lirGenRes, request.compilationResult, request.installedCodeOwner, request.factory); } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes.test/src/org/graalvm/compiler/nodes/test/IfNodeCanonicalizationTest.java 2019-03-12 08:09:42.639618962 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes.test/src/org/graalvm/compiler/nodes/test/IfNodeCanonicalizationTest.java 2019-03-12 08:09:42.279616628 +0100 @@ -37,7 +37,7 @@ import org.graalvm.compiler.nodes.StructuredGraph.AllowAssumptions; import org.graalvm.compiler.nodes.calc.SubNode; import org.graalvm.compiler.phases.common.CanonicalizerPhase; -import org.graalvm.compiler.phases.common.ConvertDeoptimizeToGuardPhase; +import org.graalvm.compiler.loop.phases.ConvertDeoptimizeToGuardPhase; import org.graalvm.compiler.phases.common.IterativeConditionalEliminationPhase; import org.graalvm.compiler.phases.tiers.PhaseContext; --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/AbstractFixedGuardNode.java 2019-03-12 08:09:43.103621972 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/AbstractFixedGuardNode.java 2019-03-12 08:09:42.735619585 +0100 @@ -126,6 +126,14 @@ try (DebugCloseable position = this.withNodeSourcePosition()) { FixedNode currentNext = next(); setNext(null); + if (currentNext instanceof AbstractBeginNode && currentNext instanceof StateSplit && ((StateSplit) currentNext).stateAfter() != null) { + // Force an extra BeginNode in case any guarded Nodes are inputs to the StateSplit + BeginNode begin = graph().add(new BeginNode()); + begin.setNodeSourcePosition(getNoDeoptSuccessorPosition()); + begin.setNext(currentNext); + currentNext = begin; + } + DeoptimizeNode deopt = graph().add(new DeoptimizeNode(action, reason, speculation)); deopt.setStateBefore(stateBefore()); IfNode ifNode; --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/ComputeObjectAddressNode.java 2019-03-12 08:09:43.579625058 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/ComputeObjectAddressNode.java 2019-03-12 08:09:43.215622697 +0100 @@ -30,6 +30,7 @@ import org.graalvm.compiler.graph.NodeClass; import org.graalvm.compiler.nodeinfo.NodeInfo; import org.graalvm.compiler.nodeinfo.NodeSize; +import org.graalvm.compiler.nodes.calc.SignExtendNode; import org.graalvm.compiler.nodes.debug.ControlFlowAnchored; import org.graalvm.compiler.nodes.spi.Lowerable; import org.graalvm.compiler.nodes.spi.LoweringTool; @@ -50,7 +51,7 @@ public ComputeObjectAddressNode(ValueNode obj, ValueNode offset) { super(TYPE, StampFactory.forKind(JavaKind.Long)); this.object = obj; - this.offset = offset; + this.offset = SignExtendNode.create(offset, 64, NodeView.DEFAULT); } @NodeIntrinsic --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/ConstantNode.java 2019-03-12 08:09:44.035628016 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/ConstantNode.java 2019-03-12 08:09:43.671625655 +0100 @@ -54,6 +54,7 @@ import jdk.vm.ci.meta.JavaKind; import jdk.vm.ci.meta.MetaAccessProvider; import jdk.vm.ci.meta.PrimitiveConstant; +import jdk.vm.ci.meta.ResolvedJavaType; /** * The {@code ConstantNode} represents a {@link Constant constant}. @@ -99,6 +100,10 @@ } } + public ConstantNode(@InjectedNodeParameter Stamp stamp, @InjectedNodeParameter ConstantReflectionProvider constantReflection, @ConstantNodeParameter ResolvedJavaType type) { + this(constantReflection.asJavaClass(type), stamp); + } + /** * @return the constant value represented by this node */ @@ -546,4 +551,7 @@ } return ConstantNode.forInt(length); } + + @NodeIntrinsic + public static native Class forClass(@ConstantNodeParameter ResolvedJavaType type); } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/EncodedGraph.java 2019-03-12 08:09:44.511631102 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/EncodedGraph.java 2019-03-12 08:09:44.147628741 +0100 @@ -82,7 +82,7 @@ return startOffset; } - protected Object[] getObjects() { + public Object[] getObjects() { return objects; } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/FrameState.java 2019-03-12 08:09:44.991634215 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/FrameState.java 2019-03-12 08:09:44.623631829 +0100 @@ -681,4 +681,8 @@ } return false; } + + public boolean isExceptionHandlingBCI() { + return bci == BytecodeFrame.AFTER_EXCEPTION_BCI || bci == BytecodeFrame.UNWIND_BCI; + } } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/GraphDecoder.java 2019-03-12 08:09:45.475637353 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/GraphDecoder.java 2019-03-12 08:09:45.111634993 +0100 @@ -132,6 +132,7 @@ nodeStartOffsets[i] = encodedGraph.getStartOffset() - reader.getUVInt(); } encodedGraph.nodeStartOffsets = nodeStartOffsets; + graph.setGuardsStage((StructuredGraph.GuardsStage) readObject(this)); } } else { reader = null; @@ -1322,7 +1323,7 @@ } protected Object readObject(MethodScope methodScope) { - return methodScope.encodedGraph.getObjects()[methodScope.reader.getUVInt()]; + return methodScope.encodedGraph.getObject(methodScope.reader.getUVInt()); } /** --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/GraphEncoder.java 2019-03-12 08:09:45.983640647 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/GraphEncoder.java 2019-03-12 08:09:45.619638287 +0100 @@ -112,9 +112,13 @@ */ public class GraphEncoder { - /** The orderId that always represents {@code null}. */ + /** + * The orderId that always represents {@code null}. + */ public static final int NULL_ORDER_ID = 0; - /** The orderId of the {@link StructuredGraph#start() start node} of the encoded graph. */ + /** + * The orderId of the {@link StructuredGraph#start() start node} of the encoded graph. + */ public static final int START_NODE_ORDER_ID = 1; /** * The orderId of the first actual node after the {@link StructuredGraph#start() start node}. @@ -148,6 +152,8 @@ /** The last snapshot of {@link #nodeClasses} that was retrieved. */ protected NodeClass[] nodeClassesArray; + protected DebugContext debug; + /** * Utility method that does everything necessary to encode a single graph. */ @@ -160,7 +166,12 @@ } public GraphEncoder(Architecture architecture) { + this(architecture, null); + } + + public GraphEncoder(Architecture architecture, DebugContext debug) { this.architecture = architecture; + this.debug = debug; objects = FrequencyEncoder.createEqualityEncoder(); nodeClasses = FrequencyEncoder.createIdentityEncoder(); writer = UnsafeArrayTypeWriter.create(architecture.supportsUnalignedMemoryAccess()); @@ -170,6 +181,7 @@ * Must be invoked before {@link #finishPrepare()} and {@link #encode}. */ public void prepare(StructuredGraph graph) { + objects.addObject(graph.getGuardsStage()); for (Node node : graph.getNodes()) { NodeClass nodeClass = node.getNodeClass(); nodeClasses.addObject(nodeClass); @@ -288,9 +300,10 @@ for (int i = 0; i < nodeCount; i++) { writer.putUV(metadataStart - nodeStartOffsets[i]); } + writeObjectId(graph.getGuardsStage()); /* Check that the decoding of the encode graph is the same as the input. */ - assert verifyEncoding(graph, new EncodedGraph(getEncoding(), metadataStart, getObjects(), getNodeClasses(), graph), architecture); + assert verifyEncoding(graph, new EncodedGraph(getEncoding(), metadataStart, getObjects(), getNodeClasses(), graph)); return metadataStart; } @@ -434,10 +447,10 @@ * original graph. */ @SuppressWarnings("try") - public static boolean verifyEncoding(StructuredGraph originalGraph, EncodedGraph encodedGraph, Architecture architecture) { - DebugContext debug = originalGraph.getDebug(); + public boolean verifyEncoding(StructuredGraph originalGraph, EncodedGraph encodedGraph) { + DebugContext debugContext = debug != null ? debug : originalGraph.getDebug(); // @formatter:off - StructuredGraph decodedGraph = new StructuredGraph.Builder(originalGraph.getOptions(), debug, AllowAssumptions.YES). + StructuredGraph decodedGraph = new StructuredGraph.Builder(originalGraph.getOptions(), debugContext, AllowAssumptions.YES). method(originalGraph.method()). setIsSubstitution(originalGraph.isSubstitution()). trackNodeSourcePosition(originalGraph.trackNodeSourcePosition()). @@ -451,9 +464,9 @@ GraphComparison.verifyGraphsEqual(originalGraph, decodedGraph); } catch (Throwable ex) { originalGraph.getDebug(); - try (DebugContext.Scope scope = debug.scope("GraphEncoder")) { - debug.dump(DebugContext.VERBOSE_LEVEL, originalGraph, "Original Graph"); - debug.dump(DebugContext.VERBOSE_LEVEL, decodedGraph, "Decoded Graph"); + try (DebugContext.Scope scope = debugContext.scope("GraphEncoder")) { + debugContext.dump(DebugContext.VERBOSE_LEVEL, originalGraph, "Original Graph"); + debugContext.dump(DebugContext.VERBOSE_LEVEL, decodedGraph, "Decoded Graph"); } throw ex; } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/IfNode.java 2019-03-12 08:09:46.471643811 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/IfNode.java 2019-03-12 08:09:46.103641426 +0100 @@ -78,6 +78,7 @@ import jdk.vm.ci.meta.PrimitiveConstant; import jdk.vm.ci.meta.ResolvedJavaMethod; import jdk.vm.ci.meta.ResolvedJavaType; +import jdk.vm.ci.meta.TriState; /** * The {@code IfNode} represents a branch that can go one of two directions depending on the outcome @@ -868,6 +869,30 @@ } } + private ValueNode canonicalizeConditionalViaImplies(ValueNode trueValue, ValueNode falseValue) { + ValueNode collapsedTrue = trueValue; + ValueNode collapsedFalse = falseValue; + boolean simplify = false; + if (trueValue instanceof ConditionalNode) { + TriState result = condition().implies(false, ((ConditionalNode) trueValue).condition()); + if (result.isKnown()) { + simplify = true; + collapsedTrue = result.toBoolean() ? ((ConditionalNode) trueValue).trueValue() : ((ConditionalNode) trueValue).falseValue(); + } + } + if (falseValue instanceof ConditionalNode) { + TriState result = condition().implies(true, ((ConditionalNode) falseValue).condition()); + if (result.isKnown()) { + simplify = true; + collapsedFalse = result.toBoolean() ? ((ConditionalNode) falseValue).trueValue() : ((ConditionalNode) falseValue).falseValue(); + } + } + if (simplify) { + return graph().unique(new ConditionalNode(condition(), collapsedTrue, collapsedFalse)); + } + return null; + } + private ValueNode canonicalizeConditionalCascade(SimplifierTool tool, ValueNode trueValue, ValueNode falseValue) { if (trueValue.getStackKind() != falseValue.getStackKind()) { return null; @@ -877,7 +902,16 @@ } if (trueValue.isConstant() && falseValue.isConstant()) { return graph().unique(new ConditionalNode(condition(), trueValue, falseValue)); - } else if (!graph().isAfterExpandLogic()) { + } + ValueNode value = canonicalizeConditionalViaImplies(trueValue, falseValue); + if (value != null) { + return value; + } + if (!graph().isAfterExpandLogic()) { + /* + * !isAfterExpandLogic() => Cannot spawn NormalizeCompareNodes after lowering in the + * ExpandLogicPhase. + */ ConditionalNode conditional = null; ValueNode constant = null; boolean negateCondition; @@ -905,8 +939,11 @@ double shortCutProbability = probability(trueSuccessor()); LogicNode newCondition = LogicNode.or(condition(), negateCondition, conditional.condition(), negateConditionalCondition, shortCutProbability); return graph().unique(new ConditionalNode(newCondition, constant, otherValue)); - } else if (constant.isJavaConstant() && conditional.trueValue().isJavaConstant() && conditional.falseValue().isJavaConstant() && condition() instanceof CompareNode && + } + + if (constant.isJavaConstant() && conditional.trueValue().isJavaConstant() && conditional.falseValue().isJavaConstant() && condition() instanceof CompareNode && conditional.condition() instanceof CompareNode) { + Condition cond1 = ((CompareNode) condition()).condition().asCondition(); if (negateCondition) { cond1 = cond1.negate(); @@ -923,27 +960,19 @@ sameVars = true; cond2 = cond2.mirror(); } - // cond2 is EQ, LT, or GT if (sameVars) { JavaKind stackKind = conditional.trueValue().stamp(NodeView.from(tool)).getStackKind(); assert !stackKind.isNumericFloat(); - long c1 = constant.asJavaConstant().asLong(); - long c2 = conditional.trueValue().asJavaConstant().asLong(); - long c3 = conditional.falseValue().asJavaConstant().asLong(); - // `x cond1 y ? c1 : (x cond2 y ? c2 : c3)` - if (cond1 == Condition.GE && cond2 == Condition.LT) { - // x >= y ? v1 : (x < y ? v2 : v3) => x >= y ? v1 : v2 - return graph().unique(new ConditionalNode(condition(), conditional.trueValue(), constant)); - } else if (cond1 == Condition.GE && cond2 == Condition.GT) { - // x >= y ? v1 : (x > y ? v2 : v3) => x >= y ? v1 : v3 - return graph().unique(new ConditionalNode(condition(), conditional.falseValue(), constant)); - } else if (cond1 == Condition.EQ && cond2 == Condition.EQ) { - // x == y ? v1 : (x == y ? v2 : v3) => x == y ? v1 : v3 - return graph().unique(new ConditionalNode(condition(), conditional.falseValue(), constant)); - } else if (cond1 == Condition.NE && cond2 == Condition.LT) { - // x != y ? v1 : (x < y ? v2 : v3) => x != y ? v1 : v3 - return graph().unique(new ConditionalNode(condition(), conditional.falseValue(), constant)); - } else if (cond1 == Condition.LT && cond2 == Condition.EQ && c1 == -1 && c2 == 0 && c3 == 1) { + + ValueNode v1 = constant; + ValueNode v2 = conditional.trueValue(); + ValueNode v3 = conditional.falseValue(); + + long c1 = v1.asJavaConstant().asLong(); + long c2 = v2.asJavaConstant().asLong(); + long c3 = v3.asJavaConstant().asLong(); + + if (cond1 == Condition.LT && cond2 == Condition.EQ && c1 == -1 && c2 == 0 && c3 == 1) { // x < y ? -1 : (x == y ? 0 : 1) => x cmp y return graph().unique(new NormalizeCompareNode(x, y, stackKind, false)); } else if (cond1 == Condition.LT && cond2 == Condition.EQ && c1 == 1 && c2 == 0 && c3 == -1) { --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/LogicNode.java 2019-03-12 08:09:46.971647053 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/LogicNode.java 2019-03-12 08:09:46.603644667 +0100 @@ -91,6 +91,18 @@ * @param other the other condition. */ public TriState implies(boolean thisNegated, LogicNode other) { + if (this == other) { + return TriState.get(!thisNegated); + } + if (other instanceof LogicNegationNode) { + return flip(this.implies(thisNegated, ((LogicNegationNode) other).getValue())); + } return TriState.UNKNOWN; } + + private static TriState flip(TriState triState) { + return triState.isUnknown() + ? triState + : TriState.get(!triState.toBoolean()); + } } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/LoopBeginNode.java 2019-03-12 08:09:47.451650166 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/LoopBeginNode.java 2019-03-12 08:09:47.083647780 +0100 @@ -53,6 +53,7 @@ protected int inversionCount; protected LoopType loopType; protected int unrollFactor; + protected boolean osrLoop; public enum LoopType { SIMPLE_LOOP, @@ -303,11 +304,6 @@ return begin instanceof LoopExitNode && ((LoopExitNode) begin).loopBegin() == this; } - public LoopExitNode getSingleLoopExit() { - assert loopExits().count() == 1; - return loopExits().first(); - } - public LoopEndNode getSingleLoopEnd() { assert loopEnds().count() == 1; return loopEnds().first(); @@ -317,12 +313,7 @@ public void removeExits() { for (LoopExitNode loopexit : loopExits().snapshot()) { try (DebugCloseable position = graph().withNodeSourcePosition(loopexit)) { - loopexit.removeProxies(); - FrameState loopStateAfter = loopexit.stateAfter(); - graph().replaceFixedWithFixed(loopexit, graph().add(new BeginNode())); - if (loopStateAfter != null) { - GraphUtil.tryKillUnused(loopStateAfter); - } + loopexit.removeExit(); } } } @@ -415,4 +406,12 @@ } } } + + public void markOsrLoop() { + osrLoop = true; + } + + public boolean isOsrLoop() { + return osrLoop; + } } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/LoopExitNode.java 2019-03-12 08:09:47.955653434 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/LoopExitNode.java 2019-03-12 08:09:47.583651022 +0100 @@ -35,6 +35,7 @@ import org.graalvm.compiler.graph.spi.Simplifiable; import org.graalvm.compiler.graph.spi.SimplifierTool; import org.graalvm.compiler.nodeinfo.NodeInfo; +import org.graalvm.compiler.nodes.util.GraphUtil; @NodeInfo(allowedUsageTypes = {Association}, cycles = CYCLES_0, size = SIZE_0) public final class LoopExitNode extends BeginStateSplitNode implements IterableNodeType, Simplifiable { @@ -102,6 +103,15 @@ }); } + public void removeExit() { + this.removeProxies(); + FrameState loopStateAfter = this.stateAfter(); + graph().replaceFixedWithFixed(this, graph().add(new BeginNode())); + if (loopStateAfter != null) { + GraphUtil.tryKillUnused(loopStateAfter); + } + } + @Override public void simplify(SimplifierTool tool) { Node prev = this.predecessor(); --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/PiNode.java 2019-03-12 08:09:48.427656494 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/PiNode.java 2019-03-12 08:09:48.059654108 +0100 @@ -32,6 +32,7 @@ import org.graalvm.compiler.core.common.type.Stamp; import org.graalvm.compiler.core.common.type.StampFactory; import org.graalvm.compiler.core.common.type.TypeReference; +import org.graalvm.compiler.debug.DebugContext; import org.graalvm.compiler.graph.IterableNodeType; import org.graalvm.compiler.graph.Node; import org.graalvm.compiler.graph.NodeClass; @@ -100,7 +101,7 @@ } public static ValueNode create(ValueNode object, Stamp stamp) { - ValueNode value = canonical(object, stamp, null); + ValueNode value = canonical(object, stamp, null, null); if (value != null) { return value; } @@ -108,7 +109,7 @@ } public static ValueNode create(ValueNode object, Stamp stamp, ValueNode guard) { - ValueNode value = canonical(object, stamp, (GuardingNode) guard); + ValueNode value = canonical(object, stamp, (GuardingNode) guard, null); if (value != null) { return value; } @@ -117,7 +118,7 @@ public static ValueNode create(ValueNode object, ValueNode guard) { Stamp stamp = AbstractPointerStamp.pointerNonNull(object.stamp(NodeView.DEFAULT)); - ValueNode value = canonical(object, stamp, (GuardingNode) guard); + ValueNode value = canonical(object, stamp, (GuardingNode) guard, null); if (value != null) { return value; } @@ -127,7 +128,7 @@ @SuppressWarnings("unused") public static boolean intrinsify(GraphBuilderContext b, ResolvedJavaMethod method, ValueNode object, ValueNode guard) { Stamp stamp = AbstractPointerStamp.pointerNonNull(object.stamp(NodeView.DEFAULT)); - ValueNode value = canonical(object, stamp, (GuardingNode) guard); + ValueNode value = canonical(object, stamp, (GuardingNode) guard, null); if (value == null) { value = new PiNode(object, stamp, guard); } @@ -139,7 +140,7 @@ public static boolean intrinsify(GraphBuilderContext b, ResolvedJavaMethod method, ValueNode object, ResolvedJavaType toType, boolean exactType, boolean nonNull) { Stamp stamp = StampFactory.object(exactType ? TypeReference.createExactTrusted(toType) : TypeReference.createWithoutAssumptions(toType), nonNull || StampTool.isPointerNonNull(object.stamp(NodeView.DEFAULT))); - ValueNode value = canonical(object, stamp, null); + ValueNode value = canonical(object, stamp, null, null); if (value == null) { value = new PiNode(object, stamp); } @@ -177,13 +178,16 @@ ValueNode alias = tool.getAlias(object()); if (alias instanceof VirtualObjectNode) { VirtualObjectNode virtual = (VirtualObjectNode) alias; - if (StampTool.typeOrNull(this) != null && StampTool.typeOrNull(this).isAssignableFrom(virtual.type())) { + ResolvedJavaType type = StampTool.typeOrNull(this, tool.getMetaAccess()); + if (type != null && type.isAssignableFrom(virtual.type())) { tool.replaceWithVirtual(virtual); + } else { + tool.getDebug().log(DebugContext.INFO_LEVEL, "could not virtualize Pi because of type mismatch: %s %s vs %s", this, type, virtual.type()); } } } - public static ValueNode canonical(ValueNode object, Stamp stamp, GuardingNode guard) { + public static ValueNode canonical(ValueNode object, Stamp stamp, GuardingNode guard, PiNode self) { // Use most up to date stamp. Stamp computedStamp = stamp.improveWith(object.stamp(NodeView.DEFAULT)); @@ -201,8 +205,9 @@ } } else { for (Node n : guard.asNode().usages()) { - if (n instanceof PiNode) { + if (n instanceof PiNode && n != self) { PiNode otherPi = (PiNode) n; + assert otherPi.guard == guard; if (object == otherPi.object() && computedStamp.equals(otherPi.stamp(NodeView.DEFAULT))) { /* * Two PiNodes with the same guard and same result, so return the one with @@ -221,7 +226,7 @@ @Override public Node canonical(CanonicalizerTool tool) { - Node value = canonical(object(), stamp(NodeView.DEFAULT), getGuard()); + Node value = canonical(object(), piStamp(), getGuard(), this); if (value != null) { return value; } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/BinaryArithmeticNode.java 2019-03-12 08:09:48.931659761 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/BinaryArithmeticNode.java 2019-03-12 08:09:48.563657375 +0100 @@ -90,6 +90,26 @@ if (result != null) { return result; } + if (forX instanceof ConditionalNode && forY.isConstant() && forX.hasExactlyOneUsage()) { + ConditionalNode conditionalNode = (ConditionalNode) forX; + BinaryOp arithmeticOp = getArithmeticOp(); + ConstantNode trueConstant = tryConstantFold(arithmeticOp, conditionalNode.trueValue(), forY, this.stamp(view), view); + if (trueConstant != null) { + ConstantNode falseConstant = tryConstantFold(arithmeticOp, conditionalNode.falseValue(), forY, this.stamp(view), view); + if (falseConstant != null) { + // @formatter:off + /* The arithmetic is folded into a constant on both sides of the conditional. + * Example: + * (cond ? -5 : 5) + 100 + * canonicalizes to: + * (cond ? 95 : 105) + */ + // @formatter:on + return ConditionalNode.create(conditionalNode.condition, trueConstant, + falseConstant, view); + } + } + } return this; } @@ -118,6 +138,10 @@ return AddNode.create(v1, v2, view); } + public static ValueNode add(ValueNode v1, ValueNode v2) { + return add(v1, v2, NodeView.DEFAULT); + } + public static ValueNode mul(StructuredGraph graph, ValueNode v1, ValueNode v2, NodeView view) { return graph.addOrUniqueWithInputs(MulNode.create(v1, v2, view)); } @@ -126,6 +150,10 @@ return MulNode.create(v1, v2, view); } + public static ValueNode mul(ValueNode v1, ValueNode v2) { + return mul(v1, v2, NodeView.DEFAULT); + } + public static ValueNode sub(StructuredGraph graph, ValueNode v1, ValueNode v2, NodeView view) { return graph.addOrUniqueWithInputs(SubNode.create(v1, v2, view)); } @@ -134,6 +162,10 @@ return SubNode.create(v1, v2, view); } + public static ValueNode sub(ValueNode v1, ValueNode v2) { + return sub(v1, v2, NodeView.DEFAULT); + } + public static ValueNode branchlessMin(ValueNode v1, ValueNode v2, NodeView view) { if (v1.isDefaultConstant() && !v2.isDefaultConstant()) { return branchlessMin(v2, v1, view); --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/IntegerDivRemNode.java 2019-03-12 08:09:49.427662977 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/IntegerDivRemNode.java 2019-03-12 08:09:49.055660565 +0100 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/IntegerEqualsNode.java 2019-03-12 08:09:49.911666115 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/IntegerEqualsNode.java 2019-03-12 08:09:49.551663781 +0100 @@ -165,15 +165,19 @@ ValueNode v1 = null; ValueNode v2 = null; if (addX.getX() == addY.getX()) { + // (x + y) == (x + z) => y == z v1 = addX.getY(); v2 = addY.getY(); } else if (addX.getX() == addY.getY()) { + // (x + y) == (z + x) => y == z v1 = addX.getY(); v2 = addY.getX(); } else if (addX.getY() == addY.getX()) { + // (y + x) == (x + z) => y == z v1 = addX.getX(); v2 = addY.getY(); } else if (addX.getY() == addY.getY()) { + // (y + x) == (z + x) => y == z v1 = addX.getX(); v2 = addY.getX(); } @@ -183,6 +187,64 @@ } } + if (forX instanceof SubNode && forY instanceof SubNode) { + SubNode subX = (SubNode) forX; + SubNode subY = (SubNode) forY; + ValueNode v1 = null; + ValueNode v2 = null; + if (subX.getX() == subY.getX()) { + // (x - y) == (x - z) => y == z + v1 = subX.getY(); + v2 = subY.getY(); + } else if (subX.getY() == subY.getY()) { + // (y - x) == (z - x) => y == z + v1 = subX.getX(); + v2 = subY.getX(); + } + if (v1 != null) { + assert v2 != null; + return create(v1, v2, view); + } + } + + if (forX instanceof AddNode) { + AddNode addNode = (AddNode) forX; + if (addNode.getX() == forY) { + // (x + y) == x => y == 0 + return create(addNode.getY(), ConstantNode.forIntegerStamp(view.stamp(addNode), 0), view); + } else if (addNode.getY() == forY) { + // (x + y) == y => x == 0 + return create(addNode.getX(), ConstantNode.forIntegerStamp(view.stamp(addNode), 0), view); + } + } + + if (forY instanceof AddNode) { + AddNode addNode = (AddNode) forY; + if (addNode.getX() == forX) { + // x == (x + y) => y == 0 + return create(addNode.getY(), ConstantNode.forIntegerStamp(view.stamp(addNode), 0), view); + } else if (addNode.getY() == forX) { + // y == (x + y) => x == 0 + return create(addNode.getX(), ConstantNode.forIntegerStamp(view.stamp(addNode), 0), view); + } + } + + if (forX instanceof SubNode) { + SubNode subNode = (SubNode) forX; + if (subNode.getX() == forY) { + // (x - y) == x => y == 0 + return create(subNode.getY(), ConstantNode.forIntegerStamp(view.stamp(subNode), 0), view); + } + } + + if (forY instanceof SubNode) { + SubNode subNode = (SubNode) forY; + if (forX == subNode.getX()) { + // x == (x - y) => y == 0 + return create(subNode.getY(), ConstantNode.forIntegerStamp(view.stamp(subNode), 0), view); + } + } + return super.canonical(constantReflection, metaAccess, options, smallestCompareWidth, condition, unorderedIsTrue, forX, forY, view); } @@ -306,4 +368,19 @@ } return TriState.UNKNOWN; } + + @Override + public TriState implies(boolean thisNegated, LogicNode other) { + // x == y => !(x < y) + // x == y => !(y < x) + if (!thisNegated && other instanceof IntegerLessThanNode) { + ValueNode otherX = ((IntegerLessThanNode) other).getX(); + ValueNode otherY = ((IntegerLessThanNode) other).getY(); + if ((getX() == otherX && getY() == otherY) || (getX() == otherY && getY() == otherX)) { + return TriState.FALSE; + } + } + + return super.implies(thisNegated, other); + } } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/IntegerLessThanNode.java 2019-03-12 08:09:50.403669305 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/IntegerLessThanNode.java 2019-03-12 08:09:50.039666945 +0100 @@ -50,6 +50,7 @@ import jdk.vm.ci.meta.JavaKind; import jdk.vm.ci.meta.MetaAccessProvider; import jdk.vm.ci.meta.PrimitiveConstant; +import jdk.vm.ci.meta.TriState; @NodeInfo(shortName = "<") public final class IntegerLessThanNode extends IntegerLowerThanNode { @@ -225,7 +226,6 @@ } } } - } } @@ -296,4 +296,29 @@ return StampFactory.forInteger(bits, cast(min, bits), cast(max, bits)); } } + + @Override + public TriState implies(boolean thisNegated, LogicNode other) { + if (!thisNegated) { + if (other instanceof IntegerLessThanNode) { + ValueNode otherX = ((IntegerLessThanNode) other).getX(); + ValueNode otherY = ((IntegerLessThanNode) other).getY(); + // x < y => !y < x + if (getX() == otherY && getY() == otherX) { + return TriState.FALSE; + } + } + + // x < y => !x == y + // x < y => !y == x + if (other instanceof IntegerEqualsNode) { + ValueNode otherX = ((IntegerEqualsNode) other).getX(); + ValueNode otherY = ((IntegerEqualsNode) other).getY(); + if ((getX() == otherX && getY() == otherY) || (getX() == otherY && getY() == otherX)) { + return TriState.FALSE; + } + } + } + return super.implies(thisNegated, other); + } } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/IsNullNode.java 2019-03-12 08:09:50.883672416 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/IsNullNode.java 2019-03-12 08:09:50.519670056 +0100 @@ -35,7 +35,6 @@ import org.graalvm.compiler.nodes.LogicConstantNode; import org.graalvm.compiler.nodes.LogicNode; import org.graalvm.compiler.nodes.NodeView; -import org.graalvm.compiler.nodes.PiNode; import org.graalvm.compiler.nodes.UnaryOpLogicNode; import org.graalvm.compiler.nodes.ValueNode; import org.graalvm.compiler.nodes.spi.LIRLowerable; @@ -43,7 +42,6 @@ import org.graalvm.compiler.nodes.spi.Virtualizable; import org.graalvm.compiler.nodes.spi.VirtualizerTool; import org.graalvm.compiler.nodes.type.StampTool; -import org.graalvm.compiler.nodes.util.GraphUtil; import jdk.vm.ci.meta.JavaConstant; import jdk.vm.ci.meta.TriState; @@ -106,11 +104,6 @@ return LogicConstantNode.contradiction(); } - if (value instanceof PiNode) { - value = GraphUtil.skipPi(value); - continue; - } - if (value instanceof ConvertNode) { ConvertNode convertNode = (ConvertNode) value; if (convertNode.mayNullCheckSkipConversion()) { --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/RightShiftNode.java 2019-03-12 08:09:51.359675502 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/RightShiftNode.java 2019-03-12 08:09:50.991673116 +0100 @@ -83,6 +83,15 @@ return new UnsignedRightShiftNode(forX, forY); } + Stamp xStampGeneric = forX.stamp(view); + if (xStampGeneric instanceof IntegerStamp) { + IntegerStamp xStamp = (IntegerStamp) xStampGeneric; + if (xStamp.lowerBound() >= -1 && xStamp.upperBound() <= 0) { + // Right shift by any amount does not change any bit. + return forX; + } + } + if (forY.isConstant()) { int amount = forY.asJavaConstant().asInt(); int originalAmout = amount; @@ -91,6 +100,16 @@ if (amount == 0) { return forX; } + + if (xStampGeneric instanceof IntegerStamp) { + IntegerStamp xStamp = (IntegerStamp) xStampGeneric; + + if (xStamp.lowerBound() >> amount == xStamp.upperBound() >> amount) { + // Right shift turns the result of the expression into a constant. + return ConstantNode.forIntegerKind(stamp.getStackKind(), xStamp.lowerBound() >> amount); + } + } + if (forX instanceof ShiftNode) { ShiftNode other = (ShiftNode) forX; if (other.getY().isConstant()) { --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/SignedDivNode.java 2019-03-12 08:09:51.839678613 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/SignedDivNode.java 2019-03-12 08:09:51.475676254 +0100 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2011, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -44,7 +44,7 @@ public static final NodeClass TYPE = NodeClass.create(SignedDivNode.class); - protected SignedDivNode(ValueNode x, ValueNode y, GuardingNode zeroCheck) { + public SignedDivNode(ValueNode x, ValueNode y, GuardingNode zeroCheck) { this(TYPE, x, y, zeroCheck); } @@ -118,8 +118,8 @@ } long abs = Math.abs(c); if (CodeUtil.isPowerOf2(abs) && forX.stamp(view) instanceof IntegerStamp) { - ValueNode dividend = forX; IntegerStamp stampX = (IntegerStamp) forX.stamp(view); + ValueNode dividend = forX; int log2 = CodeUtil.log2(abs); // no rounding if dividend is positive or if its low bits are always 0 if (stampX.canBeNegative() || (stampX.upMask() & (abs - 1)) != 0) { --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/SignedRemNode.java 2019-03-12 08:09:52.331681803 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/SignedRemNode.java 2019-03-12 08:09:51.963679417 +0100 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2011, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -26,6 +26,7 @@ import org.graalvm.compiler.core.common.type.IntegerStamp; import org.graalvm.compiler.core.common.type.Stamp; +import org.graalvm.compiler.graph.Node; import org.graalvm.compiler.graph.NodeClass; import org.graalvm.compiler.graph.spi.CanonicalizerTool; import org.graalvm.compiler.nodeinfo.NodeInfo; @@ -37,13 +38,15 @@ import org.graalvm.compiler.nodes.spi.NodeLIRBuilderTool; import jdk.vm.ci.code.CodeUtil; +import jdk.vm.ci.meta.Constant; +import jdk.vm.ci.meta.PrimitiveConstant; @NodeInfo(shortName = "%") public class SignedRemNode extends IntegerDivRemNode implements LIRLowerable { public static final NodeClass TYPE = NodeClass.create(SignedRemNode.class); - protected SignedRemNode(ValueNode x, ValueNode y, GuardingNode zeroCheck) { + public SignedRemNode(ValueNode x, ValueNode y, GuardingNode zeroCheck) { this(TYPE, x, y, zeroCheck); } @@ -53,7 +56,7 @@ public static ValueNode create(ValueNode x, ValueNode y, GuardingNode zeroCheck, NodeView view) { Stamp stamp = IntegerStamp.OPS.getRem().foldStamp(x.stamp(view), y.stamp(view)); - return canonical(null, x, y, zeroCheck, stamp, view); + return canonical(null, x, y, zeroCheck, stamp, view, null); } @Override @@ -64,10 +67,10 @@ @Override public ValueNode canonical(CanonicalizerTool tool, ValueNode forX, ValueNode forY) { NodeView view = NodeView.from(tool); - return canonical(this, forX, forY, getZeroCheck(), stamp(view), view); + return canonical(this, forX, forY, getZeroCheck(), stamp(view), view, tool); } - private static ValueNode canonical(SignedRemNode self, ValueNode forX, ValueNode forY, GuardingNode zeroCheck, Stamp stamp, NodeView view) { + private static ValueNode canonical(SignedRemNode self, ValueNode forX, ValueNode forY, GuardingNode zeroCheck, Stamp stamp, NodeView view, CanonicalizerTool tool) { if (forX.isConstant() && forY.isConstant()) { long y = forY.asJavaConstant().asLong(); if (y == 0) { @@ -81,27 +84,52 @@ IntegerStamp yStamp = (IntegerStamp) forY.stamp(view); if (constY < 0 && constY != CodeUtil.minValue(yStamp.getBits())) { Stamp newStamp = IntegerStamp.OPS.getRem().foldStamp(forX.stamp(view), forY.stamp(view)); - return canonical(null, forX, ConstantNode.forIntegerStamp(yStamp, -constY), zeroCheck, newStamp, view); + return canonical(null, forX, ConstantNode.forIntegerStamp(yStamp, -constY), zeroCheck, newStamp, view, tool); } if (constY == 1) { return ConstantNode.forIntegerStamp(stamp, 0); - } else if (CodeUtil.isPowerOf2(constY)) { - if (xStamp.isPositive()) { - // x & (y - 1) - return new AndNode(forX, ConstantNode.forIntegerStamp(stamp, constY - 1)); - } else if (xStamp.isNegative()) { - // -((-x) & (y - 1)) - return new NegateNode(new AndNode(new NegateNode(forX), ConstantNode.forIntegerStamp(stamp, constY - 1))); + } else if (CodeUtil.isPowerOf2(constY) && tool != null && tool.allUsagesAvailable()) { + if (allUsagesCompareAgainstZero(self)) { + // x % y == 0 <=> (x & (y-1)) == 0 + return new AndNode(forX, ConstantNode.forIntegerStamp(yStamp, constY - 1)); } else { - // x - ((x / y) << log2(y)) - return SubNode.create(forX, LeftShiftNode.create(SignedDivNode.canonical(forX, constY, view), ConstantNode.forInt(CodeUtil.log2(constY)), view), view); + if (xStamp.isPositive()) { + // x & (y - 1) + return new AndNode(forX, ConstantNode.forIntegerStamp(stamp, constY - 1)); + } else if (xStamp.isNegative()) { + // -((-x) & (y - 1)) + return new NegateNode(new AndNode(new NegateNode(forX), ConstantNode.forIntegerStamp(stamp, constY - 1))); + } } } } return self != null ? self : new SignedRemNode(forX, forY, zeroCheck); } + private static boolean allUsagesCompareAgainstZero(SignedRemNode self) { + int compareAgainstZero = 0; + int usageCount = self.getUsageCount(); + for (int i = 0; i < usageCount; i++) { + Node usage = self.getUsageAt(i); + if (usage instanceof IntegerEqualsNode) { + IntegerEqualsNode equalsNode = (IntegerEqualsNode) usage; + ValueNode node = equalsNode.getY(); + if (node == self) { + node = equalsNode.getX(); + } + if (node instanceof ConstantNode) { + ConstantNode constantNode = (ConstantNode) node; + Constant constant = constantNode.asConstant(); + if (constant instanceof PrimitiveConstant && ((PrimitiveConstant) constant).asLong() == 0) { + compareAgainstZero++; + } + } + } + } + return compareAgainstZero == usageCount; + } + @Override public void generate(NodeLIRBuilderTool gen) { gen.setResult(this, gen.getLIRGeneratorTool().getArithmetic().emitRem(gen.operand(getX()), gen.operand(getY()), gen.state(this))); --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/UnsignedRightShiftNode.java 2019-03-12 08:09:52.827685017 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/UnsignedRightShiftNode.java 2019-03-12 08:09:52.455682606 +0100 @@ -80,6 +80,22 @@ if (amount == 0) { return forX; } + + Stamp xStampGeneric = forX.stamp(view); + if (xStampGeneric instanceof IntegerStamp) { + IntegerStamp xStamp = (IntegerStamp) xStampGeneric; + + if (xStamp.lowerBound() >>> amount == xStamp.upperBound() >>> amount) { + // The result of the shift is constant. + return ConstantNode.forIntegerKind(stamp.getStackKind(), xStamp.lowerBound() >>> amount); + } + + if (amount == xStamp.getBits() - 1 && xStamp.lowerBound() == -1 && xStamp.upperBound() == 0) { + // Shift is equivalent to a negation, i.e., turns -1 into 1 and keeps 0 at 0. + return NegateNode.create(forX, view); + } + } + if (forX instanceof ShiftNode) { ShiftNode other = (ShiftNode) forX; if (other.getY().isConstant()) { --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/cfg/Block.java 2019-03-12 08:09:53.303688103 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/cfg/Block.java 2019-03-12 08:09:52.935685717 +0100 @@ -33,13 +33,11 @@ import org.graalvm.compiler.graph.Node; import org.graalvm.compiler.nodeinfo.Verbosity; import org.graalvm.compiler.nodes.AbstractBeginNode; -import org.graalvm.compiler.nodes.BeginNode; import org.graalvm.compiler.nodes.FixedNode; import org.graalvm.compiler.nodes.FixedWithNextNode; import org.graalvm.compiler.nodes.InvokeWithExceptionNode; import org.graalvm.compiler.nodes.LoopBeginNode; import org.graalvm.compiler.nodes.LoopEndNode; -import org.graalvm.compiler.nodes.LoopExitNode; import org.graalvm.compiler.nodes.memory.MemoryCheckpoint; import jdk.internal.vm.compiler.word.LocationIdentity; @@ -69,21 +67,6 @@ return endNode; } - /** - * Return the {@link LoopExitNode} for this block if it exists. - */ - public LoopExitNode getLoopExit() { - if (beginNode instanceof BeginNode) { - if (beginNode.next() instanceof LoopExitNode) { - return (LoopExitNode) beginNode.next(); - } - } - if (beginNode instanceof LoopExitNode) { - return (LoopExitNode) beginNode; - } - return null; - } - @Override public Loop getLoop() { return loop; --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/cfg/ControlFlowGraph.java 2019-03-12 08:09:53.775691162 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/cfg/ControlFlowGraph.java 2019-03-12 08:09:53.415688829 +0100 @@ -24,6 +24,8 @@ package org.graalvm.compiler.nodes.cfg; +import static org.graalvm.compiler.core.common.cfg.AbstractBlockBase.BLOCK_ID_COMPARATOR; + import java.util.ArrayList; import java.util.Arrays; import java.util.BitSet; @@ -49,7 +51,6 @@ import org.graalvm.compiler.nodes.LoopExitNode; import org.graalvm.compiler.nodes.MergeNode; import org.graalvm.compiler.nodes.StructuredGraph; -import org.graalvm.compiler.nodes.StructuredGraph.GuardsStage; public final class ControlFlowGraph implements AbstractControlFlowGraph { /** @@ -614,13 +615,26 @@ computeLoopBlocks(endBlock, loop, stack, true); } - if (graph.getGuardsStage() != GuardsStage.AFTER_FSA) { + // Note that at this point, due to traversal order, child loops of `loop` have + // not been discovered yet. + for (Block b : loop.getBlocks()) { + for (Block sux : b.getSuccessors()) { + if (sux.getLoop() != loop) { + assert sux.getLoopDepth() < loop.getDepth(); + loop.getNaturalExits().add(sux); + } + } + } + loop.getNaturalExits().sort(BLOCK_ID_COMPARATOR); + + if (!graph.getGuardsStage().areFrameStatesAtDeopts()) { for (LoopExitNode exit : loopBegin.loopExits()) { Block exitBlock = nodeToBlock.get(exit); assert exitBlock.getPredecessorCount() == 1; computeLoopBlocks(exitBlock.getFirstPredecessor(), loop, stack, true); - loop.addExit(exitBlock); + loop.getLoopExits().add(exitBlock); } + loop.getLoopExits().sort(BLOCK_ID_COMPARATOR); // The following loop can add new blocks to the end of the loop's block // list. @@ -630,65 +644,21 @@ for (Block sux : b.getSuccessors()) { if (sux.getLoop() != loop) { AbstractBeginNode begin = sux.getBeginNode(); - if (!(begin instanceof LoopExitNode && ((LoopExitNode) begin).loopBegin() == loopBegin)) { + if (!loopBegin.isLoopExit(begin)) { + assert !(begin instanceof LoopBeginNode); + assert sux.getLoopDepth() < loop.getDepth(); graph.getDebug().log(DebugContext.VERBOSE_LEVEL, "Unexpected loop exit with %s, including whole branch in the loop", sux); computeLoopBlocks(sux, loop, stack, false); } } } } + } else { + loop.getLoopExits().addAll(loop.getNaturalExits()); } } } } - - /* - * Compute the loop exit blocks after FSA. - */ - if (graph.getGuardsStage() == GuardsStage.AFTER_FSA) { - for (Block b : reversePostOrder) { - if (b.getLoop() != null) { - for (Block succ : b.getSuccessors()) { - // if the loop of the succ is a different one (or none) - if (b.getLoop() != succ.getLoop()) { - // and the succ loop is not a child loop of the curr one - if (succ.getLoop() == null) { - // we might exit multiple loops if b.loops is not a loop at depth 0 - Loop curr = b.getLoop(); - while (curr != null) { - curr.addExit(succ); - curr = curr.getParent(); - } - } else { - /* - * succ also has a loop, might be a child loop - * - * if it is a child loop we do not exit a loop. if it is a loop - * different than b.loop and not a child loop it must be a parent - * loop, thus we exit all loops between b.loop and succ.loop - * - * if we exit multiple loops immediately after each other the - * bytecode parser might generate loop exit nodes after another and - * the CFG will identify them as separate blocks, we just take the - * first one and exit all loops at this one - */ - if (succ.getLoop().getParent() != b.getLoop()) { - assert succ.getLoop().getDepth() < b.getLoop().getDepth(); - // b.loop must not be a transitive parent of succ.loop - assert !Loop.transitiveParentLoop(succ.getLoop(), b.getLoop()); - Loop curr = b.getLoop(); - while (curr != null && curr != succ.getLoop()) { - curr.addExit(succ); - curr = curr.getParent(); - } - } - } - } - } - } - } - } - } private static void computeLoopBlocks(Block start, Loop loop, Block[] stack, boolean usePred) { --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/extended/JavaReadNode.java 2019-03-12 08:09:54.275694403 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/extended/JavaReadNode.java 2019-03-12 08:09:53.911692044 +0100 @@ -27,6 +27,7 @@ import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_2; import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_1; +import org.graalvm.compiler.core.common.type.Stamp; import org.graalvm.compiler.core.common.type.StampFactory; import org.graalvm.compiler.graph.Node; import org.graalvm.compiler.graph.NodeClass; @@ -54,7 +55,11 @@ protected final boolean compressible; public JavaReadNode(JavaKind readKind, AddressNode address, LocationIdentity location, BarrierType barrierType, boolean compressible) { - super(TYPE, address, location, StampFactory.forKind(readKind), barrierType); + this(StampFactory.forKind(readKind), readKind, address, location, barrierType, compressible); + } + + public JavaReadNode(Stamp stamp, JavaKind readKind, AddressNode address, LocationIdentity location, BarrierType barrierType, boolean compressible) { + super(TYPE, address, location, stamp, barrierType); this.readKind = readKind; this.compressible = compressible; } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/GeneratedInvocationPlugin.java 2019-03-12 08:09:54.747697462 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/GeneratedInvocationPlugin.java 2019-03-12 08:09:54.387695129 +0100 @@ -24,6 +24,9 @@ package org.graalvm.compiler.nodes.graphbuilderconf; +import static jdk.vm.ci.services.Services.IS_BUILDING_NATIVE_IMAGE; +import static jdk.vm.ci.services.Services.IS_IN_NATIVE_IMAGE; + import java.lang.annotation.Annotation; import java.lang.reflect.Method; @@ -66,6 +69,11 @@ return true; } + if (IS_IN_NATIVE_IMAGE || IS_BUILDING_NATIVE_IMAGE) { + // The reflection here is problematic for SVM. + return true; + } + MetaAccessProvider metaAccess = b.getMetaAccess(); ResolvedJavaMethod executeMethod = metaAccess.lookupJavaMethod(getExecuteMethod()); ResolvedJavaType thisClass = metaAccess.lookupJavaType(getClass()); --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/GraphBuilderConfiguration.java 2019-03-12 08:09:55.211700470 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/GraphBuilderConfiguration.java 2019-03-12 08:09:54.851698136 +0100 @@ -25,13 +25,15 @@ package org.graalvm.compiler.nodes.graphbuilderconf; import java.util.Arrays; +import java.util.Collections; +import java.util.List; import org.graalvm.compiler.core.common.type.StampPair; import jdk.vm.ci.meta.JavaType; import jdk.vm.ci.meta.ResolvedJavaType; -public class GraphBuilderConfiguration { +public final class GraphBuilderConfiguration { public static class Plugins { private final InvocationPlugins invocationPlugins; @@ -39,7 +41,6 @@ private ParameterPlugin[] parameterPlugins; private TypePlugin[] typePlugins; private InlineInvokePlugin[] inlineInvokePlugins; - private LoopExplosionPlugin loopExplosionPlugin; private ClassInitializationPlugin classInitializationPlugin; private InvokeDynamicPlugin invokeDynamicPlugin; private ProfilingPlugin profilingPlugin; @@ -49,18 +50,21 @@ * {@code copyFrom} become the {@linkplain InvocationPlugins#getParent() default} * {@linkplain #getInvocationPlugins() invocation plugins} in this object. */ - public Plugins(Plugins copyFrom) { - this.invocationPlugins = new InvocationPlugins(copyFrom.invocationPlugins); + public Plugins(Plugins copyFrom, InvocationPlugins invocationPlugins) { + this.invocationPlugins = invocationPlugins != null ? invocationPlugins : new InvocationPlugins(copyFrom.invocationPlugins); this.nodePlugins = copyFrom.nodePlugins; this.parameterPlugins = copyFrom.parameterPlugins; this.typePlugins = copyFrom.typePlugins; this.inlineInvokePlugins = copyFrom.inlineInvokePlugins; - this.loopExplosionPlugin = copyFrom.loopExplosionPlugin; this.classInitializationPlugin = copyFrom.classInitializationPlugin; this.invokeDynamicPlugin = copyFrom.invokeDynamicPlugin; this.profilingPlugin = copyFrom.profilingPlugin; } + public Plugins(Plugins copyFrom) { + this(copyFrom, null); + } + /** * Creates a new set of plugins. * @@ -155,14 +159,6 @@ inlineInvokePlugins = new InlineInvokePlugin[0]; } - public LoopExplosionPlugin getLoopExplosionPlugin() { - return loopExplosionPlugin; - } - - public void setLoopExplosionPlugin(LoopExplosionPlugin plugin) { - this.loopExplosionPlugin = plugin; - } - public ClassInitializationPlugin getClassInitializationPlugin() { return classInitializationPlugin; } @@ -198,13 +194,11 @@ } } - private static final ResolvedJavaType[] EMPTY = new ResolvedJavaType[]{}; - private final boolean eagerResolving; private final boolean unresolvedIsError; private final BytecodeExceptionMode bytecodeExceptionMode; private final boolean omitAssertions; - private final ResolvedJavaType[] skippedExceptionTypes; + private final List skippedExceptionTypes; private final boolean insertFullInfopoints; private final boolean trackNodeSourcePosition; private final Plugins plugins; @@ -229,8 +223,14 @@ Profile } - protected GraphBuilderConfiguration(boolean eagerResolving, boolean unresolvedIsError, BytecodeExceptionMode bytecodeExceptionMode, boolean omitAssertions, boolean insertFullInfopoints, - boolean trackNodeSourcePosition, ResolvedJavaType[] skippedExceptionTypes, Plugins plugins) { + private GraphBuilderConfiguration(boolean eagerResolving, + boolean unresolvedIsError, + BytecodeExceptionMode bytecodeExceptionMode, + boolean omitAssertions, + boolean insertFullInfopoints, + boolean trackNodeSourcePosition, + List skippedExceptionTypes, + Plugins plugins) { this.eagerResolving = eagerResolving; this.unresolvedIsError = unresolvedIsError; this.bytecodeExceptionMode = bytecodeExceptionMode; @@ -248,8 +248,15 @@ */ public GraphBuilderConfiguration copy() { Plugins newPlugins = new Plugins(plugins); - GraphBuilderConfiguration result = new GraphBuilderConfiguration(eagerResolving, unresolvedIsError, bytecodeExceptionMode, omitAssertions, insertFullInfopoints, trackNodeSourcePosition, - skippedExceptionTypes, newPlugins); + GraphBuilderConfiguration result = new GraphBuilderConfiguration( + eagerResolving, + unresolvedIsError, + bytecodeExceptionMode, + omitAssertions, + insertFullInfopoints, + trackNodeSourcePosition, + skippedExceptionTypes, + newPlugins); return result; } @@ -260,43 +267,89 @@ * eagerly resolving elements. */ public GraphBuilderConfiguration withUnresolvedIsError(boolean newUnresolvedIsError) { - return new GraphBuilderConfiguration(eagerResolving, newUnresolvedIsError, bytecodeExceptionMode, omitAssertions, insertFullInfopoints, trackNodeSourcePosition, skippedExceptionTypes, + return new GraphBuilderConfiguration( + eagerResolving, + newUnresolvedIsError, + bytecodeExceptionMode, + omitAssertions, + insertFullInfopoints, + trackNodeSourcePosition, + skippedExceptionTypes, plugins); } public GraphBuilderConfiguration withEagerResolving(boolean newEagerResolving) { - return new GraphBuilderConfiguration(newEagerResolving, unresolvedIsError, bytecodeExceptionMode, omitAssertions, insertFullInfopoints, trackNodeSourcePosition, skippedExceptionTypes, + return new GraphBuilderConfiguration( + newEagerResolving, + unresolvedIsError, + bytecodeExceptionMode, + omitAssertions, + insertFullInfopoints, + trackNodeSourcePosition, + skippedExceptionTypes, plugins); } public GraphBuilderConfiguration withSkippedExceptionTypes(ResolvedJavaType[] newSkippedExceptionTypes) { - return new GraphBuilderConfiguration(eagerResolving, unresolvedIsError, bytecodeExceptionMode, omitAssertions, insertFullInfopoints, trackNodeSourcePosition, newSkippedExceptionTypes, + return new GraphBuilderConfiguration( + eagerResolving, + unresolvedIsError, + bytecodeExceptionMode, + omitAssertions, + insertFullInfopoints, + trackNodeSourcePosition, + Collections.unmodifiableList(Arrays.asList(newSkippedExceptionTypes)), plugins); } public GraphBuilderConfiguration withBytecodeExceptionMode(BytecodeExceptionMode newBytecodeExceptionMode) { - return new GraphBuilderConfiguration(eagerResolving, unresolvedIsError, newBytecodeExceptionMode, omitAssertions, insertFullInfopoints, trackNodeSourcePosition, skippedExceptionTypes, + return new GraphBuilderConfiguration(eagerResolving, + unresolvedIsError, + newBytecodeExceptionMode, + omitAssertions, + insertFullInfopoints, + trackNodeSourcePosition, + skippedExceptionTypes, plugins); } public GraphBuilderConfiguration withOmitAssertions(boolean newOmitAssertions) { - return new GraphBuilderConfiguration(eagerResolving, unresolvedIsError, bytecodeExceptionMode, newOmitAssertions, insertFullInfopoints, trackNodeSourcePosition, skippedExceptionTypes, + return new GraphBuilderConfiguration( + eagerResolving, + unresolvedIsError, + bytecodeExceptionMode, + newOmitAssertions, + insertFullInfopoints, + trackNodeSourcePosition, + skippedExceptionTypes, plugins); } public GraphBuilderConfiguration withFullInfopoints(boolean newInsertFullInfopoints) { - ResolvedJavaType[] newSkippedExceptionTypes = skippedExceptionTypes == EMPTY ? EMPTY : Arrays.copyOf(skippedExceptionTypes, skippedExceptionTypes.length); - return new GraphBuilderConfiguration(eagerResolving, unresolvedIsError, bytecodeExceptionMode, omitAssertions, newInsertFullInfopoints, trackNodeSourcePosition, newSkippedExceptionTypes, + return new GraphBuilderConfiguration( + eagerResolving, + unresolvedIsError, + bytecodeExceptionMode, + omitAssertions, + newInsertFullInfopoints, + trackNodeSourcePosition, + skippedExceptionTypes, plugins); } public GraphBuilderConfiguration withNodeSourcePosition(boolean newTrackNodeSourcePosition) { - ResolvedJavaType[] newSkippedExceptionTypes = skippedExceptionTypes == EMPTY ? EMPTY : Arrays.copyOf(skippedExceptionTypes, skippedExceptionTypes.length); - return new GraphBuilderConfiguration(eagerResolving, unresolvedIsError, bytecodeExceptionMode, omitAssertions, insertFullInfopoints, newTrackNodeSourcePosition, newSkippedExceptionTypes, + return new GraphBuilderConfiguration( + eagerResolving, + unresolvedIsError, + bytecodeExceptionMode, + omitAssertions, + insertFullInfopoints, + newTrackNodeSourcePosition, + skippedExceptionTypes, plugins); } - public ResolvedJavaType[] getSkippedExceptionTypes() { + public List getSkippedExceptionTypes() { return skippedExceptionTypes; } @@ -321,11 +374,27 @@ } public static GraphBuilderConfiguration getDefault(Plugins plugins) { - return new GraphBuilderConfiguration(false, false, BytecodeExceptionMode.Profile, false, false, false, EMPTY, plugins); + return new GraphBuilderConfiguration( + /* eagerResolving: */ false, + /* unresolvedIsError: */ false, + BytecodeExceptionMode.Profile, + /* omitAssertions: */ false, + /* insertFullInfopoints: */ false, + /* trackNodeSourcePosition: */ false, + Collections.emptyList(), + plugins); } public static GraphBuilderConfiguration getSnippetDefault(Plugins plugins) { - return new GraphBuilderConfiguration(true, true, BytecodeExceptionMode.OmitAll, false, false, false, EMPTY, plugins); + return new GraphBuilderConfiguration( + /* eagerResolving: */ true, + /* unresolvedIsError: */ true, + BytecodeExceptionMode.OmitAll, + /* omitAssertions: */ false, + /* insertFullInfopoints: */ false, + /* trackNodeSourcePosition: */ false, + Collections.emptyList(), + plugins); } /** Returns {@code true} if it is an error for a class/field/method resolution to fail. */ --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/GraphBuilderContext.java 2019-03-12 08:09:55.695703606 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/GraphBuilderContext.java 2019-03-12 08:09:55.339701299 +0100 @@ -31,7 +31,6 @@ import org.graalvm.compiler.bytecode.Bytecode; import org.graalvm.compiler.bytecode.BytecodeProvider; import org.graalvm.compiler.core.common.type.AbstractPointerStamp; -import org.graalvm.compiler.core.common.type.ObjectStamp; import org.graalvm.compiler.core.common.type.Stamp; import org.graalvm.compiler.core.common.type.StampFactory; import org.graalvm.compiler.core.common.type.StampPair; @@ -43,6 +42,7 @@ import org.graalvm.compiler.nodes.ConstantNode; import org.graalvm.compiler.nodes.DynamicPiNode; import org.graalvm.compiler.nodes.FixedGuardNode; +import org.graalvm.compiler.nodes.Invoke; import org.graalvm.compiler.nodes.LogicNode; import org.graalvm.compiler.nodes.NodeView; import org.graalvm.compiler.nodes.PiNode; @@ -92,9 +92,9 @@ } /** - * Adds a node to the graph. If the node is in the graph, returns immediately. If the node is a - * {@link StateSplit} with a null {@linkplain StateSplit#stateAfter() frame state}, the frame - * state is initialized. + * Adds a node and all its inputs to the graph. If the node is in the graph, returns + * immediately. If the node is a {@link StateSplit} with a null + * {@linkplain StateSplit#stateAfter() frame state} , the frame state is initialized. * * @param value the value to add to the graph and push to the stack. The * {@code value.getJavaKind()} kind is used when type checking this operation. @@ -108,23 +108,6 @@ return GraphBuilderContextUtil.setStateAfterIfNecessary(this, append(value)); } - /** - * Adds a node and its inputs to the graph. If the node is in the graph, returns immediately. If - * the node is a {@link StateSplit} with a null {@linkplain StateSplit#stateAfter() frame state} - * , the frame state is initialized. - * - * @param value the value to add to the graph and push to the stack. The - * {@code value.getJavaKind()} kind is used when type checking this operation. - * @return a node equivalent to {@code value} in the graph - */ - default T addWithInputs(T value) { - if (value.graph() != null) { - assert !(value instanceof StateSplit) || ((StateSplit) value).stateAfter() != null; - return value; - } - return GraphBuilderContextUtil.setStateAfterIfNecessary(this, append(value)); - } - default ValueNode addNonNullCast(ValueNode value) { AbstractPointerStamp valueStamp = (AbstractPointerStamp) value.stamp(NodeView.DEFAULT); if (valueStamp.nonNull()) { @@ -163,7 +146,7 @@ * @param forceInlineEverything specifies if all invocations encountered in the scope of * handling the replaced invoke are to be force inlined */ - void handleReplacedInvoke(InvokeKind invokeKind, ResolvedJavaMethod targetMethod, ValueNode[] args, boolean forceInlineEverything); + Invoke handleReplacedInvoke(InvokeKind invokeKind, ResolvedJavaMethod targetMethod, ValueNode[] args, boolean forceInlineEverything); void handleReplacedInvoke(CallTargetNode callTarget, JavaKind resultType); @@ -279,10 +262,8 @@ default ValueNode nullCheckedValue(ValueNode value, DeoptimizationAction action) { if (!StampTool.isPointerNonNull(value)) { LogicNode condition = getGraph().unique(IsNullNode.create(value)); - ObjectStamp receiverStamp = (ObjectStamp) value.stamp(NodeView.DEFAULT); - Stamp stamp = receiverStamp.join(objectNonNull()); FixedGuardNode fixedGuard = append(new FixedGuardNode(condition, NullCheckException, action, true)); - ValueNode nonNullReceiver = getGraph().addOrUniqueWithInputs(PiNode.create(value, stamp, fixedGuard)); + ValueNode nonNullReceiver = getGraph().addOrUniqueWithInputs(PiNode.create(value, objectNonNull(), fixedGuard)); // TODO: Propogating the non-null into the frame state would // remove subsequent null-checks on the same value. However, // it currently causes an assertion failure when merging states. --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/InvocationPlugins.java 2019-03-12 08:09:56.171706691 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/InvocationPlugins.java 2019-03-12 08:09:55.815704384 +0100 @@ -25,6 +25,8 @@ package org.graalvm.compiler.nodes.graphbuilderconf; import static java.lang.String.format; +import static jdk.vm.ci.services.Services.IS_BUILDING_NATIVE_IMAGE; +import static jdk.vm.ci.services.Services.IS_IN_NATIVE_IMAGE; import static org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugins.LateClassPlugins.CLOSED_LATE_CLASS_PLUGIN; import java.lang.reflect.Constructor; @@ -144,6 +146,9 @@ OptionalLazySymbol(String name) { this.name = name; + if (IS_BUILDING_NATIVE_IMAGE) { + resolve(); + } } @Override @@ -156,7 +161,7 @@ * resolution fails. */ public Class resolve() { - if (resolved == null) { + if (!IS_IN_NATIVE_IMAGE && resolved == null) { Class resolvedOrNull = resolveClass(name, true); resolved = resolvedOrNull == null ? MASK_NULL : resolvedOrNull; } @@ -222,6 +227,20 @@ * @param plugins where to register the plugins * @param declaringClassName the name of the class class declaring the methods for which * plugins will be registered via this object + */ + public Registration(InvocationPlugins plugins, String declaringClassName) { + this.plugins = plugins; + this.declaringType = new OptionalLazySymbol(declaringClassName); + this.methodSubstitutionBytecodeProvider = null; + } + + /** + * Creates an object for registering {@link InvocationPlugin}s for methods declared by a + * given class. + * + * @param plugins where to register the plugins + * @param declaringClassName the name of the class class declaring the methods for which + * plugins will be registered via this object * @param methodSubstitutionBytecodeProvider provider used to get the bytecodes to parse for * method substitutions */ @@ -452,8 +471,8 @@ Binding binding = new Binding(plugin, isStatic, name, argumentTypes); bindings.add(binding); - assert Checks.check(this.plugins, declaringType, binding); - assert Checks.checkResolvable(false, declaringType, binding); + assert IS_IN_NATIVE_IMAGE || Checks.check(this.plugins, declaringType, binding); + assert IS_IN_NATIVE_IMAGE || Checks.checkResolvable(false, declaringType, binding); } @Override @@ -736,8 +755,8 @@ } @SuppressWarnings("serial") - static class InvocationPlugRegistrationError extends GraalError { - InvocationPlugRegistrationError(Throwable cause) { + static class InvocationPluginRegistrationError extends GraalError { + InvocationPluginRegistrationError(Throwable cause) { super(cause); } } @@ -751,7 +770,7 @@ deferrable.run(); } deferredRegistrations = null; - } catch (InvocationPlugRegistrationError t) { + } catch (InvocationPluginRegistrationError t) { throw t; } catch (Throwable t) { /* @@ -765,7 +784,7 @@ Runnable rethrow = new Runnable() { @Override public void run() { - throw new InvocationPlugRegistrationError(t); + throw new InvocationPluginRegistrationError(t); } }; deferredRegistrations.add(rethrow); @@ -960,8 +979,8 @@ argumentTypes[0] = declaringClass; } Binding binding = put(plugin, isStatic, allowOverwrite, declaringClass, name, argumentTypes); - assert Checks.check(this, declaringClass, binding); - assert Checks.checkResolvable(isOptional, declaringClass, binding); + assert IS_IN_NATIVE_IMAGE || Checks.check(this, declaringClass, binding); + assert IS_IN_NATIVE_IMAGE || Checks.checkResolvable(isOptional, declaringClass, binding); } /** @@ -1004,10 +1023,19 @@ if (parent != null) { InvocationPlugin plugin = parent.lookupInvocation(method); if (plugin != null) { + if (IS_IN_NATIVE_IMAGE && plugin instanceof MethodSubstitutionPlugin) { + // Disable method substitutions until GR-13607 + return null; + } return plugin; } } - return get(method); + InvocationPlugin invocationPlugin = get(method); + if (IS_IN_NATIVE_IMAGE && invocationPlugin instanceof MethodSubstitutionPlugin) { + // Disable method substitutions until GR-13607 + return null; + } + return invocationPlugin; } /** @@ -1141,25 +1169,27 @@ static final Class[][] SIGS; static { - if (!Assertions.assertionsEnabled()) { + if (!Assertions.assertionsEnabled() && !IS_BUILDING_NATIVE_IMAGE) { throw new GraalError("%s must only be used in assertions", Checks.class.getName()); } ArrayList[]> sigs = new ArrayList<>(MAX_ARITY); - for (Method method : InvocationPlugin.class.getDeclaredMethods()) { - if (!Modifier.isStatic(method.getModifiers()) && method.getName().equals("apply")) { - Class[] sig = method.getParameterTypes(); - assert sig[0] == GraphBuilderContext.class; - assert sig[1] == ResolvedJavaMethod.class; - assert sig[2] == InvocationPlugin.Receiver.class; - assert Arrays.asList(sig).subList(3, sig.length).stream().allMatch(c -> c == ValueNode.class); - while (sigs.size() < sig.length - 2) { - sigs.add(null); + if (!IS_IN_NATIVE_IMAGE) { + for (Method method : InvocationPlugin.class.getDeclaredMethods()) { + if (!Modifier.isStatic(method.getModifiers()) && method.getName().equals("apply")) { + Class[] sig = method.getParameterTypes(); + assert sig[0] == GraphBuilderContext.class; + assert sig[1] == ResolvedJavaMethod.class; + assert sig[2] == InvocationPlugin.Receiver.class; + assert Arrays.asList(sig).subList(3, sig.length).stream().allMatch(c -> c == ValueNode.class); + while (sigs.size() < sig.length - 2) { + sigs.add(null); + } + sigs.set(sig.length - 3, sig); } - sigs.set(sig.length - 3, sig); } + assert sigs.indexOf(null) == -1 : format("need to add an apply() method to %s that takes %d %s arguments ", InvocationPlugin.class.getName(), sigs.indexOf(null), + ValueNode.class.getSimpleName()); } - assert sigs.indexOf(null) == -1 : format("need to add an apply() method to %s that takes %d %s arguments ", InvocationPlugin.class.getName(), sigs.indexOf(null), - ValueNode.class.getSimpleName()); SIGS = sigs.toArray(new Class[sigs.size()][]); } @@ -1276,6 +1306,9 @@ if (type instanceof OptionalLazySymbol) { return ((OptionalLazySymbol) type).resolve(); } + if (IS_IN_NATIVE_IMAGE) { + throw new GraalError("Unresolved type in native image image:" + type.getTypeName()); + } return resolveClass(type.getTypeName(), optional); } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/MethodSubstitutionPlugin.java 2019-03-12 08:09:56.663709880 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/MethodSubstitutionPlugin.java 2019-03-12 08:09:56.311707598 +0100 @@ -24,6 +24,7 @@ package org.graalvm.compiler.nodes.graphbuilderconf; +import static jdk.vm.ci.services.Services.IS_IN_NATIVE_IMAGE; import static org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugins.resolveType; import java.lang.reflect.Method; @@ -177,6 +178,10 @@ @Override public boolean execute(GraphBuilderContext b, ResolvedJavaMethod targetMethod, InvocationPlugin.Receiver receiver, ValueNode[] argsIncludingReceiver) { + if (IS_IN_NATIVE_IMAGE) { + // these are currently unimplemented + return false; + } ResolvedJavaMethod subst = getSubstitute(b.getMetaAccess()); return b.intrinsify(bytecodeProvider, targetMethod, subst, receiver, argsIncludingReceiver); } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/NodePlugin.java 2019-03-12 08:09:57.131712913 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/NodePlugin.java 2019-03-12 08:09:56.775710606 +0100 @@ -221,6 +221,9 @@ * nodes) and fixed nodes must be manually {@linkplain FixedWithNextNode#setNext added} as * successors of {@code afterExceptionLoaded}. * + * The reason for this constraint is that when this plugin runs, it's inserting instructions + * into a different block than the one currently being parsed. + * * @param graph the graph being parsed * @param afterExceptionLoaded the last fixed node after loading the exception * @return the last fixed node after instrumentation --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/java/NewArrayNode.java 2019-03-12 08:09:57.599715946 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/java/NewArrayNode.java 2019-03-12 08:09:57.247713665 +0100 @@ -98,7 +98,7 @@ ValueNode lengthAlias = tool.getAlias(length()); if (lengthAlias.asConstant() != null) { int constantLength = lengthAlias.asJavaConstant().asInt(); - if (constantLength >= 0 && constantLength < tool.getMaximumEntryCount()) { + if (constantLength >= 0 && constantLength <= tool.getMaximumEntryCount()) { ValueNode[] state = new ValueNode[constantLength]; ConstantNode defaultForKind = constantLength == 0 ? null : defaultElementValue(); for (int i = 0; i < constantLength; i++) { --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/spi/NodeLIRBuilderTool.java 2019-03-12 08:09:58.079719056 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/spi/NodeLIRBuilderTool.java 2019-03-12 08:09:57.719716723 +0100 @@ -94,4 +94,6 @@ default OptionValues getOptions() { return getLIRGeneratorTool().getResult().getLIR().getOptions(); } + + void emitReadExceptionObject(ValueNode node); } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/spi/Replacements.java 2019-03-12 08:09:58.559722167 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/spi/Replacements.java 2019-03-12 08:09:58.199719834 +0100 @@ -64,7 +64,9 @@ * @param trackNodeSourcePosition * @return the snippet graph, if any, that is derived from {@code method} */ - StructuredGraph getSnippet(ResolvedJavaMethod method, Object[] args, boolean trackNodeSourcePosition, NodeSourcePosition replaceePosition); + default StructuredGraph getSnippet(ResolvedJavaMethod method, Object[] args, boolean trackNodeSourcePosition, NodeSourcePosition replaceePosition) { + return getSnippet(method, null, args, trackNodeSourcePosition, replaceePosition); + } /** * Gets the snippet graph derived from a given method. --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/type/NarrowOopStamp.java 2019-03-12 08:09:59.031725225 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/type/NarrowOopStamp.java 2019-03-12 08:09:58.675722918 +0100 @@ -45,6 +45,13 @@ } @Override + public void accept(Visitor v) { + super.accept(v); + v.visitLong(encoding.getBase()); + v.visitInt(encoding.getShift()); + } + + @Override protected abstract AbstractObjectStamp copyWith(ResolvedJavaType type, boolean exactType, boolean nonNull, boolean alwaysNull); public Stamp uncompressed() { --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/util/GraphUtil.java 2019-03-12 08:09:59.495728232 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/util/GraphUtil.java 2019-03-12 08:09:59.143725951 +0100 @@ -1039,7 +1039,7 @@ return; } - if (newLengthInt >= tool.getMaximumEntryCount()) { + if (newLengthInt > tool.getMaximumEntryCount()) { /* The new array size is higher than maximum allowed size of virtualized objects. */ return; } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.options.processor/src/org/graalvm/compiler/options/processor/OptionProcessor.java 2019-03-12 08:09:59.967731290 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.options.processor/src/org/graalvm/compiler/options/processor/OptionProcessor.java 2019-03-12 08:09:59.615729010 +0100 @@ -174,7 +174,7 @@ } enclosing = enclosing.getEnclosingElement(); } - if (enclosingPackage == null) { + if (enclosingPackage == null || enclosingPackage.isUnnamed()) { processingEnv.getMessager().printMessage(Kind.ERROR, "Option field cannot be declared in the unnamed package", element); return; } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.options/src/org/graalvm/compiler/options/OptionDescriptor.java 2019-03-12 08:10:00.455734453 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.options/src/org/graalvm/compiler/options/OptionDescriptor.java 2019-03-12 08:10:00.099732146 +0100 @@ -34,14 +34,14 @@ */ public final class OptionDescriptor { - protected final String name; - protected final OptionType optionType; - protected final Class optionValueType; - protected final String help; - protected final List extraHelp; - protected final OptionKey optionKey; - protected final Class declaringClass; - protected final String fieldName; + private final String name; + private final OptionType optionType; + private final Class optionValueType; + private final String help; + private final List extraHelp; + private final OptionKey optionKey; + private final Class declaringClass; + private final String fieldName; private static final String[] NO_EXTRA_HELP = {}; --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.options/src/org/graalvm/compiler/options/OptionKey.java 2019-03-12 08:10:00.947737641 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.options/src/org/graalvm/compiler/options/OptionKey.java 2019-03-12 08:10:00.587735308 +0100 @@ -141,6 +141,18 @@ } /** + * Gets the value of this option in {@code values} if it is present, otherwise + * {@link #getDefaultValue()}. + */ + @SuppressWarnings("unchecked") + public T getValueOrDefault(EconomicMap, Object> values) { + if (!values.containsKey(this)) { + return defaultValue; + } + return (T) values.get(this); + } + + /** * Sets the value of this option in a given map. The * {@link #onValueUpdate(EconomicMap, Object, Object)} method is called once the value is set. * --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.options/src/org/graalvm/compiler/options/OptionValues.java 2019-03-12 08:10:01.399740569 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.options/src/org/graalvm/compiler/options/OptionValues.java 2019-03-12 08:10:01.035738211 +0100 @@ -213,7 +213,7 @@ } String name = namePrefix + e.getKey(); - String assign = containsKey(desc.optionKey) ? ":=" : "="; + String assign = containsKey(desc.getOptionKey()) ? ":=" : "="; String typeName = desc.getOptionKey() instanceof EnumOptionKey ? "String" : desc.getOptionValueType().getSimpleName(); String linePrefix = String.format("%s %s %s ", name, assign, value); int typeStartPos = PROPERTY_LINE_WIDTH - typeName.length(); --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.options/src/org/graalvm/compiler/options/OptionsParser.java 2019-03-12 08:10:01.899743809 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.options/src/org/graalvm/compiler/options/OptionsParser.java 2019-03-12 08:10:01.523741373 +0100 @@ -24,6 +24,9 @@ package org.graalvm.compiler.options; +import static jdk.vm.ci.services.Services.IS_BUILDING_NATIVE_IMAGE; +import static jdk.vm.ci.services.Services.IS_IN_NATIVE_IMAGE; + import java.util.ArrayList; import java.util.Collection; import java.util.Formatter; @@ -39,11 +42,15 @@ */ public class OptionsParser { + private static volatile List cachedOptionDescriptors; + /** - * Gets an iterable composed of the {@link ServiceLoader}s to be used when looking for - * {@link OptionDescriptors} providers. + * Gets an iterable of available {@link OptionDescriptors}. */ public static Iterable getOptionsLoader() { + if (IS_IN_NATIVE_IMAGE || cachedOptionDescriptors != null) { + return cachedOptionDescriptors; + } boolean java8OrEarlier = System.getProperty("java.specification.version").compareTo("1.9") < 0; ClassLoader loader; if (java8OrEarlier) { @@ -58,7 +65,15 @@ */ loader = ClassLoader.getSystemClassLoader(); } - return ServiceLoader.load(OptionDescriptors.class, loader); + Iterable result = ServiceLoader.load(OptionDescriptors.class, loader); + if (IS_BUILDING_NATIVE_IMAGE) { + ArrayList optionDescriptors = new ArrayList<>(); + for (OptionDescriptors descriptors : result) { + optionDescriptors.add(descriptors); + } + OptionsParser.cachedOptionDescriptors = optionDescriptors; + } + return result; } /** @@ -178,7 +193,7 @@ } } - desc.optionKey.update(values, value); + desc.getOptionKey().update(values, value); } private static long parseLong(String v) { --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/FixReadsPhase.java 2019-03-12 08:10:02.371746867 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/FixReadsPhase.java 2019-03-12 08:10:02.007744509 +0100 @@ -28,6 +28,7 @@ import jdk.internal.vm.compiler.collections.MapCursor; import org.graalvm.compiler.core.common.GraalOptions; import org.graalvm.compiler.core.common.cfg.BlockMap; +import org.graalvm.compiler.core.common.spi.ConstantFieldProvider; import org.graalvm.compiler.core.common.type.FloatStamp; import org.graalvm.compiler.core.common.type.Stamp; import org.graalvm.compiler.core.common.type.StampFactory; @@ -37,6 +38,7 @@ import org.graalvm.compiler.graph.NodeMap; import org.graalvm.compiler.graph.NodeStack; import org.graalvm.compiler.graph.Position; +import org.graalvm.compiler.graph.spi.CanonicalizerTool; import org.graalvm.compiler.nodeinfo.InputType; import org.graalvm.compiler.nodes.AbstractBeginNode; import org.graalvm.compiler.nodes.AbstractMergeNode; @@ -68,6 +70,7 @@ import org.graalvm.compiler.nodes.memory.MemoryAccess; import org.graalvm.compiler.nodes.memory.MemoryPhiNode; import org.graalvm.compiler.nodes.util.GraphUtil; +import org.graalvm.compiler.options.OptionValues; import org.graalvm.compiler.phases.BasePhase; import org.graalvm.compiler.phases.Phase; import org.graalvm.compiler.phases.graph.ScheduledNodeIterator; @@ -76,7 +79,9 @@ import org.graalvm.compiler.phases.tiers.LowTierContext; import org.graalvm.compiler.phases.tiers.PhaseContext; +import jdk.vm.ci.meta.Assumptions; import jdk.vm.ci.meta.Constant; +import jdk.vm.ci.meta.ConstantReflectionProvider; import jdk.vm.ci.meta.MetaAccessProvider; import jdk.vm.ci.meta.TriState; @@ -136,7 +141,7 @@ } - protected static class RawConditionalEliminationVisitor implements RecursiveVisitor { + public static class RawConditionalEliminationVisitor implements RecursiveVisitor { protected final NodeMap stampMap; protected final NodeStack undoOperations; @@ -147,8 +152,58 @@ private final BlockMap blockActionStart; private final EconomicMap> endMaps; private final DebugContext debug; + private final RawCanonicalizerTool rawCanonicalizerTool = new RawCanonicalizerTool(); - protected RawConditionalEliminationVisitor(StructuredGraph graph, ScheduleResult schedule, MetaAccessProvider metaAccess, boolean replaceInputsWithConstants) { + private class RawCanonicalizerTool implements NodeView, CanonicalizerTool { + + @Override + public Assumptions getAssumptions() { + return graph.getAssumptions(); + } + + @Override + public MetaAccessProvider getMetaAccess() { + return metaAccess; + } + + @Override + public ConstantReflectionProvider getConstantReflection() { + return null; + } + + @Override + public ConstantFieldProvider getConstantFieldProvider() { + return null; + } + + @Override + public boolean canonicalizeReads() { + return false; + } + + @Override + public boolean allUsagesAvailable() { + return true; + } + + @Override + public Integer smallestCompareWidth() { + return null; + } + + @Override + public OptionValues getOptions() { + return graph.getOptions(); + } + + @Override + public Stamp stamp(ValueNode node) { + return getBestStamp(node); + } + + } + + public RawConditionalEliminationVisitor(StructuredGraph graph, ScheduleResult schedule, MetaAccessProvider metaAccess, boolean replaceInputsWithConstants) { this.graph = graph; this.debug = graph.getDebug(); this.schedule = schedule; @@ -326,8 +381,22 @@ } protected void processUnary(UnaryNode node) { - Stamp newStamp = node.foldStamp(getBestStamp(node.getValue())); + ValueNode value = node.getValue(); + Stamp bestStamp = getBestStamp(value); + Stamp newStamp = node.foldStamp(bestStamp); if (!checkReplaceWithConstant(newStamp, node)) { + if (!bestStamp.equals(value.stamp(NodeView.DEFAULT))) { + ValueNode newNode = node.canonical(rawCanonicalizerTool); + if (newNode != node) { + // Canonicalization successfully triggered. + if (newNode != null && !newNode.isAlive()) { + newNode = graph.addWithoutUniqueWithInputs(newNode); + } + node.replaceAndDelete(newNode); + GraphUtil.tryKillUnused(value); + return; + } + } registerNewValueStamp(node, newStamp); } } @@ -346,10 +415,31 @@ } protected void processBinary(BinaryNode node) { - Stamp xStamp = getBestStamp(node.getX()); - Stamp yStamp = getBestStamp(node.getY()); + + ValueNode x = node.getX(); + ValueNode y = node.getY(); + + Stamp xStamp = getBestStamp(x); + Stamp yStamp = getBestStamp(y); Stamp newStamp = node.foldStamp(xStamp, yStamp); if (!checkReplaceWithConstant(newStamp, node)) { + + if (!xStamp.equals(x.stamp(NodeView.DEFAULT)) || !yStamp.equals(y.stamp(NodeView.DEFAULT))) { + // At least one of the inputs has an improved stamp => attempt to canonicalize + // based on that improvement. + ValueNode newNode = node.canonical(rawCanonicalizerTool); + if (newNode != node) { + // Canonicalization successfully triggered. + if (newNode != null && !newNode.isAlive()) { + newNode = graph.addWithoutUniqueWithInputs(newNode); + } + node.replaceAndDelete(newNode); + GraphUtil.tryKillUnused(x); + GraphUtil.tryKillUnused(y); + return; + } + } + registerNewValueStamp(node, newStamp); } } @@ -469,6 +559,10 @@ protected Stamp getBestStamp(ValueNode value) { ValueNode originalNode = value; + if (!value.isAlive()) { + return value.stamp(NodeView.DEFAULT); + } + StampElement currentStamp = stampMap.getAndGrow(originalNode); if (currentStamp == null) { return value.stamp(NodeView.DEFAULT); --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/RemoveValueProxyPhase.java 2019-03-12 08:10:02.855750002 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/RemoveValueProxyPhase.java 2019-03-12 08:10:02.491747644 +0100 @@ -26,7 +26,6 @@ import org.graalvm.compiler.nodes.FrameState; import org.graalvm.compiler.nodes.LoopExitNode; -import org.graalvm.compiler.nodes.ProxyNode; import org.graalvm.compiler.nodes.StructuredGraph; import org.graalvm.compiler.nodes.util.GraphUtil; import org.graalvm.compiler.phases.Phase; @@ -36,15 +35,14 @@ @Override protected void run(StructuredGraph graph) { for (LoopExitNode exit : graph.getNodes(LoopExitNode.TYPE)) { - for (ProxyNode vpn : exit.proxies().snapshot()) { - vpn.replaceAtUsagesAndDelete(vpn.value()); - } - FrameState stateAfter = exit.stateAfter(); - if (stateAfter != null) { + exit.removeProxies(); + FrameState frameState = exit.stateAfter(); + if (frameState != null && frameState.isExceptionHandlingBCI()) { + // The parser will create loop exits with such BCIs on the exception handling path. + // Loop optimizations must avoid duplicating such exits + // We clean them up here otherwise they could survive until code generation exit.setStateAfter(null); - if (stateAfter.hasNoUsages()) { - GraphUtil.killWithUnusedFloatingInputs(stateAfter); - } + GraphUtil.tryKillUnused(frameState); } } graph.setHasValueProxies(false); --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases/src/org/graalvm/compiler/phases/graph/ReentrantBlockIterator.java 2019-03-12 08:10:03.343753164 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases/src/org/graalvm/compiler/phases/graph/ReentrantBlockIterator.java 2019-03-12 08:10:02.979750806 +0100 @@ -77,13 +77,13 @@ EconomicMap blockEndStates = apply(closure, loop.getHeader(), initialState, block -> !(block.getLoop() == loop || block.isLoopHeader())); Block[] predecessors = loop.getHeader().getPredecessors(); - LoopInfo info = new LoopInfo<>(predecessors.length - 1, loop.getExits().size()); + LoopInfo info = new LoopInfo<>(predecessors.length - 1, loop.getLoopExits().size()); for (int i = 1; i < predecessors.length; i++) { StateT endState = blockEndStates.get(predecessors[i].getEndNode()); // make sure all end states are unique objects info.endStates.add(closure.cloneState(endState)); } - for (Block loopExit : loop.getExits()) { + for (Block loopExit : loop.getLoopExits()) { assert loopExit.getPredecessorCount() == 1; assert blockEndStates.containsKey(loopExit.getBeginNode()) : loopExit.getBeginNode() + " " + blockEndStates; StateT exitState = blockEndStates.get(loopExit.getBeginNode()); @@ -210,8 +210,8 @@ List exitStates = closure.processLoop(loop, state); int i = 0; - assert loop.getExits().size() == exitStates.size(); - for (Block exit : loop.getExits()) { + assert loop.getLoopExits().size() == exitStates.size(); + for (Block exit : loop.getLoopExits()) { states.put(exit.getBeginNode(), exitStates.get(i++)); blockQueue.addFirst(exit); } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases/src/org/graalvm/compiler/phases/util/GraphOrder.java 2019-03-12 08:10:03.819756248 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases/src/org/graalvm/compiler/phases/util/GraphOrder.java 2019-03-12 08:10:03.447753838 +0100 @@ -30,6 +30,7 @@ import jdk.internal.vm.compiler.collections.EconomicMap; import jdk.internal.vm.compiler.collections.Equivalence; import org.graalvm.compiler.core.common.cfg.Loop; +import org.graalvm.compiler.debug.DebugContext; import org.graalvm.compiler.debug.GraalError; import org.graalvm.compiler.graph.GraalGraphError; import org.graalvm.compiler.graph.Node; @@ -153,9 +154,10 @@ * This method schedules the graph and makes sure that, for every node, all inputs are available * at the position where it is scheduled. This is a very expensive assertion. */ + @SuppressWarnings("try") public static boolean assertSchedulableGraph(final StructuredGraph graph) { assert graph.getGuardsStage() != GuardsStage.AFTER_FSA : "Cannot use the BlockIteratorClosure after FrameState Assignment, HIR Loop Data Structures are no longer valid."; - try { + try (DebugContext.Scope s = graph.getDebug().scope("AssertSchedulableGraph")) { final SchedulePhase schedulePhase = new SchedulePhase(SchedulingStrategy.LATEST_OUT_OF_LOOPS, true); final EconomicMap loopEntryStates = EconomicMap.create(Equivalence.IDENTITY); schedulePhase.apply(graph, false); --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.printer/src/org/graalvm/compiler/printer/GraphPrinterDumpHandler.java 2019-03-12 08:10:04.307759410 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.printer/src/org/graalvm/compiler/printer/GraphPrinterDumpHandler.java 2019-03-12 08:10:03.943757052 +0100 @@ -43,6 +43,7 @@ import org.graalvm.compiler.debug.DebugOptions; import org.graalvm.compiler.debug.GraalError; import org.graalvm.compiler.debug.TTY; +import org.graalvm.compiler.debug.DebugOptions.PrintGraphTarget; import org.graalvm.compiler.graph.Graph; import org.graalvm.compiler.nodes.StructuredGraph; import org.graalvm.compiler.options.OptionValues; @@ -124,7 +125,7 @@ @SuppressWarnings("try") public void dump(DebugContext debug, Object object, final String format, Object... arguments) { OptionValues options = debug.getOptions(); - if (object instanceof Graph && DebugOptions.PrintGraph.getValue(options)) { + if (object instanceof Graph && DebugOptions.PrintGraph.getValue(options) != PrintGraphTarget.Disable) { final Graph graph = (Graph) object; ensureInitialized(debug, graph); if (printer == null) { --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.aarch64/src/org/graalvm/compiler/replacements/aarch64/AArch64GraphBuilderPlugins.java 2019-03-12 08:10:04.803762623 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.aarch64/src/org/graalvm/compiler/replacements/aarch64/AArch64GraphBuilderPlugins.java 2019-03-12 08:10:04.439760265 +0100 @@ -31,9 +31,9 @@ import static org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.UnaryOperation.LOG10; import static org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.UnaryOperation.SIN; import static org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.UnaryOperation.TAN; -import static org.graalvm.compiler.serviceprovider.GraalServices.JAVA_SPECIFICATION_VERSION; -import static org.graalvm.compiler.serviceprovider.GraalServices.Java11OrEarlier; -import static org.graalvm.compiler.serviceprovider.GraalServices.Java8OrEarlier; +import static org.graalvm.compiler.serviceprovider.JavaVersionUtil.JAVA_SPECIFICATION_VERSION; +import static org.graalvm.compiler.serviceprovider.JavaVersionUtil.Java11OrEarlier; +import static org.graalvm.compiler.serviceprovider.JavaVersionUtil.Java8OrEarlier; import org.graalvm.compiler.bytecode.BytecodeProvider; import org.graalvm.compiler.lir.aarch64.AArch64ArithmeticLIRGeneratorTool.RoundingMode; --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.amd64/src/org/graalvm/compiler/replacements/amd64/AMD64GraphBuilderPlugins.java 2019-03-12 08:10:05.291765784 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.amd64/src/org/graalvm/compiler/replacements/amd64/AMD64GraphBuilderPlugins.java 2019-03-12 08:10:04.927763426 +0100 @@ -32,9 +32,9 @@ import static org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.UnaryOperation.LOG10; import static org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.UnaryOperation.SIN; import static org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.UnaryOperation.TAN; -import static org.graalvm.compiler.serviceprovider.GraalServices.JAVA_SPECIFICATION_VERSION; -import static org.graalvm.compiler.serviceprovider.GraalServices.Java11OrEarlier; -import static org.graalvm.compiler.serviceprovider.GraalServices.Java8OrEarlier; +import static org.graalvm.compiler.serviceprovider.JavaVersionUtil.JAVA_SPECIFICATION_VERSION; +import static org.graalvm.compiler.serviceprovider.JavaVersionUtil.Java11OrEarlier; +import static org.graalvm.compiler.serviceprovider.JavaVersionUtil.Java8OrEarlier; import java.util.Arrays; @@ -71,7 +71,7 @@ public class AMD64GraphBuilderPlugins { - public static void register(Plugins plugins, BytecodeProvider replacementsBytecodeProvider, AMD64 arch, boolean arithmeticStubs, boolean explicitUnsafeNullChecks) { + public static void register(Plugins plugins, BytecodeProvider replacementsBytecodeProvider, AMD64 arch, boolean explicitUnsafeNullChecks) { InvocationPlugins invocationPlugins = plugins.getInvocationPlugins(); invocationPlugins.defer(new Runnable() { @Override @@ -82,10 +82,10 @@ registerPlatformSpecificUnsafePlugins(invocationPlugins, replacementsBytecodeProvider, explicitUnsafeNullChecks, new JavaKind[]{JavaKind.Int, JavaKind.Long, JavaKind.Object, JavaKind.Boolean, JavaKind.Byte, JavaKind.Short, JavaKind.Char, JavaKind.Float, JavaKind.Double}); registerUnsafePlugins(invocationPlugins, replacementsBytecodeProvider, explicitUnsafeNullChecks); - registerStringPlugins(invocationPlugins, arch, replacementsBytecodeProvider); - registerStringLatin1Plugins(invocationPlugins, arch, replacementsBytecodeProvider); - registerStringUTF16Plugins(invocationPlugins, arch, replacementsBytecodeProvider); - registerMathPlugins(invocationPlugins, arch, arithmeticStubs, replacementsBytecodeProvider); + registerStringPlugins(invocationPlugins, replacementsBytecodeProvider); + registerStringLatin1Plugins(invocationPlugins, replacementsBytecodeProvider); + registerStringUTF16Plugins(invocationPlugins, replacementsBytecodeProvider); + registerMathPlugins(invocationPlugins, arch, replacementsBytecodeProvider); registerArraysEqualsPlugins(invocationPlugins, replacementsBytecodeProvider); } }); @@ -155,21 +155,15 @@ } } - private static void registerMathPlugins(InvocationPlugins plugins, AMD64 arch, boolean arithmeticStubs, BytecodeProvider bytecodeProvider) { + private static void registerMathPlugins(InvocationPlugins plugins, AMD64 arch, BytecodeProvider bytecodeProvider) { Registration r = new Registration(plugins, Math.class, bytecodeProvider); registerUnaryMath(r, "log", LOG); registerUnaryMath(r, "log10", LOG10); registerUnaryMath(r, "exp", EXP); registerBinaryMath(r, "pow", POW); - if (arithmeticStubs) { - registerUnaryMath(r, "sin", SIN); - registerUnaryMath(r, "cos", COS); - registerUnaryMath(r, "tan", TAN); - } else { - r.registerMethodSubstitution(AMD64MathSubstitutions.class, "sin", double.class); - r.registerMethodSubstitution(AMD64MathSubstitutions.class, "cos", double.class); - r.registerMethodSubstitution(AMD64MathSubstitutions.class, "tan", double.class); - } + registerUnaryMath(r, "sin", SIN); + registerUnaryMath(r, "cos", COS); + registerUnaryMath(r, "tan", TAN); if (arch.getFeatures().contains(CPUFeature.SSE4_1)) { registerRound(r, "rint", RoundingMode.NEAREST); @@ -208,23 +202,19 @@ }); } - private static void registerStringPlugins(InvocationPlugins plugins, AMD64 arch, BytecodeProvider replacementsBytecodeProvider) { + private static void registerStringPlugins(InvocationPlugins plugins, BytecodeProvider replacementsBytecodeProvider) { if (Java8OrEarlier) { Registration r; r = new Registration(plugins, String.class, replacementsBytecodeProvider); r.setAllowOverwrite(true); - if (arch.getFeatures().contains(CPUFeature.SSE4_2)) { - r.registerMethodSubstitution(AMD64StringSubstitutions.class, "indexOf", char[].class, int.class, - int.class, char[].class, int.class, int.class, int.class); - } - if (arch.getFeatures().contains(CPUFeature.SSSE3)) { - r.registerMethodSubstitution(AMD64StringSubstitutions.class, "indexOf", Receiver.class, int.class, int.class); - } + r.registerMethodSubstitution(AMD64StringSubstitutions.class, "indexOf", char[].class, int.class, + int.class, char[].class, int.class, int.class, int.class); + r.registerMethodSubstitution(AMD64StringSubstitutions.class, "indexOf", Receiver.class, int.class, int.class); r.registerMethodSubstitution(AMD64StringSubstitutions.class, "compareTo", Receiver.class, String.class); } } - private static void registerStringLatin1Plugins(InvocationPlugins plugins, AMD64 arch, BytecodeProvider replacementsBytecodeProvider) { + private static void registerStringLatin1Plugins(InvocationPlugins plugins, BytecodeProvider replacementsBytecodeProvider) { if (JAVA_SPECIFICATION_VERSION >= 9) { Registration r = new Registration(plugins, "java.lang.StringLatin1", replacementsBytecodeProvider); r.setAllowOverwrite(true); @@ -232,14 +222,12 @@ r.registerMethodSubstitution(AMD64StringLatin1Substitutions.class, "compareToUTF16", byte[].class, byte[].class); r.registerMethodSubstitution(AMD64StringLatin1Substitutions.class, "inflate", byte[].class, int.class, char[].class, int.class, int.class); r.registerMethodSubstitution(AMD64StringLatin1Substitutions.class, "inflate", byte[].class, int.class, byte[].class, int.class, int.class); - - if (arch.getFeatures().contains(CPUFeature.SSSE3)) { - r.registerMethodSubstitution(AMD64StringLatin1Substitutions.class, "indexOf", byte[].class, int.class, int.class); - } + r.registerMethodSubstitution(AMD64StringLatin1Substitutions.class, "indexOf", byte[].class, int.class, int.class); + r.registerMethodSubstitution(AMD64StringLatin1Substitutions.class, "indexOf", byte[].class, int.class, byte[].class, int.class, int.class); } } - private static void registerStringUTF16Plugins(InvocationPlugins plugins, AMD64 arch, BytecodeProvider replacementsBytecodeProvider) { + private static void registerStringUTF16Plugins(InvocationPlugins plugins, BytecodeProvider replacementsBytecodeProvider) { if (JAVA_SPECIFICATION_VERSION >= 9) { Registration r = new Registration(plugins, "java.lang.StringUTF16", replacementsBytecodeProvider); r.setAllowOverwrite(true); @@ -247,10 +235,9 @@ r.registerMethodSubstitution(AMD64StringUTF16Substitutions.class, "compareToLatin1", byte[].class, byte[].class); r.registerMethodSubstitution(AMD64StringUTF16Substitutions.class, "compress", char[].class, int.class, byte[].class, int.class, int.class); r.registerMethodSubstitution(AMD64StringUTF16Substitutions.class, "compress", byte[].class, int.class, byte[].class, int.class, int.class); - - if (arch.getFeatures().contains(CPUFeature.SSSE3)) { - r.registerMethodSubstitution(AMD64StringUTF16Substitutions.class, "indexOfCharUnsafe", byte[].class, int.class, int.class, int.class); - } + r.registerMethodSubstitution(AMD64StringUTF16Substitutions.class, "indexOfCharUnsafe", byte[].class, int.class, int.class, int.class); + r.registerMethodSubstitution(AMD64StringUTF16Substitutions.class, "indexOfUnsafe", byte[].class, int.class, byte[].class, int.class, int.class); + r.registerMethodSubstitution(AMD64StringUTF16Substitutions.class, "indexOfLatin1Unsafe", byte[].class, int.class, byte[].class, int.class, int.class); } } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.amd64/src/org/graalvm/compiler/replacements/amd64/AMD64StringLatin1Substitutions.java 2019-03-12 08:10:05.787768997 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.amd64/src/org/graalvm/compiler/replacements/amd64/AMD64StringLatin1Substitutions.java 2019-03-12 08:10:05.427766665 +0100 @@ -30,6 +30,7 @@ import org.graalvm.compiler.api.replacements.MethodSubstitution; import org.graalvm.compiler.nodes.DeoptimizeNode; import org.graalvm.compiler.replacements.nodes.ArrayCompareToNode; +import org.graalvm.compiler.replacements.nodes.ArrayRegionEqualsNode; import org.graalvm.compiler.word.Word; import jdk.internal.vm.compiler.word.Pointer; @@ -89,7 +90,15 @@ return ArrayCompareToNode.compareTo(value, other, value.length, other.length, JavaKind.Byte, JavaKind.Char); } - @MethodSubstitution(optional = true) + private static Word pointer(byte[] target) { + return Word.objectToTrackedPointer(target).add(byteArrayBaseOffset(INJECTED)); + } + + private static Word byteOffsetPointer(byte[] source, int offset) { + return pointer(source).add(offset * byteArrayIndexScale(INJECTED)); + } + + @MethodSubstitution public static int indexOf(byte[] value, int ch, int origFromIndex) { int fromIndex = origFromIndex; if (ch >>> 8 != 0) { @@ -103,7 +112,7 @@ // Note: fromIndex might be near -1>>>1. return -1; } - Pointer sourcePointer = Word.objectToTrackedPointer(value).add(byteArrayBaseOffset(INJECTED)).add(fromIndex); + Pointer sourcePointer = byteOffsetPointer(value, fromIndex); int result = AMD64ArrayIndexOf.indexOf1Byte(sourcePointer, length - fromIndex, (byte) ch); if (result != -1) { return result + fromIndex; @@ -111,6 +120,59 @@ return result; } + @MethodSubstitution + public static int indexOf(byte[] source, int sourceCount, byte[] target, int targetCount, int origFromIndex) { + int fromIndex = origFromIndex; + if (fromIndex >= sourceCount) { + return (targetCount == 0 ? sourceCount : -1); + } + if (fromIndex < 0) { + fromIndex = 0; + } + if (targetCount == 0) { + // The empty string is in every string. + return fromIndex; + } + if (sourceCount - fromIndex < targetCount) { + // The empty string contains nothing except the empty string. + return -1; + } + int totalOffset = fromIndex; + if (targetCount == 1) { + Pointer sourcePointer = byteOffsetPointer(source, totalOffset); + int indexOfResult = AMD64ArrayIndexOf.indexOf1Byte(sourcePointer, sourceCount - fromIndex, target[0]); + if (indexOfResult >= 0) { + return indexOfResult + totalOffset; + } + return indexOfResult; + } else if (targetCount == 2) { + Pointer sourcePointer = byteOffsetPointer(source, totalOffset); + int indexOfResult = AMD64ArrayIndexOf.indexOfTwoConsecutiveBytes(sourcePointer, sourceCount - fromIndex, target[0], target[1]); + if (indexOfResult >= 0) { + return indexOfResult + totalOffset; + } + return indexOfResult; + } else { + int haystackLength = sourceCount - (fromIndex + (targetCount - 2)); + while (haystackLength > 0) { + Pointer sourcePointer = byteOffsetPointer(source, totalOffset); + int indexOfResult = AMD64ArrayIndexOf.indexOfTwoConsecutiveBytes(sourcePointer, haystackLength, target[0], target[1]); + if (indexOfResult < 0) { + return -1; + } + totalOffset += indexOfResult; + haystackLength -= (indexOfResult + 1); + Pointer cmpSourcePointer = byteOffsetPointer(source, totalOffset); + Pointer targetPointer = pointer(target); + if (ArrayRegionEqualsNode.regionEquals(cmpSourcePointer, targetPointer, targetCount, JavaKind.Byte)) { + return totalOffset; + } + totalOffset++; + } + return -1; + } + } + /** * Intrinsic for {@code java.lang.StringLatin1.inflate([BI[CII)V}. * --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.amd64/src/org/graalvm/compiler/replacements/amd64/AMD64StringSubstitutions.java 2019-03-12 08:10:06.267772106 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.amd64/src/org/graalvm/compiler/replacements/amd64/AMD64StringSubstitutions.java 2019-03-12 08:10:05.907769775 +0100 @@ -82,7 +82,6 @@ // The empty string contains nothing except the empty string. return -1; } - assert sourceCount - fromIndex > 0 && targetCount > 0; if (targetCount == 1) { Pointer sourcePointer = Word.objectToTrackedPointer(source).add(charArrayBaseOffset(INJECTED)).add(totalOffset * charArrayIndexScale(INJECTED)); --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.amd64/src/org/graalvm/compiler/replacements/amd64/AMD64StringUTF16Substitutions.java 2019-03-12 08:10:06.767775345 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.amd64/src/org/graalvm/compiler/replacements/amd64/AMD64StringUTF16Substitutions.java 2019-03-12 08:10:06.399772962 +0100 @@ -24,25 +24,27 @@ package org.graalvm.compiler.replacements.amd64; +import jdk.vm.ci.meta.DeoptimizationAction; +import jdk.vm.ci.meta.DeoptimizationReason; +import jdk.vm.ci.meta.JavaKind; +import jdk.vm.ci.meta.MetaAccessProvider; import org.graalvm.compiler.api.replacements.ClassSubstitution; import org.graalvm.compiler.api.replacements.Fold; import org.graalvm.compiler.api.replacements.Fold.InjectedParameter; import org.graalvm.compiler.api.replacements.MethodSubstitution; import org.graalvm.compiler.nodes.DeoptimizeNode; +import org.graalvm.compiler.replacements.ReplacementsUtil; +import org.graalvm.compiler.replacements.StringUTF16Substitutions; import org.graalvm.compiler.replacements.nodes.ArrayCompareToNode; +import org.graalvm.compiler.replacements.nodes.ArrayRegionEqualsNode; import org.graalvm.compiler.word.Word; import jdk.internal.vm.compiler.word.Pointer; -import jdk.vm.ci.meta.DeoptimizationAction; -import jdk.vm.ci.meta.DeoptimizationReason; -import jdk.vm.ci.meta.JavaKind; -import jdk.vm.ci.meta.MetaAccessProvider; - // JaCoCo Exclude /** * Substitutions for {@code java.lang.StringUTF16} methods. - * + *

* Since JDK 9. */ @ClassSubstitution(className = "java.lang.StringUTF16", optional = true) @@ -68,9 +70,15 @@ return metaAccess.getArrayIndexScale(JavaKind.Char); } - /** Marker value for the {@link InjectedParameter} injected parameter. */ + /** + * Marker value for the {@link InjectedParameter} injected parameter. + */ static final MetaAccessProvider INJECTED = null; + public static int length(byte[] value) { + return value.length >> 1; + } + /** * @param value is char[] * @param other is char[] @@ -93,9 +101,9 @@ return ArrayCompareToNode.compareTo(other, value, other.length, value.length, JavaKind.Char, JavaKind.Byte); } - @MethodSubstitution(optional = true) + @MethodSubstitution public static int indexOfCharUnsafe(byte[] value, int ch, int fromIndex, int max) { - Pointer sourcePointer = Word.objectToTrackedPointer(value).add(byteArrayBaseOffset(INJECTED)).add(fromIndex * charArrayIndexScale(INJECTED)); + Pointer sourcePointer = charOffsetPointer(value, fromIndex); int result = AMD64ArrayIndexOf.indexOf1Char(sourcePointer, max - fromIndex, (char) ch); if (result != -1) { return result + fromIndex; @@ -103,6 +111,100 @@ return result; } + private static Word pointer(byte[] target) { + return Word.objectToTrackedPointer(target).add(byteArrayBaseOffset(INJECTED)); + } + + private static Word charOffsetPointer(byte[] value, int offset) { + return pointer(value).add(offset * charArrayIndexScale(INJECTED)); + } + + @MethodSubstitution + public static int indexOfUnsafe(byte[] source, int sourceCount, byte[] target, int targetCount, int fromIndex) { + ReplacementsUtil.runtimeAssert(fromIndex >= 0, "StringUTF16.indexOfUnsafe invalid args: fromIndex negative"); + ReplacementsUtil.runtimeAssert(targetCount > 0, "StringUTF16.indexOfUnsafe invalid args: targetCount <= 0"); + ReplacementsUtil.runtimeAssert(targetCount <= length(target), "StringUTF16.indexOfUnsafe invalid args: targetCount > length(target)"); + ReplacementsUtil.runtimeAssert(sourceCount >= targetCount, "StringUTF16.indexOfUnsafe invalid args: sourceCount < targetCount"); + int totalOffset = fromIndex; + if (targetCount == 1) { + Pointer sourcePointer = charOffsetPointer(source, totalOffset); + int indexOfResult = AMD64ArrayIndexOf.indexOf1Char(sourcePointer, sourceCount - fromIndex, StringUTF16Substitutions.getChar(target, 0)); + if (indexOfResult >= 0) { + return indexOfResult + totalOffset; + } + return indexOfResult; + } else if (targetCount == 2) { + Pointer sourcePointer = charOffsetPointer(source, totalOffset); + int indexOfResult = AMD64ArrayIndexOf.indexOfTwoConsecutiveChars(sourcePointer, sourceCount - fromIndex, StringUTF16Substitutions.getChar(target, 0), + StringUTF16Substitutions.getChar(target, 1)); + if (indexOfResult >= 0) { + return indexOfResult + totalOffset; + } + return indexOfResult; + } else { + int haystackLength = sourceCount - (fromIndex + (targetCount - 2)); + while (haystackLength > 0) { + Pointer sourcePointer = charOffsetPointer(source, totalOffset); + int indexOfResult = AMD64ArrayIndexOf.indexOfTwoConsecutiveChars(sourcePointer, haystackLength, StringUTF16Substitutions.getChar(target, 0), + StringUTF16Substitutions.getChar(target, 1)); + if (indexOfResult < 0) { + return -1; + } + totalOffset += indexOfResult; + haystackLength -= (indexOfResult + 1); + Pointer cmpSourcePointer = charOffsetPointer(source, totalOffset); + Pointer targetPointer = pointer(target); + if (ArrayRegionEqualsNode.regionEquals(cmpSourcePointer, targetPointer, targetCount, JavaKind.Char)) { + return totalOffset; + } + totalOffset++; + } + return -1; + } + } + + @MethodSubstitution + public static int indexOfLatin1Unsafe(byte[] source, int sourceCount, byte[] target, int targetCount, int fromIndex) { + ReplacementsUtil.runtimeAssert(fromIndex >= 0, "StringUTF16.indexOfLatin1Unsafe invalid args: fromIndex negative"); + ReplacementsUtil.runtimeAssert(targetCount > 0, "StringUTF16.indexOfLatin1Unsafe invalid args: targetCount <= 0"); + ReplacementsUtil.runtimeAssert(targetCount <= target.length, "StringUTF16.indexOfLatin1Unsafe invalid args: targetCount > length(target)"); + ReplacementsUtil.runtimeAssert(sourceCount >= targetCount, "StringUTF16.indexOfLatin1Unsafe invalid args: sourceCount < targetCount"); + int totalOffset = fromIndex; + if (targetCount == 1) { + Pointer sourcePointer = charOffsetPointer(source, totalOffset); + int indexOfResult = AMD64ArrayIndexOf.indexOf1Char(sourcePointer, sourceCount - fromIndex, (char) Byte.toUnsignedInt(target[0])); + if (indexOfResult >= 0) { + return indexOfResult + totalOffset; + } + return indexOfResult; + } else if (targetCount == 2) { + Pointer sourcePointer = charOffsetPointer(source, totalOffset); + int indexOfResult = AMD64ArrayIndexOf.indexOfTwoConsecutiveChars(sourcePointer, sourceCount - fromIndex, (char) Byte.toUnsignedInt(target[0]), (char) Byte.toUnsignedInt(target[1])); + if (indexOfResult >= 0) { + return indexOfResult + totalOffset; + } + return indexOfResult; + } else { + int haystackLength = sourceCount - (fromIndex + (targetCount - 2)); + while (haystackLength > 0) { + Pointer sourcePointer = charOffsetPointer(source, totalOffset); + int indexOfResult = AMD64ArrayIndexOf.indexOfTwoConsecutiveChars(sourcePointer, haystackLength, (char) Byte.toUnsignedInt(target[0]), (char) Byte.toUnsignedInt(target[1])); + if (indexOfResult < 0) { + return -1; + } + totalOffset += indexOfResult; + haystackLength -= (indexOfResult + 1); + Pointer cmpSourcePointer = charOffsetPointer(source, totalOffset); + Pointer targetPointer = pointer(target); + if (ArrayRegionEqualsNode.regionEquals(cmpSourcePointer, targetPointer, targetCount, JavaKind.Char, JavaKind.Byte)) { + return totalOffset; + } + totalOffset++; + } + return -1; + } + } + /** * Intrinsic for {@code java.lang.StringUTF16.compress([CI[BII)I}. * @@ -129,7 +231,7 @@ * @HotSpotIntrinsicCandidate * public static int compress(byte[] src, int src_indx, byte[] dst, int dst_indx, int len) * - * + *

* In this variant {@code dest} refers to a byte array containing 2 byte per char so * {@code srcIndex} and {@code len} are in terms of char elements and have to be scaled by 2 * when referring to {@code src}. --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.processor/src/org/graalvm/compiler/replacements/processor/GeneratedPlugin.java 2019-03-12 08:10:07.263778558 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.processor/src/org/graalvm/compiler/replacements/processor/GeneratedPlugin.java 2019-03-12 08:10:06.891776149 +0100 @@ -208,7 +208,11 @@ out.printf(" %s arg%d;\n", getErasedType(type), argIdx); out.printf(" if (args[%d].isConstant()) {\n", nodeIdx); if (type.equals(processor.getType("jdk.vm.ci.meta.ResolvedJavaType"))) { - out.printf(" arg%d = %s.asJavaType(args[%d].asConstant());\n", argIdx, deps.use(WellKnownDependency.CONSTANT_REFLECTION), nodeIdx); + out.printf(" jdk.vm.ci.meta.JavaConstant cst = args[%d].asJavaConstant();\n", nodeIdx); + out.printf(" arg%d = %s.asJavaType(cst);\n", argIdx, deps.use(WellKnownDependency.CONSTANT_REFLECTION)); + out.printf(" if (arg%d == null) {\n", argIdx); + out.printf(" arg%d = %s.asObject(jdk.vm.ci.meta.ResolvedJavaType.class, cst);\n", argIdx, deps.use(WellKnownDependency.SNIPPET_REFLECTION)); + out.printf(" }\n"); } else { switch (type.getKind()) { case BOOLEAN: --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/DeoptimizeOnIntegerExactTest.java 2019-03-12 08:10:07.731781589 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/DeoptimizeOnIntegerExactTest.java 2019-03-12 08:10:07.371779258 +0100 @@ -134,6 +134,7 @@ @Override protected InstalledCode addMethod(DebugContext debug, final ResolvedJavaMethod method, final CompilationResult compilationResult) { - return getBackend().createInstalledCode(debug, method, compilationResult, speculationLog, null, false); + assert speculationLog == compilationResult.getSpeculationLog(); + return getBackend().createInstalledCode(debug, method, compilationResult, null, false); } } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/IntegerExactFoldTest.java 2019-03-12 08:10:08.211784699 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/IntegerExactFoldTest.java 2019-03-12 08:10:07.847782341 +0100 @@ -40,23 +40,21 @@ import org.graalvm.compiler.nodes.ReturnNode; import org.graalvm.compiler.nodes.StructuredGraph; import org.graalvm.compiler.nodes.StructuredGraph.AllowAssumptions; -import org.graalvm.compiler.nodes.StructuredGraph.GuardsStage; import org.graalvm.compiler.nodes.ValueNode; import org.graalvm.compiler.nodes.spi.LoweringTool; import org.graalvm.compiler.phases.common.CanonicalizerPhase; +import org.graalvm.compiler.phases.common.GuardLoweringPhase; import org.graalvm.compiler.phases.common.LoweringPhase; import org.graalvm.compiler.phases.tiers.HighTierContext; -import org.graalvm.compiler.phases.tiers.PhaseContext; +import org.graalvm.compiler.phases.tiers.MidTierContext; import org.graalvm.compiler.replacements.nodes.arithmetic.IntegerExactArithmeticNode; import org.graalvm.compiler.replacements.nodes.arithmetic.IntegerExactArithmeticSplitNode; import org.junit.Assert; -import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; import org.junit.runners.Parameterized.Parameters; -@Ignore @RunWith(Parameterized.class) public class IntegerExactFoldTest extends GraalCompilerTest { private final long lowerBoundA; @@ -97,6 +95,7 @@ assertNotNull("original node must be in the graph", originalNode); new CanonicalizerPhase().apply(graph, getDefaultHighTierContext()); + ValueNode node = findNode(graph); boolean overflowExpected = node instanceof IntegerExactArithmeticNode; @@ -110,17 +109,19 @@ Node originalNode = graph.getNodes().filter(x -> x instanceof IntegerExactArithmeticNode).first(); assertNotNull("original node must be in the graph", originalNode); - - graph.setGuardsStage(GuardsStage.FIXED_DEOPTS); CanonicalizerPhase canonicalizer = new CanonicalizerPhase(); - PhaseContext context = new PhaseContext(getProviders()); - new LoweringPhase(canonicalizer, LoweringTool.StandardLoweringStage.HIGH_TIER).apply(graph, context); + HighTierContext highTierContext = getDefaultHighTierContext(); + new LoweringPhase(canonicalizer, LoweringTool.StandardLoweringStage.HIGH_TIER).apply(graph, highTierContext); + MidTierContext midTierContext = getDefaultMidTierContext(); + new GuardLoweringPhase().apply(graph, midTierContext); + new CanonicalizerPhase().apply(graph, midTierContext); + IntegerExactArithmeticSplitNode loweredNode = graph.getNodes().filter(IntegerExactArithmeticSplitNode.class).first(); assertNotNull("the lowered node must be in the graph", loweredNode); loweredNode.getX().setStamp(StampFactory.forInteger(bits, lowerBoundA, upperBoundA)); loweredNode.getY().setStamp(StampFactory.forInteger(bits, lowerBoundB, upperBoundB)); - new CanonicalizerPhase().apply(graph, context); + new CanonicalizerPhase().apply(graph, midTierContext); ValueNode node = findNode(graph); boolean overflowExpected = node instanceof IntegerExactArithmeticSplitNode; --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/ReplacementsParseTest.java 2019-03-12 08:10:08.695787833 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/ReplacementsParseTest.java 2019-03-12 08:10:08.327785450 +0100 @@ -24,6 +24,7 @@ package org.graalvm.compiler.replacements.test; +import static org.graalvm.compiler.debug.DebugOptions.DumpOnError; import static org.graalvm.compiler.java.BytecodeParserOptions.InlinePartialIntrinsicExitDuringParsing; import java.util.function.Function; @@ -40,15 +41,19 @@ import org.graalvm.compiler.core.common.type.Stamp; import org.graalvm.compiler.core.common.type.StampFactory; import org.graalvm.compiler.debug.DebugContext; +import org.graalvm.compiler.debug.GraalError; import org.graalvm.compiler.graph.GraalGraphError; import org.graalvm.compiler.graph.Node.ConstantNodeParameter; import org.graalvm.compiler.graph.Node.NodeIntrinsic; +import org.graalvm.compiler.java.BytecodeParser.BytecodeParserError; import org.graalvm.compiler.nodes.FrameState; import org.graalvm.compiler.nodes.PiNode; import org.graalvm.compiler.nodes.StructuredGraph; +import org.graalvm.compiler.nodes.StructuredGraph.AllowAssumptions; +import org.graalvm.compiler.nodes.StructuredGraph.Builder; import org.graalvm.compiler.nodes.ValueNode; -import org.graalvm.compiler.nodes.extended.OpaqueNode; import org.graalvm.compiler.nodes.extended.ForeignCallNode; +import org.graalvm.compiler.nodes.extended.OpaqueNode; import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderContext; import org.graalvm.compiler.nodes.graphbuilderconf.InlineInvokePlugin; import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugin.Receiver; @@ -64,8 +69,10 @@ import org.graalvm.compiler.phases.common.GuardLoweringPhase; import org.graalvm.compiler.phases.common.LoweringPhase; import org.graalvm.compiler.phases.tiers.HighTierContext; +import org.graalvm.compiler.test.GraalTest; import jdk.internal.vm.compiler.word.LocationIdentity; import org.junit.Assert; +import org.junit.Assume; import org.junit.BeforeClass; import org.junit.Test; @@ -190,6 +197,10 @@ static int nonVoidIntrinsicWithOptimizedSplit(int x) { return x; } + + static int div(int x, int y) { + return x / y; + } } @ClassSubstitution(TestObject.class) @@ -287,6 +298,12 @@ return x; } + @MethodSubstitution + static int div(int x, int y) { + assert y != 0; + return x / y; + } + public static void nonVoidIntrinsicWithCallStub(int zLen) { nonVoidIntrinsicWithCallStub(STUB_CALL, zLen); } @@ -311,6 +328,7 @@ r.registerMethodSubstitution(TestObjectSubstitutions.class, "copyFirstL2R", byte[].class, byte[].class); r.registerMethodSubstitution(TestObjectSubstitutions.class, "nonVoidIntrinsicWithCall", int.class, int.class); r.registerMethodSubstitution(TestObjectSubstitutions.class, "nonVoidIntrinsicWithOptimizedSplit", int.class); + r.registerMethodSubstitution(TestObjectSubstitutions.class, "div", int.class, int.class); if (replacementBytecodeProvider.supportsInvokedynamic()) { r.registerMethodSubstitution(TestObjectSubstitutions.class, "identity", String.class); @@ -357,6 +375,7 @@ @Test public void testNextAfter() { + Assume.assumeFalse(GraalTest.Java8OrEarlier); double[] inArray = new double[1024]; double[] outArray = new double[1024]; for (int i = 0; i < inArray.length; i++) { @@ -590,6 +609,23 @@ testGraph("nonVoidIntrinsicWithOptimizedSplit"); } + public static int div(int x, int y) { + return TestObject.div(x, y); + } + + @Test + public void testAssertionInMethodSubstitution() { + try { + ResolvedJavaMethod method = getResolvedJavaMethod("div"); + // avoid dumping graphs and printing exception since and exception is expected + OptionValues options = new OptionValues(getInitialOptions(), DumpOnError, false); + parse(new Builder(options, getDebugContext(options, null, method), AllowAssumptions.YES).method(method).compilationId(getCompilationId(method)), getEagerGraphBuilderSuite()); + throw GraalError.shouldNotReachHere("BytecodeParser should have complained about using assertion in an intrinsic."); + } catch (BytecodeParserError e) { + // Expected behavior + } + } + @SuppressWarnings("try") private void testGraph(String name) { StructuredGraph graph = parseEager(name, StructuredGraph.AllowAssumptions.YES); --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/StringCompareToTest.java 2019-03-12 08:10:09.183790994 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/StringCompareToTest.java 2019-03-12 08:10:08.815788611 +0100 @@ -24,21 +24,23 @@ package org.graalvm.compiler.replacements.test; -import jdk.vm.ci.aarch64.AArch64; -import jdk.vm.ci.amd64.AMD64; -import jdk.vm.ci.meta.ResolvedJavaMethod; +import static org.graalvm.compiler.core.common.GraalOptions.RemoveNeverExecutedCode; + +import java.util.List; + import org.graalvm.compiler.graph.Node; import org.graalvm.compiler.nodes.StructuredGraph; import org.graalvm.compiler.options.OptionValues; import org.graalvm.compiler.replacements.nodes.ArrayCompareToNode; import org.graalvm.compiler.serviceprovider.GraalServices; +import org.graalvm.compiler.serviceprovider.JavaVersionUtil; import org.junit.Assert; import org.junit.Assume; import org.junit.Test; -import java.util.List; - -import static org.graalvm.compiler.core.common.GraalOptions.RemoveNeverExecutedCode; +import jdk.vm.ci.aarch64.AArch64; +import jdk.vm.ci.amd64.AMD64; +import jdk.vm.ci.meta.ResolvedJavaMethod; /** * Tests compareTo method intrinsic. @@ -89,7 +91,7 @@ OptionValues options; boolean needCheckNode = true; - if (GraalServices.Java8OrEarlier) { + if (JavaVersionUtil.Java8OrEarlier) { needCheckNode = false; } else { List vmArgs = GraalServices.getInputArguments(); --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/StringIndexOfTestBase.java 2019-03-12 08:10:09.655794051 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/StringIndexOfTestBase.java 2019-03-12 08:10:09.295791720 +0100 @@ -24,6 +24,8 @@ package org.graalvm.compiler.replacements.test; +import static org.junit.Assume.assumeFalse; + import org.graalvm.compiler.core.test.GraalCompilerTest; import org.junit.Test; import org.junit.runner.RunWith; @@ -31,6 +33,7 @@ import java.util.ArrayList; import java.util.Collection; +import jdk.vm.ci.aarch64.AArch64; @RunWith(value = Parameterized.class) public abstract class StringIndexOfTestBase extends GraalCompilerTest { @@ -44,6 +47,25 @@ String[] utf16targets = new String[]{"grga " + ((char) 0x10D) + "varak", "grga", ((char) 0x10D) + "varak"}; addTargets(tests, targets); addTargets(tests, utf16targets); + + // Check long targets + // Checkstyle: stop + String lipsum = "Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata "; + // Checkstyle: resume + String lipsumUTF16 = lipsum + ((char) 0x10D); + int[] subStringLengths = {7, 8, 15, 16, 31, 32, 63, 64}; + for (int len : subStringLengths) { + String target = lipsum.substring(50, 50 + len); + tests.add(new Object[]{lipsum, target}); + tests.add(new Object[]{lipsum, target + "X"}); + tests.add(new Object[]{lipsumUTF16, target}); + tests.add(new Object[]{lipsumUTF16, target + "X"}); + tests.add(new Object[]{lipsumUTF16, target + ((char) 0x10D)}); + } + tests.add(new Object[]{ + "\u0100\u0101\u0102\u0103\u0104\u0105\u0106\u0107\u00f9\u00fa\u00fb\u00fc\u00fd\u00fe\u00ff\u0108\u0109\u010a\u010b\u010c", + "\u00f9\u00fa\u00fb\u00fc\u00fd\u00fe\u00ff"}); + return tests; } @@ -99,6 +121,7 @@ @Test public void testStringBuilderIndexOfConstant() { + assumeFalse("Disabled on AArch64 due to issues on AArch64; see GR-13100 or JDK-8215792", getTarget().arch instanceof AArch64); /* * Put a copy of the target string in the space after the current string to detect cases * where we search too far. @@ -111,6 +134,7 @@ @Test public void testStringBuilderIndexOfConstantOffset() { + assumeFalse("Disabled on AArch64 due to issues on AArch64; see GR-13100 or JDK-8215792", getTarget().arch instanceof AArch64); /* * Put a copy of the target string in the space after the current string to detect cases * where we search too far. --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/classfile/ClassfileBytecodeProviderTest.java 2019-03-12 08:10:10.143797212 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/classfile/ClassfileBytecodeProviderTest.java 2019-03-12 08:10:09.775794829 +0100 @@ -195,6 +195,10 @@ } protected void checkClass(MetaAccessProvider metaAccess, SnippetReflectionProvider snippetReflection, String className) throws ClassNotFoundException { + if (className.equals("jdk.vm.ci.services.JVMCIClassLoaderFactory")) { + // JVMCIClassLoaderFactory must only be initialized by the VM + return; + } Class c = Class.forName(className, true, getClass().getClassLoader()); ClassfileBytecodeProvider cbp = new ClassfileBytecodeProvider(metaAccess, snippetReflection); for (Method method : c.getDeclaredMethods()) { @@ -205,15 +209,21 @@ private static void checkMethod(ClassfileBytecodeProvider cbp, MetaAccessProvider metaAccess, Executable executable) { ResolvedJavaMethod method = metaAccess.lookupJavaMethod(executable); if (method.hasBytecodes()) { - ResolvedJavaMethodBytecode expected = new ResolvedJavaMethodBytecode(method); Bytecode actual = getBytecode(cbp, method); - new BytecodeComparer(expected, actual).compare(); + if (actual != null) { + ResolvedJavaMethodBytecode expected = new ResolvedJavaMethodBytecode(method); + new BytecodeComparer(expected, actual).compare(); + } } } protected static Bytecode getBytecode(ClassfileBytecodeProvider cbp, ResolvedJavaMethod method) { try { return cbp.getBytecode(method); + } catch (UnsupportedClassVersionError e) { + // This can happen when a library containing old class files + // is bundled into a Graal jar (GR-12672). + return null; } catch (Throwable e) { throw new AssertionError(String.format("Error getting bytecode for %s", method.format("%H.%n(%p)")), e); } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/CachingPEGraphDecoder.java 2019-03-12 08:10:10.619800295 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/CachingPEGraphDecoder.java 2019-03-12 08:10:10.259797963 +0100 @@ -44,7 +44,7 @@ import org.graalvm.compiler.nodes.graphbuilderconf.ParameterPlugin; import org.graalvm.compiler.phases.OptimisticOptimizations; import org.graalvm.compiler.phases.common.CanonicalizerPhase; -import org.graalvm.compiler.phases.common.ConvertDeoptimizeToGuardPhase; +import org.graalvm.compiler.loop.phases.ConvertDeoptimizeToGuardPhase; import org.graalvm.compiler.phases.tiers.PhaseContext; import org.graalvm.compiler.phases.util.Providers; --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/DefaultJavaLoweringProvider.java 2019-03-12 08:10:11.107803455 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/DefaultJavaLoweringProvider.java 2019-03-12 08:10:10.743801098 +0100 @@ -47,7 +47,6 @@ import org.graalvm.compiler.api.replacements.Snippet; import org.graalvm.compiler.api.replacements.SnippetReflectionProvider; import org.graalvm.compiler.core.common.LIRKind; -import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor; import org.graalvm.compiler.core.common.spi.ForeignCallsProvider; import org.graalvm.compiler.core.common.type.IntegerStamp; import org.graalvm.compiler.core.common.type.ObjectStamp; @@ -123,6 +122,7 @@ import org.graalvm.compiler.nodes.memory.ReadNode; import org.graalvm.compiler.nodes.memory.WriteNode; import org.graalvm.compiler.nodes.memory.address.AddressNode; +import org.graalvm.compiler.nodes.memory.address.IndexAddressNode; import org.graalvm.compiler.nodes.memory.address.OffsetAddressNode; import org.graalvm.compiler.nodes.spi.Lowerable; import org.graalvm.compiler.nodes.spi.LoweringProvider; @@ -138,9 +138,7 @@ import org.graalvm.compiler.phases.util.Providers; import org.graalvm.compiler.replacements.SnippetLowerableMemoryNode.SnippetLowering; import org.graalvm.compiler.replacements.nodes.BinaryMathIntrinsicNode; -import org.graalvm.compiler.replacements.nodes.BinaryMathIntrinsicNode.BinaryOperation; import org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode; -import org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.UnaryOperation; import jdk.internal.vm.compiler.word.LocationIdentity; import jdk.vm.ci.code.CodeUtil; @@ -206,6 +204,8 @@ lowerLoadIndexedNode((LoadIndexedNode) n, tool); } else if (n instanceof StoreIndexedNode) { lowerStoreIndexedNode((StoreIndexedNode) n, tool); + } else if (n instanceof IndexAddressNode) { + lowerIndexAddressNode((IndexAddressNode) n); } else if (n instanceof ArrayLengthNode) { lowerArrayLengthNode((ArrayLengthNode) n, tool); } else if (n instanceof LoadHubNode) { @@ -321,27 +321,20 @@ ResolvedJavaMethod method = math.graph().method(); if (method != null) { if (method.getAnnotation(Snippet.class) != null) { - /* - * In the context of the snippet use the LIR lowering instead of the Node lowering. - */ + // In the context of SnippetStub, i.e., Graal-generated stubs, use the LIR + // lowering to emit the stub assembly code instead of the Node lowering. return; } if (method.getName().equalsIgnoreCase(math.getOperation().name()) && tool.getMetaAccess().lookupJavaType(Math.class).equals(method.getDeclaringClass())) { - /* - * A root compilation of the intrinsic method should emit the full assembly - * implementation. - */ + // A root compilation of the intrinsic method should emit the full assembly + // implementation. return; } - - } - ForeignCallDescriptor foreignCall = toForeignCall(math.getOperation()); - if (foreignCall != null) { - StructuredGraph graph = math.graph(); - ForeignCallNode call = graph.add(new ForeignCallNode(foreignCalls, toForeignCall(math.getOperation()), math.getX(), math.getY())); - graph.addAfterFixed(tool.lastFixedNode(), call); - math.replaceAtUsages(call); } + StructuredGraph graph = math.graph(); + ForeignCallNode call = graph.add(new ForeignCallNode(foreignCalls, math.getOperation().foreignCallDescriptor, math.getX(), math.getY())); + graph.addAfterFixed(tool.lastFixedNode(), call); + math.replaceAtUsages(call); } private void lowerUnaryMath(UnaryMathIntrinsicNode math, LoweringTool tool) { @@ -350,36 +343,16 @@ } ResolvedJavaMethod method = math.graph().method(); if (method != null) { - if (method.getAnnotation(Snippet.class) != null) { - /* - * In the context of the snippet use the LIR lowering instead of the Node lowering. - */ - return; - } if (method.getName().equalsIgnoreCase(math.getOperation().name()) && tool.getMetaAccess().lookupJavaType(Math.class).equals(method.getDeclaringClass())) { - /* - * A root compilation of the intrinsic method should emit the full assembly - * implementation. - */ + // A root compilation of the intrinsic method should emit the full assembly + // implementation. return; } - - } - ForeignCallDescriptor foreignCall = toForeignCall(math.getOperation()); - if (foreignCall != null) { - StructuredGraph graph = math.graph(); - ForeignCallNode call = math.graph().add(new ForeignCallNode(foreignCalls, foreignCall, math.getValue())); - graph.addAfterFixed(tool.lastFixedNode(), call); - math.replaceAtUsages(call); } - } - - protected ForeignCallDescriptor toForeignCall(UnaryOperation operation) { - return operation.foreignCallDescriptor; - } - - protected ForeignCallDescriptor toForeignCall(BinaryOperation operation) { - return operation.foreignCallDescriptor; + StructuredGraph graph = math.graph(); + ForeignCallNode call = math.graph().add(new ForeignCallNode(foreignCalls, math.getOperation().foreignCallDescriptor, math.getValue())); + graph.addAfterFixed(tool.lastFixedNode(), call); + math.replaceAtUsages(call); } protected void lowerVerifyHeap(VerifyHeapNode n) { @@ -476,6 +449,11 @@ return graph.unique(new OffsetAddressNode(array, offset)); } + protected void lowerIndexAddressNode(IndexAddressNode indexAddress) { + AddressNode lowered = createArrayAddress(indexAddress.graph(), indexAddress.getArray(), indexAddress.getElementKind(), indexAddress.getIndex()); + indexAddress.replaceAndDelete(lowered); + } + protected void lowerLoadIndexedNode(LoadIndexedNode loadIndexed, LoweringTool tool) { StructuredGraph graph = loadIndexed.graph(); ValueNode array = loadIndexed.array(); --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/GraphKit.java 2019-03-12 08:10:11.627806823 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/GraphKit.java 2019-03-12 08:10:11.251804388 +0100 @@ -24,6 +24,7 @@ package org.graalvm.compiler.replacements; +import static jdk.vm.ci.services.Services.IS_IN_NATIVE_IMAGE; import static org.graalvm.compiler.nodes.graphbuilderconf.IntrinsicContext.CompilationContext.INLINE_AFTER_PARSING; import java.lang.reflect.Method; @@ -358,13 +359,18 @@ Plugins plugins = new Plugins(graphBuilderPlugins); GraphBuilderConfiguration config = GraphBuilderConfiguration.getSnippetDefault(plugins); - StructuredGraph calleeGraph = new StructuredGraph.Builder(invoke.getOptions(), invoke.getDebug()).method(method).trackNodeSourcePosition( - invoke.graph().trackNodeSourcePosition()).setIsSubstitution(true).build(); - IntrinsicContext initialReplacementContext = new IntrinsicContext(method, method, providers.getReplacements().getDefaultReplacementBytecodeProvider(), INLINE_AFTER_PARSING); - GraphBuilderPhase.Instance instance = createGraphBuilderInstance(metaAccess, providers.getStampProvider(), providers.getConstantReflection(), providers.getConstantFieldProvider(), config, - OptimisticOptimizations.NONE, - initialReplacementContext); - instance.apply(calleeGraph); + StructuredGraph calleeGraph; + if (IS_IN_NATIVE_IMAGE) { + calleeGraph = providers.getReplacements().getSnippet(method, null, false, null); + } else { + calleeGraph = new StructuredGraph.Builder(invoke.getOptions(), invoke.getDebug()).method(method).trackNodeSourcePosition(invoke.graph().trackNodeSourcePosition()).setIsSubstitution( + true).build(); + IntrinsicContext initialReplacementContext = new IntrinsicContext(method, method, providers.getReplacements().getDefaultReplacementBytecodeProvider(), INLINE_AFTER_PARSING); + GraphBuilderPhase.Instance instance = createGraphBuilderInstance(metaAccess, providers.getStampProvider(), providers.getConstantReflection(), providers.getConstantFieldProvider(), config, + OptimisticOptimizations.NONE, + initialReplacementContext); + instance.apply(calleeGraph); + } // Remove all frame states from inlinee calleeGraph.clearAllStateAfter(); --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/IntrinsicGraphBuilder.java 2019-03-12 08:10:12.099809880 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/IntrinsicGraphBuilder.java 2019-03-12 08:10:11.735807522 +0100 @@ -40,6 +40,7 @@ import org.graalvm.compiler.nodes.FixedNode; import org.graalvm.compiler.nodes.FixedWithNextNode; import org.graalvm.compiler.nodes.FrameState; +import org.graalvm.compiler.nodes.Invoke; import org.graalvm.compiler.nodes.ParameterNode; import org.graalvm.compiler.nodes.ReturnNode; import org.graalvm.compiler.nodes.StateSplit; @@ -163,7 +164,7 @@ } @Override - public void handleReplacedInvoke(InvokeKind invokeKind, ResolvedJavaMethod targetMethod, ValueNode[] args, boolean forceInlineEverything) { + public Invoke handleReplacedInvoke(InvokeKind invokeKind, ResolvedJavaMethod targetMethod, ValueNode[] args, boolean forceInlineEverything) { throw GraalError.shouldNotReachHere(); } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/PEGraphDecoder.java 2019-03-12 08:10:12.567812910 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/PEGraphDecoder.java 2019-03-12 08:10:12.203810553 +0100 @@ -353,7 +353,7 @@ } @Override - public void handleReplacedInvoke(InvokeKind invokeKind, ResolvedJavaMethod targetMethod, ValueNode[] args, boolean inlineEverything) { + public Invoke handleReplacedInvoke(InvokeKind invokeKind, ResolvedJavaMethod targetMethod, ValueNode[] args, boolean inlineEverything) { throw unimplemented(); } @@ -589,13 +589,19 @@ } } + @SuppressWarnings("try") public void decode(ResolvedJavaMethod method, boolean isSubstitution, boolean trackNodeSourcePosition) { - PEMethodScope methodScope = new PEMethodScope(graph, null, null, lookupEncodedGraph(method, null, null, isSubstitution, trackNodeSourcePosition), method, null, 0, loopExplosionPlugin, null); - decode(createInitialLoopScope(methodScope, null)); - cleanupGraph(methodScope); - - debug.dump(DebugContext.VERBOSE_LEVEL, graph, "After graph cleanup"); - assert graph.verify(); + try (DebugContext.Scope scope = debug.scope("PEGraphDecode", graph)) { + EncodedGraph encodedGraph = lookupEncodedGraph(method, null, null, isSubstitution, trackNodeSourcePosition); + PEMethodScope methodScope = new PEMethodScope(graph, null, null, encodedGraph, method, null, 0, loopExplosionPlugin, null); + decode(createInitialLoopScope(methodScope, null)); + cleanupGraph(methodScope); + + debug.dump(DebugContext.VERBOSE_LEVEL, graph, "After graph cleanup"); + assert graph.verify(); + } catch (Throwable t) { + throw debug.handle(t); + } try { /* Check that the control flow graph can be computed, to catch problems early. */ @@ -737,11 +743,19 @@ Invoke invoke = invokeData.invoke; ResolvedJavaMethod targetMethod = callTarget.targetMethod(); + if (loopScope.methodScope.encodedGraph.isCallToOriginal(targetMethod)) { + return false; + } + InvocationPlugin invocationPlugin = getInvocationPlugin(targetMethod); if (invocationPlugin == null) { return false; } + if (loopScope.methodScope.encodedGraph.isCallToOriginal(targetMethod)) { + return false; + } + ValueNode[] arguments = callTarget.arguments().toArray(new ValueNode[0]); FixedWithNextNode invokePredecessor = (FixedWithNextNode) invoke.asNode().predecessor(); --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/ReplacementsImpl.java 2019-03-12 08:10:13.079816225 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/ReplacementsImpl.java 2019-03-12 08:10:12.711813842 +0100 @@ -81,10 +81,13 @@ import org.graalvm.compiler.nodes.java.MethodCallTargetNode; import org.graalvm.compiler.nodes.spi.Replacements; import org.graalvm.compiler.nodes.spi.StampProvider; +import org.graalvm.compiler.options.Option; +import org.graalvm.compiler.options.OptionKey; +import org.graalvm.compiler.options.OptionType; import org.graalvm.compiler.options.OptionValues; import org.graalvm.compiler.phases.OptimisticOptimizations; import org.graalvm.compiler.phases.common.CanonicalizerPhase; -import org.graalvm.compiler.phases.common.ConvertDeoptimizeToGuardPhase; +import org.graalvm.compiler.loop.phases.ConvertDeoptimizeToGuardPhase; import org.graalvm.compiler.phases.common.DeadCodeEliminationPhase; import org.graalvm.compiler.phases.tiers.PhaseContext; import org.graalvm.compiler.phases.util.Providers; @@ -99,6 +102,13 @@ public class ReplacementsImpl implements Replacements, InlineInvokePlugin { + public static class Options { + // @formatter:off + @Option(help = "This is a testing option to exercise the SymbolicSnippetEncoder", type = OptionType.Expert) + public static final OptionKey UseEncodedSnippets = new OptionKey<>(false); + // @formatter:on + } + protected final OptionValues options; public Providers getProviders() { @@ -223,11 +233,6 @@ private static final TimerKey SnippetPreparationTime = DebugContext.timer("SnippetPreparationTime"); - @Override - public StructuredGraph getSnippet(ResolvedJavaMethod method, Object[] args, boolean trackNodeSourcePosition, NodeSourcePosition replaceePosition) { - return getSnippet(method, null, args, trackNodeSourcePosition, replaceePosition); - } - private static final AtomicInteger nextDebugContextId = new AtomicInteger(); public DebugContext openDebugContext(String idPrefix, ResolvedJavaMethod method) { --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/SnippetCounter.java 2019-03-12 08:10:13.571819412 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/SnippetCounter.java 2019-03-12 08:10:13.203817029 +0100 @@ -35,6 +35,9 @@ * snippet specific metrics. */ public final class SnippetCounter implements Comparable { + + public static final SnippetCounter DISABLED_COUNTER = new SnippetCounter(null, "Disabled", "Disabled"); + /** * A group of related counters. */ @@ -130,7 +133,7 @@ * compile-time constant {@link SnippetCounter} object. */ public void inc() { - if (group != null) { + if (getGroup() != null) { SnippetCounterNode.increment(this); } } @@ -140,7 +143,7 @@ * compile-time constant {@link SnippetCounter} object. */ public void add(int increment) { - if (group != null) { + if (getGroup() != null) { SnippetCounterNode.add(this, increment); } } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/SnippetIntegerHistogram.java 2019-03-12 08:10:14.059822571 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/SnippetIntegerHistogram.java 2019-03-12 08:10:13.695820214 +0100 @@ -29,6 +29,8 @@ * gathering snippet specific metrics. */ public final class SnippetIntegerHistogram { + public static final SnippetIntegerHistogram DISABLED_COUNTER = new SnippetIntegerHistogram(null, 1, "Disabled", "Disabled"); + private final SnippetCounter.Group group; private final String name; --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/SnippetTemplate.java 2019-03-12 08:10:14.539825679 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/SnippetTemplate.java 2019-03-12 08:10:14.179823348 +0100 @@ -25,6 +25,7 @@ package org.graalvm.compiler.replacements; import static java.util.FormattableFlags.ALTERNATE; +import static jdk.vm.ci.services.Services.IS_IN_NATIVE_IMAGE; import static org.graalvm.compiler.debug.DebugContext.DEFAULT_LOG_STREAM; import static org.graalvm.compiler.debug.DebugContext.applyFormattingFlagsAndWidth; import static org.graalvm.compiler.debug.DebugOptions.DebugStubsAndSnippets; @@ -212,8 +213,9 @@ constantParameters[0] = true; } - // Retrieve the names only when assertions are turned on. - assert initNames(method, count); + // Retrieve the names only when assertions are turned on. Parameter annotations are + // unsupported in the native image. + assert IS_IN_NATIVE_IMAGE || initNames(method, count); } final boolean[] constantParameters; --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/StandardGraphBuilderPlugins.java 2019-03-12 08:10:15.035828891 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/StandardGraphBuilderPlugins.java 2019-03-12 08:10:14.667826508 +0100 @@ -33,8 +33,8 @@ import static jdk.vm.ci.code.MemoryBarriers.STORE_LOAD; import static jdk.vm.ci.code.MemoryBarriers.STORE_STORE; import static org.graalvm.compiler.nodes.NamedLocationIdentity.OFF_HEAP_LOCATION; -import static org.graalvm.compiler.serviceprovider.GraalServices.Java11OrEarlier; -import static org.graalvm.compiler.serviceprovider.GraalServices.Java8OrEarlier; +import static org.graalvm.compiler.serviceprovider.JavaVersionUtil.Java11OrEarlier; +import static org.graalvm.compiler.serviceprovider.JavaVersionUtil.Java8OrEarlier; import java.lang.reflect.Array; import java.lang.reflect.Field; @@ -54,8 +54,6 @@ import org.graalvm.compiler.graph.Edges; import org.graalvm.compiler.graph.Node; import org.graalvm.compiler.graph.NodeList; -import org.graalvm.compiler.java.IntegerExactOpSpeculation; -import org.graalvm.compiler.java.IntegerExactOpSpeculation.IntegerExactOp; import org.graalvm.compiler.nodes.AbstractBeginNode; import org.graalvm.compiler.nodes.BeginNode; import org.graalvm.compiler.nodes.ConstantNode; @@ -66,6 +64,7 @@ import org.graalvm.compiler.nodes.IfNode; import org.graalvm.compiler.nodes.LogicNode; import org.graalvm.compiler.nodes.MergeNode; +import org.graalvm.compiler.nodes.NamedLocationIdentity; import org.graalvm.compiler.nodes.NodeView; import org.graalvm.compiler.nodes.StateSplit; import org.graalvm.compiler.nodes.StructuredGraph; @@ -93,6 +92,9 @@ import org.graalvm.compiler.nodes.extended.BranchProbabilityNode; import org.graalvm.compiler.nodes.extended.BytecodeExceptionNode.BytecodeExceptionKind; import org.graalvm.compiler.nodes.extended.GetClassNode; +import org.graalvm.compiler.nodes.extended.GuardingNode; +import org.graalvm.compiler.nodes.extended.JavaReadNode; +import org.graalvm.compiler.nodes.extended.JavaWriteNode; import org.graalvm.compiler.nodes.extended.MembarNode; import org.graalvm.compiler.nodes.extended.OpaqueNode; import org.graalvm.compiler.nodes.extended.RawLoadNode; @@ -113,6 +115,8 @@ import org.graalvm.compiler.nodes.java.RegisterFinalizerNode; import org.graalvm.compiler.nodes.java.UnsafeCompareAndExchangeNode; import org.graalvm.compiler.nodes.java.UnsafeCompareAndSwapNode; +import org.graalvm.compiler.nodes.memory.HeapAccess; +import org.graalvm.compiler.nodes.memory.address.IndexAddressNode; import org.graalvm.compiler.nodes.type.StampTool; import org.graalvm.compiler.nodes.util.GraphUtil; import org.graalvm.compiler.nodes.virtual.EnsureVirtualizedNode; @@ -120,13 +124,16 @@ import org.graalvm.compiler.replacements.nodes.ReverseBytesNode; import org.graalvm.compiler.replacements.nodes.VirtualizableInvokeMacroNode; import org.graalvm.compiler.replacements.nodes.arithmetic.IntegerAddExactNode; +import org.graalvm.compiler.replacements.nodes.arithmetic.IntegerAddExactOverflowNode; import org.graalvm.compiler.replacements.nodes.arithmetic.IntegerAddExactSplitNode; -import org.graalvm.compiler.replacements.nodes.arithmetic.IntegerExactArithmeticNode; import org.graalvm.compiler.replacements.nodes.arithmetic.IntegerExactArithmeticSplitNode; import org.graalvm.compiler.replacements.nodes.arithmetic.IntegerMulExactNode; +import org.graalvm.compiler.replacements.nodes.arithmetic.IntegerMulExactOverflowNode; import org.graalvm.compiler.replacements.nodes.arithmetic.IntegerMulExactSplitNode; import org.graalvm.compiler.replacements.nodes.arithmetic.IntegerSubExactNode; +import org.graalvm.compiler.replacements.nodes.arithmetic.IntegerSubExactOverflowNode; import org.graalvm.compiler.replacements.nodes.arithmetic.IntegerSubExactSplitNode; +import org.graalvm.compiler.serviceprovider.SpeculationReasonGroup; import jdk.internal.vm.compiler.word.LocationIdentity; import jdk.vm.ci.code.BytecodePosition; @@ -197,8 +204,10 @@ public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver) { if (receiver.isConstant()) { String s = snippetReflection.asObject(String.class, (JavaConstant) receiver.get().asConstant()); - b.addPush(JavaKind.Int, b.add(ConstantNode.forInt(s.hashCode()))); - return true; + if (s != null) { + b.addPush(JavaKind.Int, b.add(ConstantNode.forInt(s.hashCode()))); + return true; + } } return false; } @@ -221,12 +230,32 @@ }); } else { r.registerMethodSubstitution(JDK9StringSubstitutions.class, "equals", Receiver.class, Object.class); + Registration utf16sub = new Registration(plugins, StringUTF16Substitutions.class, bytecodeProvider); + utf16sub.register2("getCharDirect", byte[].class, int.class, new InvocationPlugin() { + @Override + public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode arg1, ValueNode arg2) { + b.addPush(JavaKind.Char, new JavaReadNode(JavaKind.Char, new IndexAddressNode(arg1, arg2, JavaKind.Byte), NamedLocationIdentity.getArrayLocation(JavaKind.Byte), + HeapAccess.BarrierType.NONE, false)); + return true; + } + }); + utf16sub.register3("putCharDirect", byte[].class, int.class, int.class, new InvocationPlugin() { + @Override + public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode arg1, ValueNode arg2, ValueNode arg3) { + b.add(new JavaWriteNode(JavaKind.Char, new IndexAddressNode(arg1, arg2, JavaKind.Byte), NamedLocationIdentity.getArrayLocation(JavaKind.Byte), arg3, + HeapAccess.BarrierType.NONE, false)); + return true; + } + }); final Registration latin1r = new Registration(plugins, "java.lang.StringLatin1", bytecodeProvider); latin1r.register5("indexOf", byte[].class, int.class, byte[].class, int.class, int.class, new StringLatin1IndexOfConstantPlugin()); final Registration utf16r = new Registration(plugins, "java.lang.StringUTF16", bytecodeProvider); utf16r.register5("indexOfUnsafe", byte[].class, int.class, byte[].class, int.class, int.class, new StringUTF16IndexOfConstantPlugin()); + utf16r.setAllowOverwrite(true); + utf16r.registerMethodSubstitution(StringUTF16Substitutions.class, "getChar", byte[].class, int.class); + utf16r.registerMethodSubstitution(StringUTF16Substitutions.class, "putChar", byte[].class, int.class, int.class); Registration sr = new Registration(plugins, JDK9StringSubstitutions.class); sr.register1("getValue", String.class, new InvocationPlugin() { @@ -529,16 +558,47 @@ }); } - private static ValueNode createIntegerExactArithmeticNode(ValueNode x, ValueNode y, SpeculationReason speculation, IntegerExactOp op) { + public enum IntegerExactOp { + INTEGER_ADD_EXACT, + INTEGER_INCREMENT_EXACT, + INTEGER_SUBTRACT_EXACT, + INTEGER_DECREMENT_EXACT, + INTEGER_MULTIPLY_EXACT + } + + private static GuardingNode createIntegerExactArithmeticGuardNode(GraphBuilderContext b, ValueNode x, ValueNode y, IntegerExactOp op) { + LogicNode overflowCheck; + switch (op) { + case INTEGER_ADD_EXACT: + case INTEGER_INCREMENT_EXACT: { + overflowCheck = new IntegerAddExactOverflowNode(x, y); + break; + } + case INTEGER_SUBTRACT_EXACT: + case INTEGER_DECREMENT_EXACT: { + overflowCheck = new IntegerSubExactOverflowNode(x, y); + break; + } + case INTEGER_MULTIPLY_EXACT: { + overflowCheck = new IntegerMulExactOverflowNode(x, y); + break; + } + default: + throw GraalError.shouldNotReachHere("Unknown integer exact operation."); + } + return b.add(new FixedGuardNode(overflowCheck, DeoptimizationReason.ArithmeticException, DeoptimizationAction.InvalidateRecompile, true)); + } + + private static ValueNode createIntegerExactArithmeticNode(GraphBuilderContext b, ValueNode x, ValueNode y, IntegerExactOp op) { switch (op) { case INTEGER_ADD_EXACT: case INTEGER_INCREMENT_EXACT: - return new IntegerAddExactNode(x, y, speculation); + return new IntegerAddExactNode(x, y, createIntegerExactArithmeticGuardNode(b, x, y, op)); case INTEGER_SUBTRACT_EXACT: case INTEGER_DECREMENT_EXACT: - return new IntegerSubExactNode(x, y, speculation); + return new IntegerSubExactNode(x, y, createIntegerExactArithmeticGuardNode(b, x, y, op)); case INTEGER_MULTIPLY_EXACT: - return new IntegerMulExactNode(x, y, speculation); + return new IntegerMulExactNode(x, y, createIntegerExactArithmeticGuardNode(b, x, y, op)); default: throw GraalError.shouldNotReachHere("Unknown integer exact operation."); } @@ -559,15 +619,15 @@ } } - private static boolean createIntegerExactOperation(GraphBuilderContext b, JavaKind kind, ValueNode x, ValueNode y, IntegerExactOp op) { - BytecodeExceptionKind exceptionKind = kind == JavaKind.Int ? BytecodeExceptionKind.INTEGER_EXACT_OVERFLOW : BytecodeExceptionKind.LONG_EXACT_OVERFLOW; - AbstractBeginNode exceptionEdge = b.genExplicitExceptionEdge(exceptionKind); - if (exceptionEdge != null) { + private static void createIntegerExactOperation(GraphBuilderContext b, JavaKind kind, ValueNode x, ValueNode y, IntegerExactOp op) { + if (b.needsExplicitException()) { + BytecodeExceptionKind exceptionKind = kind == JavaKind.Int ? BytecodeExceptionKind.INTEGER_EXACT_OVERFLOW : BytecodeExceptionKind.LONG_EXACT_OVERFLOW; + AbstractBeginNode exceptionEdge = b.genExplicitExceptionEdge(exceptionKind); IntegerExactArithmeticSplitNode split = b.addPush(kind, createIntegerExactSplit(x, y, exceptionEdge, op)); split.setNext(b.add(new BeginNode())); - return true; + } else { + b.addPush(kind, createIntegerExactArithmeticNode(b, x, y, op)); } - return false; } private static void registerMathPlugins(InvocationPlugins plugins, boolean allowDeoptimization) { @@ -575,12 +635,12 @@ if (allowDeoptimization) { for (JavaKind kind : new JavaKind[]{JavaKind.Int, JavaKind.Long}) { Class type = kind.toJavaClass(); - r.register1("decrementExact", type, new InvocationPlugin() { @Override public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode x) { ConstantNode y = b.add(ConstantNode.forIntegerKind(kind, 1)); - return createIntegerExactOperation(b, kind, x, y, IntegerExactOp.INTEGER_DECREMENT_EXACT); + createIntegerExactOperation(b, kind, x, y, IntegerExactOp.INTEGER_DECREMENT_EXACT); + return true; } }); @@ -588,29 +648,31 @@ @Override public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode x) { ConstantNode y = b.add(ConstantNode.forIntegerKind(kind, 1)); - return createIntegerExactOperation(b, kind, x, y, IntegerExactOp.INTEGER_INCREMENT_EXACT); + createIntegerExactOperation(b, kind, x, y, IntegerExactOp.INTEGER_INCREMENT_EXACT); + return true; } }); - r.register2("addExact", type, type, new InvocationPlugin() { @Override public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode x, ValueNode y) { - return createIntegerExactOperation(b, kind, x, y, IntegerExactOp.INTEGER_ADD_EXACT); + createIntegerExactOperation(b, kind, x, y, IntegerExactOp.INTEGER_ADD_EXACT); + return true; } }); - r.register2("subtractExact", type, type, new InvocationPlugin() { @Override public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode x, ValueNode y) { - return createIntegerExactOperation(b, kind, x, y, IntegerExactOp.INTEGER_SUBTRACT_EXACT); + createIntegerExactOperation(b, kind, x, y, IntegerExactOp.INTEGER_SUBTRACT_EXACT); + return true; } }); r.register2("multiplyExact", type, type, new InvocationPlugin() { @Override public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode x, ValueNode y) { - return createIntegerExactOperation(b, kind, x, y, IntegerExactOp.INTEGER_MULTIPLY_EXACT); - } + createIntegerExactOperation(b, kind, x, y, IntegerExactOp.INTEGER_MULTIPLY_EXACT); + return true; + } }); } } @@ -1100,23 +1162,7 @@ } } - private static final class DirectiveSpeculationReason implements SpeculationLog.SpeculationReason { - private final BytecodePosition pos; - - private DirectiveSpeculationReason(BytecodePosition pos) { - this.pos = pos; - } - - @Override - public int hashCode() { - return pos.hashCode(); - } - - @Override - public boolean equals(Object obj) { - return obj instanceof DirectiveSpeculationReason && ((DirectiveSpeculationReason) obj).pos.equals(this.pos); - } - } + private static final SpeculationReasonGroup DIRECTIVE_SPECULATIONS = new SpeculationReasonGroup("GraalDirective", BytecodePosition.class); private static void registerGraalDirectivesPlugins(InvocationPlugins plugins) { Registration r = new Registration(plugins, GraalDirectives.class); @@ -1139,9 +1185,9 @@ r.register0("deoptimizeAndInvalidateWithSpeculation", new InvocationPlugin() { @Override public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver) { - GraalError.guarantee(b.getGraph().getSpeculationLog() != null, "A speculation log is need to use `deoptimizeAndInvalidateWithSpeculation`"); + GraalError.guarantee(b.getGraph().getSpeculationLog() != null, "A speculation log is needed to use `deoptimizeAndInvalidateWithSpeculation`"); BytecodePosition pos = new BytecodePosition(null, b.getMethod(), b.bci()); - DirectiveSpeculationReason reason = new DirectiveSpeculationReason(pos); + SpeculationReason reason = DIRECTIVE_SPECULATIONS.createSpeculationReason(pos); Speculation speculation; if (b.getGraph().getSpeculationLog().maySpeculate(reason)) { speculation = b.getGraph().getSpeculationLog().speculate(reason); @@ -1324,7 +1370,7 @@ b.add(new DeoptimizeNode(DeoptimizationAction.InvalidateReprofile, DeoptimizationReason.TransferToInterpreter)); } else if (falseCount == 0 || trueCount == 0) { boolean expected = falseCount == 0; - LogicNode condition = b.addWithInputs( + LogicNode condition = b.add( IntegerEqualsNode.create(b.getConstantReflection(), b.getMetaAccess(), b.getOptions(), null, result, b.add(ConstantNode.forBoolean(!expected)), NodeView.DEFAULT)); b.append(new FixedGuardNode(condition, DeoptimizationReason.UnreachedCode, DeoptimizationAction.InvalidateReprofile, true)); --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/arraycopy/ArrayCopyCallNode.java 2019-03-12 08:10:15.551832232 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/arraycopy/ArrayCopyCallNode.java 2019-03-12 08:10:15.191829902 +0100 @@ -201,14 +201,19 @@ arraycopy(src, srcPos, dest, destPos, length, JavaKind.Object, LocationIdentity.any(), false, false, false, heapWordSize); } - public static void arraycopy(Object src, int srcPos, Object dest, int destPos, int length, @ConstantNodeParameter JavaKind elementKind, @ConstantNodeParameter int heapWordSize) { - arraycopy(src, srcPos, dest, destPos, length, elementKind, false, false, false, heapWordSize); + public static void arraycopy(Object src, int srcPos, Object dest, int destPos, int length, @ConstantNodeParameter JavaKind elementKind, @ConstantNodeParameter LocationIdentity locationIdentity, + @ConstantNodeParameter int heapWordSize) { + arraycopy(src, srcPos, dest, destPos, length, elementKind, locationIdentity, false, false, false, heapWordSize); } public static void disjointArraycopy(Object src, int srcPos, Object dest, int destPos, int length, @ConstantNodeParameter JavaKind elementKind, @ConstantNodeParameter int heapWordSize) { arraycopy(src, srcPos, dest, destPos, length, elementKind, false, true, false, heapWordSize); } + public static void disjointArraycopyKillsAny(Object src, int srcPos, Object dest, int destPos, int length, @ConstantNodeParameter JavaKind elementKind, @ConstantNodeParameter int heapWordSize) { + arraycopy(src, srcPos, dest, destPos, length, elementKind, LocationIdentity.any(), false, true, false, heapWordSize); + } + public static void disjointUninitializedArraycopy(Object src, int srcPos, Object dest, int destPos, int length, @ConstantNodeParameter JavaKind elementKind, @ConstantNodeParameter int heapWordSize) { arraycopy(src, srcPos, dest, destPos, length, elementKind, false, true, true, heapWordSize); --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/arraycopy/ArrayCopyNode.java 2019-03-12 08:10:16.035835366 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/arraycopy/ArrayCopyNode.java 2019-03-12 08:10:15.675833035 +0100 @@ -35,23 +35,30 @@ import org.graalvm.compiler.replacements.nodes.BasicArrayCopyNode; import jdk.internal.vm.compiler.word.LocationIdentity; -import jdk.vm.ci.meta.JavaKind; - @NodeInfo public final class ArrayCopyNode extends BasicArrayCopyNode implements Lowerable { public static final NodeClass TYPE = NodeClass.create(ArrayCopyNode.class); - private JavaKind elementKind; + protected final boolean forceAnyLocation; public ArrayCopyNode(int bci, ValueNode src, ValueNode srcPos, ValueNode dst, ValueNode dstPos, ValueNode length) { + this(bci, src, srcPos, dst, dstPos, length, false); + } + + public ArrayCopyNode(int bci, ValueNode src, ValueNode srcPos, ValueNode dst, ValueNode dstPos, ValueNode length, boolean forceAnyLocation) { super(TYPE, src, srcPos, dst, dstPos, length, null, bci); - elementKind = ArrayCopySnippets.Templates.selectComponentKind(this); + this.forceAnyLocation = forceAnyLocation; + if (!forceAnyLocation) { + elementKind = ArrayCopySnippets.Templates.selectComponentKind(this); + } else { + assert elementKind == null; + } } @Override public LocationIdentity getLocationIdentity() { - if (elementKind == null) { + if (!forceAnyLocation && elementKind == null) { elementKind = ArrayCopySnippets.Templates.selectComponentKind(this); } if (elementKind != null) { @@ -64,4 +71,8 @@ public void lower(LoweringTool tool) { tool.getLowerer().lower(this, tool); } + + public boolean killsAnyLocation() { + return forceAnyLocation; + } } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/arraycopy/ArrayCopySnippets.java 2019-03-12 08:10:16.527838552 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/arraycopy/ArrayCopySnippets.java 2019-03-12 08:10:16.159836169 +0100 @@ -24,6 +24,7 @@ package org.graalvm.compiler.replacements.arraycopy; +import static jdk.vm.ci.services.Services.IS_BUILDING_NATIVE_IMAGE; import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FREQUENT_PROBABILITY; import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.LIKELY_PROBABILITY; import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.NOT_FREQUENT_PROBABILITY; @@ -34,7 +35,6 @@ import jdk.internal.vm.compiler.collections.UnmodifiableEconomicMap; import org.graalvm.compiler.api.directives.GraalDirectives; -import org.graalvm.compiler.api.replacements.Fold; import org.graalvm.compiler.api.replacements.Fold.InjectedParameter; import org.graalvm.compiler.api.replacements.Snippet; import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter; @@ -120,8 +120,8 @@ @Snippet public void arraycopyExactSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck, - @ConstantParameter JavaKind elementKind, @ConstantParameter SnippetCounter elementKindCounter, @ConstantParameter SnippetCounter elementKindCopiedCounter, - @ConstantParameter Counters counters) { + @ConstantParameter JavaKind elementKind, @ConstantParameter LocationIdentity locationIdentity, + @ConstantParameter SnippetCounter elementKindCounter, @ConstantParameter SnippetCounter elementKindCopiedCounter, @ConstantParameter Counters counters) { Object nonNullSrc = GraalDirectives.guardingNonNull(src); Object nonNullDest = GraalDirectives.guardingNonNull(dest); checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck); @@ -130,19 +130,19 @@ elementKindCounter.inc(); elementKindCopiedCounter.add(length); - ArrayCopyCallNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, elementKind, heapWordSize()); + ArrayCopyCallNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, elementKind, locationIdentity, heapWordSize()); } @Snippet public void arraycopyUnrolledSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck, - @ConstantParameter JavaKind elementKind, @ConstantParameter int unrolledLength, @ConstantParameter Counters counters) { + @ConstantParameter JavaKind elementKind, @ConstantParameter LocationIdentity locationIdentity, @ConstantParameter int unrolledLength, @ConstantParameter Counters counters) { Object nonNullSrc = GraalDirectives.guardingNonNull(src); Object nonNullDest = GraalDirectives.guardingNonNull(dest); checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck); checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters); incrementLengthCounter(length, counters); - unrolledArraycopyWork(nonNullSrc, srcPos, nonNullDest, destPos, unrolledLength, elementKind); + unrolledArraycopyWork(nonNullSrc, srcPos, nonNullDest, destPos, unrolledLength, elementKind, locationIdentity); } @Snippet @@ -179,15 +179,9 @@ System.arraycopy(src, srcPos, dest, destPos, length); } - @Fold - static LocationIdentity getArrayLocation(JavaKind kind) { - return NamedLocationIdentity.getArrayLocation(kind); - } - - private static void unrolledArraycopyWork(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, int length, JavaKind elementKind) { + private static void unrolledArraycopyWork(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, int length, JavaKind elementKind, LocationIdentity arrayLocation) { int scale = ReplacementsUtil.arrayIndexScale(INJECTED_META_ACCESS, elementKind); int arrayBaseOffset = ReplacementsUtil.getArrayBaseOffset(INJECTED_META_ACCESS, elementKind); - LocationIdentity arrayLocation = getArrayLocation(elementKind); long sourceOffset = arrayBaseOffset + (long) srcPos * scale; long destOffset = arrayBaseOffset + (long) destPos * scale; @@ -258,7 +252,9 @@ } private static void incrementLengthCounter(int length, Counters counters) { - counters.lengthHistogram.inc(length); + if (!IS_BUILDING_NATIVE_IMAGE) { + counters.lengthHistogram.inc(length); + } } private static void checkLimits(Object src, int srcPos, Object dest, int destPos, int length, Counters counters) { @@ -466,13 +462,16 @@ assert arrayTypeCheck != ArrayCopyTypeCheck.UNDEFINED_ARRAY_TYPE_CHECK; args.addConst("arrayTypeCheck", arrayTypeCheck); } + Object locationIdentity = arraycopy.killsAnyLocation() ? LocationIdentity.any() : NamedLocationIdentity.getArrayLocation(elementKind); if (snippetInfo == arraycopyUnrolledSnippet) { args.addConst("elementKind", elementKind != null ? elementKind : JavaKind.Illegal); + args.addConst("locationIdentity", locationIdentity); args.addConst("unrolledLength", arraycopy.getLength().asJavaConstant().asInt()); } if (snippetInfo == arraycopyExactSnippet) { assert elementKind != null; args.addConst("elementKind", elementKind); + args.addConst("locationIdentity", locationIdentity); args.addConst("elementKindCounter", counters.arraycopyCallCounters.get(elementKind)); args.addConst("elementKindCopiedCounter", counters.arraycopyCallCopiedCounters.get(elementKind)); } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/ArrayCompareToNode.java 2019-03-12 08:10:17.027841789 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/ArrayCompareToNode.java 2019-03-12 08:10:16.663839432 +0100 @@ -125,7 +125,7 @@ @Override public LocationIdentity getLocationIdentity() { - return NamedLocationIdentity.getArrayLocation(kind1); + return kind1 != kind2 ? LocationIdentity.ANY_LOCATION : NamedLocationIdentity.getArrayLocation(kind1); } @Override --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/ArrayRegionEqualsNode.java 2019-03-12 08:10:17.519844974 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/ArrayRegionEqualsNode.java 2019-03-12 08:10:17.155842618 +0100 @@ -55,7 +55,8 @@ public static final NodeClass TYPE = NodeClass.create(ArrayRegionEqualsNode.class); /** {@link JavaKind} of the arrays to compare. */ - private final JavaKind kind; + private final JavaKind kind1; + private final JavaKind kind2; /** Pointer to first array region to be tested for equality. */ @Input private ValueNode array1; @@ -68,16 +69,21 @@ @OptionalInput(Memory) private MemoryNode lastLocationAccess; - public ArrayRegionEqualsNode(ValueNode array1, ValueNode array2, ValueNode length, @ConstantNodeParameter JavaKind kind) { + public ArrayRegionEqualsNode(ValueNode array1, ValueNode array2, ValueNode length, @ConstantNodeParameter JavaKind kind1, @ConstantNodeParameter JavaKind kind2) { super(TYPE, StampFactory.forKind(JavaKind.Boolean)); - this.kind = kind; + this.kind1 = kind1; + this.kind2 = kind2; this.array1 = array1; this.array2 = array2; this.length = length; } + public static boolean regionEquals(Pointer array1, Pointer array2, int length, @ConstantNodeParameter JavaKind kind) { + return regionEquals(array1, array2, length, kind, kind); + } + @NodeIntrinsic - public static native boolean regionEquals(Pointer array1, Pointer array2, int length, @ConstantNodeParameter JavaKind kind); + public static native boolean regionEquals(Pointer array1, Pointer array2, int length, @ConstantNodeParameter JavaKind kind1, @ConstantNodeParameter JavaKind kind2); @Override public void generate(NodeLIRBuilderTool gen) { @@ -85,13 +91,18 @@ if (length.isConstant()) { constantLength = length.asJavaConstant().asInt(); } - Value result = gen.getLIRGeneratorTool().emitArrayEquals(kind, gen.operand(array1), gen.operand(array2), gen.operand(length), constantLength, true); + Value result; + if (kind1 == kind2) { + result = gen.getLIRGeneratorTool().emitArrayEquals(kind1, gen.operand(array1), gen.operand(array2), gen.operand(length), constantLength, true); + } else { + result = gen.getLIRGeneratorTool().emitArrayEquals(kind1, kind2, gen.operand(array1), gen.operand(array2), gen.operand(length), constantLength, true); + } gen.setResult(this, result); } @Override public LocationIdentity getLocationIdentity() { - return NamedLocationIdentity.getArrayLocation(kind); + return kind1 != kind2 ? LocationIdentity.ANY_LOCATION : NamedLocationIdentity.getArrayLocation(kind1); } @Override --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/BasicArrayCopyNode.java 2019-03-12 08:10:18.003848107 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/BasicArrayCopyNode.java 2019-03-12 08:10:17.643845777 +0100 @@ -83,15 +83,12 @@ public BasicArrayCopyNode(NodeClass type, ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, JavaKind elementKind, int bci) { super(type, StampFactory.forKind(JavaKind.Void)); this.bci = bci; - args = new NodeInputList<>(this, new ValueNode[]{src, srcPos, dest, destPos, length}); + this.args = new NodeInputList<>(this, new ValueNode[]{src, srcPos, dest, destPos, length}); this.elementKind = elementKind != JavaKind.Illegal ? elementKind : null; } public BasicArrayCopyNode(NodeClass type, ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, JavaKind elementKind) { - super(type, StampFactory.forKind(JavaKind.Void)); - this.bci = BytecodeFrame.INVALID_FRAMESTATE_BCI; - args = new NodeInputList<>(this, new ValueNode[]{src, srcPos, dest, destPos, length}); - this.elementKind = elementKind != JavaKind.Illegal ? elementKind : null; + this(type, src, srcPos, dest, destPos, length, elementKind, BytecodeFrame.INVALID_FRAMESTATE_BCI); } public ValueNode getSource() { --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/BasicObjectCloneNode.java 2019-03-12 08:10:18.487851241 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/BasicObjectCloneNode.java 2019-03-12 08:10:18.127848910 +0100 @@ -154,7 +154,7 @@ return; } int constantLength = lengthAlias.asJavaConstant().asInt(); - if (constantLength >= 0 && constantLength < tool.getMaximumEntryCount()) { + if (constantLength >= 0 && constantLength <= tool.getMaximumEntryCount()) { ValueNode[] state = new ValueNode[constantLength]; ResolvedJavaType componentType = type.getComponentType(); for (int i = 0; i < constantLength; i++) { --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/BinaryMathIntrinsicNode.java 2019-03-12 08:10:18.983854452 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/BinaryMathIntrinsicNode.java 2019-03-12 08:10:18.619852096 +0100 @@ -26,6 +26,8 @@ import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_UNKNOWN; import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_1; + +import jdk.vm.ci.meta.Value; import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor; import org.graalvm.compiler.core.common.type.FloatStamp; import org.graalvm.compiler.core.common.type.PrimitiveStamp; @@ -46,10 +48,9 @@ import org.graalvm.compiler.nodes.spi.ArithmeticLIRLowerable; import org.graalvm.compiler.nodes.spi.Lowerable; import org.graalvm.compiler.nodes.spi.LoweringTool; -import org.graalvm.compiler.nodes.spi.NodeLIRBuilderTool; import jdk.vm.ci.meta.JavaKind; -import jdk.vm.ci.meta.Value; +import org.graalvm.compiler.nodes.spi.NodeLIRBuilderTool; @NodeInfo(nameTemplate = "MathIntrinsic#{p#operation/s}", cycles = CYCLES_UNKNOWN, size = SIZE_1) public final class BinaryMathIntrinsicNode extends BinaryNode implements ArithmeticLIRLowerable, Lowerable { @@ -106,6 +107,7 @@ @Override public void generate(NodeLIRBuilderTool nodeValueMap, ArithmeticLIRGeneratorTool gen) { + // We can only reach here in the math stubs Value xValue = nodeValueMap.operand(getX()); Value yValue = nodeValueMap.operand(getY()); Value result; --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/MacroNode.java 2019-03-12 08:10:19.447857456 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/MacroNode.java 2019-03-12 08:10:19.083855100 +0100 @@ -25,6 +25,7 @@ package org.graalvm.compiler.replacements.nodes; import static jdk.vm.ci.code.BytecodeFrame.isPlaceholderBci; +import static jdk.vm.ci.services.Services.IS_IN_NATIVE_IMAGE; import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_UNKNOWN; import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_UNKNOWN; @@ -205,13 +206,15 @@ if (invoke.stateAfter() == null) { ResolvedJavaMethod method = graph().method(); - if (method.getAnnotation(MethodSubstitution.class) != null || method.getAnnotation(Snippet.class) != null) { - // One cause for this is that a MacroNode is created for a method that - // no longer needs a MacroNode. For example, Class.getComponentType() - // only needs a MacroNode prior to JDK9 as it was given a non-native - // implementation in JDK9. - throw new GraalError("%s macro created for call to %s in %s must be lowerable to a snippet or intrinsic graph. " + - "Maybe a macro node is not needed for this method in the current JDK?", getClass().getSimpleName(), targetMethod.format("%h.%n(%p)"), graph()); + if (!IS_IN_NATIVE_IMAGE) { + if (method.getAnnotation(MethodSubstitution.class) != null || method.getAnnotation(Snippet.class) != null) { + // One cause for this is that a MacroNode is created for a method that + // no longer needs a MacroNode. For example, Class.getComponentType() + // only needs a MacroNode prior to JDK9 as it was given a non-native + // implementation in JDK9. + throw new GraalError("%s macro created for call to %s in %s must be lowerable to a snippet or intrinsic graph. " + + "Maybe a macro node is not needed for this method in the current JDK?", getClass().getSimpleName(), targetMethod.format("%h.%n(%p)"), graph()); + } } throw new GraalError("%s: cannot lower to invoke without state: %s", graph(), this); } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/UnaryMathIntrinsicNode.java 2019-03-12 08:10:19.939860641 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/UnaryMathIntrinsicNode.java 2019-03-12 08:10:19.571858258 +0100 @@ -27,6 +27,7 @@ import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_64; import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_1; +import jdk.vm.ci.meta.Value; import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor; import org.graalvm.compiler.core.common.type.FloatStamp; import org.graalvm.compiler.core.common.type.PrimitiveStamp; @@ -44,10 +45,9 @@ import org.graalvm.compiler.nodes.spi.ArithmeticLIRLowerable; import org.graalvm.compiler.nodes.spi.Lowerable; import org.graalvm.compiler.nodes.spi.LoweringTool; -import org.graalvm.compiler.nodes.spi.NodeLIRBuilderTool; import jdk.vm.ci.meta.JavaKind; -import jdk.vm.ci.meta.Value; +import org.graalvm.compiler.nodes.spi.NodeLIRBuilderTool; @NodeInfo(nameTemplate = "MathIntrinsic#{p#operation/s}", cycles = CYCLES_64, size = SIZE_1) public final class UnaryMathIntrinsicNode extends UnaryNode implements ArithmeticLIRLowerable, Lowerable { @@ -87,6 +87,43 @@ throw new GraalError("unknown op %s", this); } } + + public Stamp computeStamp(Stamp valueStamp) { + if (valueStamp instanceof FloatStamp) { + FloatStamp floatStamp = (FloatStamp) valueStamp; + switch (this) { + case COS: + case SIN: { + boolean nonNaN = floatStamp.lowerBound() != Double.NEGATIVE_INFINITY && floatStamp.upperBound() != Double.POSITIVE_INFINITY && floatStamp.isNonNaN(); + return StampFactory.forFloat(JavaKind.Double, -1.0, 1.0, nonNaN); + } + case TAN: { + boolean nonNaN = floatStamp.lowerBound() != Double.NEGATIVE_INFINITY && floatStamp.upperBound() != Double.POSITIVE_INFINITY && floatStamp.isNonNaN(); + return StampFactory.forFloat(JavaKind.Double, Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY, nonNaN); + } + case LOG: + case LOG10: { + double lowerBound = compute(floatStamp.lowerBound()); + double upperBound = compute(floatStamp.upperBound()); + if (floatStamp.contains(0.0)) { + // 0.0 and -0.0 infinity produces -Inf + lowerBound = Double.NEGATIVE_INFINITY; + } + boolean nonNaN = floatStamp.lowerBound() >= 0.0 && floatStamp.isNonNaN(); + return StampFactory.forFloat(JavaKind.Double, lowerBound, upperBound, nonNaN); + } + case EXP: { + double lowerBound = Math.exp(floatStamp.lowerBound()); + double upperBound = Math.exp(floatStamp.upperBound()); + boolean nonNaN = floatStamp.isNonNaN(); + return StampFactory.forFloat(JavaKind.Double, lowerBound, upperBound, nonNaN); + } + + } + } + return StampFactory.forKind(JavaKind.Double); + } + } public UnaryOperation getOperation() { @@ -109,50 +146,14 @@ } protected UnaryMathIntrinsicNode(ValueNode value, UnaryOperation op) { - super(TYPE, computeStamp(value.stamp(NodeView.DEFAULT), op), value); + super(TYPE, op.computeStamp(value.stamp(NodeView.DEFAULT)), value); assert value.stamp(NodeView.DEFAULT) instanceof FloatStamp && PrimitiveStamp.getBits(value.stamp(NodeView.DEFAULT)) == 64; this.operation = op; } @Override public Stamp foldStamp(Stamp valueStamp) { - return computeStamp(valueStamp, getOperation()); - } - - static Stamp computeStamp(Stamp valueStamp, UnaryOperation op) { - if (valueStamp instanceof FloatStamp) { - FloatStamp floatStamp = (FloatStamp) valueStamp; - switch (op) { - case COS: - case SIN: { - boolean nonNaN = floatStamp.lowerBound() != Double.NEGATIVE_INFINITY && floatStamp.upperBound() != Double.POSITIVE_INFINITY && floatStamp.isNonNaN(); - return StampFactory.forFloat(JavaKind.Double, -1.0, 1.0, nonNaN); - } - case TAN: { - boolean nonNaN = floatStamp.lowerBound() != Double.NEGATIVE_INFINITY && floatStamp.upperBound() != Double.POSITIVE_INFINITY && floatStamp.isNonNaN(); - return StampFactory.forFloat(JavaKind.Double, Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY, nonNaN); - } - case LOG: - case LOG10: { - double lowerBound = op.compute(floatStamp.lowerBound()); - double upperBound = op.compute(floatStamp.upperBound()); - if (floatStamp.contains(0.0)) { - // 0.0 and -0.0 infinity produces -Inf - lowerBound = Double.NEGATIVE_INFINITY; - } - boolean nonNaN = floatStamp.lowerBound() >= 0.0 && floatStamp.isNonNaN(); - return StampFactory.forFloat(JavaKind.Double, lowerBound, upperBound, nonNaN); - } - case EXP: { - double lowerBound = Math.exp(floatStamp.lowerBound()); - double upperBound = Math.exp(floatStamp.upperBound()); - boolean nonNaN = floatStamp.isNonNaN(); - return StampFactory.forFloat(JavaKind.Double, lowerBound, upperBound, nonNaN); - } - - } - } - return StampFactory.forKind(JavaKind.Double); + return getOperation().computeStamp(valueStamp); } @Override @@ -162,6 +163,7 @@ @Override public void generate(NodeLIRBuilderTool nodeValueMap, ArithmeticLIRGeneratorTool gen) { + // We can only reach here in the math stubs Value input = nodeValueMap.operand(getValue()); Value result; switch (getOperation()) { --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/arithmetic/IntegerAddExactNode.java 2019-03-12 08:10:20.407863670 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/arithmetic/IntegerAddExactNode.java 2019-03-12 08:10:20.047861340 +0100 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2013, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -37,35 +37,32 @@ import org.graalvm.compiler.graph.spi.CanonicalizerTool; import org.graalvm.compiler.nodeinfo.InputType; import org.graalvm.compiler.nodeinfo.NodeInfo; -import org.graalvm.compiler.nodes.AbstractBeginNode; import org.graalvm.compiler.nodes.ConstantNode; import org.graalvm.compiler.nodes.NodeView; import org.graalvm.compiler.nodes.ValueNode; import org.graalvm.compiler.nodes.calc.AddNode; -import org.graalvm.compiler.nodes.extended.AnchoringNode; -import org.graalvm.compiler.nodes.spi.LoweringTool; import jdk.vm.ci.code.CodeUtil; import jdk.vm.ci.meta.JavaConstant; import jdk.vm.ci.meta.JavaKind; -import jdk.vm.ci.meta.SpeculationLog.SpeculationReason; +import org.graalvm.compiler.nodes.extended.GuardedNode; +import org.graalvm.compiler.nodes.extended.GuardingNode; /** * Node representing an exact integer addition that will throw an {@link ArithmeticException} in * case the addition would overflow the 32 bit range. */ @NodeInfo(cycles = CYCLES_2, size = SIZE_2) -public final class IntegerAddExactNode extends AddNode implements IntegerExactArithmeticNode { +public final class IntegerAddExactNode extends AddNode implements GuardedNode, IntegerExactArithmeticNode { public static final NodeClass TYPE = NodeClass.create(IntegerAddExactNode.class); - @OptionalInput(InputType.Anchor) protected AnchoringNode anchor; - protected final SpeculationReason speculation; + @Input(InputType.Guard) protected GuardingNode guard; - public IntegerAddExactNode(ValueNode x, ValueNode y, SpeculationReason speculation) { + public IntegerAddExactNode(ValueNode x, ValueNode y, GuardingNode guard) { super(TYPE, x, y); setStamp(x.stamp(NodeView.DEFAULT).unrestricted()); assert x.stamp(NodeView.DEFAULT).isCompatible(y.stamp(NodeView.DEFAULT)) && x.stamp(NodeView.DEFAULT) instanceof IntegerStamp; - this.speculation = speculation; + this.guard = guard; } @Override @@ -129,13 +126,10 @@ @Override public ValueNode canonical(CanonicalizerTool tool, ValueNode forX, ValueNode forY) { if (forX.isConstant() && !forY.isConstant()) { - return new IntegerAddExactNode(forY, forX, speculation).canonical(tool); + return new IntegerAddExactNode(forY, forX, guard).canonical(tool); } - if (forX.isConstant()) { - ConstantNode constantNode = canonicalXconstant(forX, forY); - if (constantNode != null) { - return constantNode; - } + if (forX.isConstant() && forY.isConstant()) { + return canonicalXYconstant(forX, forY); } else if (forY.isConstant()) { long c = forY.asJavaConstant().asLong(); if (c == 0) { @@ -148,48 +142,31 @@ return this; } - private static ConstantNode canonicalXconstant(ValueNode forX, ValueNode forY) { + private ValueNode canonicalXYconstant(ValueNode forX, ValueNode forY) { JavaConstant xConst = forX.asJavaConstant(); JavaConstant yConst = forY.asJavaConstant(); - if (xConst != null && yConst != null) { - assert xConst.getJavaKind() == yConst.getJavaKind(); - try { - if (xConst.getJavaKind() == JavaKind.Int) { - return ConstantNode.forInt(Math.addExact(xConst.asInt(), yConst.asInt())); - } else { - assert xConst.getJavaKind() == JavaKind.Long; - return ConstantNode.forLong(Math.addExact(xConst.asLong(), yConst.asLong())); - } - } catch (ArithmeticException ex) { - // The operation will result in an overflow exception, so do not canonicalize. + assert xConst.getJavaKind() == yConst.getJavaKind(); + try { + if (xConst.getJavaKind() == JavaKind.Int) { + return ConstantNode.forInt(Math.addExact(xConst.asInt(), yConst.asInt())); + } else { + assert xConst.getJavaKind() == JavaKind.Long; + return ConstantNode.forLong(Math.addExact(xConst.asLong(), yConst.asLong())); } + } catch (ArithmeticException ex) { + // The operation will result in an overflow exception, so do not canonicalize. } - return null; - } - - @Override - public IntegerExactArithmeticSplitNode createSplit(AbstractBeginNode next, AbstractBeginNode deopt) { - return graph().add(new IntegerAddExactSplitNode(stamp(NodeView.DEFAULT), getX(), getY(), next, deopt)); - } - - @Override - public SpeculationReason getSpeculation() { - return speculation; - } - - @Override - public AnchoringNode getAnchor() { - return anchor; + return this; } @Override - public void setAnchor(AnchoringNode x) { - updateUsagesInterface(this.anchor, x); - this.anchor = x; + public GuardingNode getGuard() { + return guard; } @Override - public void lower(LoweringTool tool) { - IntegerExactArithmeticSplitNode.lower(tool, this); + public void setGuard(GuardingNode guard) { + updateUsagesInterface(this.guard, guard); + this.guard = guard; } } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/arithmetic/IntegerExactArithmeticNode.java 2019-03-12 08:10:20.895866830 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/arithmetic/IntegerExactArithmeticNode.java 2019-03-12 08:10:20.523864421 +0100 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2013, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -24,20 +24,5 @@ package org.graalvm.compiler.replacements.nodes.arithmetic; -import org.graalvm.compiler.nodes.AbstractBeginNode; -import org.graalvm.compiler.nodes.extended.AnchoringNode; -import org.graalvm.compiler.nodes.spi.Lowerable; - -import jdk.vm.ci.meta.SpeculationLog.SpeculationReason; - -public interface IntegerExactArithmeticNode extends Lowerable { - - IntegerExactArithmeticSplitNode createSplit(AbstractBeginNode next, AbstractBeginNode deopt); - - SpeculationReason getSpeculation(); - - AnchoringNode getAnchor(); - - void setAnchor(AnchoringNode x); - +public interface IntegerExactArithmeticNode { } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/arithmetic/IntegerExactArithmeticSplitNode.java 2019-03-12 08:10:21.379869962 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/arithmetic/IntegerExactArithmeticSplitNode.java 2019-03-12 08:10:21.011867580 +0100 @@ -32,21 +32,11 @@ import org.graalvm.compiler.graph.spi.Simplifiable; import org.graalvm.compiler.nodeinfo.NodeInfo; import org.graalvm.compiler.nodes.AbstractBeginNode; -import org.graalvm.compiler.nodes.BeginNode; import org.graalvm.compiler.nodes.ControlSplitNode; -import org.graalvm.compiler.nodes.DeoptimizeNode; -import org.graalvm.compiler.nodes.FixedNode; -import org.graalvm.compiler.nodes.FixedWithNextNode; -import org.graalvm.compiler.nodes.StructuredGraph; import org.graalvm.compiler.nodes.ValueNode; -import org.graalvm.compiler.nodes.calc.FloatingNode; import org.graalvm.compiler.nodes.spi.LIRLowerable; -import org.graalvm.compiler.nodes.spi.LoweringTool; import org.graalvm.compiler.nodes.spi.NodeLIRBuilderTool; -import jdk.vm.ci.meta.DeoptimizationAction; -import jdk.vm.ci.meta.DeoptimizationReason; -import jdk.vm.ci.meta.SpeculationLog; import jdk.vm.ci.meta.Value; @NodeInfo(cycles = CYCLES_2, cyclesRationale = "add+cmp", size = SIZE_2) @@ -117,23 +107,6 @@ protected abstract Value generateArithmetic(NodeLIRBuilderTool generator); - static void lower(LoweringTool tool, IntegerExactArithmeticNode node) { - if (node.asNode().graph().getGuardsStage() == StructuredGraph.GuardsStage.FIXED_DEOPTS) { - FloatingNode floatingNode = (FloatingNode) node; - FixedWithNextNode previous = tool.lastFixedNode(); - FixedNode next = previous.next(); - previous.setNext(null); - StructuredGraph graph = floatingNode.graph(); - DeoptimizeNode deopt = graph.add(new DeoptimizeNode(DeoptimizationAction.InvalidateReprofile, DeoptimizationReason.ArithmeticException, - node.getSpeculation() == null ? SpeculationLog.NO_SPECULATION : graph.getSpeculationLog().speculate(node.getSpeculation()))); - AbstractBeginNode normalBegin = graph.add(new BeginNode()); - normalBegin.setNext(next); - IntegerExactArithmeticSplitNode split = node.createSplit(normalBegin, BeginNode.begin(deopt)); - previous.setNext(split); - floatingNode.replaceAndDelete(split); - } - } - @Override public int getSuccessorCount() { return 2; --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/arithmetic/IntegerMulExactNode.java 2019-03-12 08:10:21.843872966 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/arithmetic/IntegerMulExactNode.java 2019-03-12 08:10:21.475870584 +0100 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2013, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -32,34 +32,31 @@ import org.graalvm.compiler.graph.spi.CanonicalizerTool; import org.graalvm.compiler.nodeinfo.InputType; import org.graalvm.compiler.nodeinfo.NodeInfo; -import org.graalvm.compiler.nodes.AbstractBeginNode; import org.graalvm.compiler.nodes.ConstantNode; import org.graalvm.compiler.nodes.NodeView; import org.graalvm.compiler.nodes.ValueNode; import org.graalvm.compiler.nodes.calc.MulNode; -import org.graalvm.compiler.nodes.extended.AnchoringNode; -import org.graalvm.compiler.nodes.spi.LoweringTool; +import org.graalvm.compiler.nodes.extended.GuardedNode; +import org.graalvm.compiler.nodes.extended.GuardingNode; import jdk.vm.ci.meta.JavaConstant; import jdk.vm.ci.meta.JavaKind; -import jdk.vm.ci.meta.SpeculationLog.SpeculationReason; /** * Node representing an exact integer multiplication that will throw an {@link ArithmeticException} * in case the addition would overflow the 32 bit range. */ @NodeInfo(cycles = CYCLES_4, cyclesRationale = "mul+cmp", size = SIZE_2) -public final class IntegerMulExactNode extends MulNode implements IntegerExactArithmeticNode { +public final class IntegerMulExactNode extends MulNode implements GuardedNode, IntegerExactArithmeticNode { public static final NodeClass TYPE = NodeClass.create(IntegerMulExactNode.class); - @OptionalInput(InputType.Anchor) protected AnchoringNode anchor; - protected final SpeculationReason speculation; + @Input(InputType.Guard) protected GuardingNode guard; - public IntegerMulExactNode(ValueNode x, ValueNode y, SpeculationReason speculation) { + public IntegerMulExactNode(ValueNode x, ValueNode y, GuardingNode guard) { super(TYPE, x, y); setStamp(x.stamp(NodeView.DEFAULT).unrestricted()); assert x.stamp(NodeView.DEFAULT).isCompatible(y.stamp(NodeView.DEFAULT)) && x.stamp(NodeView.DEFAULT) instanceof IntegerStamp; - this.speculation = speculation; + this.guard = guard; } @Override @@ -76,10 +73,10 @@ @Override public ValueNode canonical(CanonicalizerTool tool, ValueNode forX, ValueNode forY) { if (forX.isConstant() && !forY.isConstant()) { - return new IntegerMulExactNode(forY, forX, speculation).canonical(tool); + return new IntegerMulExactNode(forY, forX, guard).canonical(tool); } - if (forX.isConstant()) { - return canonicalXconstant(forX, forY); + if (forX.isConstant() && forY.isConstant()) { + return canonicalXYconstant(forX, forY); } else if (forY.isConstant()) { long c = forY.asJavaConstant().asLong(); if (c == 1) { @@ -95,7 +92,7 @@ return this; } - private ValueNode canonicalXconstant(ValueNode forX, ValueNode forY) { + private ValueNode canonicalXYconstant(ValueNode forX, ValueNode forY) { JavaConstant xConst = forX.asJavaConstant(); JavaConstant yConst = forY.asJavaConstant(); assert xConst.getJavaKind() == yConst.getJavaKind(); @@ -113,28 +110,13 @@ } @Override - public IntegerExactArithmeticSplitNode createSplit(AbstractBeginNode next, AbstractBeginNode deopt) { - return graph().add(new IntegerMulExactSplitNode(stamp(NodeView.DEFAULT), getX(), getY(), next, deopt)); + public GuardingNode getGuard() { + return guard; } @Override - public SpeculationReason getSpeculation() { - return speculation; - } - - @Override - public AnchoringNode getAnchor() { - return anchor; - } - - @Override - public void setAnchor(AnchoringNode x) { - updateUsagesInterface(this.anchor, x); - this.anchor = x; - } - - @Override - public void lower(LoweringTool tool) { - IntegerExactArithmeticSplitNode.lower(tool, this); + public void setGuard(GuardingNode guard) { + updateUsagesInterface(this.guard, guard); + this.guard = guard; } } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/arithmetic/IntegerSubExactNode.java 2019-03-12 08:10:22.315876021 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/arithmetic/IntegerSubExactNode.java 2019-03-12 08:10:21.951873665 +0100 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2013, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -32,35 +32,32 @@ import org.graalvm.compiler.graph.spi.CanonicalizerTool; import org.graalvm.compiler.nodeinfo.InputType; import org.graalvm.compiler.nodeinfo.NodeInfo; -import org.graalvm.compiler.nodes.AbstractBeginNode; import org.graalvm.compiler.nodes.ConstantNode; import org.graalvm.compiler.nodes.NodeView; import org.graalvm.compiler.nodes.ValueNode; import org.graalvm.compiler.nodes.calc.SubNode; -import org.graalvm.compiler.nodes.extended.AnchoringNode; -import org.graalvm.compiler.nodes.spi.LoweringTool; +import org.graalvm.compiler.nodes.extended.GuardedNode; +import org.graalvm.compiler.nodes.extended.GuardingNode; import org.graalvm.compiler.nodes.util.GraphUtil; import jdk.vm.ci.meta.JavaConstant; import jdk.vm.ci.meta.JavaKind; -import jdk.vm.ci.meta.SpeculationLog.SpeculationReason; /** * Node representing an exact integer substraction that will throw an {@link ArithmeticException} in * case the addition would overflow the 32 bit range. */ @NodeInfo(cycles = CYCLES_2, size = SIZE_2) -public final class IntegerSubExactNode extends SubNode implements IntegerExactArithmeticNode { +public final class IntegerSubExactNode extends SubNode implements GuardedNode, IntegerExactArithmeticNode { public static final NodeClass TYPE = NodeClass.create(IntegerSubExactNode.class); - @OptionalInput(InputType.Anchor) protected AnchoringNode anchor; - protected final SpeculationReason speculation; + @Input(InputType.Guard) protected GuardingNode guard; - public IntegerSubExactNode(ValueNode x, ValueNode y, SpeculationReason speculation) { + public IntegerSubExactNode(ValueNode x, ValueNode y, GuardingNode guard) { super(TYPE, x, y); setStamp(x.stamp(NodeView.DEFAULT).unrestricted()); assert x.stamp(NodeView.DEFAULT).isCompatible(y.stamp(NodeView.DEFAULT)) && x.stamp(NodeView.DEFAULT) instanceof IntegerStamp; - this.speculation = speculation; + this.guard = guard; } @Override @@ -111,28 +108,13 @@ } @Override - public IntegerExactArithmeticSplitNode createSplit(AbstractBeginNode next, AbstractBeginNode deopt) { - return graph().add(new IntegerSubExactSplitNode(stamp(NodeView.DEFAULT), getX(), getY(), next, deopt)); + public GuardingNode getGuard() { + return guard; } @Override - public SpeculationReason getSpeculation() { - return speculation; - } - - @Override - public AnchoringNode getAnchor() { - return anchor; - } - - @Override - public void setAnchor(AnchoringNode x) { - updateUsagesInterface(this.anchor, x); - this.anchor = x; - } - - @Override - public void lower(LoweringTool tool) { - IntegerExactArithmeticSplitNode.lower(tool, this); + public void setGuard(GuardingNode guard) { + updateUsagesInterface(this.guard, guard); + this.guard = guard; } } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.serviceprovider/src/org/graalvm/compiler/serviceprovider/GraalServices.java 2019-03-12 08:10:22.811879231 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.serviceprovider/src/org/graalvm/compiler/serviceprovider/GraalServices.java 2019-03-12 08:10:22.447876876 +0100 @@ -28,12 +28,14 @@ import java.io.IOException; import java.io.InputStream; +import java.util.Arrays; import java.util.Iterator; import java.util.List; import java.util.ServiceConfigurationError; import java.util.ServiceLoader; import java.util.concurrent.atomic.AtomicLong; +import jdk.vm.ci.meta.SpeculationLog.SpeculationReason; import jdk.vm.ci.runtime.JVMCI; import jdk.vm.ci.services.JVMCIPermission; import jdk.vm.ci.services.Services; @@ -43,30 +45,6 @@ */ public final class GraalServices { - private static int getJavaSpecificationVersion() { - String value = System.getProperty("java.specification.version"); - if (value.startsWith("1.")) { - value = value.substring(2); - } - return Integer.parseInt(value); - } - - /** - * The integer value corresponding to the value of the {@code java.specification.version} system - * property after any leading {@code "1."} has been stripped. - */ - public static final int JAVA_SPECIFICATION_VERSION = getJavaSpecificationVersion(); - - /** - * Determines if the Java runtime is version 8 or earlier. - */ - public static final boolean Java8OrEarlier = JAVA_SPECIFICATION_VERSION <= 8; - - /** - * Determines if the Java runtime is version 11 or earlier. - */ - public static final boolean Java11OrEarlier = JAVA_SPECIFICATION_VERSION <= 11; - private GraalServices() { } @@ -195,6 +173,44 @@ } /** + * An implementation of {@link SpeculationReason} based on direct, unencoded values. + */ + static final class DirectSpeculationReason implements SpeculationReason { + final int groupId; + final String groupName; + final Object[] context; + + DirectSpeculationReason(int groupId, String groupName, Object[] context) { + this.groupId = groupId; + this.groupName = groupName; + this.context = context; + } + + @Override + public boolean equals(Object obj) { + if (obj instanceof DirectSpeculationReason) { + DirectSpeculationReason that = (DirectSpeculationReason) obj; + return this.groupId == that.groupId && Arrays.equals(this.context, that.context); + } + return false; + } + + @Override + public int hashCode() { + return groupId + Arrays.hashCode(this.context); + } + + @Override + public String toString() { + return String.format("%s@%d%s", groupName, groupId, Arrays.toString(context)); + } + } + + static SpeculationReason createSpeculationReason(int groupId, String groupName, Object... context) { + return new DirectSpeculationReason(groupId, groupName, context); + } + + /** * Gets a unique identifier for this execution such as a process ID or a * {@linkplain #getGlobalTimeStamp() fixed timestamp}. */ --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.test/src/org/graalvm/compiler/test/GraalTest.java 2019-03-12 08:10:23.291882337 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.test/src/org/graalvm/compiler/test/GraalTest.java 2019-03-12 08:10:22.931880008 +0100 @@ -44,6 +44,7 @@ import org.graalvm.compiler.debug.GlobalMetrics; import org.graalvm.compiler.options.OptionValues; import org.graalvm.compiler.serviceprovider.GraalServices; +import org.graalvm.compiler.serviceprovider.JavaVersionUtil; import org.junit.After; import org.junit.Assert; import org.junit.AssumptionViolatedException; @@ -72,8 +73,8 @@ } } - public static final boolean Java8OrEarlier = GraalServices.Java8OrEarlier; - public static final boolean Java11OrEarlier = GraalServices.Java11OrEarlier; + public static final boolean Java8OrEarlier = JavaVersionUtil.Java8OrEarlier; + public static final boolean Java11OrEarlier = JavaVersionUtil.Java11OrEarlier; protected Method getMethod(String methodName) { return getMethod(getClass(), methodName); --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.test/src/org/graalvm/compiler/test/SubprocessUtil.java 2019-03-12 08:10:23.771885444 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.test/src/org/graalvm/compiler/test/SubprocessUtil.java 2019-03-12 08:10:23.407883089 +0100 @@ -37,7 +37,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.graalvm.compiler.serviceprovider.GraalServices; +import org.graalvm.compiler.serviceprovider.JavaVersionUtil; import org.graalvm.util.CollectionsUtil; import org.junit.Assume; @@ -249,7 +249,7 @@ return new Subprocess(command, process.waitFor(), output); } - private static final boolean isJava8OrEarlier = GraalServices.Java8OrEarlier; + private static final boolean isJava8OrEarlier = JavaVersionUtil.Java8OrEarlier; private static boolean hasArg(String optionName) { if (optionName.equals("-cp") || optionName.equals("-classpath")) { --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.virtual/src/org/graalvm/compiler/virtual/phases/ea/EffectsClosure.java 2019-03-12 08:10:24.267888656 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.virtual/src/org/graalvm/compiler/virtual/phases/ea/EffectsClosure.java 2019-03-12 08:10:23.911886351 +0100 @@ -295,7 +295,7 @@ protected final List processLoop(Loop loop, BlockT initialState) { if (initialState.isDead()) { ArrayList states = new ArrayList<>(); - for (int i = 0; i < loop.getExits().size(); i++) { + for (int i = 0; i < loop.getLoopExits().size(); i++) { states.add(initialState); } return states; @@ -347,7 +347,7 @@ blockEffects.get(loop.getHeader()).insertAll(mergeProcessor.mergeEffects, 0); loopMergeEffects.put(loop, mergeProcessor.afterMergeEffects); - assert info.exitStates.size() == loop.getExits().size(); + assert info.exitStates.size() == loop.getLoopExits().size(); loopEntryStates.put((LoopBeginNode) loop.getHeader().getBeginNode(), loopEntryState); assert assertExitStatesNonEmpty(loop, info); @@ -412,8 +412,8 @@ } private boolean assertExitStatesNonEmpty(Loop loop, LoopInfo info) { - for (int i = 0; i < loop.getExits().size(); i++) { - assert info.exitStates.get(i) != null : "no loop exit state at " + loop.getExits().get(i) + " / " + loop.getHeader(); + for (int i = 0; i < loop.getLoopExits().size(); i++) { + assert info.exitStates.get(i) != null : "no loop exit state at " + loop.getLoopExits().get(i) + " / " + loop.getHeader(); } return true; } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.graphio/src/org/graalvm/graphio/GraphOutput.java 2019-03-12 08:10:24.767891891 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.graphio/src/org/graalvm/graphio/GraphOutput.java 2019-03-12 08:10:24.407889561 +0100 @@ -28,6 +28,7 @@ import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; +import java.nio.ByteBuffer; import java.nio.channels.WritableByteChannel; import java.util.Collections; import java.util.Map; @@ -37,8 +38,9 @@ * * @param the type of graph this instance handles * @param the type of methods this instance handles + * @since 1.0 a {@link WritableByteChannel} is implemented */ -public final class GraphOutput implements Closeable { +public final class GraphOutput implements Closeable, WritableByteChannel { private final GraphProtocol printer; private GraphOutput(GraphProtocol p) { @@ -107,6 +109,30 @@ } /** + * Checks if the {@link GraphOutput} is open. + * + * @return true if the {@link GraphOutput} is open. + * @since 1.0 + */ + @Override + public boolean isOpen() { + return printer.isOpen(); + } + + /** + * Writes raw bytes into {@link GraphOutput}. + * + * @param src the bytes to write + * @return the number of bytes written, possibly zero + * @throws IOException in case of IO error + * @since 1.0 + */ + @Override + public int write(ByteBuffer src) throws IOException { + return printer.write(src); + } + + /** * Builder to configure and create an instance of {@link GraphOutput}. * * @param the type of the (root element of) graph @@ -121,6 +147,7 @@ private GraphBlocks blocks = DefaultGraphBlocks.empty(); private int major = 4; private int minor = 0; + private boolean embeddedGraphOutput; Builder(GraphStructure structure) { this.structure = structure; @@ -143,6 +170,22 @@ } /** + * Sets {@link GraphOutput} as embedded. The embedded {@link GraphOutput} shares + * {@link WritableByteChannel channel} with another already open non parent + * {@link GraphOutput}. The embedded {@link GraphOutput} flushes data after each + * {@link GraphOutput#print print}, {@link GraphOutput#beginGroup beginGroup} and + * {@link GraphOutput#endGroup endGroup} call. + * + * @param embedded if {@code true} the builder creates an embedded {@link GraphOutput} + * @return this builder + * @since 1.0 + */ + public Builder embedded(boolean embedded) { + this.embeddedGraphOutput = embedded; + return this; + } + + /** * Associates different implementation of types. * * @param graphTypes implementation of types and enum recognition @@ -226,7 +269,7 @@ private GraphOutput buildImpl(ElementsAndLocations e, WritableByteChannel channel) throws IOException { // @formatter:off ProtocolImpl p = new ProtocolImpl<>( - major, minor, structure, types, blocks, + major, minor, embeddedGraphOutput, structure, types, blocks, e == null ? null : e.elements, e == null ? null : e.locations, channel ); --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.graphio/src/org/graalvm/graphio/GraphProtocol.java 2019-03-12 08:10:25.259895076 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.graphio/src/org/graalvm/graphio/GraphProtocol.java 2019-03-12 08:10:24.899892746 +0100 @@ -78,10 +78,12 @@ private final ConstantPool constantPool; private final ByteBuffer buffer; private final WritableByteChannel channel; + private final boolean embedded; final int versionMajor; final int versionMinor; + private boolean printing; - GraphProtocol(WritableByteChannel channel, int major, int minor) throws IOException { + GraphProtocol(WritableByteChannel channel, int major, int minor, boolean embedded) throws IOException { if (major > 6 || (major == 6 && minor > 0)) { throw new IllegalArgumentException("Unrecognized version " + major + "." + minor); } @@ -90,7 +92,11 @@ this.constantPool = new ConstantPool(); this.buffer = ByteBuffer.allocateDirect(256 * 1024); this.channel = channel; - writeVersion(); + this.embedded = embedded; + if (!embedded) { + writeVersion(); + flushEmbedded(); + } } GraphProtocol(GraphProtocol parent) { @@ -99,36 +105,67 @@ this.constantPool = parent.constantPool; this.buffer = parent.buffer; this.channel = parent.channel; + this.embedded = parent.embedded; } @SuppressWarnings("all") public final void print(Graph graph, Map properties, int id, String format, Object... args) throws IOException { - writeByte(BEGIN_GRAPH); - if (versionMajor >= 3) { - writeInt(id); - writeString(format); - writeInt(args.length); - for (Object a : args) { - writePropertyObject(graph, a); + printing = true; + try { + writeByte(BEGIN_GRAPH); + if (versionMajor >= 3) { + writeInt(id); + writeString(format); + writeInt(args.length); + for (Object a : args) { + writePropertyObject(graph, a); + } + } else { + writePoolObject(formatTitle(graph, id, format, args)); } - } else { - writePoolObject(formatTitle(graph, id, format, args)); + writeGraph(graph, properties); + flushEmbedded(); + flush(); + } finally { + printing = false; } - writeGraph(graph, properties); - flush(); } public final void beginGroup(Graph noGraph, String name, String shortName, ResolvedJavaMethod method, int bci, Map properties) throws IOException { - writeByte(BEGIN_GROUP); - writePoolObject(name); - writePoolObject(shortName); - writePoolObject(method); - writeInt(bci); - writeProperties(noGraph, properties); + printing = true; + try { + writeByte(BEGIN_GROUP); + writePoolObject(name); + writePoolObject(shortName); + writePoolObject(method); + writeInt(bci); + writeProperties(noGraph, properties); + flushEmbedded(); + } finally { + printing = false; + } } public final void endGroup() throws IOException { - writeByte(CLOSE_GROUP); + printing = true; + try { + writeByte(CLOSE_GROUP); + flushEmbedded(); + } finally { + printing = false; + } + } + + final int write(ByteBuffer src) throws IOException { + if (printing) { + throw new IllegalStateException("Trying to write during graph print."); + } + constantPool.reset(); + return writeBytesRaw(src); + } + + final boolean isOpen() { + return channel.isOpen(); } @Override @@ -280,6 +317,13 @@ writeByte(versionMinor); } + private void flushEmbedded() throws IOException { + if (embedded) { + flush(); + constantPool.reset(); + } + } + private void flush() throws IOException { buffer.flip(); /* @@ -358,6 +402,23 @@ } } + private int writeBytesRaw(ByteBuffer b) throws IOException { + int limit = b.limit(); + int written = 0; + while (b.position() < limit) { + int toWrite = Math.min(limit - b.position(), buffer.capacity()); + ensureAvailable(toWrite); + b.limit(b.position() + toWrite); + try { + buffer.put(b); + written += toWrite; + } finally { + b.limit(limit); + } + } + return written; + } + private void writeInts(int[] b) throws IOException { if (b == null) { writeInt(-1); @@ -818,6 +879,12 @@ put(obj, id); return id; } + + void reset() { + clear(); + availableIds.clear(); + nextId = 0; + } } } --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.graphio/src/org/graalvm/graphio/ProtocolImpl.java 2019-03-12 08:10:25.735898156 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.graphio/src/org/graalvm/graphio/ProtocolImpl.java 2019-03-12 08:10:25.375895826 +0100 @@ -40,11 +40,11 @@ private final GraphElements elements; private final GraphLocations locations; - ProtocolImpl(int major, int minor, GraphStructure structure, GraphTypes enums, GraphBlocks blocks, + ProtocolImpl(int major, int minor, boolean embedded, GraphStructure structure, GraphTypes enums, GraphBlocks blocks, GraphElements elements, GraphLocations locs, WritableByteChannel channel) throws IOException { - super(channel, major, minor); + super(channel, major, minor, embedded); this.structure = structure; this.types = enums; this.blocks = blocks; --- old/test/hotspot/jtreg/ProblemList-graal.txt 2019-03-12 08:10:26.215901263 +0100 +++ new/test/hotspot/jtreg/ProblemList-graal.txt 2019-03-12 08:10:25.851898906 +0100 @@ -226,4 +226,3 @@ org.graalvm.compiler.hotspot.test.ReservedStackAccessTest 8213567 windows-all -org.graalvm.compiler.hotspot.test.CheckGraalIntrinsics 8218074 --- old/test/hotspot/jtreg/compiler/graalunit/TestPackages.txt 2019-03-12 08:10:26.687904316 +0100 +++ new/test/hotspot/jtreg/compiler/graalunit/TestPackages.txt 2019-03-12 08:10:26.327901987 +0100 @@ -11,6 +11,7 @@ Debug org.graalvm.compiler.debug.test Graph org.graalvm.compiler.graph.test @requires vm.graal.enabled HotspotAmd64 org.graalvm.compiler.hotspot.amd64.test +HotspotJdk9 org.graalvm.compiler.hotspot.jdk9.test HotspotSparc org.graalvm.compiler.hotspot.sparc.test @requires vm.simpleArch == "sparcv9" HotspotLir org.graalvm.compiler.hotspot.lir.test Hotspot org.graalvm.compiler.hotspot.test --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm/src/org/graalvm/compiler/asm/BranchTargetOutOfBoundsException.java 2019-03-12 08:10:26.803905068 +0100 @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.asm; + +import org.graalvm.compiler.core.common.GraalBailoutException; + +@SuppressWarnings("serial") +public class BranchTargetOutOfBoundsException extends GraalBailoutException { + + public BranchTargetOutOfBoundsException(boolean permanent, String format, Object... args) { + super(permanent, format, args); + } + + public BranchTargetOutOfBoundsException(String format, Object... args) { + super(format, args); + } + + public BranchTargetOutOfBoundsException(Throwable cause, String format, Object... args) { + super(cause, format, args); + } + +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.aarch64.test/src/org/graalvm/compiler/core/aarch64/test/AArch64CbzTest.java 2019-03-12 08:10:27.291908226 +0100 @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, Arm Limited. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + + +package org.graalvm.compiler.core.aarch64.test; + +import org.graalvm.compiler.lir.LIRInstruction; +import org.graalvm.compiler.lir.aarch64.AArch64ControlFlow; +import org.junit.Test; + +import java.util.function.Predicate; + +public class AArch64CbzTest extends AArch64MatchRuleTest { + private static final Predicate predicate = op -> (op instanceof AArch64ControlFlow.CompareBranchZeroOp); + + public static int equalsTo(int x) { + if (x == 0) { + return 1; + } else { + return x - 1; + } + } + + public static int notEqualsTo(int x) { + if (x != 0) { + return x + 2; + } else { + return 3; + } + } + + public static String isNull(String s) { + if (s == null) { + return "abc"; + } else { + return s + "abc"; + } + } + + public static String isNotNull(String s) { + if (s != null) { + return s + "abc"; + } else { + return "abc"; + } + } + + public static String objectEqualsNull(String s1, String s2) { + if (s1.equals(null)) { + return s1 + "abc"; + } else { + return s2 + "abd"; + } + } + + public static String objectEquals(String s1, String s2) { + if (s1.equals(s2)) { + return s1 + "abc"; + } else { + return s2 + "abd"; + } + } + + @Test + public void testEqualsTo() { + test("equalsTo", 0); + test("equalsTo", 1); + checkLIR("equalsTo", predicate, 1); + } + + @Test + public void testNotEqualsTo() { + test("notEqualsTo", 0); + test("notEqualsTo", 1); + checkLIR("notEqualsTo", predicate, 1); + } + + @Test + public void testIsNull() { + test("isNull", new Object[]{null}); + test("isNull", "abc"); + checkLIR("isNull", predicate, 1); + } + + @Test + public void testIsNotNull() { + test("isNotNull", new Object[]{null}); + test("isNotNull", "abc"); + checkLIR("isNotNull", predicate, 1); + } + + @Test + public void testObjectEqualsNull() { + test("objectEqualsNull", "ab", "ac"); + test("objectEqualsNull", "abc", "abc"); + checkLIR("objectEqualsNull", predicate, 1); + } + + @Test + public void testObjectEquals() { + test("objectEquals", "ab", "ac"); + test("objectEquals", "abc", "abc"); + checkLIR("objectEquals", predicate, 0); + } +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.aarch64.test/src/org/graalvm/compiler/core/aarch64/test/AArch64MembarOpTest.java 2019-03-12 08:10:27.779911383 +0100 @@ -0,0 +1,130 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2019, Arm Limited. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.core.aarch64.test; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assume.assumeTrue; + +import java.util.ArrayList; +import java.util.List; + +import jdk.internal.vm.compiler.collections.Pair; +import org.graalvm.compiler.asm.aarch64.AArch64Assembler.BarrierKind; +import org.graalvm.compiler.asm.aarch64.AArch64MacroAssembler; +import org.graalvm.compiler.code.CompilationResult; +import org.graalvm.compiler.core.gen.LIRGenerationProvider; +import org.graalvm.compiler.core.test.backend.BackendTest; +import org.graalvm.compiler.lir.aarch64.AArch64Move.MembarOp; +import org.graalvm.compiler.lir.asm.CompilationResultBuilder; +import org.graalvm.compiler.lir.asm.CompilationResultBuilderFactory; +import org.graalvm.compiler.lir.gen.LIRGenerationResult; +import org.graalvm.compiler.nodes.StructuredGraph; +import org.junit.Before; +import org.junit.Test; + +import jdk.vm.ci.aarch64.AArch64; +import jdk.vm.ci.code.MemoryBarriers; +import jdk.vm.ci.runtime.JVMCI; +import jdk.vm.ci.runtime.JVMCIBackend; + +public class AArch64MembarOpTest extends BackendTest { + + private final JVMCIBackend providers; + private final CompilationResultBuilder crb; + + public AArch64MembarOpTest() { + this.providers = JVMCI.getRuntime().getHostJVMCIBackend(); + + final StructuredGraph graph = parseEager("stub", StructuredGraph.AllowAssumptions.YES); + LIRGenerationResult lirGenRes = getLIRGenerationResult(graph); + CompilationResult compResult = new CompilationResult(graph.compilationId()); + this.crb = ((LIRGenerationProvider) getBackend()).newCompilationResultBuilder(lirGenRes, lirGenRes.getFrameMap(), compResult, CompilationResultBuilderFactory.Default); + } + + public void stub() { + } + + @Before + public void checkAArch64() { + assumeTrue("skipping AArch64 specific test", JVMCI.getRuntime().getHostJVMCIBackend().getTarget().arch instanceof AArch64); + } + + @Test + public void runNormalMembarTests() { + List> cases = new ArrayList<>(); + cases.add(Pair.create(MemoryBarriers.LOAD_LOAD, BarrierKind.LOAD_LOAD)); + cases.add(Pair.create(MemoryBarriers.LOAD_STORE, BarrierKind.LOAD_LOAD)); + cases.add(Pair.create(MemoryBarriers.LOAD_LOAD | MemoryBarriers.LOAD_STORE, BarrierKind.LOAD_LOAD)); + cases.add(Pair.create(MemoryBarriers.STORE_LOAD, BarrierKind.ANY_ANY)); + cases.add(Pair.create(MemoryBarriers.STORE_LOAD | MemoryBarriers.LOAD_LOAD, BarrierKind.ANY_ANY)); + cases.add(Pair.create(MemoryBarriers.STORE_LOAD | MemoryBarriers.LOAD_STORE, BarrierKind.ANY_ANY)); + cases.add(Pair.create(MemoryBarriers.STORE_LOAD | MemoryBarriers.LOAD_LOAD | MemoryBarriers.LOAD_STORE, BarrierKind.ANY_ANY)); + cases.add(Pair.create(MemoryBarriers.STORE_STORE, BarrierKind.STORE_STORE)); + cases.add(Pair.create(MemoryBarriers.STORE_STORE | MemoryBarriers.LOAD_LOAD, BarrierKind.ANY_ANY)); + cases.add(Pair.create(MemoryBarriers.STORE_STORE | MemoryBarriers.LOAD_STORE, BarrierKind.ANY_ANY)); + cases.add(Pair.create(MemoryBarriers.STORE_STORE | MemoryBarriers.LOAD_LOAD | MemoryBarriers.LOAD_STORE, BarrierKind.ANY_ANY)); + cases.add(Pair.create(MemoryBarriers.STORE_STORE | MemoryBarriers.STORE_LOAD, BarrierKind.ANY_ANY)); + cases.add(Pair.create(MemoryBarriers.STORE_STORE | MemoryBarriers.STORE_LOAD | MemoryBarriers.LOAD_LOAD, BarrierKind.ANY_ANY)); + cases.add(Pair.create(MemoryBarriers.STORE_STORE | MemoryBarriers.STORE_LOAD | MemoryBarriers.LOAD_STORE, BarrierKind.ANY_ANY)); + cases.add(Pair.create(MemoryBarriers.STORE_STORE | MemoryBarriers.STORE_LOAD | MemoryBarriers.LOAD_STORE | MemoryBarriers.LOAD_LOAD, BarrierKind.ANY_ANY)); + + for (Pair c : cases) { + assertArrayEquals(new MembarOpActual(c.getLeft()).emit(new AArch64MacroAssembler(providers.getTarget())), + new MembarOpExpected(c.getRight()).emit(new AArch64MacroAssembler(providers.getTarget()))); + } + } + + @Test(expected = AssertionError.class) + public void runExceptionalTests() { + new MembarOpActual(16).emit(new AArch64MacroAssembler(providers.getTarget())); + } + + private class MembarOpActual { + private MembarOp op; + + MembarOpActual(int barriers) { + op = new MembarOp(barriers); + } + + byte[] emit(AArch64MacroAssembler masm) { + op.emitCode(crb, masm); + return masm.close(false); + } + } + + private class MembarOpExpected { + private BarrierKind barrierKind; + + MembarOpExpected(BarrierKind barrierKind) { + this.barrierKind = barrierKind; + } + + byte[] emit(AArch64MacroAssembler masm) { + masm.dmb(barrierKind); + return masm.close(false); + } + } +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.aarch64.test/src/org/graalvm/compiler/core/aarch64/test/AArch64TestBitAndBranchTest.java 2019-03-12 08:10:28.251914438 +0100 @@ -0,0 +1,190 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.core.aarch64.test; + +import static org.junit.Assume.assumeTrue; + +import java.util.function.Predicate; + +import org.graalvm.compiler.api.directives.GraalDirectives; +import org.graalvm.compiler.asm.aarch64.AArch64MacroAssembler; +import org.graalvm.compiler.lir.LIR; +import org.graalvm.compiler.lir.LIRInstruction; +import org.graalvm.compiler.lir.LIRInstructionClass; +import org.graalvm.compiler.lir.aarch64.AArch64ControlFlow; +import org.graalvm.compiler.lir.aarch64.AArch64LIRInstruction; +import org.graalvm.compiler.lir.asm.CompilationResultBuilder; +import org.graalvm.compiler.lir.gen.LIRGenerationResult; +import org.graalvm.compiler.lir.gen.LIRGeneratorTool; +import org.graalvm.compiler.lir.jtt.LIRTest; +import org.graalvm.compiler.lir.jtt.LIRTestSpecification; +import org.graalvm.compiler.lir.phases.LIRPhase; +import org.graalvm.compiler.lir.phases.LIRSuites; +import org.graalvm.compiler.lir.phases.PreAllocationOptimizationPhase.PreAllocationOptimizationContext; +import org.graalvm.compiler.options.OptionValues; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import jdk.vm.ci.aarch64.AArch64; +import jdk.vm.ci.code.TargetDescription; +import jdk.vm.ci.meta.Value; + +public class AArch64TestBitAndBranchTest extends LIRTest { + private static final Predicate checkForBitTestAndBranchOp = op -> (op instanceof AArch64ControlFlow.BitTestAndBranchOp); + private LIR lir; + + @Before + public void checkAArch64() { + assumeTrue("skipping AArch64 specific test", getTarget().arch instanceof AArch64); + } + + public static long testBit42Snippet(long a, long b, long c) { + if ((a & (1 << 42)) == 0) { + return b; + } else { + return c; + } + } + + @Test + public void testBit42() { + test("testBit42Snippet", 1L << 42L, Long.MAX_VALUE, Long.MIN_VALUE); + test("testBit42Snippet", ~(1L << 42L), Long.MAX_VALUE, Long.MIN_VALUE); + checkLIR("testBit42Snippet", checkForBitTestAndBranchOp, 1); + } + + private static final LargeOpSpec largeOpSingleNop = new LargeOpSpec((1 << 14 - 2), 2); + + /** + * Tests the graceful case, where the estimation for + * {@link CompilationResultBuilder#labelWithinRange(LIRInstruction, org.graalvm.compiler.asm.Label, int)} + * holds. + */ + public static int testBitTestAndBranchSingleSnippet(int a) { + int res; + if (a % 2 == 0) { + res = fillOps(largeOpSingleNop, 1); + } else { + res = fillOps(largeOpSingleNop, 2); + } + return GraalDirectives.opaque(res); + } + + @Test + public void testBitTestAndBranchSingle() { + runTest("testBitTestAndBranchSingleSnippet", 1); + checkLIR("testBitTestAndBranchSingleSnippet", checkForBitTestAndBranchOp, 1); + } + + private static final LargeOpSpec largeOpFourNop = new LargeOpSpec((1 << 14 - 2), 8); + + /** + * Tests the case, where the estimation for + * {@link CompilationResultBuilder#labelWithinRange(LIRInstruction, org.graalvm.compiler.asm.Label, int)} + * does not hold and the code generation must be redone with large branches. + */ + public static int testBitTestAndBranchFourSnippet(int a) { + int res; + if (a % 2 == 0) { + res = fillOps(largeOpFourNop, 1); + } else { + res = fillOps(largeOpFourNop, 2); + } + return GraalDirectives.opaque(res); + } + + @Test + public void testBitTestAndBranchFour() { + runTest("testBitTestAndBranchFourSnippet", 1); + checkLIR("testBitTestAndBranchFourSnippet", checkForBitTestAndBranchOp, 1); + } + + private static class LargeOpSpec extends LIRTestSpecification { + private final int n; + private final int nopCount; + + LargeOpSpec(int n, int nopCount) { + super(); + this.n = n; + this.nopCount = nopCount; + } + + @Override + public void generate(LIRGeneratorTool gen, Value a) { + for (int i = 0; i < n; i++) { + gen.append(new NoOp(nopCount)); + } + setResult(a); + } + } + + public static class NoOp extends AArch64LIRInstruction { + private static final LIRInstructionClass TYPE = LIRInstructionClass.create(NoOp.class); + private final int nopCount; + + public NoOp(int nopCount) { + super(TYPE); + this.nopCount = nopCount; + } + + @Override + protected void emitCode(CompilationResultBuilder crb, AArch64MacroAssembler masm) { + for (int i = 0; i < nopCount; i++) { + masm.nop(); + } + } + } + + @LIRIntrinsic + public static int fillOps(@SuppressWarnings("unused") LargeOpSpec s, int a) { + return a; + } + + @Override + protected LIRSuites createLIRSuites(OptionValues options) { + LIRSuites suites = super.createLIRSuites(options); + suites.getPreAllocationOptimizationStage().appendPhase(new CheckPhase()); + return suites; + } + + public class CheckPhase extends LIRPhase { + @Override + protected void run(TargetDescription target, LIRGenerationResult lirGenRes, PreAllocationOptimizationContext context) { + lir = lirGenRes.getLIR(); + } + } + + protected void checkLIR(String methodName, Predicate predicate, int expected) { + compile(getResolvedJavaMethod(methodName), null); + int actualOpNum = 0; + for (LIRInstruction ins : lir.getLIRforBlock(lir.codeEmittingOrder()[0])) { + if (predicate.test(ins)) { + actualOpNum++; + } + } + Assert.assertEquals(expected, actualOpNum); + } +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/doc-files/MitigateSpeculativeExecutionAttacksHelp.txt 2019-03-12 08:10:28.743917621 +0100 @@ -0,0 +1,9 @@ +Select a strategy to mitigate speculative execution attacks (e.g., SPECTRE). +The accepted values are: + None - No mitigations are used in JIT compiled code. + AllTargets - All branches are protected against speculative attacks. + This has a significant performance impact. + GuardTargets - Only branches that preserve Java memory safety are protected. + This has less performance impact than AllTargets. + NonDeoptGuardTargets - GuardTargets except that branches which deoptimize are not + protected since they can not be executed repeatedly. --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/type/SymbolicJVMCIReference.java 2019-03-12 08:10:29.231920780 +0100 @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.core.common.type; + +import jdk.vm.ci.meta.ResolvedJavaType; + +/** + * This interface represents an object which contains a symbolic reference to a JVMCI type or method + * that can be converted back into the original object by looking up types relative to an + * {@code accessingClass}. + */ +public interface SymbolicJVMCIReference { + T resolve(ResolvedJavaType accessingClass); +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/CountedLoopTest2.java 2019-03-12 08:10:29.727923989 +0100 @@ -0,0 +1,130 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.core.test; + +import org.graalvm.compiler.api.directives.GraalDirectives; +import org.graalvm.compiler.graph.Node; +import org.graalvm.compiler.graph.iterators.FilteredNodeIterable; +import org.graalvm.compiler.loop.LoopEx; +import org.graalvm.compiler.loop.LoopsData; +import org.graalvm.compiler.nodes.DeoptimizingNode; +import org.graalvm.compiler.nodes.StructuredGraph; +import org.graalvm.compiler.nodes.StructuredGraph.AllowAssumptions; +import org.junit.Assert; +import org.junit.Test; + +public class CountedLoopTest2 extends GraalCompilerTest { + public static float countedDeoptLoop0(int n) { + float v = 0; + for (int i = 0; i < n; i++) { + v += 2.1f * i; + GraalDirectives.controlFlowAnchor(); + } + GraalDirectives.deoptimizeAndInvalidate(); + return v; + } + + @Test + public void test0() { + test("countedDeoptLoop0"); + } + + public static float countedDeoptLoop1(int n) { + float v = 0; + for (int i = 0; i < n; i++) { + v += 2.1f * i; + GraalDirectives.controlFlowAnchor(); + } + if (v > 0) { + if (v / 55 < 3) { + v -= 2; + GraalDirectives.controlFlowAnchor(); + } else { + v += 6; + GraalDirectives.controlFlowAnchor(); + } + } else { + v += 1; + GraalDirectives.controlFlowAnchor(); + } + GraalDirectives.deoptimizeAndInvalidate(); + return v; + } + + @Test + public void test1() { + test("countedDeoptLoop1"); + } + + public static float countedDeoptLoop2(int n, float init) { + float v = init; + if (v > 0) { + if (v / 55 < 3) { + for (int i = 0; i < n; i++) { + v += 2.1f * i; + GraalDirectives.controlFlowAnchor(); + } + } else { + for (int i = 0; i < n; i++) { + v += 1.1f * i; + GraalDirectives.controlFlowAnchor(); + } + } + } else { + for (int i = 0; i < n; i++) { + v += -0.1f * i; + GraalDirectives.controlFlowAnchor(); + } + } + GraalDirectives.deoptimizeAndInvalidate(); + return v; + } + + @Test + public void test2() { + test("countedDeoptLoop2", 3); + } + + private void test(String methodName) { + test(methodName, 1); + } + + private void test(String methodName, int nLoops) { + StructuredGraph graph = parseEager(methodName, AllowAssumptions.YES); + LoopsData loops = new LoopsData(graph); + Assert.assertEquals(nLoops, loops.loops().size()); + for (LoopEx loop : loops.loops()) { + Assert.assertTrue(loop.detectCounted()); + } + + StructuredGraph finalGraph = getFinalGraph(methodName); + loops = new LoopsData(finalGraph); + Assert.assertEquals(nLoops, loops.loops().size()); + FilteredNodeIterable nonStartDeopts = finalGraph.getNodes().filter(n -> { + return n instanceof DeoptimizingNode.DeoptBefore && ((DeoptimizingNode.DeoptBefore) n).stateBefore().bci > 0; + }); + Assert.assertTrue(nonStartDeopts.isNotEmpty()); + } +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/IntegerDivRemConstantTest.java 2019-03-12 08:10:30.207927095 +0100 @@ -0,0 +1,293 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2019, Arm Limited and affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + + +package org.graalvm.compiler.core.test; + +import org.junit.Test; + +public class IntegerDivRemConstantTest extends GraalCompilerTest { + public static int intDivPositiveConstant(int val) { + return val / 5; + } + + @Test + public void testIntDivPositiveConstant() { + test("intDivPositiveConstant", -10); + test("intDivPositiveConstant", 0); + test("intDivPositiveConstant", 4256); + test("intDivPositiveConstant", Integer.MAX_VALUE); + test("intDivPositiveConstant", Integer.MIN_VALUE); + } + + public static int intDivIntegerMax(int val) { + return val / Integer.MAX_VALUE; + } + + @Test + public void testIntDivIntegerMax() { + test("intDivIntegerMax", -10); + test("intDivIntegerMax", 0); + test("intDivIntegerMax", 4256); + test("intDivIntegerMax", Integer.MAX_VALUE); + test("intDivIntegerMax", Integer.MIN_VALUE); + } + + public static int intDivNegativeConstant(int val) { + return val / -1234; + } + + @Test + public void testIntDivNegativeConstant() { + test("intDivNegativeConstant", -123); + test("intDivNegativeConstant", 0); + test("intDivNegativeConstant", 123); + test("intDivNegativeConstant", Integer.MAX_VALUE); + test("intDivNegativeConstant", Integer.MIN_VALUE); + } + + public static int intDivIntegerMinOdd(int val) { + return val / (Integer.MIN_VALUE + 1); + } + + @Test + public void testIntDivIntegerMinOdd() { + test("intDivIntegerMinOdd", -123); + test("intDivIntegerMinOdd", 0); + test("intDivIntegerMinOdd", 123); + test("intDivIntegerMinOdd", Integer.MAX_VALUE); + test("intDivIntegerMinOdd", Integer.MIN_VALUE); + } + + public static long longDivPositiveConstant(long val) { + return val / 35170432; + } + + @Test + public void testLongDivPositiveConstant() { + test("longDivPositiveConstant", -1234L); + test("longDivPositiveConstant", 0L); + test("longDivPositiveConstant", 214423L); + test("longDivPositiveConstant", Long.MAX_VALUE); + test("longDivPositiveConstant", Long.MIN_VALUE); + } + + public static long longDivLongMax(long val) { + return val / Long.MAX_VALUE; + } + + @Test + public void testLongDivLongMax() { + test("longDivLongMax", -1234L); + test("longDivLongMax", 0L); + test("longDivLongMax", 214423L); + test("longDivLongMax", Long.MAX_VALUE); + test("longDivLongMax", Long.MIN_VALUE); + } + + public static long longDivNegativeConstant(long val) { + return val / -413; + } + + @Test + public void testLongDivNegativeConstant() { + test("longDivNegativeConstant", -43L); + test("longDivNegativeConstant", 0L); + test("longDivNegativeConstant", 147065L); + test("longDivNegativeConstant", Long.MAX_VALUE); + test("longDivNegativeConstant", Long.MIN_VALUE); + } + + public static long longDivLongMinOdd(long val) { + return val / (Long.MIN_VALUE + 1); + } + + @Test + public void testLongDivLongMinOdd() { + test("longDivLongMinOdd", -1234L); + test("longDivLongMinOdd", 0L); + test("longDivLongMinOdd", 214423L); + test("longDivLongMinOdd", Long.MAX_VALUE); + test("longDivLongMinOdd", Long.MIN_VALUE); + } + + public static int intRemPositiveConstant(int val) { + return val % 139968; + } + + @Test + public void testIntRemPositiveConstant() { + test("intRemPositiveConstant", -10); + test("intRemPositiveConstant", 0); + test("intRemPositiveConstant", 4256); + test("intRemPositiveConstant", Integer.MAX_VALUE); + test("intRemPositiveConstant", Integer.MIN_VALUE); + } + + public static int intRemNegativeConstant(int val) { + return val % -139968; + } + + @Test + public void testIntRemNegativeConstant() { + test("intRemNegativeConstant", -10); + test("intRemNegativeConstant", 0); + test("intRemNegativeConstant", 4256); + test("intRemNegativeConstant", Integer.MAX_VALUE); + test("intRemNegativeConstant", Integer.MIN_VALUE); + } + + @SuppressWarnings("divzero") + public static int intRemZero(int val) { + return val % 0; + } + + @Test + public void testIntRemZero() { + test("intRemZero", -10); + test("intRemZero", 0); + test("intRemZero", 4256); + test("intRemZero", Integer.MAX_VALUE); + test("intRemZero", Integer.MIN_VALUE); + } + + public static int intRemMax(int val) { + return val % Integer.MAX_VALUE; + } + + @Test + public void testIntRemMax() { + test("intRemMax", -10); + test("intRemMax", 0); + test("intRemMax", 4256); + test("intRemMax", Integer.MAX_VALUE); + test("intRemMax", Integer.MIN_VALUE); + } + + public static int intRemMin(int val) { + return val % Integer.MIN_VALUE; + } + + @Test + public void testIntRemMin() { + test("intRemMin", -10); + test("intRemMin", 0); + test("intRemMin", 4256); + test("intRemMin", Integer.MAX_VALUE); + test("intRemMin", Integer.MIN_VALUE); + } + + public static long longRemPositiveConstant(long val) { + return val % 35170432; + } + + public static int intRemPowerOf2(int val) { + return val % 4; + } + + @Test + public void testIntRemPowerOf2() { + test("intRemPowerOf2", -10); + test("intRemPowerOf2", 0); + test("intRemPowerOf2", 4256); + test("intRemPowerOf2", Integer.MAX_VALUE); + test("intRemPowerOf2", Integer.MIN_VALUE); + } + + @Test + public void testLongRemPositiveConstant() { + test("longRemPositiveConstant", -1234L); + test("longRemPositiveConstant", 0L); + test("longRemPositiveConstant", 214423L); + test("longRemPositiveConstant", Long.MAX_VALUE); + test("longRemPositiveConstant", Long.MIN_VALUE); + } + + public static long longRemNegativeConstant(long val) { + return val % -413; + } + + @Test + public void testLongRemNegativeConstant() { + test("longRemNegativeConstant", -43L); + test("longRemNegativeConstant", 0L); + test("longRemNegativeConstant", 147065L); + test("longRemNegativeConstant", Long.MAX_VALUE); + test("longRemNegativeConstant", Long.MIN_VALUE); + } + + @SuppressWarnings("divzero") + public static long longRemZero(long val) { + return val % 0; + } + + @Test + public void testLongRemZero() { + test("longRemZero", -43L); + test("longRemZero", 0L); + test("longRemZero", 147065L); + test("longRemZero", Long.MAX_VALUE); + test("longRemZero", Long.MIN_VALUE); + } + + public static long longRemMax(long val) { + return val % Long.MAX_VALUE; + } + + @Test + public void testLongRemMax() { + test("longRemMax", -43L); + test("longRemMax", 0L); + test("longRemMax", 147065L); + test("longRemMax", Long.MAX_VALUE); + test("longRemMax", Long.MIN_VALUE); + } + + public static long longRemMin(long val) { + return val % Long.MIN_VALUE; + } + + @Test + public void testLongRemMin() { + test("longRemMin", -43L); + test("longRemMin", 0L); + test("longRemMin", 147065L); + test("longRemMin", Long.MAX_VALUE); + test("longRemMin", Long.MIN_VALUE); + } + + public static long longRemPowerOf2(long val) { + return val % 4L; + } + + @Test + public void testLongRemPowerOf2() { + test("longRemPowerOf2", -43L); + test("longRemPowerOf2", 0L); + test("longRemPowerOf2", 147065L); + test("longRemPowerOf2", Long.MAX_VALUE); + test("longRemPowerOf2", Long.MIN_VALUE); + } +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/SwitchCanonicalizerTest.java 2019-03-12 08:10:30.695930252 +0100 @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.core.test; + +import org.graalvm.compiler.nodes.StructuredGraph; +import org.graalvm.compiler.nodes.extended.IntegerSwitchNode; +import org.graalvm.compiler.phases.common.CanonicalizerPhase; +import org.junit.Test; + +public class SwitchCanonicalizerTest extends GraalCompilerTest { + + public int divByPowerOf2(int n) { + switch (n / 8) { + case Integer.MAX_VALUE / 8 + 1: + return hashCode(); + default: + return 1; + } + } + + @Test + public void testDivByPowerOf2() { + shouldFoldSwitch("divByPowerOf2"); + } + + public int divByNonPowerOf2(int n) { + switch (n / 7) { + case Integer.MAX_VALUE / 7 + 1: + return hashCode(); + default: + return 1; + } + } + + @Test + public void testDivByNonPowerOf2() { + shouldFoldSwitch("divByNonPowerOf2"); + } + + public int remByPowerOf2(int n) { + switch (n % 8) { + case 9: + return hashCode(); + default: + return 1; + } + } + + @Test + public void testRemByPowerOf2() { + shouldFoldSwitch("remByPowerOf2"); + } + + public int remByPowerOf2PositiveX(int n) { + int n0 = n > 0 ? 8 : 9; + switch (n0 % 8) { + case 9: + return hashCode(); + default: + return 1; + } + } + + @Test + public void testRemByPowerOf2PositiveX() { + shouldFoldSwitch("remByPowerOf2PositiveX"); + } + + public int remByPowerOf2NegativeX(int n) { + int n0 = n > 0 ? -8 : -9; + switch (n0 % 8) { + case 9: + return hashCode(); + default: + return 1; + } + } + + @Test + public void testRemByPowerOf2NegativeX() { + shouldFoldSwitch("remByPowerOf2NegativeX"); + } + + public int remByNonPowerOf2(int n) { + switch (n % 7) { + case 9: + return hashCode(); + default: + return 1; + } + } + + @Test + public void testRemByNonPowerOf2() { + shouldFoldSwitch("remByNonPowerOf2"); + } + + private void shouldFoldSwitch(String methodName) { + StructuredGraph graph = parseForCompile(getResolvedJavaMethod(methodName)); + new CanonicalizerPhase().apply(graph, getDefaultHighTierContext()); + assertFalse(graph.hasNode(IntegerSwitchNode.TYPE)); + } + +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core/src/org/graalvm/compiler/core/gen/LIRCompilerBackend.java 2019-03-12 08:10:31.191933462 +0100 @@ -0,0 +1,245 @@ +/* + * Copyright (c) 2017, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.core.gen; + +import java.util.Collection; +import java.util.List; + +import jdk.internal.vm.compiler.collections.EconomicSet; +import org.graalvm.compiler.code.CompilationResult; +import org.graalvm.compiler.core.LIRGenerationPhase; +import org.graalvm.compiler.core.LIRGenerationPhase.LIRGenerationContext; +import org.graalvm.compiler.core.common.GraalOptions; +import org.graalvm.compiler.core.common.alloc.ComputeBlockOrder; +import org.graalvm.compiler.core.common.alloc.RegisterAllocationConfig; +import org.graalvm.compiler.core.common.cfg.AbstractBlockBase; +import org.graalvm.compiler.core.target.Backend; +import org.graalvm.compiler.debug.CounterKey; +import org.graalvm.compiler.debug.DebugCloseable; +import org.graalvm.compiler.debug.DebugContext; +import org.graalvm.compiler.debug.GraalError; +import org.graalvm.compiler.debug.TimerKey; +import org.graalvm.compiler.lir.LIR; +import org.graalvm.compiler.lir.alloc.OutOfRegistersException; +import org.graalvm.compiler.lir.asm.CompilationResultBuilder; +import org.graalvm.compiler.lir.asm.CompilationResultBuilderFactory; +import org.graalvm.compiler.lir.framemap.FrameMap; +import org.graalvm.compiler.lir.gen.LIRGenerationResult; +import org.graalvm.compiler.lir.gen.LIRGeneratorTool; +import org.graalvm.compiler.lir.phases.AllocationPhase.AllocationContext; +import org.graalvm.compiler.lir.phases.LIRSuites; +import org.graalvm.compiler.lir.phases.PostAllocationOptimizationPhase.PostAllocationOptimizationContext; +import org.graalvm.compiler.lir.phases.PreAllocationOptimizationPhase.PreAllocationOptimizationContext; +import org.graalvm.compiler.nodes.StructuredGraph; +import org.graalvm.compiler.nodes.StructuredGraph.ScheduleResult; +import org.graalvm.compiler.nodes.cfg.Block; +import org.graalvm.compiler.nodes.spi.NodeLIRBuilderTool; + +import jdk.vm.ci.code.RegisterConfig; +import jdk.vm.ci.code.TargetDescription; +import jdk.vm.ci.code.site.ConstantReference; +import jdk.vm.ci.code.site.DataPatch; +import jdk.vm.ci.meta.Assumptions; +import jdk.vm.ci.meta.JavaConstant; +import jdk.vm.ci.meta.JavaKind; +import jdk.vm.ci.meta.ResolvedJavaField; +import jdk.vm.ci.meta.ResolvedJavaMethod; +import jdk.vm.ci.meta.SpeculationLog; +import jdk.vm.ci.meta.VMConstant; + +public class LIRCompilerBackend { + private static final TimerKey EmitLIR = DebugContext.timer("EmitLIR").doc("Time spent generating LIR from HIR."); + private static final TimerKey EmitCode = DebugContext.timer("EmitCode").doc("Time spent generating machine code from LIR."); + private static final TimerKey BackEnd = DebugContext.timer("BackEnd").doc("Time spent in EmitLIR and EmitCode."); + + @SuppressWarnings("try") + public static void emitBackEnd(StructuredGraph graph, Object stub, ResolvedJavaMethod installedCodeOwner, Backend backend, T compilationResult, + CompilationResultBuilderFactory factory, RegisterConfig registerConfig, LIRSuites lirSuites) { + DebugContext debug = graph.getDebug(); + try (DebugContext.Scope s = debug.scope("BackEnd", graph.getLastSchedule()); DebugCloseable a = BackEnd.start(debug)) { + LIRGenerationResult lirGen = null; + lirGen = emitLIR(backend, graph, stub, registerConfig, lirSuites); + try (DebugContext.Scope s2 = debug.scope("CodeGen", lirGen, lirGen.getLIR())) { + int bytecodeSize = graph.method() == null ? 0 : graph.getBytecodeSize(); + compilationResult.setHasUnsafeAccess(graph.hasUnsafeAccess()); + emitCode(backend, + graph.getAssumptions(), + graph.method(), + graph.getMethods(), + graph.getFields(), + graph.getSpeculationLog(), + bytecodeSize, + lirGen, + compilationResult, + installedCodeOwner, + factory); + } catch (Throwable e) { + throw debug.handle(e); + } + } catch (Throwable e) { + throw debug.handle(e); + } finally { + graph.checkCancellation(); + } + } + + @SuppressWarnings("try") + public static LIRGenerationResult emitLIR(Backend backend, StructuredGraph graph, Object stub, RegisterConfig registerConfig, LIRSuites lirSuites) { + String registerPressure = GraalOptions.RegisterPressure.getValue(graph.getOptions()); + String[] allocationRestrictedTo = registerPressure == null ? null : registerPressure.split(","); + try { + return emitLIR0(backend, graph, stub, registerConfig, lirSuites, allocationRestrictedTo); + } catch (OutOfRegistersException e) { + if (allocationRestrictedTo != null) { + allocationRestrictedTo = null; + return emitLIR0(backend, graph, stub, registerConfig, lirSuites, allocationRestrictedTo); + } + /* If the re-execution fails we convert the exception into a "hard" failure */ + throw new GraalError(e); + } finally { + graph.checkCancellation(); + } + } + + @SuppressWarnings("try") + private static LIRGenerationResult emitLIR0(Backend backend, StructuredGraph graph, Object stub, RegisterConfig registerConfig, LIRSuites lirSuites, + String[] allocationRestrictedTo) { + DebugContext debug = graph.getDebug(); + try (DebugContext.Scope ds = debug.scope("EmitLIR"); DebugCloseable a = EmitLIR.start(debug)) { + assert !graph.hasValueProxies(); + + ScheduleResult schedule = graph.getLastSchedule(); + Block[] blocks = schedule.getCFG().getBlocks(); + Block startBlock = schedule.getCFG().getStartBlock(); + assert startBlock != null; + assert startBlock.getPredecessorCount() == 0; + + AbstractBlockBase[] codeEmittingOrder = ComputeBlockOrder.computeCodeEmittingOrder(blocks.length, startBlock); + AbstractBlockBase[] linearScanOrder = ComputeBlockOrder.computeLinearScanOrder(blocks.length, startBlock); + LIR lir = new LIR(schedule.getCFG(), linearScanOrder, codeEmittingOrder, graph.getOptions(), graph.getDebug()); + + LIRGenerationProvider lirBackend = (LIRGenerationProvider) backend; + LIRGenerationResult lirGenRes = lirBackend.newLIRGenerationResult(graph.compilationId(), lir, registerConfig, graph, stub); + LIRGeneratorTool lirGen = lirBackend.newLIRGenerator(lirGenRes); + NodeLIRBuilderTool nodeLirGen = lirBackend.newNodeLIRBuilder(graph, lirGen); + + // LIR generation + LIRGenerationContext context = new LIRGenerationContext(lirGen, nodeLirGen, graph, schedule); + new LIRGenerationPhase().apply(backend.getTarget(), lirGenRes, context); + + try (DebugContext.Scope s = debug.scope("LIRStages", nodeLirGen, lirGenRes, lir)) { + // Dump LIR along with HIR (the LIR is looked up from context) + debug.dump(DebugContext.BASIC_LEVEL, graph.getLastSchedule(), "After LIR generation"); + LIRGenerationResult result = emitLowLevel(backend.getTarget(), lirGenRes, lirGen, lirSuites, backend.newRegisterAllocationConfig(registerConfig, allocationRestrictedTo)); + return result; + } catch (Throwable e) { + throw debug.handle(e); + } + } catch (Throwable e) { + throw debug.handle(e); + } finally { + graph.checkCancellation(); + } + } + + private static LIRGenerationResult emitLowLevel(TargetDescription target, LIRGenerationResult lirGenRes, LIRGeneratorTool lirGen, LIRSuites lirSuites, + RegisterAllocationConfig registerAllocationConfig) { + DebugContext debug = lirGenRes.getLIR().getDebug(); + PreAllocationOptimizationContext preAllocOptContext = new PreAllocationOptimizationContext(lirGen); + lirSuites.getPreAllocationOptimizationStage().apply(target, lirGenRes, preAllocOptContext); + debug.dump(DebugContext.BASIC_LEVEL, lirGenRes.getLIR(), "After PreAllocationOptimizationStage"); + + AllocationContext allocContext = new AllocationContext(lirGen.getSpillMoveFactory(), registerAllocationConfig); + lirSuites.getAllocationStage().apply(target, lirGenRes, allocContext); + debug.dump(DebugContext.BASIC_LEVEL, lirGenRes.getLIR(), "After AllocationStage"); + + PostAllocationOptimizationContext postAllocOptContext = new PostAllocationOptimizationContext(lirGen); + lirSuites.getPostAllocationOptimizationStage().apply(target, lirGenRes, postAllocOptContext); + debug.dump(DebugContext.BASIC_LEVEL, lirGenRes.getLIR(), "After PostAllocationOptimizationStage"); + + return lirGenRes; + } + + @SuppressWarnings("try") + public static void emitCode(Backend backend, + Assumptions assumptions, + ResolvedJavaMethod rootMethod, + Collection inlinedMethods, + EconomicSet accessedFields, + SpeculationLog speculationLog, + int bytecodeSize, + LIRGenerationResult lirGenRes, + CompilationResult compilationResult, + ResolvedJavaMethod installedCodeOwner, + CompilationResultBuilderFactory factory) { + DebugContext debug = lirGenRes.getLIR().getDebug(); + try (DebugCloseable a = EmitCode.start(debug)) { + LIRGenerationProvider lirBackend = (LIRGenerationProvider) backend; + + FrameMap frameMap = lirGenRes.getFrameMap(); + CompilationResultBuilder crb = lirBackend.newCompilationResultBuilder(lirGenRes, frameMap, compilationResult, factory); + lirBackend.emitCode(crb, lirGenRes.getLIR(), installedCodeOwner); + if (assumptions != null && !assumptions.isEmpty()) { + compilationResult.setAssumptions(assumptions.toArray()); + } + if (rootMethod != null) { + compilationResult.setMethods(rootMethod, inlinedMethods); + compilationResult.setFields(accessedFields); + compilationResult.setBytecodeSize(bytecodeSize); + } + if (speculationLog != null) { + compilationResult.setSpeculationLog(speculationLog); + } + crb.finish(); + if (debug.isCountEnabled()) { + List ldp = compilationResult.getDataPatches(); + JavaKind[] kindValues = JavaKind.values(); + CounterKey[] dms = new CounterKey[kindValues.length]; + for (int i = 0; i < dms.length; i++) { + dms[i] = DebugContext.counter("DataPatches-%s", kindValues[i]); + } + + for (DataPatch dp : ldp) { + JavaKind kind = JavaKind.Illegal; + if (dp.reference instanceof ConstantReference) { + VMConstant constant = ((ConstantReference) dp.reference).getConstant(); + if (constant instanceof JavaConstant) { + kind = ((JavaConstant) constant).getJavaKind(); + } + } + dms[kind.ordinal()].add(debug, 1); + } + + DebugContext.counter("CompilationResults").increment(debug); + DebugContext.counter("CodeBytesEmitted").add(debug, compilationResult.getTargetCodeSize()); + DebugContext.counter("InfopointsEmitted").add(debug, compilationResult.getInfopoints().size()); + DebugContext.counter("DataPatches").add(debug, ldp.size()); + DebugContext.counter("ExceptionHandlersEmitted").add(debug, compilationResult.getExceptionHandlers().size()); + } + + debug.dump(DebugContext.BASIC_LEVEL, compilationResult, "After code generation"); + } + } +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core/src/org/graalvm/compiler/core/gen/LIRGenerationProvider.java 2019-03-12 08:10:31.683936645 +0100 @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2017, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.core.gen; + +import org.graalvm.compiler.code.CompilationResult; +import org.graalvm.compiler.core.common.CompilationIdentifier; +import org.graalvm.compiler.lir.LIR; +import org.graalvm.compiler.lir.asm.CompilationResultBuilder; +import org.graalvm.compiler.lir.asm.CompilationResultBuilderFactory; +import org.graalvm.compiler.lir.framemap.FrameMap; +import org.graalvm.compiler.lir.gen.LIRGenerationResult; +import org.graalvm.compiler.lir.gen.LIRGeneratorTool; +import org.graalvm.compiler.nodes.StructuredGraph; +import org.graalvm.compiler.nodes.spi.NodeLIRBuilderTool; + +import jdk.vm.ci.code.RegisterConfig; +import jdk.vm.ci.meta.ResolvedJavaMethod; + +/** + * Provides compiler backend-specific generation helpers for the {@link LIRCompilerBackend}. + */ +public interface LIRGenerationProvider { + LIRGeneratorTool newLIRGenerator(LIRGenerationResult lirGenRes); + + LIRGenerationResult newLIRGenerationResult(CompilationIdentifier compilationId, LIR lir, RegisterConfig registerConfig, StructuredGraph graph, + Object stub); + + NodeLIRBuilderTool newNodeLIRBuilder(StructuredGraph graph, LIRGeneratorTool lirGen); + + /** + * Creates the object used to fill in the details of a given compilation result. + */ + CompilationResultBuilder newCompilationResultBuilder(LIRGenerationResult lirGenResult, FrameMap frameMap, CompilationResult compilationResult, + CompilationResultBuilderFactory factory); + + /** + * Emits the code for a given graph. + * + * @param installedCodeOwner the method the compiled code will be associated with once + * installed. This argument can be null. + */ + void emitCode(CompilationResultBuilder crb, LIR lir, ResolvedJavaMethod installedCodeOwner); +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.debug/src/org/graalvm/compiler/debug/doc-files/PrintGraphHelp.txt 2019-03-12 08:10:32.171939802 +0100 @@ -0,0 +1,6 @@ +Where IdealGraphVisualizer graph dumps triggered by Dump or DumpOnError should be written. +The accepted values are: + File - Dump IGV graphs to the local file system (see DumpPath). + Network - Dump IGV graphs to the network destination specified by PrintGraphHost and PrintGraphPort. + If a network connection cannot be opened, dumping falls back to file dumping. + Disable - Do not dump IGV graphs. \ No newline at end of file --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.graph.test/src/org/graalvm/compiler/graph/test/graphio/GraphOutputTest.java 2019-03-12 08:10:32.659942958 +0100 @@ -0,0 +1,284 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.graph.test.graphio; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.Channels; +import java.nio.channels.WritableByteChannel; +import java.util.Collection; +import java.util.Collections; +import java.util.Map; +import java.util.Objects; +import org.graalvm.graphio.GraphOutput; +import org.graalvm.graphio.GraphStructure; +import org.junit.Test; + +public final class GraphOutputTest { + + @Test + @SuppressWarnings("static-method") + public void testWritableByteChannel() throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + WritableByteChannel channel = Channels.newChannel(out); + ByteBuffer data = generateData(1 << 17); + GraphOutput graphOutput = GraphOutput.newBuilder(new MockGraphStructure()).protocolVersion(6, 0).embedded(true).build(channel); + try (GraphOutput closable = graphOutput) { + assertTrue(closable.isOpen()); + closable.write(data); + } + assertFalse(graphOutput.isOpen()); + assertArrayEquals(data.array(), out.toByteArray()); + } + + @Test + @SuppressWarnings("static-method") + public void testWriteDuringPrint() throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + WritableByteChannel channel = Channels.newChannel(out); + class Action implements Runnable { + GraphOutput out; + + @Override + public void run() { + try { + ByteBuffer data = ByteBuffer.allocate(16); + data.limit(16); + out.write(data); + } catch (IOException ioe) { + throw new RuntimeException(ioe); + } + } + } + Action action = new Action(); + try (GraphOutput graphOutput = GraphOutput.newBuilder(new MockGraphStructure(action)).protocolVersion(6, 0).build(channel)) { + action.out = graphOutput; + try { + graphOutput.print(new MockGraph(), Collections.emptyMap(), 0, "Mock Graph"); + fail("Expected IllegalStateException"); + } catch (IllegalStateException ise) { + // expected exception + } + } + } + + @Test + @SuppressWarnings("static-method") + public void testEmbeddedWritableByteChannel() throws IOException { + ByteArrayOutputStream expected = new ByteArrayOutputStream(); + WritableByteChannel expectedChannel = Channels.newChannel(expected); + Map properties = Collections.singletonMap("version.id", 1); + try (GraphOutput graphOutput = GraphOutput.newBuilder(new MockGraphStructure()).protocolVersion(6, 0).build(expectedChannel)) { + graphOutput.print(new MockGraph(), properties, 1, "Graph 1"); + graphOutput.write(ByteBuffer.allocate(0)); + graphOutput.print(new MockGraph(), properties, 2, "Graph 1"); + } + ByteArrayOutputStream embedded = new ByteArrayOutputStream(); + SharedWritableByteChannel embeddChannel = new SharedWritableByteChannel(Channels.newChannel(embedded)); + try { + try (GraphOutput baseOutput = GraphOutput.newBuilder(new MockGraphStructure()).protocolVersion(6, 0).build(embeddChannel)) { + try (GraphOutput embeddedOutput = GraphOutput.newBuilder(new MockGraphStructure()).protocolVersion(6, 0).embedded(true).build((WritableByteChannel) baseOutput)) { + embeddedOutput.print(new MockGraph(), properties, 1, "Graph 1"); + baseOutput.print(new MockGraph(), properties, 2, "Graph 1"); + } + } + } finally { + embeddChannel.realClose(); + } + assertArrayEquals(expected.toByteArray(), embedded.toByteArray()); + } + + private static ByteBuffer generateData(int size) { + ByteBuffer buffer = ByteBuffer.allocate(size); + for (int i = 0; i < size; i++) { + buffer.put(i, (byte) i); + } + buffer.limit(size); + return buffer; + } + + private static final class SharedWritableByteChannel implements WritableByteChannel { + + private final WritableByteChannel delegate; + + SharedWritableByteChannel(WritableByteChannel delegate) { + Objects.requireNonNull(delegate, "Delegate must be non null."); + this.delegate = delegate; + } + + @Override + public int write(ByteBuffer bb) throws IOException { + return delegate.write(bb); + } + + @Override + public boolean isOpen() { + return delegate.isOpen(); + } + + @Override + public void close() throws IOException { + } + + void realClose() throws IOException { + delegate.close(); + } + } + + private static final class MockGraphStructure implements GraphStructure { + + private final Runnable enterAction; + + MockGraphStructure() { + this.enterAction = null; + } + + MockGraphStructure(Runnable enterAction) { + this.enterAction = enterAction; + } + + @Override + public MockGraph graph(MockGraph currentGraph, Object obj) { + onEnter(); + return null; + } + + @Override + public Iterable nodes(MockGraph graph) { + onEnter(); + return Collections.emptySet(); + } + + @Override + public int nodesCount(MockGraph graph) { + onEnter(); + return 0; + } + + @Override + public int nodeId(Void node) { + onEnter(); + return 0; + } + + @Override + public boolean nodeHasPredecessor(Void node) { + onEnter(); + return false; + } + + @Override + public void nodeProperties(MockGraph graph, Void node, Map properties) { + onEnter(); + } + + @Override + public Void node(Object obj) { + onEnter(); + return null; + } + + @Override + public Void nodeClass(Object obj) { + onEnter(); + return null; + } + + @Override + public Void classForNode(Void node) { + onEnter(); + return null; + } + + @Override + public String nameTemplate(Void nodeClass) { + onEnter(); + return null; + } + + @Override + public Object nodeClassType(Void nodeClass) { + onEnter(); + return null; + } + + @Override + public Void portInputs(Void nodeClass) { + onEnter(); + return null; + } + + @Override + public Void portOutputs(Void nodeClass) { + onEnter(); + return null; + } + + @Override + public int portSize(Void port) { + onEnter(); + return 0; + } + + @Override + public boolean edgeDirect(Void port, int index) { + onEnter(); + return false; + } + + @Override + public String edgeName(Void port, int index) { + onEnter(); + return null; + } + + @Override + public Object edgeType(Void port, int index) { + onEnter(); + return null; + } + + @Override + public Collection edgeNodes(MockGraph graph, Void node, Void port, int index) { + onEnter(); + return null; + } + + private void onEnter() { + if (enterAction != null) { + enterAction.run(); + } + } + } + + private static final class MockGraph { + } +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64.test/src/org/graalvm/compiler/hotspot/amd64/test/BinaryMathStubTest.java 2019-03-12 08:10:33.139946065 +0100 @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.hotspot.amd64.test; + +import static org.junit.Assume.assumeTrue; + +import java.util.ArrayList; +import java.util.List; + +import org.graalvm.compiler.api.test.Graal; +import org.graalvm.compiler.core.test.GraalCompilerTest; +import org.graalvm.compiler.runtime.RuntimeProvider; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import jdk.vm.ci.amd64.AMD64; +import jdk.vm.ci.code.Architecture; + +@RunWith(Parameterized.class) +public class BinaryMathStubTest extends GraalCompilerTest { + + @Parameterized.Parameters(name = "{0}") + public static List data() { + ArrayList ret = new ArrayList<>(); + ret.add(new Object[]{"pow"}); + return ret; + } + + private static final double[] inputs = {0.0D, Math.PI / 2, Math.PI, -1.0D, Double.MAX_VALUE, Double.MIN_VALUE, Double.NaN, Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY}; + private final String stub; + + public BinaryMathStubTest(String stub) { + this.stub = stub; + } + + @Before + public void checkAMD64() { + Architecture arch = Graal.getRequiredCapability(RuntimeProvider.class).getHostBackend().getTarget().arch; + assumeTrue("skipping AMD64 specific test", arch instanceof AMD64); + } + + public static double pow(double x, double y) { + return Math.pow(x, y); + } + + @Test + public void testStub() { + for (double x : inputs) { + for (double y : inputs) { + test(stub, x, y); + } + } + } +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64.test/src/org/graalvm/compiler/hotspot/amd64/test/UnaryMathStubTest.java 2019-03-12 08:10:33.627949221 +0100 @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.hotspot.amd64.test; + +import static org.junit.Assume.assumeTrue; + +import java.util.ArrayList; +import java.util.List; + +import org.graalvm.compiler.api.test.Graal; +import org.graalvm.compiler.core.test.GraalCompilerTest; +import org.graalvm.compiler.runtime.RuntimeProvider; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import jdk.vm.ci.amd64.AMD64; +import jdk.vm.ci.code.Architecture; + +@RunWith(Parameterized.class) +public class UnaryMathStubTest extends GraalCompilerTest { + + @Parameterized.Parameters(name = "{0}") + public static List data() { + ArrayList ret = new ArrayList<>(); + ret.add(new Object[]{"sin"}); + ret.add(new Object[]{"cos"}); + ret.add(new Object[]{"tan"}); + ret.add(new Object[]{"exp"}); + ret.add(new Object[]{"log"}); + ret.add(new Object[]{"log10"}); + return ret; + } + + private static final double[] inputs = {0.0D, Math.PI / 2, Math.PI, -1.0D, Double.MAX_VALUE, Double.MIN_VALUE, Double.NaN, Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY}; + private final String stub; + + public UnaryMathStubTest(String stub) { + this.stub = stub; + } + + @Before + public void checkAMD64() { + Architecture arch = Graal.getRequiredCapability(RuntimeProvider.class).getHostBackend().getTarget().arch; + assumeTrue("skipping AMD64 specific test", arch instanceof AMD64); + } + + public static double sin(double value) { + return Math.sin(value); + } + + public static double cos(double value) { + return Math.cos(value); + } + + public static double tan(double value) { + return Math.tan(value); + } + + public static double exp(double value) { + return Math.exp(value); + } + + public static double log(double value) { + return Math.log(value); + } + + public static double log10(double value) { + return Math.log10(value); + } + + @Test + public void testStub() { + for (double input : inputs) { + test(stub, input); + } + } +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64X87MathIntrinsicNode.java 2019-03-12 08:10:34.111952353 +0100 @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2012, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.hotspot.amd64; + +import static org.graalvm.compiler.hotspot.amd64.AMD64HotSpotMathIntrinsicOp.IntrinsicOpcode.COS; +import static org.graalvm.compiler.hotspot.amd64.AMD64HotSpotMathIntrinsicOp.IntrinsicOpcode.LOG; +import static org.graalvm.compiler.hotspot.amd64.AMD64HotSpotMathIntrinsicOp.IntrinsicOpcode.LOG10; +import static org.graalvm.compiler.hotspot.amd64.AMD64HotSpotMathIntrinsicOp.IntrinsicOpcode.SIN; +import static org.graalvm.compiler.hotspot.amd64.AMD64HotSpotMathIntrinsicOp.IntrinsicOpcode.TAN; +import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_64; +import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_1; + +import org.graalvm.compiler.core.common.LIRKind; +import org.graalvm.compiler.core.common.type.FloatStamp; +import org.graalvm.compiler.core.common.type.PrimitiveStamp; +import org.graalvm.compiler.core.common.type.Stamp; +import org.graalvm.compiler.debug.GraalError; +import org.graalvm.compiler.graph.NodeClass; +import org.graalvm.compiler.graph.spi.CanonicalizerTool; +import org.graalvm.compiler.lir.Variable; +import org.graalvm.compiler.lir.gen.LIRGeneratorTool; +import org.graalvm.compiler.nodeinfo.NodeInfo; +import org.graalvm.compiler.nodes.ConstantNode; +import org.graalvm.compiler.nodes.NodeView; +import org.graalvm.compiler.nodes.ValueNode; +import org.graalvm.compiler.nodes.calc.UnaryNode; +import org.graalvm.compiler.nodes.spi.LIRLowerable; +import org.graalvm.compiler.nodes.spi.NodeLIRBuilderTool; +import org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.UnaryOperation; + +import jdk.vm.ci.meta.Value; + +@NodeInfo(nameTemplate = "X87MathIntrinsic#{p#operation/s}", cycles = CYCLES_64, size = SIZE_1) +public final class AMD64X87MathIntrinsicNode extends UnaryNode implements LIRLowerable { + + public static final NodeClass TYPE = NodeClass.create(AMD64X87MathIntrinsicNode.class); + protected final UnaryOperation operation; + + protected AMD64X87MathIntrinsicNode(ValueNode value, UnaryOperation op) { + super(TYPE, op.computeStamp(value.stamp(NodeView.DEFAULT)), value); + assert value.stamp(NodeView.DEFAULT) instanceof FloatStamp && PrimitiveStamp.getBits(value.stamp(NodeView.DEFAULT)) == 64; + this.operation = op; + } + + @Override + public Stamp foldStamp(Stamp valueStamp) { + return operation.computeStamp(valueStamp); + } + + @Override + public void generate(NodeLIRBuilderTool generator) { + LIRGeneratorTool gen = generator.getLIRGeneratorTool(); + Value input = generator.operand(getValue()); + Variable result = gen.newVariable(LIRKind.combine(input)); + + switch (operation) { + case SIN: + gen.append(new AMD64HotSpotMathIntrinsicOp(SIN, result, gen.asAllocatable(input))); + break; + case COS: + gen.append(new AMD64HotSpotMathIntrinsicOp(COS, result, gen.asAllocatable(input))); + break; + case TAN: + gen.append(new AMD64HotSpotMathIntrinsicOp(TAN, result, gen.asAllocatable(input))); + break; + case LOG: + gen.append(new AMD64HotSpotMathIntrinsicOp(LOG, result, gen.asAllocatable(input))); + break; + case LOG10: + gen.append(new AMD64HotSpotMathIntrinsicOp(LOG10, result, gen.asAllocatable(input))); + break; + default: + throw GraalError.shouldNotReachHere(); + } + generator.setResult(this, result); + } + + @Override + public ValueNode canonical(CanonicalizerTool tool, ValueNode forValue) { + if (forValue.isConstant()) { + return ConstantNode.forDouble(operation.compute(forValue.asJavaConstant().asDouble())); + } + return this; + } + + @NodeIntrinsic + public static native double compute(double value, @ConstantNodeParameter UnaryOperation op); + +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64X87MathSnippets.java 2019-03-12 08:10:34.595955484 +0100 @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.hotspot.amd64; + +import static org.graalvm.compiler.replacements.SnippetTemplate.DEFAULT_REPLACER; + +import org.graalvm.compiler.api.replacements.Snippet; +import org.graalvm.compiler.api.replacements.SnippetReflectionProvider; +import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor; +import org.graalvm.compiler.debug.DebugHandlersFactory; +import org.graalvm.compiler.debug.GraalError; +import org.graalvm.compiler.graph.Node.ConstantNodeParameter; +import org.graalvm.compiler.graph.Node.NodeIntrinsic; +import org.graalvm.compiler.nodes.extended.ForeignCallNode; +import org.graalvm.compiler.nodes.spi.LoweringTool; +import org.graalvm.compiler.options.OptionValues; +import org.graalvm.compiler.phases.util.Providers; +import org.graalvm.compiler.replacements.SnippetTemplate.AbstractTemplates; +import org.graalvm.compiler.replacements.SnippetTemplate.Arguments; +import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo; +import org.graalvm.compiler.replacements.Snippets; +import org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode; +import org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.UnaryOperation; + +import jdk.vm.ci.code.TargetDescription; + +public class AMD64X87MathSnippets implements Snippets { + + private static final double PI_4 = Math.PI / 4; + + @Snippet + public static double sin(double input) { + if (Math.abs(input) < PI_4) { + return AMD64X87MathIntrinsicNode.compute(input, UnaryOperation.SIN); + } + return callDouble1(UnaryOperation.SIN.foreignCallDescriptor, input); + } + + @Snippet + public static double cos(double input) { + if (Math.abs(input) < PI_4) { + return AMD64X87MathIntrinsicNode.compute(input, UnaryOperation.COS); + } + return callDouble1(UnaryOperation.COS.foreignCallDescriptor, input); + } + + @Snippet + public static double tan(double input) { + if (Math.abs(input) < PI_4) { + return AMD64X87MathIntrinsicNode.compute(input, UnaryOperation.TAN); + } + return callDouble1(UnaryOperation.TAN.foreignCallDescriptor, input); + } + + @NodeIntrinsic(value = ForeignCallNode.class) + private static native double callDouble1(@ConstantNodeParameter ForeignCallDescriptor descriptor, double value); + + public static class Templates extends AbstractTemplates { + + private final SnippetInfo sin; + private final SnippetInfo cos; + private final SnippetInfo tan; + + public Templates(OptionValues options, Iterable factories, Providers providers, SnippetReflectionProvider snippetReflection, TargetDescription target) { + super(options, factories, providers, snippetReflection, target); + + sin = snippet(AMD64X87MathSnippets.class, "sin"); + cos = snippet(AMD64X87MathSnippets.class, "cos"); + tan = snippet(AMD64X87MathSnippets.class, "tan"); + } + + public void lower(UnaryMathIntrinsicNode mathIntrinsicNode, LoweringTool tool) { + SnippetInfo info; + + switch (mathIntrinsicNode.getOperation()) { + case SIN: + info = sin; + break; + case COS: + info = cos; + break; + case TAN: + info = tan; + break; + default: + throw GraalError.shouldNotReachHere("Snippet not found for math intrinsic " + mathIntrinsicNode.getOperation().name()); + } + + Arguments args = new Arguments(info, mathIntrinsicNode.graph().getGuardsStage(), tool.getLoweringStage()); + args.add("input", mathIntrinsicNode.getValue()); + template(mathIntrinsicNode, args).instantiate(providers.getMetaAccess(), mathIntrinsicNode, DEFAULT_REPLACER, tool, args); + mathIntrinsicNode.safeDelete(); + } + } +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.jdk9.test/src/org/graalvm/compiler/hotspot/jdk9/test/StringUTF16ToBytesGetCharsTest.java 2019-03-12 08:10:35.091958693 +0100 @@ -0,0 +1,142 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.hotspot.jdk9.test; + +import static org.junit.Assume.assumeFalse; + +import jdk.vm.ci.code.InstalledCode; +import jdk.vm.ci.meta.ResolvedJavaMethod; +import org.graalvm.compiler.core.common.CompilationIdentifier; +import org.graalvm.compiler.hotspot.replacements.StringUTF16Substitutions; +import org.graalvm.compiler.nodes.StructuredGraph; +import org.graalvm.compiler.nodes.java.NewArrayNode; +import org.graalvm.compiler.replacements.arraycopy.ArrayCopyCallNode; +import org.graalvm.compiler.replacements.test.MethodSubstitutionTest; +import org.graalvm.compiler.test.AddExports; +import org.junit.Before; +import org.junit.Test; + +/** + * Test substitutions for (innate) methods StringUTF16.toBytes and StringUTF16.getChars provided by + * {@link StringUTF16Substitutions}. + */ +@AddExports({"java.base/java.lang"}) +public final class StringUTF16ToBytesGetCharsTest extends MethodSubstitutionTest { + + private static final int N = 1000; + private static final int N_OVERFLOW = 10; + + @Before + public void checkAMD64() { + assumeFalse(Java8OrEarlier); + } + + @Test + public void testStringUTF16ToBytes() throws ClassNotFoundException { + Class javaclass = Class.forName("java.lang.StringUTF16"); + + ResolvedJavaMethod caller = getResolvedJavaMethod(javaclass, "toBytes", char[].class, int.class, int.class); + StructuredGraph graph = getReplacements().getIntrinsicGraph(caller, CompilationIdentifier.INVALID_COMPILATION_ID, getDebugContext()); + assertInGraph(graph, NewArrayNode.class); + assertInGraph(graph, ArrayCopyCallNode.class); + + InstalledCode code = getCode(caller, graph); + + for (int srcOffset = 0; srcOffset < 2; srcOffset++) { + for (int i = 0; i < N; i++) { + int length = i2sz(i); + char[] src = fillUTF16Chars(new char[length]); + int copiedLength = Math.max(0, length - srcOffset); + int srcDelta = Math.min(srcOffset, copiedLength); + byte[] dst = (byte[]) invokeSafe(caller, null, src, srcDelta, copiedLength); + assert dst.length == copiedLength * 2; + byte[] dst2 = (byte[]) executeVarargsSafe(code, src, srcDelta, copiedLength); + assertDeepEquals(dst, dst2); + } + } + for (int srcOff = 0; srcOff < N_OVERFLOW; ++srcOff) { + for (int len = 0; len < N_OVERFLOW; ++len) { + char[] src = fillUTF16Chars(new char[N_OVERFLOW]); + test(caller, null, src, srcOff, len); + } + } + } + + @Test + public void testStringUTF16getChars() throws ClassNotFoundException { + Class javaclass = Class.forName("java.lang.StringUTF16"); + + ResolvedJavaMethod caller = getResolvedJavaMethod(javaclass, "getChars", byte[].class, int.class, int.class, char[].class, int.class); + StructuredGraph graph = getReplacements().getIntrinsicGraph(caller, CompilationIdentifier.INVALID_COMPILATION_ID, getDebugContext()); + assertInGraph(graph, ArrayCopyCallNode.class); + + InstalledCode code = getCode(caller, graph); + + for (int dstOffset = 0; dstOffset < 2; dstOffset++) { + for (int srcOffset = 0; srcOffset < 2; srcOffset++) { + for (int i = 0; i < N; i++) { + int length = i2sz(i); + byte[] src = fillUTF16Bytes(new byte[length * 2]); + char[] dst = new char[length]; + int copiedLength = Math.max(0, length - Math.max(dstOffset, srcOffset)); + int dstDelta = Math.min(dstOffset, copiedLength); + int srcDelta = Math.min(srcOffset, copiedLength); + invokeSafe(caller, null, src, srcDelta, srcDelta + copiedLength, dst, dstDelta); + char[] dst2 = new char[length]; + executeVarargsSafe(code, src, srcDelta, srcDelta + copiedLength, dst2, dstDelta); + assertDeepEquals(dst, dst2); + } + } + } + for (int srcOff = 0; srcOff < N_OVERFLOW; ++srcOff) { + for (int dstOff = 0; dstOff < N_OVERFLOW; ++dstOff) { + for (int len = 0; len < N_OVERFLOW; ++len) { + byte[] src = fillUTF16Bytes(new byte[N_OVERFLOW]); + char[] dst = new char[N_OVERFLOW]; + test(caller, null, src, srcOff, len, dst, dstOff); + } + } + } + } + + private static char[] fillUTF16Chars(char[] v) { + for (int ch = 0, i = 0; i < v.length; i++, ch += 0x101) { + v[i] = (char) ch; + } + return v; + } + + private static byte[] fillUTF16Bytes(byte[] v) { + for (int ch = 1, i = 0; i < v.length; i += 2, ch++) { + v[i] = (byte) (ch - 1); + v[i + 1] = (byte) ch; + } + return v; + } + + private static int i2sz(int i) { + return i * 3; + } +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/EliminateRedundantInitializationPhaseTest.java 2019-03-12 08:10:35.579961848 +0100 @@ -0,0 +1,224 @@ +/* + * Copyright (c) 2017, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.hotspot.test; + +import org.graalvm.compiler.core.test.GraalCompilerTest; +import org.graalvm.compiler.hotspot.meta.HotSpotClassInitializationPlugin; +import org.graalvm.compiler.hotspot.nodes.aot.InitializeKlassNode; +import org.graalvm.compiler.hotspot.phases.aot.EliminateRedundantInitializationPhase; +import org.graalvm.compiler.nodes.StructuredGraph; +import org.graalvm.compiler.nodes.StructuredGraph.AllowAssumptions; +import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderConfiguration.Plugins; +import org.graalvm.compiler.phases.tiers.HighTierContext; +import org.junit.Assert; +import org.junit.Test; + +public class EliminateRedundantInitializationPhaseTest extends GraalCompilerTest { + @Override + protected Plugins getDefaultGraphBuilderPlugins() { + Plugins plugins = super.getDefaultGraphBuilderPlugins(); + plugins.setClassInitializationPlugin(new HotSpotClassInitializationPlugin()); + return plugins; + } + + public static class X { + public static int x; + public static int y; + public static int z; + } + + public static class Y extends X { + public static int a; + public static int b; + } + + public static void assignFields() { + X.x = 1; + X.y = 2; + X.z = 3; + } + + public static void assignFieldsConditionally(boolean choice) { + X.x = 1; + if (choice) { + X.y = 2; + } else { + X.z = 3; + } + } + + public static void assignFieldsSubclassDominates() { + Y.a = 1; + X.x = 2; + X.y = 3; + X.z = 4; + } + + public static void assignFieldsConditionallySubclassDominates(boolean choice) { + Y.a = 1; + if (choice) { + X.x = 2; + } else { + X.y = 3; + } + Y.z = 4; + } + + public static void assignFieldsSubclassPostdominates() { + X.x = 1; + Y.a = 2; + } + + public static void assignFieldsConditionallySubclassPostdominates(boolean choice) { + X.x = 1; + if (choice) { + X.y = 2; + } else { + X.z = 3; + } + Y.a = 4; + } + + public static void assignFieldsConditionallyMixed(boolean choice) { + X.x = 1; + if (choice) { + Y.a = 2; + } else { + X.z = 3; + } + Y.b = 4; + } + + public static void assignFieldsInLoop() { + X.x = 1; + for (int i = 0; i < 10; i++) { + X.y += X.z; + } + } + + public static void assignFieldsInBranches(boolean choice) { + if (choice) { + X.x = 1; + } else { + X.y = 2; + } + X.z = 3; + } + + public static void assignFieldsInBranchesMixed(boolean choice) { + if (choice) { + X.x = 1; + } else { + Y.a = 2; + } + X.z = 3; + } + + static class SomeClass { + @BytecodeParserNeverInline + static void method() { + } + + @BytecodeParserForceInline + static void inlinedMethod() { + } + } + + public static void invokestatic() { + SomeClass.method(); + } + + public static void invokestaticInlined() { + SomeClass.inlinedMethod(); + } + + private void test(String name, int initNodesAfterParse, int initNodesAfterOpt) { + StructuredGraph graph = parseEager(name, AllowAssumptions.NO); + Assert.assertEquals(initNodesAfterParse, graph.getNodes().filter(InitializeKlassNode.class).count()); + HighTierContext highTierContext = getDefaultHighTierContext(); + new EliminateRedundantInitializationPhase().apply(graph, highTierContext); + Assert.assertEquals(initNodesAfterOpt, graph.getNodes().filter(InitializeKlassNode.class).count()); + } + + @Test + public void test1() { + test("assignFields", 3, 1); + } + + @Test + public void test2() { + test("assignFieldsConditionally", 3, 1); + } + + @Test + public void test3() { + test("assignFieldsSubclassDominates", 4, 1); + } + + @Test + public void test4() { + test("assignFieldsConditionallySubclassDominates", 4, 1); + } + + @Test + public void test5() { + test("assignFieldsSubclassPostdominates", 2, 2); + } + + @Test + public void test6() { + test("assignFieldsConditionallySubclassPostdominates", 4, 2); + } + + @Test + public void test7() { + test("assignFieldsConditionallyMixed", 4, 3); + } + + @Test + public void test8() { + test("assignFieldsInLoop", 4, 1); + } + + @Test + public void test9() { + test("assignFieldsInBranches", 3, 2); + } + + @Test + public void test10() { + test("assignFieldsInBranchesMixed", 3, 2); + } + + @Test + public void test11() { + test("invokestatic", 1, 0); + } + + @Test + public void test12() { + test("invokestaticInlined", 1, 1); + } +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/SymbolicSnippetEncoder.java 2019-03-12 08:10:36.071965032 +0100 @@ -0,0 +1,1034 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.hotspot; + +import static jdk.vm.ci.runtime.JVMCI.getRuntime; +import static jdk.vm.ci.services.Services.IS_BUILDING_NATIVE_IMAGE; +import static jdk.vm.ci.services.Services.IS_IN_NATIVE_IMAGE; +import static org.graalvm.compiler.nodes.graphbuilderconf.InlineInvokePlugin.InlineInfo.createIntrinsicInlineInfo; +import static org.graalvm.compiler.replacements.ReplacementsImpl.Options.UseEncodedSnippets; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +import org.graalvm.compiler.api.replacements.Fold; +import org.graalvm.compiler.api.replacements.MethodSubstitution; +import org.graalvm.compiler.api.replacements.Snippet; +import org.graalvm.compiler.api.replacements.SnippetReflectionProvider; +import org.graalvm.compiler.api.runtime.GraalJVMCICompiler; +import org.graalvm.compiler.api.runtime.GraalRuntime; +import org.graalvm.compiler.bytecode.BytecodeProvider; +import org.graalvm.compiler.core.common.spi.ConstantFieldProvider; +import org.graalvm.compiler.core.common.type.AbstractObjectStamp; +import org.graalvm.compiler.core.common.type.Stamp; +import org.graalvm.compiler.core.common.type.StampPair; +import org.graalvm.compiler.core.common.type.SymbolicJVMCIReference; +import org.graalvm.compiler.debug.DebugContext; +import org.graalvm.compiler.debug.GraalError; +import org.graalvm.compiler.graph.Node; +import org.graalvm.compiler.graph.NodeClass; +import org.graalvm.compiler.graph.NodeMap; +import org.graalvm.compiler.graph.NodeSourcePosition; +import org.graalvm.compiler.hotspot.meta.HotSpotForeignCallsProvider; +import org.graalvm.compiler.hotspot.meta.HotSpotProviders; +import org.graalvm.compiler.java.BytecodeParser; +import org.graalvm.compiler.java.GraphBuilderPhase; +import org.graalvm.compiler.nodeinfo.Verbosity; +import org.graalvm.compiler.nodes.CallTargetNode; +import org.graalvm.compiler.nodes.ConstantNode; +import org.graalvm.compiler.nodes.EncodedGraph; +import org.graalvm.compiler.nodes.FrameState; +import org.graalvm.compiler.nodes.FullInfopointNode; +import org.graalvm.compiler.nodes.GraphEncoder; +import org.graalvm.compiler.nodes.ParameterNode; +import org.graalvm.compiler.nodes.ProxyNode; +import org.graalvm.compiler.nodes.StructuredGraph; +import org.graalvm.compiler.nodes.ValueNode; +import org.graalvm.compiler.nodes.cfg.Block; +import org.graalvm.compiler.nodes.graphbuilderconf.GeneratedInvocationPlugin; +import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderConfiguration; +import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderContext; +import org.graalvm.compiler.nodes.graphbuilderconf.InlineInvokePlugin; +import org.graalvm.compiler.nodes.graphbuilderconf.IntrinsicContext; +import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugin; +import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugins; +import org.graalvm.compiler.nodes.graphbuilderconf.MethodSubstitutionPlugin; +import org.graalvm.compiler.nodes.graphbuilderconf.NodePlugin; +import org.graalvm.compiler.nodes.graphbuilderconf.ParameterPlugin; +import org.graalvm.compiler.nodes.java.AccessFieldNode; +import org.graalvm.compiler.nodes.java.MethodCallTargetNode; +import org.graalvm.compiler.nodes.spi.DelegatingReplacements; +import org.graalvm.compiler.nodes.spi.StampProvider; +import org.graalvm.compiler.nodes.virtual.VirtualObjectNode; +import org.graalvm.compiler.options.OptionValues; +import org.graalvm.compiler.phases.OptimisticOptimizations; +import org.graalvm.compiler.phases.schedule.SchedulePhase; +import org.graalvm.compiler.phases.util.Providers; +import org.graalvm.compiler.replacements.ConstantBindingParameterPlugin; +import org.graalvm.compiler.replacements.PEGraphDecoder; +import org.graalvm.compiler.replacements.ReplacementsImpl; +import org.graalvm.compiler.replacements.SnippetCounter; +import org.graalvm.compiler.replacements.SnippetIntegerHistogram; + +import jdk.vm.ci.code.Architecture; +import jdk.vm.ci.code.TargetDescription; +import jdk.vm.ci.hotspot.HotSpotJVMCIRuntime; +import jdk.vm.ci.hotspot.HotSpotResolvedJavaField; +import jdk.vm.ci.hotspot.HotSpotResolvedJavaMethod; +import jdk.vm.ci.hotspot.HotSpotResolvedJavaType; +import jdk.vm.ci.meta.Constant; +import jdk.vm.ci.meta.ConstantReflectionProvider; +import jdk.vm.ci.meta.JavaConstant; +import jdk.vm.ci.meta.JavaKind; +import jdk.vm.ci.meta.JavaType; +import jdk.vm.ci.meta.MemoryAccessProvider; +import jdk.vm.ci.meta.MetaAccessProvider; +import jdk.vm.ci.meta.MethodHandleAccessProvider; +import jdk.vm.ci.meta.ResolvedJavaField; +import jdk.vm.ci.meta.ResolvedJavaMethod; +import jdk.vm.ci.meta.ResolvedJavaType; +import jdk.vm.ci.meta.UnresolvedJavaField; +import jdk.vm.ci.meta.UnresolvedJavaMethod; +import jdk.vm.ci.meta.UnresolvedJavaType; + +/** + * This class performs graph encoding using {@link GraphEncoder} but also converts JVMCI type and + * method references into a symbolic form that can be resolved at graph decode time using + * {@link SymbolicJVMCIReference}. + */ +public class SymbolicSnippetEncoder extends DelegatingReplacements { + + /** + * This is a customized HotSpotReplacementsImpl intended only for parsing snippets and method + * substitutions for graph encoding. + */ + private final HotSpotSnippetReplacementsImpl replacements; + + /** + * The set of all snippet methods that have been encoded. + */ + private final Set snippetMethods = Collections.synchronizedSet(new HashSet<>()); + + /** + * A mapping from the method substitution method to the original method name. The string key and + * values are produced using {@link #methodKey(ResolvedJavaMethod)}. + */ + private final Map originalMethods = new ConcurrentHashMap<>(); + + /** + * The current count of graphs encoded. Used to detect when new graphs have been enqueued for + * encoding. + */ + int encodedGraphs = 0; + + /** + * All the graphs parsed so far. + */ + private Map preparedSnippetGraphs = new HashMap<>(); + + /** + * The invocation plugins which were delayed during graph preparation. + */ + private Set delayedInvocationPluginMethods = new HashSet<>(); + + void addDelayedInvocationPluginMethod(ResolvedJavaMethod method) { + delayedInvocationPluginMethods.add(method); + } + + Set getSnippetMethods() { + return snippetMethods; + } + + protected class SnippetInlineInvokePlugin implements InlineInvokePlugin { + + @Override + public InlineInfo shouldInlineInvoke(GraphBuilderContext b, ResolvedJavaMethod method, ValueNode[] args) { + if (method.getAnnotation(Fold.class) != null) { + delayedInvocationPluginMethods.add(method); + return InlineInfo.DO_NOT_INLINE_NO_EXCEPTION; + } + + if (getIntrinsifyingPlugin(method) != null) { + delayedInvocationPluginMethods.add(method); + return InlineInfo.DO_NOT_INLINE_NO_EXCEPTION; + } + + // Force inlining when parsing replacements + return createIntrinsicInlineInfo(method, null, getDefaultReplacementBytecodeProvider()); + } + + @Override + public void notifyAfterInline(ResolvedJavaMethod methodToInline) { + assert methodToInline.getAnnotation(Fold.class) == null : methodToInline; + } + } + + public static class SnippetInvocationPlugins extends InvocationPlugins { + + SnippetInvocationPlugins(InvocationPlugins invocationPlugins) { + super(invocationPlugins); + } + + @Override + public InvocationPlugin lookupInvocation(ResolvedJavaMethod method) { + if (method.getAnnotation(Fold.class) != null) { + return null; + } + return super.lookupInvocation(method); + } + } + + /** + * This plugin disables the snippet counter machinery. + */ + private class SnippetCounterPlugin implements NodePlugin { + String snippetCounterName = 'L' + SnippetCounter.class.getName().replace('.', '/') + ';'; + String snippetIntegerHistogramName = 'L' + SnippetIntegerHistogram.class.getName().replace('.', '/') + ';'; + + @Override + public boolean handleLoadField(GraphBuilderContext b, ValueNode object, ResolvedJavaField field) { + if (field.getName().equals("group") && field.getDeclaringClass().getName().equals(snippetCounterName)) { + b.addPush(JavaKind.Object, ConstantNode.forConstant(JavaConstant.NULL_POINTER, b.getMetaAccess())); + return true; + } + if (field.getType().getName().equals(snippetCounterName)) { + b.addPush(JavaKind.Object, ConstantNode.forConstant(replacements.snippetReflection.forObject(SnippetCounter.DISABLED_COUNTER), b.getMetaAccess())); + return true; + } + + if (field.getType().getName().equals(snippetIntegerHistogramName)) { + b.addPush(JavaKind.Object, ConstantNode.forConstant(replacements.snippetReflection.forObject(SnippetIntegerHistogram.DISABLED_COUNTER), b.getMetaAccess())); + return true; + } + return false; + } + } + + /** + * Generate a String name for a method including all type information. Used as a symbolic key + * for lookup. + */ + private static String methodKey(ResolvedJavaMethod method) { + return method.format("%f %H.%n(%P)"); + } + + SymbolicSnippetEncoder(HotSpotReplacementsImpl replacements) { + super(replacements); + + GraphBuilderConfiguration.Plugins plugins = replacements.getGraphBuilderPlugins(); + SnippetInvocationPlugins invocationPlugins = new SnippetInvocationPlugins(plugins.getInvocationPlugins()); + GraphBuilderConfiguration.Plugins copy = new GraphBuilderConfiguration.Plugins(plugins, invocationPlugins); + copy.clearInlineInvokePlugins(); + copy.appendInlineInvokePlugin(new SnippetInlineInvokePlugin()); + copy.appendNodePlugin(new SnippetCounterPlugin()); + HotSpotProviders providers = (HotSpotProviders) replacements.getProviders().copyWith(new HotSpotSubstrateConstantReflectionProvider(replacements.getProviders().getConstantReflection())); + this.replacements = new HotSpotSnippetReplacementsImpl(replacements, providers.copyWith(copy)); + this.replacements.setGraphBuilderPlugins(copy); + } + + @Override + public GraphBuilderConfiguration.Plugins getGraphBuilderPlugins() { + return replacements.getGraphBuilderPlugins(); + } + + /** + * Compiles the snippet and stores the graph. + */ + public void registerMethodSubstitution(ResolvedJavaMethod method, ResolvedJavaMethod original) { + assert method.getAnnotation(MethodSubstitution.class) != null : "MethodSubstitution must be annotated with @" + MethodSubstitution.class.getSimpleName(); + buildGraph(method, original, null, false, false); + snippetMethods.add(method); + } + + static class EncodedSnippets { + private byte[] snippetEncoding; + private Object[] snippetObjects; + private NodeClass[] snippetNodeClasses; + private Map snippetStartOffsets; + private Map originalMethods; + + EncodedSnippets(byte[] snippetEncoding, Object[] snippetObjects, NodeClass[] snippetNodeClasses, Map snippetStartOffsets, Map originalMethods) { + this.snippetEncoding = snippetEncoding; + this.snippetObjects = snippetObjects; + this.snippetNodeClasses = snippetNodeClasses; + this.snippetStartOffsets = snippetStartOffsets; + this.originalMethods = originalMethods; + } + + public StructuredGraph getMethodSubstitutionGraph(MethodSubstitutionPlugin plugin, ResolvedJavaMethod original, ReplacementsImpl replacements) { + Integer startOffset = snippetStartOffsets.get(plugin.toString()); + if (startOffset == null) { + throw GraalError.shouldNotReachHere("plugin graph not found: " + plugin); + } + + return decodeGraph(original, null, startOffset, replacements); + } + + @SuppressWarnings("try") + private StructuredGraph decodeGraph(ResolvedJavaMethod method, Object[] args, int startOffset, ReplacementsImpl replacements) { + OptionValues options = replacements.getOptions(); + SnippetReflectionProvider snippetReflection = replacements.snippetReflection; + ParameterPlugin parameterPlugin = null; + Providers providers = replacements.getProviders(); + if (args != null) { + parameterPlugin = new ConstantBindingParameterPlugin(args, providers.getMetaAccess(), snippetReflection); + } + + EncodedGraph encodedGraph = new SymbolicEncodedGraph(snippetEncoding, startOffset, snippetObjects, snippetNodeClasses, method.getDeclaringClass(), + originalMethods.get(methodKey(method))); + try (DebugContext debug = replacements.openDebugContext("SVMSnippet_", method)) { + StructuredGraph result = new StructuredGraph.Builder(options, debug).method(method).setIsSubstitution(true).build(); + PEGraphDecoder graphDecoder = new PEGraphDecoder( + providers.getCodeCache().getTarget().arch, + result, + providers.getMetaAccess(), + providers.getConstantReflection(), + providers.getConstantFieldProvider(), + providers.getStampProvider(), + null, // loopExplosionPlugin + replacements.getGraphBuilderPlugins().getInvocationPlugins(), + new InlineInvokePlugin[0], + parameterPlugin, + null, // nodePlugins + null, // callInlinedMethod + null // sourceLanguagePositionProvider + ) { + @Override + protected EncodedGraph lookupEncodedGraph(ResolvedJavaMethod lookupMethod, + ResolvedJavaMethod originalMethod, + BytecodeProvider intrinsicBytecodeProvider, + boolean isSubstitution, + boolean trackNodeSourcePosition) { + if (lookupMethod.equals(method)) { + return encodedGraph; + } else { + throw GraalError.shouldNotReachHere(method.format("%H.%n(%p)")); + } + } + }; + + graphDecoder.decode(method, result.isSubstitution(), encodedGraph.trackNodeSourcePosition()); + + assert result.verify(); + return result; + } + } + + StructuredGraph getEncodedSnippet(ResolvedJavaMethod method, ReplacementsImpl replacements, Object[] args) { + Integer startOffset = null; + if (snippetStartOffsets != null) { + startOffset = snippetStartOffsets.get(methodKey(method)); + } + if (startOffset == null) { + if (IS_IN_NATIVE_IMAGE) { + throw GraalError.shouldNotReachHere("snippet not found: " + method.format("%H.%n(%p)")); + } else { + return null; + } + } + + SymbolicEncodedGraph encodedGraph = new SymbolicEncodedGraph(snippetEncoding, startOffset, snippetObjects, snippetNodeClasses, method.getDeclaringClass(), + originalMethods.get(methodKey(method))); + return decodeSnippetGraph(encodedGraph, method, replacements, args, HotSpotJVMCIRuntime.runtime().getHostJVMCIBackend().getTarget().arch); + } + + } + + private StructuredGraph buildGraph(ResolvedJavaMethod method, ResolvedJavaMethod original, Object receiver, boolean requireInlining, boolean trackNodeSourcePosition) { + assert method.hasBytecodes() : "Snippet must not be abstract or native"; + Object[] args = null; + if (receiver != null) { + args = new Object[method.getSignature().getParameterCount(true)]; + args[0] = receiver; + } + try (DebugContext debug = openDebugContext("Snippet_", method)) { + StructuredGraph graph = replacements.makeGraph(debug, replacements.getDefaultReplacementBytecodeProvider(), method, args, original, trackNodeSourcePosition, null); + + // Check if all methods which should be inlined are really inlined. + for (MethodCallTargetNode callTarget : graph.getNodes(MethodCallTargetNode.TYPE)) { + ResolvedJavaMethod callee = callTarget.targetMethod(); + if (requireInlining && !delayedInvocationPluginMethods.contains(callee) && !Objects.equals(callee, original)) { + throw GraalError.shouldNotReachHere("method " + callee.format("%H.%n") + " not inlined in snippet " + method.getName() + " (maybe not final?)"); + } + } + assert verifySnippetEncodeDecode(method, original, trackNodeSourcePosition, graph); + debug.dump(DebugContext.VERBOSE_LEVEL, graph, "After buildGraph"); + return graph; + } + } + + @SuppressWarnings("try") + static StructuredGraph decodeSnippetGraph(SymbolicEncodedGraph encodedGraph, ResolvedJavaMethod method, ReplacementsImpl replacements, Object[] args, Architecture architecture) { + Providers providers = replacements.getProviders(); + ParameterPlugin parameterPlugin = null; + if (args != null) { + parameterPlugin = new ConstantBindingParameterPlugin(args, providers.getMetaAccess(), replacements.snippetReflection); + } + + try (DebugContext debug = replacements.openDebugContext("SVMSnippet_", method)) { + // @formatter:off + StructuredGraph result = new StructuredGraph.Builder(replacements.getOptions(), debug) + .method(method) + .trackNodeSourcePosition(encodedGraph.trackNodeSourcePosition()) + .setIsSubstitution(true) + .build(); + // @formatter:on + try (DebugContext.Scope scope = debug.scope("DecodeSnippetGraph", result)) { + PEGraphDecoder graphDecoder = new PEGraphDecoder( + architecture, + result, + providers.getMetaAccess(), + providers.getConstantReflection(), + providers.getConstantFieldProvider(), + providers.getStampProvider(), + null, + replacements.getGraphBuilderPlugins().getInvocationPlugins(), + new InlineInvokePlugin[0], + parameterPlugin, + null, + null, + null) { + @Override + protected EncodedGraph lookupEncodedGraph( + ResolvedJavaMethod lookupMethod, + ResolvedJavaMethod originalMethod, + BytecodeProvider intrinsicBytecodeProvider, + boolean isSubstitution, + boolean track) { + if (lookupMethod.equals(method)) { + assert !track || encodedGraph.trackNodeSourcePosition(); + return encodedGraph; + } else { + throw GraalError.shouldNotReachHere(method.format("%H.%n(%p)")); + } + } + }; + + graphDecoder.decode(method, result.isSubstitution(), encodedGraph.trackNodeSourcePosition()); + debug.dump(DebugContext.VERBOSE_LEVEL, result, "After decoding"); + + assert result.verify(); + return result; + } catch (Throwable t) { + throw debug.handle(t); + } + } + } + + @SuppressWarnings("try") + private boolean verifySnippetEncodeDecode(ResolvedJavaMethod method, ResolvedJavaMethod original, boolean trackNodeSourcePosition, StructuredGraph structuredGraph) { + // Verify the encoding and decoding process + EncodedGraph encodedGraph = GraphEncoder.encodeSingleGraph(structuredGraph, HotSpotJVMCIRuntime.runtime().getHostJVMCIBackend().getTarget().arch); + + Architecture arch = HotSpotJVMCIRuntime.runtime().getHostJVMCIBackend().getTarget().arch; + + try (DebugContext debug = replacements.openDebugContext("VerifySnippetEncodeDecode_", method)) { + HotSpotProviders originalProvider = (HotSpotProviders) replacements.getProviders(); + + SnippetReflectionProvider snippetReflection = originalProvider.getSnippetReflection(); + SymbolicSnippetEncoder.HotSpotSubstrateConstantReflectionProvider constantReflection = new SymbolicSnippetEncoder.HotSpotSubstrateConstantReflectionProvider( + originalProvider.getConstantReflection()); + HotSpotProviders newProviders = new HotSpotProviders(originalProvider.getMetaAccess(), originalProvider.getCodeCache(), constantReflection, + originalProvider.getConstantFieldProvider(), originalProvider.getForeignCalls(), originalProvider.getLowerer(), null, originalProvider.getSuites(), + originalProvider.getRegisters(), snippetReflection, originalProvider.getWordTypes(), originalProvider.getGraphBuilderPlugins()); + HotSpotSnippetReplacementsImpl filteringReplacements = new HotSpotSnippetReplacementsImpl(getOptions(), newProviders, snippetReflection, + originalProvider.getReplacements().getDefaultReplacementBytecodeProvider(), + originalProvider.getCodeCache().getTarget()); + filteringReplacements.setGraphBuilderPlugins(originalProvider.getReplacements().getGraphBuilderPlugins()); + try (DebugContext.Scope scaope = debug.scope("VerifySnippetEncodeDecode", structuredGraph)) { + for (int i = 0; i < encodedGraph.getNumObjects(); i++) { + filterSnippetObject(encodedGraph.getObject(i)); + } + StructuredGraph snippet = filteringReplacements.makeGraph(debug, filteringReplacements.getDefaultReplacementBytecodeProvider(), method, null, original, + trackNodeSourcePosition, null); + SymbolicEncodedGraph symbolicGraph = new SymbolicEncodedGraph(encodedGraph, method.getDeclaringClass(), original != null ? methodKey(original) : null); + StructuredGraph decodedSnippet = decodeSnippetGraph(symbolicGraph, method, replacements, null, arch); + String snippetString = getCanonicalGraphString(snippet, true, false); + String decodedSnippetString = getCanonicalGraphString(decodedSnippet, true, false); + if (snippetString.equals(decodedSnippetString)) { + debug.log("Snippet decode for %s produces exactly same graph", method); + debug.dump(DebugContext.INFO_LEVEL, decodedSnippet, "Decoded snippet graph for %s", method); + } else { + debug.log("Snippet decode for %s produces different graph", method); + debug.log("%s", compareGraphStrings(snippet, snippetString, decodedSnippet, decodedSnippetString)); + debug.dump(DebugContext.INFO_LEVEL, snippet, "Snippet graph for %s", method); + debug.dump(DebugContext.INFO_LEVEL, structuredGraph, "Encoded snippet graph for %s", method); + debug.dump(DebugContext.INFO_LEVEL, decodedSnippet, "Decoded snippet graph for %s", method); + } + } catch (Throwable t) { + throw debug.handle(t); + } + } + return true; + } + + /** + * If there are new graphs waiting to be encoded, reencode all the graphs and return the result. + */ + @SuppressWarnings("try") + synchronized EncodedSnippets maybeEncodeSnippets() { + Map graphs = this.preparedSnippetGraphs; + if (encodedGraphs != graphs.size()) { + DebugContext debug = openDebugContext("SnippetEncoder", null); + try (DebugContext.Scope scope = debug.scope("SnippetSupportEncode")) { + encodedGraphs = graphs.size(); + for (StructuredGraph graph : graphs.values()) { + for (Node node : graph.getNodes()) { + node.setNodeSourcePosition(null); + } + } + return encodeSnippets(debug); + } + } + return null; + } + + @Override + public void registerSnippet(ResolvedJavaMethod method, ResolvedJavaMethod original, Object receiver, boolean trackNodeSourcePosition) { + if (IS_BUILDING_NATIVE_IMAGE || UseEncodedSnippets.getValue(getOptions())) { + assert method.getAnnotation(Snippet.class) != null : "Snippet must be annotated with @" + Snippet.class.getSimpleName(); + String key = methodKey(method); + if (!preparedSnippetGraphs.containsKey(key)) { + if (original != null) { + originalMethods.put(key, methodKey(original)); + } + StructuredGraph snippet = buildGraph(method, original, receiver, true, trackNodeSourcePosition); + snippetMethods.add(method); + preparedSnippetGraphs.put(key, snippet); + } + } + } + + EncodedSnippets encodeSnippets(DebugContext debug) { + GraphEncoder encoder = new GraphEncoder(HotSpotJVMCIRuntime.runtime().getHostJVMCIBackend().getTarget().arch, debug); + for (StructuredGraph graph : preparedSnippetGraphs.values()) { + encoder.prepare(graph); + } + encoder.finishPrepare(); + + byte[] snippetEncoding; + Object[] snippetObjects; + NodeClass[] snippetNodeClasses; + Map snippetStartOffsets; + + snippetStartOffsets = new HashMap<>(); + for (Map.Entry entry : preparedSnippetGraphs.entrySet()) { + snippetStartOffsets.put(entry.getKey(), encoder.encode(entry.getValue())); + } + snippetEncoding = encoder.getEncoding(); + snippetObjects = encoder.getObjects(); + snippetNodeClasses = encoder.getNodeClasses(); + for (int i = 0; i < snippetObjects.length; i++) { + Object o = filterSnippetObject(snippetObjects[i]); + debug.log("snippetObjects[%d] = %s -> %s", i, o != null ? o.getClass().getSimpleName() : null, o); + snippetObjects[i] = o; + } + debug.log("Encoded %d snippet preparedSnippetGraphs using %d bytes with %d objects", snippetStartOffsets.size(), snippetEncoding.length, snippetObjects.length); + return new EncodedSnippets(snippetEncoding, snippetObjects, snippetNodeClasses, snippetStartOffsets, originalMethods); + } + + /** + * Encode any outstanding graphs and return true if any work was done. + */ + @SuppressWarnings("try") + public boolean encode() { + EncodedSnippets encodedSnippets = maybeEncodeSnippets(); + if (encodedSnippets != null) { + HotSpotReplacementsImpl.setEncodedSnippets(encodedSnippets); + return true; + } + return false; + } + + private DebugContext openDebugContext(String idPrefix, ResolvedJavaMethod method) { + return replacements.openDebugContext(idPrefix, method); + } + + static class SymbolicEncodedGraph extends EncodedGraph { + + private final ResolvedJavaType accessingClass; + private final String originalMethod; + + SymbolicEncodedGraph(byte[] encoding, int startOffset, Object[] objects, NodeClass[] types, ResolvedJavaType accessingClass, String originalMethod) { + super(encoding, startOffset, objects, types, null, null, null, false, false); + this.accessingClass = accessingClass; + this.originalMethod = originalMethod; + } + + SymbolicEncodedGraph(EncodedGraph encodedGraph, ResolvedJavaType declaringClass, String originalMethod) { + this(encodedGraph.getEncoding(), encodedGraph.getStartOffset(), encodedGraph.getObjects(), encodedGraph.getNodeClasses(), declaringClass, originalMethod); + } + + @Override + public Object getObject(int i) { + Object o = objects[i]; + if (o instanceof SymbolicJVMCIReference) { + objects[i] = o = ((SymbolicJVMCIReference) o).resolve(accessingClass); + } else if (o instanceof UnresolvedJavaType) { + objects[i] = o = ((UnresolvedJavaType) o).resolve(accessingClass); + } else if (o instanceof UnresolvedJavaMethod) { + throw new InternalError(o.toString()); + } else if (o instanceof UnresolvedJavaField) { + objects[i] = o = ((UnresolvedJavaField) o).resolve(accessingClass); + } else if (o instanceof GraalCapability) { + objects[i] = o = ((GraalCapability) o).resolve(((GraalJVMCICompiler) getRuntime().getCompiler()).getGraalRuntime()); + } + return o; + } + + @Override + public boolean isCallToOriginal(ResolvedJavaMethod callTarget) { + if (originalMethod != null && originalMethod.equals(methodKey(callTarget))) { + return true; + } + return super.isCallToOriginal(callTarget); + } + } + + /** + * Symbolic reference to an object which can be retrieved from + * {@link GraalRuntime#getCapability(Class)}. + */ + static class GraalCapability { + final Class capabilityClass; + + GraalCapability(Class capabilityClass) { + this.capabilityClass = capabilityClass; + } + + public Object resolve(GraalRuntime runtime) { + Object capability = runtime.getCapability(this.capabilityClass); + if (capability != null) { + assert capability.getClass() == capabilityClass; + return capability; + } + throw new InternalError(this.capabilityClass.getName()); + } + } + + static class SymbolicResolvedJavaMethod implements SymbolicJVMCIReference { + final UnresolvedJavaType type; + final String methodName; + final String signature; + + SymbolicResolvedJavaMethod(HotSpotResolvedJavaMethod method) { + this.type = UnresolvedJavaType.create(method.getDeclaringClass().getName()); + this.methodName = method.getName(); + this.signature = method.getSignature().toMethodDescriptor(); + } + + @Override + public String toString() { + return "SymbolicResolvedJavaMethod{" + + "declaringType='" + type.getName() + '\'' + + ", methodName='" + methodName + '\'' + + ", signature='" + signature + '\'' + + '}'; + } + + @Override + public ResolvedJavaMethod resolve(ResolvedJavaType accessingClass) { + ResolvedJavaType resolvedType = type.resolve(accessingClass); + for (ResolvedJavaMethod method : methodName.equals("") ? resolvedType.getDeclaredConstructors() : resolvedType.getDeclaredMethods()) { + if (method.getName().equals(methodName) && method.getSignature().toMethodDescriptor().equals(signature)) { + return method; + } + } + throw new InternalError("Could not resolve " + this + " in context of " + accessingClass.toJavaName()); + } + } + + static class SymbolicResolvedJavaField implements SymbolicJVMCIReference { + final UnresolvedJavaType declaringType; + final String name; + final UnresolvedJavaType signature; + private final boolean isStatic; + + SymbolicResolvedJavaField(HotSpotResolvedJavaField field) { + this.declaringType = UnresolvedJavaType.create(field.getDeclaringClass().getName()); + this.name = field.getName(); + this.signature = UnresolvedJavaType.create(field.getType().getName()); + this.isStatic = field.isStatic(); + } + + @Override + public ResolvedJavaField resolve(ResolvedJavaType accessingClass) { + ResolvedJavaType resolvedType = declaringType.resolve(accessingClass); + ResolvedJavaType resolvedFieldType = signature.resolve(accessingClass); + ResolvedJavaField[] fields = isStatic ? resolvedType.getStaticFields() : resolvedType.getInstanceFields(true); + for (ResolvedJavaField field : fields) { + if (field.getName().equals(name)) { + if (field.getType().equals(resolvedFieldType)) { + return field; + } + } + } + throw new InternalError("Could not resolve " + this + " in context of " + accessingClass.toJavaName()); + } + + @Override + public String toString() { + return "SymbolicResolvedJavaField{" + + signature.getName() + ' ' + + declaringType.getName() + '.' + + name + + '}'; + } + } + + static class SymbolicStampPair implements SymbolicJVMCIReference { + Object trustedStamp; + Object uncheckdStamp; + + SymbolicStampPair(StampPair stamp) { + this.trustedStamp = maybeMakeSymbolic(stamp.getTrustedStamp()); + this.uncheckdStamp = maybeMakeSymbolic(stamp.getUncheckedStamp()); + } + + @Override + public StampPair resolve(ResolvedJavaType accessingClass) { + return StampPair.create(resolveStamp(accessingClass, trustedStamp), resolveStamp(accessingClass, uncheckdStamp)); + } + } + + private static Object maybeMakeSymbolic(Stamp trustedStamp) { + if (trustedStamp != null) { + SymbolicJVMCIReference symbolicJVMCIReference = trustedStamp.makeSymbolic(); + if (symbolicJVMCIReference != null) { + return symbolicJVMCIReference; + } + } + return trustedStamp; + } + + private static Stamp resolveStamp(ResolvedJavaType accessingClass, Object stamp) { + if (stamp == null) { + return null; + } + if (stamp instanceof Stamp) { + return (Stamp) stamp; + } + return (Stamp) ((SymbolicJVMCIReference) stamp).resolve(accessingClass); + } + + public static class HotSpotSubstrateConstantReflectionProvider implements ConstantReflectionProvider { + + private final ConstantReflectionProvider constantReflection; + + HotSpotSubstrateConstantReflectionProvider(ConstantReflectionProvider constantReflection) { + this.constantReflection = constantReflection; + } + + HashSet safeConstants = new HashSet<>(); + + @Override + public Boolean constantEquals(Constant x, Constant y) { + return constantReflection.constantEquals(x, y); + } + + @Override + public Integer readArrayLength(JavaConstant array) { + return constantReflection.readArrayLength(array); + } + + @Override + public JavaConstant readArrayElement(JavaConstant array, int index) { + return constantReflection.readArrayElement(array, index); + } + + @Override + public JavaConstant readFieldValue(ResolvedJavaField field, JavaConstant receiver) { + JavaConstant javaConstant = constantReflection.readFieldValue(field, receiver); + if (!safeConstants.contains(receiver) && !field.getDeclaringClass().getName().contains("graalvm") && !field.getDeclaringClass().getName().contains("jdk/vm/ci/") && + !field.getName().equals("TYPE")) { + // Only permit constant reflection on compiler classes. This is necessary primarily + // because of the boxing snippets which are compiled as snippets but are really just + // regular JDK java sources that are being compiled like a snippet. These shouldn't + // permit constant folding during graph preparation as that embeds constants from + // the runtime into a compiler graph. + return null; + } + if (javaConstant.getJavaKind() == JavaKind.Object) { + safeConstants.add(javaConstant); + } + return javaConstant; + } + + @Override + public JavaConstant boxPrimitive(JavaConstant source) { + return constantReflection.boxPrimitive(source); + } + + @Override + public JavaConstant unboxPrimitive(JavaConstant source) { + return constantReflection.unboxPrimitive(source); + } + + @Override + public JavaConstant forString(String value) { + return constantReflection.forString(value); + } + + @Override + public ResolvedJavaType asJavaType(Constant constant) { + return constantReflection.asJavaType(constant); + } + + @Override + public MethodHandleAccessProvider getMethodHandleAccess() { + return constantReflection.getMethodHandleAccess(); + } + + @Override + public MemoryAccessProvider getMemoryAccessProvider() { + return constantReflection.getMemoryAccessProvider(); + } + + @Override + public JavaConstant asJavaClass(ResolvedJavaType type) { + return constantReflection.asJavaClass(type); + } + + @Override + public Constant asObjectHub(ResolvedJavaType type) { + return constantReflection.asObjectHub(type); + } + } + + /** + * Objects embedded in encoded graphs might need to converted into a symbolic form so convert + * the object or pass it through. + */ + static Object filterSnippetObject(Object o) { + if (o instanceof HotSpotResolvedJavaMethod) { + return new SymbolicResolvedJavaMethod((HotSpotResolvedJavaMethod) o); + } else if (o instanceof HotSpotResolvedJavaField) { + return new SymbolicResolvedJavaField((HotSpotResolvedJavaField) o); + } else if (o instanceof HotSpotResolvedJavaType) { + return UnresolvedJavaType.create(((HotSpotResolvedJavaType) o).getName()); + } else if (o instanceof NodeSourcePosition) { + // Filter these out for now. These can't easily be handled because these positions + // description snippet methods which might not be available in the runtime. + return null; + } else if (o instanceof HotSpotForeignCallsProvider || o instanceof GraalHotSpotVMConfig) { + return new GraalCapability(o.getClass()); + } else if (o instanceof Stamp) { + SymbolicJVMCIReference ref = ((Stamp) o).makeSymbolic(); + if (ref != null) { + return ref; + } + return o; + } else if (o instanceof StampPair) { + if (((StampPair) o).getTrustedStamp() instanceof AbstractObjectStamp) { + return new SymbolicStampPair((StampPair) o); + } + } + return o; + } + + static String compareGraphStrings(StructuredGraph expectedGraph, String expectedString, StructuredGraph actualGraph, String actualString) { + if (!expectedString.equals(actualString)) { + String[] expectedLines = expectedString.split("\n"); + String[] actualLines = actualString.split("\n"); + int diffIndex = -1; + int limit = Math.min(actualLines.length, expectedLines.length); + String marker = " <<<"; + for (int i = 0; i < limit; i++) { + if (!expectedLines[i].equals(actualLines[i])) { + diffIndex = i; + break; + } + } + if (diffIndex == -1) { + // Prefix is the same so add some space after the prefix + diffIndex = limit; + if (actualLines.length == limit) { + actualLines = Arrays.copyOf(actualLines, limit + 1); + actualLines[diffIndex] = ""; + } else { + assert expectedLines.length == limit; + expectedLines = Arrays.copyOf(expectedLines, limit + 1); + expectedLines[diffIndex] = ""; + } + } + // Place a marker next to the first line that differs + expectedLines[diffIndex] = expectedLines[diffIndex] + marker; + actualLines[diffIndex] = actualLines[diffIndex] + marker; + String ediff = String.join("\n", expectedLines); + String adiff = String.join("\n", actualLines); + return "mismatch in preparedSnippetGraphs:\n========= expected (" + expectedGraph + ") =========\n" + ediff + "\n\n========= actual (" + actualGraph + ") =========\n" + adiff; + } else { + return "mismatch in preparedSnippetGraphs"; + } + } + + static String getCanonicalGraphString(StructuredGraph graph, boolean excludeVirtual, boolean checkConstants) { + SchedulePhase schedule = new SchedulePhase(SchedulePhase.SchedulingStrategy.EARLIEST); + schedule.apply(graph); + StructuredGraph.ScheduleResult scheduleResult = graph.getLastSchedule(); + + NodeMap canonicalId = graph.createNodeMap(); + int nextId = 0; + + List constantsLines = new ArrayList<>(); + + StringBuilder result = new StringBuilder(); + for (Block block : scheduleResult.getCFG().getBlocks()) { + result.append("Block ").append(block).append(' '); + if (block == scheduleResult.getCFG().getStartBlock()) { + result.append("* "); + } + result.append("-> "); + for (Block succ : block.getSuccessors()) { + result.append(succ).append(' '); + } + result.append('\n'); + for (Node node : scheduleResult.getBlockToNodesMap().get(block)) { + if (node instanceof ValueNode && node.isAlive()) { + if (!excludeVirtual || !(node instanceof VirtualObjectNode || node instanceof ProxyNode || node instanceof FullInfopointNode || node instanceof ParameterNode)) { + if (node instanceof ConstantNode) { + if (checkConstants) { + String name = node.toString(Verbosity.Name); + if (excludeVirtual) { + constantsLines.add(name); + } else { + constantsLines.add(name + " (" + filteredUsageCount(node) + ")"); + } + } + } else { + int id; + if (canonicalId.get(node) != null) { + id = canonicalId.get(node); + } else { + id = nextId++; + canonicalId.set(node, id); + } + String name = node.getClass().getSimpleName(); + result.append(" ").append(id).append('|').append(name); + if (node instanceof AccessFieldNode) { + result.append('#'); + result.append(((AccessFieldNode) node).field()); + } + if (!excludeVirtual) { + result.append(" ("); + result.append(filteredUsageCount(node)); + result.append(')'); + } + result.append('\n'); + } + } + } + } + } + + StringBuilder constantsLinesResult = new StringBuilder(); + if (checkConstants) { + constantsLinesResult.append(constantsLines.size()).append(" constants:\n"); + } + Collections.sort(constantsLines); + for (String s : constantsLines) { + constantsLinesResult.append(s); + constantsLinesResult.append('\n'); + } + + return constantsLinesResult.toString() + result.toString(); + } + + private static int filteredUsageCount(Node node) { + return node.usages().filter(n -> !(n instanceof FrameState)).count(); + } + + /** + * This horror show of classes exists solely get {@link HotSpotSnippetBytecodeParser} to be used + * as the parser for these snippets. + */ + static class HotSpotSnippetReplacementsImpl extends HotSpotReplacementsImpl { + HotSpotSnippetReplacementsImpl(HotSpotReplacementsImpl replacements, Providers providers) { + super(replacements, providers); + } + + HotSpotSnippetReplacementsImpl(OptionValues options, Providers providers, SnippetReflectionProvider snippetReflection, BytecodeProvider bytecodeProvider, TargetDescription target) { + super(options, providers, snippetReflection, bytecodeProvider, target); + } + + @Override + protected GraphMaker createGraphMaker(ResolvedJavaMethod substitute, ResolvedJavaMethod original) { + return new SnippetGraphMaker(this, substitute, original); + } + } + + static class SnippetGraphMaker extends ReplacementsImpl.GraphMaker { + SnippetGraphMaker(ReplacementsImpl replacements, ResolvedJavaMethod substitute, ResolvedJavaMethod substitutedMethod) { + super(replacements, substitute, substitutedMethod); + } + + @Override + protected GraphBuilderPhase.Instance createGraphBuilder(MetaAccessProvider metaAccess, StampProvider stampProvider, ConstantReflectionProvider constantReflection, + ConstantFieldProvider constantFieldProvider, GraphBuilderConfiguration graphBuilderConfig, OptimisticOptimizations optimisticOpts, IntrinsicContext initialIntrinsicContext) { + return new HotSpotSnippetGraphBuilderPhase(metaAccess, stampProvider, constantReflection, constantFieldProvider, graphBuilderConfig, optimisticOpts, + initialIntrinsicContext); + } + } + + static class HotSpotSnippetGraphBuilderPhase extends GraphBuilderPhase.Instance { + HotSpotSnippetGraphBuilderPhase(MetaAccessProvider metaAccess, StampProvider stampProvider, ConstantReflectionProvider constantReflection, ConstantFieldProvider constantFieldProvider, + GraphBuilderConfiguration graphBuilderConfig, OptimisticOptimizations optimisticOpts, IntrinsicContext initialIntrinsicContext) { + super(metaAccess, stampProvider, constantReflection, constantFieldProvider, graphBuilderConfig, optimisticOpts, initialIntrinsicContext); + } + + @Override + protected BytecodeParser createBytecodeParser(StructuredGraph graph, BytecodeParser parent, ResolvedJavaMethod method, int entryBCI, IntrinsicContext intrinsicContext) { + return new HotSpotSnippetBytecodeParser(this, graph, parent, method, entryBCI, intrinsicContext); + } + } + + static class HotSpotSnippetBytecodeParser extends BytecodeParser { + HotSpotSnippetBytecodeParser(GraphBuilderPhase.Instance graphBuilderInstance, StructuredGraph graph, BytecodeParser parent, ResolvedJavaMethod method, int entryBCI, + IntrinsicContext intrinsicContext) { + super(graphBuilderInstance, graph, parent, method, entryBCI, intrinsicContext); + } + + @Override + public boolean canDeferPlugin(GeneratedInvocationPlugin plugin) { + return plugin.getSource().equals(Fold.class) || plugin.getSource().equals(Node.NodeIntrinsic.class); + } + + @Override + protected boolean tryInvocationPlugin(CallTargetNode.InvokeKind invokeKind, ValueNode[] args, ResolvedJavaMethod targetMethod, JavaKind resultType, JavaType returnType) { + if (intrinsicContext != null && intrinsicContext.isCallToOriginal(targetMethod)) { + return false; + } + return super.tryInvocationPlugin(invokeKind, args, targetMethod, resultType, returnType); + } + } +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/gc/g1/G1ArrayRangePostWriteBarrier.java 2019-03-12 08:10:36.555968162 +0100 @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2013, 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + + +package org.graalvm.compiler.hotspot.gc.g1; + +import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_64; +import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_64; + +import org.graalvm.compiler.graph.NodeClass; +import org.graalvm.compiler.hotspot.gc.shared.ArrayRangeWriteBarrier; +import org.graalvm.compiler.nodeinfo.NodeInfo; +import org.graalvm.compiler.nodes.ValueNode; +import org.graalvm.compiler.nodes.memory.address.AddressNode; + +@NodeInfo(cycles = CYCLES_64, size = SIZE_64) +public class G1ArrayRangePostWriteBarrier extends ArrayRangeWriteBarrier { + public static final NodeClass TYPE = NodeClass.create(G1ArrayRangePostWriteBarrier.class); + + public G1ArrayRangePostWriteBarrier(AddressNode address, ValueNode length, int elementStride) { + super(TYPE, address, length, elementStride); + } + +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/gc/g1/G1ArrayRangePreWriteBarrier.java 2019-03-12 08:10:37.055971397 +0100 @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2013, 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + + +package org.graalvm.compiler.hotspot.gc.g1; + +import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_64; +import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_64; + +import org.graalvm.compiler.graph.NodeClass; +import org.graalvm.compiler.hotspot.gc.shared.ArrayRangeWriteBarrier; +import org.graalvm.compiler.nodeinfo.NodeInfo; +import org.graalvm.compiler.nodes.ValueNode; +import org.graalvm.compiler.nodes.memory.address.AddressNode; + +@NodeInfo(cycles = CYCLES_64, size = SIZE_64) +public final class G1ArrayRangePreWriteBarrier extends ArrayRangeWriteBarrier { + public static final NodeClass TYPE = NodeClass.create(G1ArrayRangePreWriteBarrier.class); + + public G1ArrayRangePreWriteBarrier(AddressNode address, ValueNode length, int elementStride) { + super(TYPE, address, length, elementStride); + } + +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/gc/g1/G1BarrierSet.java 2019-03-12 08:10:37.551974605 +0100 @@ -0,0 +1,135 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2019, Red Hat Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.hotspot.gc.g1; + +import org.graalvm.compiler.debug.GraalError; +import org.graalvm.compiler.hotspot.gc.shared.BarrierSet; +import org.graalvm.compiler.nodes.StructuredGraph; +import org.graalvm.compiler.nodes.ValueNode; +import org.graalvm.compiler.nodes.extended.ArrayRangeWrite; +import org.graalvm.compiler.nodes.java.AbstractCompareAndSwapNode; +import org.graalvm.compiler.nodes.java.LoweredAtomicReadAndWriteNode; +import org.graalvm.compiler.nodes.memory.FixedAccessNode; +import org.graalvm.compiler.nodes.memory.HeapAccess; +import org.graalvm.compiler.nodes.memory.ReadNode; +import org.graalvm.compiler.nodes.memory.WriteNode; +import org.graalvm.compiler.nodes.memory.address.AddressNode; +import org.graalvm.compiler.nodes.type.StampTool; + +public class G1BarrierSet extends BarrierSet { + + @Override + public void addReadNodeBarriers(ReadNode node, StructuredGraph graph) { + if (node.getBarrierType() != HeapAccess.BarrierType.NONE) { + assert (node.getBarrierType() == HeapAccess.BarrierType.PRECISE); + G1ReferentFieldReadBarrier barrier = graph.add(new G1ReferentFieldReadBarrier(node.getAddress(), node, false)); + graph.addAfterFixed(node, barrier); + } + } + + @Override + public void addWriteNodeBarriers(WriteNode node, StructuredGraph graph) { + HeapAccess.BarrierType barrierType = node.getBarrierType(); + switch (barrierType) { + case NONE: + // nothing to do + break; + case IMPRECISE: + case PRECISE: + boolean precise = barrierType == HeapAccess.BarrierType.PRECISE; + if (!node.getLocationIdentity().isInit()) { + // The pre barrier does nothing if the value being read is null, so it can + // be explicitly skipped when this is an initializing store. + addG1PreWriteBarrier(node, node.getAddress(), null, true, node.getNullCheck(), graph); + } + addG1PostWriteBarrier(node, node.getAddress(), node.value(), precise, graph); + break; + default: + throw new GraalError("unexpected barrier type: " + barrierType); + } + } + + @Override + public void addAtomicReadWriteNodeBarriers(LoweredAtomicReadAndWriteNode node, StructuredGraph graph) { + HeapAccess.BarrierType barrierType = node.getBarrierType(); + switch (barrierType) { + case NONE: + // nothing to do + break; + case IMPRECISE: + case PRECISE: + boolean precise = barrierType == HeapAccess.BarrierType.PRECISE; + addG1PreWriteBarrier(node, node.getAddress(), null, true, node.getNullCheck(), graph); + addG1PostWriteBarrier(node, node.getAddress(), node.getNewValue(), precise, graph); + break; + default: + throw new GraalError("unexpected barrier type: " + barrierType); + } + } + + @Override + public void addCASBarriers(AbstractCompareAndSwapNode node, StructuredGraph graph) { + HeapAccess.BarrierType barrierType = node.getBarrierType(); + switch (barrierType) { + case NONE: + // nothing to do + break; + case IMPRECISE: + case PRECISE: + boolean precise = barrierType == HeapAccess.BarrierType.PRECISE; + addG1PreWriteBarrier(node, node.getAddress(), node.getExpectedValue(), false, false, graph); + addG1PostWriteBarrier(node, node.getAddress(), node.getNewValue(), precise, graph); + break; + default: + throw new GraalError("unexpected barrier type: " + barrierType); + } + } + + @Override + public void addArrayRangeBarriers(ArrayRangeWrite write, StructuredGraph graph) { + if (!write.isInitialization()) { + // The pre barrier does nothing if the value being read is null, so it can + // be explicitly skipped when this is an initializing store. + G1ArrayRangePreWriteBarrier g1ArrayRangePreWriteBarrier = graph.add(new G1ArrayRangePreWriteBarrier(write.getAddress(), write.getLength(), write.getElementStride())); + graph.addBeforeFixed(write.asNode(), g1ArrayRangePreWriteBarrier); + } + G1ArrayRangePostWriteBarrier g1ArrayRangePostWriteBarrier = graph.add(new G1ArrayRangePostWriteBarrier(write.getAddress(), write.getLength(), write.getElementStride())); + graph.addAfterFixed(write.asNode(), g1ArrayRangePostWriteBarrier); + } + + private static void addG1PreWriteBarrier(FixedAccessNode node, AddressNode address, ValueNode value, boolean doLoad, boolean nullCheck, StructuredGraph graph) { + G1PreWriteBarrier preBarrier = graph.add(new G1PreWriteBarrier(address, value, doLoad, nullCheck)); + preBarrier.setStateBefore(node.stateBefore()); + node.setNullCheck(false); + node.setStateBefore(null); + graph.addBeforeFixed(node, preBarrier); + } + + private static void addG1PostWriteBarrier(FixedAccessNode node, AddressNode address, ValueNode value, boolean precise, StructuredGraph graph) { + final boolean alwaysNull = StampTool.isPointerAlwaysNull(value); + graph.addAfterFixed(node, graph.add(new G1PostWriteBarrier(address, value, precise, alwaysNull))); + } +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/gc/g1/G1PostWriteBarrier.java 2019-03-12 08:10:38.031977710 +0100 @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2013, 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.hotspot.gc.g1; + +import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_64; +import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_64; + +import org.graalvm.compiler.graph.NodeClass; +import org.graalvm.compiler.hotspot.gc.shared.ObjectWriteBarrier; +import org.graalvm.compiler.nodeinfo.NodeInfo; +import org.graalvm.compiler.nodes.ValueNode; +import org.graalvm.compiler.nodes.memory.address.AddressNode; + +@NodeInfo(cycles = CYCLES_64, size = SIZE_64) +public class G1PostWriteBarrier extends ObjectWriteBarrier { + + public static final NodeClass TYPE = NodeClass.create(G1PostWriteBarrier.class); + protected final boolean alwaysNull; + + public G1PostWriteBarrier(AddressNode address, ValueNode value, boolean precise, boolean alwaysNull) { + this(TYPE, address, value, precise, alwaysNull); + } + + private G1PostWriteBarrier(NodeClass c, AddressNode address, ValueNode value, boolean precise, boolean alwaysNull) { + super(c, address, value, precise); + this.alwaysNull = alwaysNull; + } + + public boolean alwaysNull() { + return alwaysNull; + } +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/gc/g1/G1PreWriteBarrier.java 2019-03-12 08:10:38.511980814 +0100 @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2013, 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + + +package org.graalvm.compiler.hotspot.gc.g1; + +import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_64; +import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_64; + +import org.graalvm.compiler.graph.NodeClass; +import org.graalvm.compiler.hotspot.gc.shared.ObjectWriteBarrier; +import org.graalvm.compiler.nodeinfo.InputType; +import org.graalvm.compiler.nodeinfo.NodeInfo; +import org.graalvm.compiler.nodes.DeoptimizingNode; +import org.graalvm.compiler.nodes.FrameState; +import org.graalvm.compiler.nodes.ValueNode; +import org.graalvm.compiler.nodes.memory.address.AddressNode; + +@NodeInfo(cycles = CYCLES_64, size = SIZE_64) +public final class G1PreWriteBarrier extends ObjectWriteBarrier implements DeoptimizingNode.DeoptBefore { + + public static final NodeClass TYPE = NodeClass.create(G1PreWriteBarrier.class); + + @OptionalInput(InputType.State) private FrameState stateBefore; + private final boolean nullCheck; + private final boolean doLoad; + + public G1PreWriteBarrier(AddressNode address, ValueNode expectedObject, boolean doLoad, boolean nullCheck) { + super(TYPE, address, expectedObject, true); + this.doLoad = doLoad; + this.nullCheck = nullCheck; + } + + public ValueNode getExpectedObject() { + return getValue(); + } + + public boolean doLoad() { + return doLoad; + } + + public boolean getNullCheck() { + return nullCheck; + } + + @Override + public boolean canDeoptimize() { + return nullCheck; + } + + @Override + public FrameState stateBefore() { + return stateBefore; + } + + @Override + public void setStateBefore(FrameState state) { + updateUsages(stateBefore, state); + stateBefore = state; + } +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/gc/g1/G1ReferentFieldReadBarrier.java 2019-03-12 08:10:38.995983945 +0100 @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2013, 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.hotspot.gc.g1; + +import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_64; +import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_64; + +import org.graalvm.compiler.graph.NodeClass; +import org.graalvm.compiler.hotspot.gc.shared.ObjectWriteBarrier; +import org.graalvm.compiler.nodeinfo.NodeInfo; +import org.graalvm.compiler.nodes.ValueNode; +import org.graalvm.compiler.nodes.memory.address.AddressNode; + +/** + * The {@code G1ReferentFieldReadBarrier} is added when a read access is performed to the referent + * field of a {@link java.lang.ref.Reference} object (through a {@code LoadFieldNode} or an + * {@code UnsafeLoadNode}). The return value of the read is passed to the snippet implementing the + * read barrier and consequently is added to the SATB queue if the concurrent marker is enabled. + */ +@NodeInfo(cycles = CYCLES_64, size = SIZE_64) +public final class G1ReferentFieldReadBarrier extends ObjectWriteBarrier { + public static final NodeClass TYPE = NodeClass.create(G1ReferentFieldReadBarrier.class); + + private final boolean doLoad; + + public G1ReferentFieldReadBarrier(AddressNode address, ValueNode expectedObject, boolean doLoad) { + super(TYPE, address, expectedObject, true); + this.doLoad = doLoad; + } + + public ValueNode getExpectedObject() { + return getValue(); + } + + public boolean doLoad() { + return doLoad; + } +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/gc/shared/ArrayRangeWriteBarrier.java 2019-03-12 08:10:39.487987126 +0100 @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2013, 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.hotspot.gc.shared; + +import org.graalvm.compiler.graph.NodeClass; +import org.graalvm.compiler.hotspot.nodes.WriteBarrier; +import org.graalvm.compiler.nodeinfo.InputType; +import org.graalvm.compiler.nodeinfo.NodeInfo; +import org.graalvm.compiler.nodes.ValueNode; +import org.graalvm.compiler.nodes.memory.address.AddressNode; +import org.graalvm.compiler.nodes.spi.Lowerable; + +@NodeInfo +public abstract class ArrayRangeWriteBarrier extends WriteBarrier implements Lowerable { + + public static final NodeClass TYPE = NodeClass.create(ArrayRangeWriteBarrier.class); + @Input(InputType.Association) AddressNode address; + @Input ValueNode length; + + private final int elementStride; + + protected ArrayRangeWriteBarrier(NodeClass c, AddressNode address, ValueNode length, int elementStride) { + super(c); + this.address = address; + this.length = length; + this.elementStride = elementStride; + } + + public AddressNode getAddress() { + return address; + } + + public ValueNode getLength() { + return length; + } + + public int getElementStride() { + return elementStride; + } +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/gc/shared/BarrierSet.java 2019-03-12 08:10:39.963990206 +0100 @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2019, Red Hat Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.hotspot.gc.shared; + +import org.graalvm.compiler.nodes.StructuredGraph; +import org.graalvm.compiler.nodes.extended.ArrayRangeWrite; +import org.graalvm.compiler.nodes.java.AbstractCompareAndSwapNode; +import org.graalvm.compiler.nodes.java.LoweredAtomicReadAndWriteNode; +import org.graalvm.compiler.nodes.memory.ReadNode; +import org.graalvm.compiler.nodes.memory.WriteNode; + +public abstract class BarrierSet { + public abstract void addReadNodeBarriers(ReadNode node, StructuredGraph graph); + + public abstract void addWriteNodeBarriers(WriteNode node, StructuredGraph graph); + + public abstract void addAtomicReadWriteNodeBarriers(LoweredAtomicReadAndWriteNode node, StructuredGraph graph); + + public abstract void addCASBarriers(AbstractCompareAndSwapNode node, StructuredGraph graph); + + public abstract void addArrayRangeBarriers(ArrayRangeWrite write, StructuredGraph graph); +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/gc/shared/CardTableBarrierSet.java 2019-03-12 08:10:40.451993362 +0100 @@ -0,0 +1,113 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2019, Red Hat Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.hotspot.gc.shared; + +import org.graalvm.compiler.debug.GraalError; +import org.graalvm.compiler.nodes.StructuredGraph; +import org.graalvm.compiler.nodes.ValueNode; +import org.graalvm.compiler.nodes.extended.ArrayRangeWrite; +import org.graalvm.compiler.nodes.java.AbstractCompareAndSwapNode; +import org.graalvm.compiler.nodes.java.LoweredAtomicReadAndWriteNode; +import org.graalvm.compiler.nodes.memory.FixedAccessNode; +import org.graalvm.compiler.nodes.memory.HeapAccess; +import org.graalvm.compiler.nodes.memory.ReadNode; +import org.graalvm.compiler.nodes.memory.WriteNode; +import org.graalvm.compiler.nodes.memory.address.AddressNode; +import org.graalvm.compiler.nodes.type.StampTool; + +public class CardTableBarrierSet extends BarrierSet { + + @Override + public void addReadNodeBarriers(ReadNode node, StructuredGraph graph) { + assert node.getBarrierType() == HeapAccess.BarrierType.NONE : "Non precise read barrier has been attached to read node."; + } + + @Override + public void addWriteNodeBarriers(WriteNode node, StructuredGraph graph) { + HeapAccess.BarrierType barrierType = node.getBarrierType(); + switch (barrierType) { + case NONE: + // nothing to do + break; + case IMPRECISE: + case PRECISE: + boolean precise = barrierType == HeapAccess.BarrierType.PRECISE; + addSerialPostWriteBarrier(node, node.getAddress(), node.value(), precise, graph); + break; + default: + throw new GraalError("unexpected barrier type: " + barrierType); + } + } + + @Override + public void addAtomicReadWriteNodeBarriers(LoweredAtomicReadAndWriteNode node, StructuredGraph graph) { + HeapAccess.BarrierType barrierType = node.getBarrierType(); + switch (barrierType) { + case NONE: + // nothing to do + break; + case IMPRECISE: + case PRECISE: + boolean precise = barrierType == HeapAccess.BarrierType.PRECISE; + addSerialPostWriteBarrier(node, node.getAddress(), node.getNewValue(), precise, graph); + break; + default: + throw new GraalError("unexpected barrier type: " + barrierType); + } + } + + @Override + public void addCASBarriers(AbstractCompareAndSwapNode node, StructuredGraph graph) { + HeapAccess.BarrierType barrierType = node.getBarrierType(); + switch (barrierType) { + case NONE: + // nothing to do + break; + case IMPRECISE: + case PRECISE: + boolean precise = barrierType == HeapAccess.BarrierType.PRECISE; + addSerialPostWriteBarrier(node, node.getAddress(), node.getNewValue(), precise, graph); + break; + default: + throw new GraalError("unexpected barrier type: " + barrierType); + } + } + + @Override + public void addArrayRangeBarriers(ArrayRangeWrite write, StructuredGraph graph) { + SerialArrayRangeWriteBarrier serialArrayRangeWriteBarrier = graph.add(new SerialArrayRangeWriteBarrier(write.getAddress(), write.getLength(), write.getElementStride())); + graph.addAfterFixed(write.asNode(), serialArrayRangeWriteBarrier); + } + + protected void addSerialPostWriteBarrier(FixedAccessNode node, AddressNode address, ValueNode value, boolean precise, StructuredGraph graph) { + final boolean alwaysNull = StampTool.isPointerAlwaysNull(value); + if (alwaysNull) { + // Serial barrier isn't needed for null value + return; + } + graph.addAfterFixed(node, graph.add(new SerialWriteBarrier(address, precise))); + } +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/gc/shared/ObjectWriteBarrier.java 2019-03-12 08:10:40.935996491 +0100 @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.hotspot.gc.shared; + +import org.graalvm.compiler.graph.NodeClass; +import org.graalvm.compiler.hotspot.nodes.WriteBarrier; +import org.graalvm.compiler.nodeinfo.InputType; +import org.graalvm.compiler.nodeinfo.NodeInfo; +import org.graalvm.compiler.nodes.ValueNode; +import org.graalvm.compiler.nodes.memory.address.AddressNode; + +@NodeInfo +public abstract class ObjectWriteBarrier extends WriteBarrier { + + public static final NodeClass TYPE = NodeClass.create(ObjectWriteBarrier.class); + @Input(InputType.Association) protected AddressNode address; + @OptionalInput protected ValueNode value; + protected final boolean precise; + + protected ObjectWriteBarrier(NodeClass c, AddressNode address, ValueNode value, boolean precise) { + super(c); + this.address = address; + this.value = value; + this.precise = precise; + } + + public ValueNode getValue() { + return value; + } + + public AddressNode getAddress() { + return address; + } + + public boolean usePrecise() { + return precise; + } +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/gc/shared/SerialArrayRangeWriteBarrier.java 2019-03-12 08:10:41.423999648 +0100 @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.hotspot.gc.shared; + +import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_8; +import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_8; + +import org.graalvm.compiler.graph.NodeClass; +import org.graalvm.compiler.nodeinfo.NodeInfo; +import org.graalvm.compiler.nodes.ValueNode; +import org.graalvm.compiler.nodes.memory.address.AddressNode; + +@NodeInfo(cycles = CYCLES_8, size = SIZE_8) +public final class SerialArrayRangeWriteBarrier extends ArrayRangeWriteBarrier { + public static final NodeClass TYPE = NodeClass.create(SerialArrayRangeWriteBarrier.class); + + public SerialArrayRangeWriteBarrier(AddressNode address, ValueNode length, int elementStride) { + super(TYPE, address, length, elementStride); + } + +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/gc/shared/SerialWriteBarrier.java 2019-03-12 08:10:41.920002855 +0100 @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.hotspot.gc.shared; + +import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_8; +import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_4; + +import org.graalvm.compiler.graph.NodeClass; +import org.graalvm.compiler.nodeinfo.NodeInfo; +import org.graalvm.compiler.nodes.memory.address.AddressNode; + +@NodeInfo(cycles = CYCLES_8, size = SIZE_4) +public class SerialWriteBarrier extends ObjectWriteBarrier { + + public static final NodeClass TYPE = NodeClass.create(SerialWriteBarrier.class); + + public SerialWriteBarrier(AddressNode address, boolean precise) { + this(TYPE, address, precise); + } + + protected SerialWriteBarrier(NodeClass c, AddressNode address, boolean precise) { + super(c, address, null, precise); + } +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/ArraysSupportSubstitutions.java 2019-03-12 08:10:42.428006140 +0100 @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2019, Red Hat Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + + +package org.graalvm.compiler.hotspot.replacements; + +import org.graalvm.compiler.api.replacements.ClassSubstitution; +import org.graalvm.compiler.api.replacements.MethodSubstitution; +import org.graalvm.compiler.hotspot.HotSpotBackend; +import org.graalvm.compiler.nodes.ComputeObjectAddressNode; +import org.graalvm.compiler.word.Word; +import jdk.internal.vm.compiler.word.WordFactory; + +@ClassSubstitution(className = "jdk.internal.util.ArraysSupport", optional = true) +public class ArraysSupportSubstitutions { + + @SuppressWarnings("unused") + @MethodSubstitution(isStatic = true) + static int vectorizedMismatch(Object a, long aOffset, Object b, long bOffset, int length, int log2ArrayIndexScale) { + Word aAddr = WordFactory.unsigned(ComputeObjectAddressNode.get(a, aOffset)); + Word bAddr = WordFactory.unsigned(ComputeObjectAddressNode.get(b, bOffset)); + + return HotSpotBackend.vectorizedMismatch(aAddr, bAddr, length, log2ArrayIndexScale); + } +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/CounterModeSubstitutions.java 2019-03-12 08:10:42.916009297 +0100 @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.hotspot.replacements; + +import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfigBase.INJECTED_INTRINSIC_CONTEXT; +import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfigBase.INJECTED_METAACCESS; +import static org.graalvm.compiler.hotspot.replacements.CipherBlockChainingSubstitutions.aesCryptType; +import static org.graalvm.compiler.hotspot.replacements.CipherBlockChainingSubstitutions.embeddedCipherOffset; +import static org.graalvm.compiler.nodes.PiNode.piCastNonNull; + +import org.graalvm.compiler.api.replacements.ClassSubstitution; +import org.graalvm.compiler.api.replacements.Fold; +import org.graalvm.compiler.api.replacements.Fold.InjectedParameter; +import org.graalvm.compiler.api.replacements.MethodSubstitution; +import org.graalvm.compiler.hotspot.HotSpotBackend; +import org.graalvm.compiler.nodes.ComputeObjectAddressNode; +import org.graalvm.compiler.nodes.extended.RawLoadNode; +import org.graalvm.compiler.nodes.graphbuilderconf.IntrinsicContext; +import org.graalvm.compiler.replacements.ReplacementsUtil; +import org.graalvm.compiler.word.Word; +import jdk.internal.vm.compiler.word.LocationIdentity; +import jdk.internal.vm.compiler.word.WordFactory; + +import jdk.vm.ci.meta.JavaKind; +import jdk.vm.ci.meta.ResolvedJavaType; + +@ClassSubstitution(className = "com.sun.crypto.provider.CounterMode", optional = true) +public class CounterModeSubstitutions { + + @MethodSubstitution(isStatic = false) + static int implCrypt(Object receiver, byte[] in, int inOff, int len, byte[] out, int outOff) { + Object realReceiver = piCastNonNull(receiver, HotSpotReplacementsUtil.methodHolderClass(INJECTED_INTRINSIC_CONTEXT)); + Object embeddedCipher = RawLoadNode.load(realReceiver, embeddedCipherOffset(INJECTED_INTRINSIC_CONTEXT), JavaKind.Object, LocationIdentity.any()); + Object aesCipher = piCastNonNull(embeddedCipher, aesCryptType(INJECTED_INTRINSIC_CONTEXT)); + + Word srcAddr = WordFactory.unsigned(ComputeObjectAddressNode.get(in, ReplacementsUtil.getArrayBaseOffset(INJECTED_METAACCESS, JavaKind.Byte) + inOff)); + Word dstAddr = WordFactory.unsigned(ComputeObjectAddressNode.get(out, ReplacementsUtil.getArrayBaseOffset(INJECTED_METAACCESS, JavaKind.Byte) + outOff)); + Word usedPtr = WordFactory.unsigned(ComputeObjectAddressNode.get(realReceiver, usedOffset(INJECTED_INTRINSIC_CONTEXT))); + + int cntOffset = counterOffset(INJECTED_INTRINSIC_CONTEXT); + int encCntOffset = encCounterOffset(INJECTED_INTRINSIC_CONTEXT); + Object kObject = RawLoadNode.load(aesCipher, AESCryptSubstitutions.kOffset(INJECTED_INTRINSIC_CONTEXT), JavaKind.Object, LocationIdentity.any()); + Object cntObj = RawLoadNode.load(realReceiver, cntOffset, JavaKind.Object, LocationIdentity.any()); + Object encCntObj = RawLoadNode.load(realReceiver, encCntOffset, JavaKind.Object, LocationIdentity.any()); + + Word kPtr = Word.objectToTrackedPointer(kObject).add(ReplacementsUtil.getArrayBaseOffset(INJECTED_METAACCESS, JavaKind.Int)); + Word cntPtr = Word.objectToTrackedPointer(cntObj).add(ReplacementsUtil.getArrayBaseOffset(INJECTED_METAACCESS, JavaKind.Byte)); + Word encCntPtr = Word.objectToTrackedPointer(encCntObj).add(ReplacementsUtil.getArrayBaseOffset(INJECTED_METAACCESS, JavaKind.Byte)); + + return HotSpotBackend.counterModeAESCrypt(srcAddr, dstAddr, kPtr, cntPtr, len, encCntPtr, usedPtr); + } + + static ResolvedJavaType counterModeType(IntrinsicContext context) { + return HotSpotReplacementsUtil.getType(context, "Lcom/sun/crypto/provider/CounterMode;"); + } + + @Fold + static int counterOffset(@InjectedParameter IntrinsicContext context) { + return HotSpotReplacementsUtil.getFieldOffset(counterModeType(context), "counter"); + } + + @Fold + static int encCounterOffset(@InjectedParameter IntrinsicContext context) { + return HotSpotReplacementsUtil.getFieldOffset(counterModeType(context), "encryptedCounter"); + } + + @Fold + static int usedOffset(@InjectedParameter IntrinsicContext context) { + return HotSpotReplacementsUtil.getFieldOffset(counterModeType(context), "used"); + } +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/DigestBaseSubstitutions.java 2019-03-12 08:10:43.408012478 +0100 @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.hotspot.replacements; + +import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_METAACCESS; +import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfigBase.INJECTED_INTRINSIC_CONTEXT; +import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfigBase.INJECTED_VMCONFIG; +import static org.graalvm.compiler.nodes.java.InstanceOfNode.doInstanceof; + +import org.graalvm.compiler.api.replacements.ClassSubstitution; +import org.graalvm.compiler.api.replacements.Fold; +import org.graalvm.compiler.api.replacements.MethodSubstitution; +import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig; +import org.graalvm.compiler.hotspot.HotSpotBackend; +import org.graalvm.compiler.nodes.ComputeObjectAddressNode; +import org.graalvm.compiler.nodes.PiNode; +import org.graalvm.compiler.nodes.extended.RawLoadNode; +import org.graalvm.compiler.replacements.ReplacementsUtil; +import org.graalvm.compiler.word.Word; +import jdk.internal.vm.compiler.word.LocationIdentity; +import jdk.internal.vm.compiler.word.WordFactory; + +import jdk.vm.ci.meta.JavaKind; +import jdk.vm.ci.meta.ResolvedJavaType; + +@ClassSubstitution(className = "sun.security.provider.DigestBase", optional = true) +public class DigestBaseSubstitutions { + + @MethodSubstitution(isStatic = false) + static int implCompressMultiBlock0(Object receiver, byte[] buf, int ofs, int limit) { + Object realReceiver = PiNode.piCastNonNull(receiver, HotSpotReplacementsUtil.methodHolderClass(INJECTED_INTRINSIC_CONTEXT)); + ResolvedJavaType sha1type = HotSpotReplacementsUtil.getType(INJECTED_INTRINSIC_CONTEXT, "Lsun/security/provider/SHA;"); + ResolvedJavaType sha256type = HotSpotReplacementsUtil.getType(INJECTED_INTRINSIC_CONTEXT, "Lsun/security/provider/SHA2;"); + ResolvedJavaType sha512type = HotSpotReplacementsUtil.getType(INJECTED_INTRINSIC_CONTEXT, "Lsun/security/provider/SHA5;"); + + Word bufAddr = WordFactory.unsigned(ComputeObjectAddressNode.get(buf, ReplacementsUtil.getArrayBaseOffset(INJECTED_METAACCESS, JavaKind.Byte) + ofs)); + if (useSHA1Intrinsics(INJECTED_VMCONFIG) && doInstanceof(sha1type, realReceiver)) { + Object sha1obj = PiNode.piCastNonNull(realReceiver, sha1type); + Object state = RawLoadNode.load(sha1obj, HotSpotReplacementsUtil.getFieldOffset(sha1type, "state"), JavaKind.Object, LocationIdentity.any()); + Word stateAddr = WordFactory.unsigned(ComputeObjectAddressNode.get(state, ReplacementsUtil.getArrayBaseOffset(INJECTED_METAACCESS, JavaKind.Int))); + return HotSpotBackend.shaImplCompressMBStub(bufAddr, stateAddr, ofs, limit); + } else if (useSHA256Intrinsics(INJECTED_VMCONFIG) && doInstanceof(sha256type, realReceiver)) { + Object sha256obj = PiNode.piCastNonNull(realReceiver, sha256type); + Object state = RawLoadNode.load(sha256obj, HotSpotReplacementsUtil.getFieldOffset(sha256type, "state"), JavaKind.Object, LocationIdentity.any()); + Word stateAddr = WordFactory.unsigned(ComputeObjectAddressNode.get(state, ReplacementsUtil.getArrayBaseOffset(INJECTED_METAACCESS, JavaKind.Int))); + return HotSpotBackend.sha2ImplCompressMBStub(bufAddr, stateAddr, ofs, limit); + } else if (useSHA512Intrinsics(INJECTED_VMCONFIG) && doInstanceof(sha512type, realReceiver)) { + Object sha512obj = PiNode.piCastNonNull(realReceiver, sha512type); + Object state = RawLoadNode.load(sha512obj, HotSpotReplacementsUtil.getFieldOffset(sha512type, "state"), JavaKind.Object, LocationIdentity.any()); + Word stateAddr = WordFactory.unsigned(ComputeObjectAddressNode.get(state, ReplacementsUtil.getArrayBaseOffset(INJECTED_METAACCESS, JavaKind.Int))); + return HotSpotBackend.sha5ImplCompressMBStub(bufAddr, stateAddr, ofs, limit); + } else { + return implCompressMultiBlock0(realReceiver, buf, ofs, limit); + } + } + + @Fold + public static boolean useSHA1Intrinsics(@Fold.InjectedParameter GraalHotSpotVMConfig config) { + return config.useSHA1Intrinsics(); + } + + @Fold + public static boolean useSHA256Intrinsics(@Fold.InjectedParameter GraalHotSpotVMConfig config) { + return config.useSHA256Intrinsics(); + } + + @Fold + public static boolean useSHA512Intrinsics(@Fold.InjectedParameter GraalHotSpotVMConfig config) { + return config.useSHA512Intrinsics(); + } +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/StringUTF16Substitutions.java 2019-03-12 08:10:43.896015634 +0100 @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.hotspot.replacements; + +import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfigBase.INJECTED_VMCONFIG; +import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY; +import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability; + +import jdk.vm.ci.meta.DeoptimizationAction; +import jdk.vm.ci.meta.DeoptimizationReason; +import jdk.vm.ci.meta.JavaKind; +import org.graalvm.compiler.api.replacements.ClassSubstitution; +import org.graalvm.compiler.api.replacements.MethodSubstitution; +import org.graalvm.compiler.nodes.DeoptimizeNode; +import org.graalvm.compiler.nodes.java.NewArrayNode; +import org.graalvm.compiler.replacements.arraycopy.ArrayCopyCallNode; + +// JaCoCo Exclude + +/** + * Substitutions for {@code StringUTF16} methods for JDK9 and later. + */ +@ClassSubstitution(className = "java.lang.StringUTF16", optional = true) +public class StringUTF16Substitutions { + + private static final int MAX_LENGTH = Integer.MAX_VALUE >> 1; + + @MethodSubstitution + public static byte[] toBytes(char[] value, int srcBegin, int length) { + if (probability(SLOW_PATH_PROBABILITY, srcBegin < 0) || + probability(SLOW_PATH_PROBABILITY, length < 0) || + probability(SLOW_PATH_PROBABILITY, length > MAX_LENGTH) || + probability(SLOW_PATH_PROBABILITY, srcBegin > value.length - length)) { + DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.BoundsCheckException); + } + byte[] val = (byte[]) NewArrayNode.newUninitializedArray(Byte.TYPE, length << 1); + // the intrinsic does not perform bounds/type checks, so it can be used here. + // Using KillsAny variant since we are reading and writing 2 different types. + ArrayCopyCallNode.disjointArraycopyKillsAny(value, srcBegin, val, 0, length, JavaKind.Char, HotSpotReplacementsUtil.getHeapWordSize(INJECTED_VMCONFIG)); + return val; + } + + @MethodSubstitution + public static void getChars(byte[] value, int srcBegin, int srcEnd, char[] dst, int dstBegin) { + int length = srcEnd - srcBegin; + if (probability(SLOW_PATH_PROBABILITY, srcBegin < 0) || + probability(SLOW_PATH_PROBABILITY, length < 0) || + probability(SLOW_PATH_PROBABILITY, srcBegin > (value.length >> 1) - length) || + probability(SLOW_PATH_PROBABILITY, dstBegin > dst.length - length)) { + DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.BoundsCheckException); + } + // The intrinsic does not perform bounds/type checks, so it can be used here. + // Using KillsAny variant since we are reading and writing 2 different types. + ArrayCopyCallNode.disjointArraycopyKillsAny(value, srcBegin, dst, dstBegin, length, JavaKind.Char, HotSpotReplacementsUtil.getHeapWordSize(INJECTED_VMCONFIG)); + } +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.jtt/src/org/graalvm/compiler/jtt/optimize/IfNodeCanonicalizationsTest.java 2019-03-12 08:10:44.380018764 +0100 @@ -0,0 +1,366 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.jtt.optimize; + +import static org.junit.runners.Parameterized.Parameter; +import static org.junit.runners.Parameterized.Parameters; + +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import org.graalvm.compiler.jtt.JTTTest; +import org.graalvm.compiler.nodes.ValueNode; +import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderContext; +import org.graalvm.compiler.nodes.graphbuilderconf.InlineInvokePlugin; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import jdk.vm.ci.meta.ResolvedJavaMethod; + +@RunWith(Parameterized.class) +public class IfNodeCanonicalizationsTest extends JTTTest { + + @Override + protected InlineInvokePlugin.InlineInfo bytecodeParserShouldInlineInvoke(GraphBuilderContext b, ResolvedJavaMethod method, ValueNode[] args) { + if (method.getDeclaringClass().getUnqualifiedName().equals(IfNodeCanonicalizationsTest.class.getSimpleName()) && method.getName().startsWith("compare")) { + return InlineInvokePlugin.InlineInfo.createStandardInlineInfo(method); + } + return super.bytecodeParserShouldInlineInvoke(b, method, args); + } + + @Parameter(value = 0) public String testName; + @Parameter(value = 1) public int x; + @Parameter(value = 2) public int y; + + public static int compare0(int a, int b) { + return (a < b) ? 1 : ((a < b) ? 2 : 3); + } + + public static int compare1(int a, int b) { + return (a < b) ? ((a < b) ? 1 : 2) : 3; + } + + public static int compare2(int a, int b) { + return (a < b) ? 1 : ((a > b) ? 2 : 3); + } + + public static int compare3(int a, int b) { + return (a < b) ? ((a > b) ? 1 : 2) : 3; + } + + public static int compare4(int a, int b) { + return (a < b) ? 1 : ((a <= b) ? 2 : 3); + } + + public static int compare5(int a, int b) { + return (a < b) ? ((a <= b) ? 1 : 2) : 3; + } + + public static int compare6(int a, int b) { + return (a < b) ? 1 : ((a >= b) ? 2 : 3); + } + + public static int compare7(int a, int b) { + return (a < b) ? ((a >= b) ? 1 : 2) : 3; + } + + public static int compare8(int a, int b) { + return (a < b) ? 1 : ((a == b) ? 2 : 3); + } + + public static int compare9(int a, int b) { + return (a < b) ? ((a == b) ? 1 : 2) : 3; + } + + public static int compare10(int a, int b) { + return (a < b) ? 1 : ((a != b) ? 2 : 3); + } + + public static int compare11(int a, int b) { + return (a < b) ? ((a != b) ? 1 : 2) : 3; + } + + public static int compare12(int a, int b) { + return (a > b) ? 1 : ((a < b) ? 2 : 3); + } + + public static int compare13(int a, int b) { + return (a > b) ? ((a < b) ? 1 : 2) : 3; + } + + public static int compare14(int a, int b) { + return (a > b) ? 1 : ((a > b) ? 2 : 3); + } + + public static int compare15(int a, int b) { + return (a > b) ? ((a > b) ? 1 : 2) : 3; + } + + public static int compare16(int a, int b) { + return (a > b) ? 1 : ((a <= b) ? 2 : 3); + } + + public static int compare17(int a, int b) { + return (a > b) ? ((a <= b) ? 1 : 2) : 3; + } + + public static int compare18(int a, int b) { + return (a > b) ? 1 : ((a >= b) ? 2 : 3); + } + + public static int compare19(int a, int b) { + return (a > b) ? ((a >= b) ? 1 : 2) : 3; + } + + public static int compare20(int a, int b) { + return (a > b) ? 1 : ((a == b) ? 2 : 3); + } + + public static int compare21(int a, int b) { + return (a > b) ? ((a == b) ? 1 : 2) : 3; + } + + public static int compare22(int a, int b) { + return (a > b) ? 1 : ((a != b) ? 2 : 3); + } + + public static int compare23(int a, int b) { + return (a > b) ? ((a != b) ? 1 : 2) : 3; + } + + public static int compare24(int a, int b) { + return (a <= b) ? 1 : ((a < b) ? 2 : 3); + } + + public static int compare25(int a, int b) { + return (a <= b) ? ((a < b) ? 1 : 2) : 3; + } + + public static int compare26(int a, int b) { + return (a <= b) ? 1 : ((a > b) ? 2 : 3); + } + + public static int compare27(int a, int b) { + return (a <= b) ? ((a > b) ? 1 : 2) : 3; + } + + public static int compare28(int a, int b) { + return (a <= b) ? 1 : ((a <= b) ? 2 : 3); + } + + public static int compare29(int a, int b) { + return (a <= b) ? ((a <= b) ? 1 : 2) : 3; + } + + public static int compare30(int a, int b) { + return (a <= b) ? 1 : ((a >= b) ? 2 : 3); + } + + public static int compare31(int a, int b) { + return (a <= b) ? ((a >= b) ? 1 : 2) : 3; + } + + public static int compare32(int a, int b) { + return (a <= b) ? 1 : ((a == b) ? 2 : 3); + } + + public static int compare33(int a, int b) { + return (a <= b) ? ((a == b) ? 1 : 2) : 3; + } + + public static int compare34(int a, int b) { + return (a <= b) ? 1 : ((a != b) ? 2 : 3); + } + + public static int compare35(int a, int b) { + return (a <= b) ? ((a != b) ? 1 : 2) : 3; + } + + public static int compare36(int a, int b) { + return (a >= b) ? 1 : ((a < b) ? 2 : 3); + } + + public static int compare37(int a, int b) { + return (a >= b) ? ((a < b) ? 1 : 2) : 3; + } + + public static int compare38(int a, int b) { + return (a >= b) ? 1 : ((a > b) ? 2 : 3); + } + + public static int compare39(int a, int b) { + return (a >= b) ? ((a > b) ? 1 : 2) : 3; + } + + public static int compare40(int a, int b) { + return (a >= b) ? 1 : ((a <= b) ? 2 : 3); + } + + public static int compare41(int a, int b) { + return (a >= b) ? ((a <= b) ? 1 : 2) : 3; + } + + public static int compare42(int a, int b) { + return (a >= b) ? 1 : ((a >= b) ? 2 : 3); + } + + public static int compare43(int a, int b) { + return (a >= b) ? ((a >= b) ? 1 : 2) : 3; + } + + public static int compare44(int a, int b) { + return (a >= b) ? 1 : ((a == b) ? 2 : 3); + } + + public static int compare45(int a, int b) { + return (a >= b) ? ((a == b) ? 1 : 2) : 3; + } + + public static int compare46(int a, int b) { + return (a >= b) ? 1 : ((a != b) ? 2 : 3); + } + + public static int compare47(int a, int b) { + return (a >= b) ? ((a != b) ? 1 : 2) : 3; + } + + public static int compare48(int a, int b) { + return (a == b) ? 1 : ((a < b) ? 2 : 3); + } + + public static int compare49(int a, int b) { + return (a == b) ? ((a < b) ? 1 : 2) : 3; + } + + public static int compare50(int a, int b) { + return (a == b) ? 1 : ((a > b) ? 2 : 3); + } + + public static int compare51(int a, int b) { + return (a == b) ? ((a > b) ? 1 : 2) : 3; + } + + public static int compare52(int a, int b) { + return (a == b) ? 1 : ((a <= b) ? 2 : 3); + } + + public static int compare53(int a, int b) { + return (a == b) ? ((a <= b) ? 1 : 2) : 3; + } + + public static int compare54(int a, int b) { + return (a == b) ? 1 : ((a >= b) ? 2 : 3); + } + + public static int compare55(int a, int b) { + return (a == b) ? ((a >= b) ? 1 : 2) : 3; + } + + public static int compare56(int a, int b) { + return (a == b) ? 1 : ((a == b) ? 2 : 3); + } + + public static int compare57(int a, int b) { + return (a == b) ? ((a == b) ? 1 : 2) : 3; + } + + public static int compare58(int a, int b) { + return (a == b) ? 1 : ((a != b) ? 2 : 3); + } + + public static int compare59(int a, int b) { + return (a == b) ? ((a != b) ? 1 : 2) : 3; + } + + public static int compare60(int a, int b) { + return (a != b) ? 1 : ((a < b) ? 2 : 3); + } + + public static int compare61(int a, int b) { + return (a != b) ? ((a < b) ? 1 : 2) : 3; + } + + public static int compare62(int a, int b) { + return (a != b) ? 1 : ((a > b) ? 2 : 3); + } + + public static int compare63(int a, int b) { + return (a != b) ? ((a > b) ? 1 : 2) : 3; + } + + public static int compare64(int a, int b) { + return (a != b) ? 1 : ((a <= b) ? 2 : 3); + } + + public static int compare65(int a, int b) { + return (a != b) ? ((a <= b) ? 1 : 2) : 3; + } + + public static int compare66(int a, int b) { + return (a != b) ? 1 : ((a >= b) ? 2 : 3); + } + + public static int compare67(int a, int b) { + return (a != b) ? ((a >= b) ? 1 : 2) : 3; + } + + public static int compare68(int a, int b) { + return (a != b) ? 1 : ((a == b) ? 2 : 3); + } + + public static int compare69(int a, int b) { + return (a != b) ? ((a == b) ? 1 : 2) : 3; + } + + public static int compare70(int a, int b) { + return (a != b) ? 1 : ((a != b) ? 2 : 3); + } + + public static int compare71(int a, int b) { + return (a != b) ? ((a != b) ? 1 : 2) : 3; + } + + @Test + public void runNamedTest() { + runTest(testName, x, y); + } + + @Parameters(name = "{0}(a = {1}, b = {2})") + public static Collection data() { + List tests = new ArrayList<>(); + for (Method m : IfNodeCanonicalizationsTest.class.getDeclaredMethods()) { + if (m.getName().startsWith("compare") && Modifier.isStatic(m.getModifiers())) { + tests.add(new Object[]{m.getName(), 0, 0}); + tests.add(new Object[]{m.getName(), 0, 1}); + tests.add(new Object[]{m.getName(), 1, 0}); + } + } + return tests; + } +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.jtt/src/org/graalvm/compiler/jtt/optimize/SwitchHashTableTest.java 2019-03-12 08:10:44.872021945 +0100 @@ -0,0 +1,635 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.jtt.optimize; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Random; +import java.util.Set; +import java.util.stream.Collectors; + +import org.graalvm.compiler.jtt.JTTTest; +import org.graalvm.compiler.lir.hashing.HashFunction; +import org.graalvm.compiler.lir.hashing.Hasher; +import org.junit.Test; + +import jdk.vm.ci.meta.JavaConstant; + +/* + * Tests optimization of hash table switches. + * Code generated by `SwitchHashTableTest.TestGenerator.main` + */ +public class SwitchHashTableTest extends JTTTest { + @Test + public void checkHashFunctionInstances() { + List coveredByTestCases = Arrays.asList("val >> min", "val", "val >> (val & min)", "(val >> min) ^ val", "val - min", "rotateRight(val, prime)", "rotateRight(val, prime) ^ val", + "rotateRight(val, prime) + val", "(val >> min) * val", "(val * prime) >> min"); + Set functions = HashFunction.instances().stream().map(Object::toString).collect(Collectors.toSet()); + functions.removeAll(coveredByTestCases); + assertTrue("The following hash functions are not covered by the `Switch03` test: " + functions + + ". Re-run the `Switch03.TestGenerator.main` and update the test class.", functions.isEmpty()); + } + + // Hasher[function=rotateRight(val, prime), effort=4, cardinality=16] + public static int test1(int arg) { + switch (arg) { + case 3080012: + return 3080012; + case 3080017: + return 3080017; + case 3080029: + return 3080029; + case 3080037: + return 3080037; + case 3080040: + return 3080040; + case 3080054: + return 3080054; + case 3080060: + return 3080060; + case 3080065: + return 3080065; + case 3080073: + return 3080073; + case 3080082: + return 3080082; + case 3080095: + return 3080095; + case 3080103: + return 3080103; + case 3080116: + return 3080116; + case 3080127: + return 3080127; + case 3080130: + return 3080130; + default: + return -1; + } + } + + @Test + public void run1() throws Throwable { + runTest("test1", 0); // zero + runTest("test1", 3080011); // bellow + runTest("test1", 3080012); // first + runTest("test1", 3080065); // middle + runTest("test1", 3080130); // last + runTest("test1", 3080131); // above + runTest("test1", 3080013); // miss + } + + // Hasher[function=rotateRight(val, prime) ^ val, effort=5, cardinality=28] + public static int test2(int arg) { + switch (arg) { + case 718707335: + return 718707335; + case 718707336: + return 718707336; + case 718707347: + return 718707347; + case 718707359: + return 718707359; + case 718707366: + return 718707366; + case 718707375: + return 718707375; + case 718707378: + return 718707378; + case 718707386: + return 718707386; + case 718707396: + return 718707396; + case 718707401: + return 718707401; + case 718707408: + return 718707408; + case 718707409: + return 718707409; + case 718707420: + return 718707420; + case 718707431: + return 718707431; + case 718707436: + return 718707436; + default: + return -1; + } + } + + @Test + public void run2() throws Throwable { + runTest("test2", 0); // zero + runTest("test2", 718707334); // bellow + runTest("test2", 718707335); // first + runTest("test2", 718707386); // middle + runTest("test2", 718707436); // last + runTest("test2", 718707437); // above + runTest("test2", 718707337); // miss + } + + // Hasher[function=(val * prime) >> min, effort=4, cardinality=16] + public static int test3(int arg) { + switch (arg) { + case 880488712: + return 880488712; + case 880488723: + return 880488723; + case 880488737: + return 880488737; + case 880488744: + return 880488744; + case 880488752: + return 880488752; + case 880488757: + return 880488757; + case 880488767: + return 880488767; + case 880488777: + return 880488777; + case 880488781: + return 880488781; + case 880488794: + return 880488794; + case 880488795: + return 880488795; + case 880488807: + return 880488807; + case 880488814: + return 880488814; + case 880488821: + return 880488821; + case 880488831: + return 880488831; + default: + return -1; + } + } + + @Test + public void run3() throws Throwable { + runTest("test3", 0); // zero + runTest("test3", 880488711); // bellow + runTest("test3", 880488712); // first + runTest("test3", 880488777); // middle + runTest("test3", 880488831); // last + runTest("test3", 880488832); // above + runTest("test3", 880488713); // miss + } + + // Hasher[function=rotateRight(val, prime) + val, effort=5, cardinality=28] + public static int test4(int arg) { + switch (arg) { + case 189404658: + return 189404658; + case 189404671: + return 189404671; + case 189404678: + return 189404678; + case 189404680: + return 189404680; + case 189404687: + return 189404687; + case 189404698: + return 189404698; + case 189404699: + return 189404699; + case 189404711: + return 189404711; + case 189404724: + return 189404724; + case 189404725: + return 189404725; + case 189404732: + return 189404732; + case 189404739: + return 189404739; + case 189404748: + return 189404748; + case 189404754: + return 189404754; + case 189404765: + return 189404765; + default: + return -1; + } + } + + @Test + public void run4() throws Throwable { + runTest("test4", 0); // zero + runTest("test4", 189404657); // bellow + runTest("test4", 189404658); // first + runTest("test4", 189404711); // middle + runTest("test4", 189404765); // last + runTest("test4", 189404766); // above + runTest("test4", 189404659); // miss + } + + // Hasher[function=val - min, effort=2, cardinality=24] + public static int test5(int arg) { + switch (arg) { + case 527674226: + return 527674226; + case 527674235: + return 527674235; + case 527674236: + return 527674236; + case 527674247: + return 527674247; + case 527674251: + return 527674251; + case 527674253: + return 527674253; + case 527674257: + return 527674257; + case 527674263: + return 527674263; + case 527674265: + return 527674265; + case 527674272: + return 527674272; + case 527674286: + return 527674286; + case 527674293: + return 527674293; + case 527674294: + return 527674294; + case 527674306: + return 527674306; + case 527674308: + return 527674308; + default: + return -1; + } + } + + @Test + public void run5() throws Throwable { + runTest("test5", 0); // zero + runTest("test5", 527674225); // bellow + runTest("test5", 527674226); // first + runTest("test5", 527674263); // middle + runTest("test5", 527674308); // last + runTest("test5", 527674309); // above + runTest("test5", 527674227); // miss + } + + // Hasher[function=val, effort=1, cardinality=24] + public static int test6(int arg) { + switch (arg) { + case 676979121: + return 676979121; + case 676979128: + return 676979128; + case 676979135: + return 676979135; + case 676979146: + return 676979146; + case 676979148: + return 676979148; + case 676979156: + return 676979156; + case 676979158: + return 676979158; + case 676979169: + return 676979169; + case 676979175: + return 676979175; + case 676979179: + return 676979179; + case 676979182: + return 676979182; + case 676979194: + return 676979194; + case 676979200: + return 676979200; + case 676979205: + return 676979205; + case 676979219: + return 676979219; + default: + return -1; + } + } + + @Test + public void run6() throws Throwable { + runTest("test6", 0); // zero + runTest("test6", 676979120); // bellow + runTest("test6", 676979121); // first + runTest("test6", 676979169); // middle + runTest("test6", 676979219); // last + runTest("test6", 676979220); // above + runTest("test6", 676979122); // miss + } + + // Hasher[function=(val >> min) ^ val, effort=3, cardinality=16] + public static int test7(int arg) { + switch (arg) { + case 634218696: + return 634218696; + case 634218710: + return 634218710; + case 634218715: + return 634218715; + case 634218720: + return 634218720; + case 634218724: + return 634218724; + case 634218732: + return 634218732; + case 634218737: + return 634218737; + case 634218749: + return 634218749; + case 634218751: + return 634218751; + case 634218758: + return 634218758; + case 634218767: + return 634218767; + case 634218772: + return 634218772; + case 634218786: + return 634218786; + case 634218792: + return 634218792; + case 634218795: + return 634218795; + default: + return -1; + } + } + + @Test + public void run7() throws Throwable { + runTest("test7", 0); // zero + runTest("test7", 634218695); // bellow + runTest("test7", 634218696); // first + runTest("test7", 634218749); // middle + runTest("test7", 634218795); // last + runTest("test7", 634218796); // above + runTest("test7", 634218697); // miss + } + + // Hasher[function=val >> min, effort=2, cardinality=16] + public static int test8(int arg) { + switch (arg) { + case 473982403: + return 473982403; + case 473982413: + return 473982413; + case 473982416: + return 473982416; + case 473982425: + return 473982425; + case 473982439: + return 473982439; + case 473982445: + return 473982445; + case 473982459: + return 473982459; + case 473982468: + return 473982468; + case 473982479: + return 473982479; + case 473982482: + return 473982482; + case 473982494: + return 473982494; + case 473982501: + return 473982501; + case 473982505: + return 473982505; + case 473982519: + return 473982519; + case 473982523: + return 473982523; + default: + return -1; + } + } + + @Test + public void run8() throws Throwable { + runTest("test8", 0); // zero + runTest("test8", 473982402); // bellow + runTest("test8", 473982403); // first + runTest("test8", 473982468); // middle + runTest("test8", 473982523); // last + runTest("test8", 473982524); // above + runTest("test8", 473982404); // miss + } + + // Hasher[function=val >> (val & min), effort=3, cardinality=16] + public static int test9(int arg) { + switch (arg) { + case 15745090: + return 15745090; + case 15745093: + return 15745093; + case 15745102: + return 15745102; + case 15745108: + return 15745108; + case 15745122: + return 15745122; + case 15745131: + return 15745131; + case 15745132: + return 15745132; + case 15745146: + return 15745146; + case 15745151: + return 15745151; + case 15745163: + return 15745163; + case 15745169: + return 15745169; + case 15745182: + return 15745182; + case 15745191: + return 15745191; + case 15745198: + return 15745198; + case 15745207: + return 15745207; + default: + return -1; + } + } + + @Test + public void run9() throws Throwable { + runTest("test9", 0); // zero + runTest("test9", 15745089); // bellow + runTest("test9", 15745090); // first + runTest("test9", 15745146); // middle + runTest("test9", 15745207); // last + runTest("test9", 15745208); // above + runTest("test9", 15745091); // miss + } + + // Hasher[function=(val >> min) * val, effort=4, cardinality=28] + public static int test10(int arg) { + switch (arg) { + case 989358996: + return 989358996; + case 989359010: + return 989359010; + case 989359022: + return 989359022; + case 989359030: + return 989359030; + case 989359038: + return 989359038; + case 989359047: + return 989359047; + case 989359053: + return 989359053; + case 989359059: + return 989359059; + case 989359061: + return 989359061; + case 989359072: + return 989359072; + case 989359073: + return 989359073; + case 989359087: + return 989359087; + case 989359097: + return 989359097; + case 989359099: + return 989359099; + case 989359108: + return 989359108; + default: + return -1; + } + } + + @Test + public void run10() throws Throwable { + runTest("test10", 0); // zero + runTest("test10", 989358995); // bellow + runTest("test10", 989358996); // first + runTest("test10", 989359059); // middle + runTest("test10", 989359108); // last + runTest("test10", 989359109); // above + runTest("test10", 989358997); // miss + } + + public static class TestGenerator { + + private static int nextId = 0; + private static final int size = 15; + private static double minDensity = 0.5; + + // test code generator + public static void main(String[] args) { + + Random r = new Random(0); + Set seen = new HashSet<>(); + Set all = HashFunction.instances().stream().map(Object::toString).collect(Collectors.toSet()); + + println("@Test"); + println("public void checkHashFunctionInstances() {"); + println(" List coveredByTestCases = Arrays.asList(" + String.join(", ", all.stream().map(s -> "\"" + s + "\"").collect(Collectors.toSet())) + ");"); + println(" Set functions = HashFunction.instances().stream().map(Object::toString).collect(Collectors.toSet());"); + println(" functions.removeAll(coveredByTestCases);"); + println(" assertTrue(\"The following hash functions are not covered by the `Switch03` test: \" + functions +"); + println(" \". Re-run the `Switch03.TestGenerator.main` and update the test class.\", functions.isEmpty());"); + println("}"); + + while (seen.size() < all.size()) { + int v = r.nextInt(Integer.MAX_VALUE / 2); + List keys = new ArrayList<>(); + while (keys.size() < 15) { + keys.add(v); + v += r.nextInt(15); + } + keys.sort(Integer::compare); + double density = ((double) keys.size() + 1) / (keys.get(keys.size() - 1) - keys.get(0)); + if (density < minDensity) { + Hasher.forKeys(toConstants(keys), minDensity).ifPresent(h -> { + String f = h.function().toString(); + if (!seen.contains(f)) { + gen(keys, h); + seen.add(f); + } + }); + } + } + } + + private static void gen(List keys, Hasher hasher) { + int id = ++nextId; + + println("// " + hasher + ""); + println("public static int test" + id + "(int arg) {"); + println(" switch (arg) {"); + + for (Integer key : keys) { + println(" case " + key + ": return " + key + ";"); + } + + println(" default: return -1;"); + println(" }"); + println("}"); + + int miss = keys.get(0) + 1; + while (keys.contains(miss)) { + miss++; + } + + println("@Test"); + println("public void run" + id + "() throws Throwable {"); + println(" runTest(\"test" + id + "\", 0); // zero "); + println(" runTest(\"test" + id + "\", " + (keys.get(0) - 1) + "); // bellow "); + println(" runTest(\"test" + id + "\", " + keys.get(0) + "); // first "); + println(" runTest(\"test" + id + "\", " + keys.get(size / 2) + "); // middle "); + println(" runTest(\"test" + id + "\", " + keys.get(size - 1) + "); // last "); + println(" runTest(\"test" + id + "\", " + (keys.get(size - 1) + 1) + "); // above "); + println(" runTest(\"test" + id + "\", " + miss + "); // miss "); + println("}"); + } + + private static void println(String s) { + System.out.println(s); + } + + private static JavaConstant[] toConstants(List keys) { + JavaConstant[] ckeys = new JavaConstant[keys.size()]; + + for (int i = 0; i < keys.size(); i++) { + ckeys[i] = JavaConstant.forInt(keys.get(i)); + } + return ckeys; + } + } +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64HotSpotHelper.java 2019-03-12 08:10:45.352025048 +0100 @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.lir.amd64; + +import org.graalvm.compiler.asm.amd64.AMD64Address; +import org.graalvm.compiler.core.common.LIRKind; +import org.graalvm.compiler.debug.GraalError; +import org.graalvm.compiler.lir.asm.ArrayDataPointerConstant; +import org.graalvm.compiler.lir.asm.CompilationResultBuilder; + +import jdk.vm.ci.amd64.AMD64; +import jdk.vm.ci.amd64.AMD64Kind; +import jdk.vm.ci.code.Register; +import jdk.vm.ci.meta.Value; + +public final class AMD64HotSpotHelper { + + private AMD64HotSpotHelper() { + } + + protected static Value[] registersToValues(Register[] registers) { + Value[] temps = new Value[registers.length]; + for (int i = 0; i < registers.length; i++) { + Register register = registers[i]; + if (AMD64.CPU.equals(register.getRegisterCategory())) { + temps[i] = register.asValue(LIRKind.value(AMD64Kind.QWORD)); + } else if (AMD64.XMM.equals(register.getRegisterCategory())) { + temps[i] = register.asValue(LIRKind.value(AMD64Kind.DOUBLE)); + } else { + throw GraalError.shouldNotReachHere("Unsupported register type in math stubs."); + } + } + return temps; + } + + protected static AMD64Address recordExternalAddress(CompilationResultBuilder crb, ArrayDataPointerConstant ptr) { + return (AMD64Address) crb.recordDataReferenceInCode(ptr); + } + + protected static ArrayDataPointerConstant pointerConstant(int alignment, int[] ints) { + return new ArrayDataPointerConstant(ints, alignment); + } +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathCosOp.java 2019-03-12 08:10:45.820028075 +0100 @@ -0,0 +1,869 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, Intel Corporation. All rights reserved. + * Intel Math Library (LIBM) Source Code + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.lir.amd64; + +import static jdk.vm.ci.amd64.AMD64.r10; +import static jdk.vm.ci.amd64.AMD64.r11; +import static jdk.vm.ci.amd64.AMD64.r8; +import static jdk.vm.ci.amd64.AMD64.r9; +import static jdk.vm.ci.amd64.AMD64.rax; +import static jdk.vm.ci.amd64.AMD64.rbx; +import static jdk.vm.ci.amd64.AMD64.rcx; +import static jdk.vm.ci.amd64.AMD64.rdi; +import static jdk.vm.ci.amd64.AMD64.rdx; +import static jdk.vm.ci.amd64.AMD64.rsi; +import static jdk.vm.ci.amd64.AMD64.rsp; +import static jdk.vm.ci.amd64.AMD64.xmm0; +import static jdk.vm.ci.amd64.AMD64.xmm1; +import static jdk.vm.ci.amd64.AMD64.xmm2; +import static jdk.vm.ci.amd64.AMD64.xmm3; +import static jdk.vm.ci.amd64.AMD64.xmm4; +import static jdk.vm.ci.amd64.AMD64.xmm5; +import static jdk.vm.ci.amd64.AMD64.xmm6; +import static jdk.vm.ci.amd64.AMD64.xmm7; +import static org.graalvm.compiler.lir.amd64.AMD64HotSpotHelper.pointerConstant; +import static org.graalvm.compiler.lir.amd64.AMD64HotSpotHelper.recordExternalAddress; + +import org.graalvm.compiler.asm.Label; +import org.graalvm.compiler.asm.amd64.AMD64Address; +import org.graalvm.compiler.asm.amd64.AMD64Assembler.ConditionFlag; +import org.graalvm.compiler.asm.amd64.AMD64MacroAssembler; +import org.graalvm.compiler.lir.LIRInstructionClass; +import org.graalvm.compiler.lir.asm.ArrayDataPointerConstant; +import org.graalvm.compiler.lir.asm.CompilationResultBuilder; + +/** + * 

+ *                     ALGORITHM DESCRIPTION - COS()
+ *                     ---------------------
+ *
+ *     1. RANGE REDUCTION
+ *
+ *     We perform an initial range reduction from X to r with
+ *
+ *          X =~= N * pi/32 + r
+ *
+ *     so that |r| <= pi/64 + epsilon. We restrict inputs to those
+ *     where |N| <= 932560. Beyond this, the range reduction is
+ *     insufficiently accurate. For extremely small inputs,
+ *     denormalization can occur internally, impacting performance.
+ *     This means that the main path is actually only taken for
+ *     2^-252 <= |X| < 90112.
+ *
+ *     To avoid branches, we perform the range reduction to full
+ *     accuracy each time.
+ *
+ *          X - N * (P_1 + P_2 + P_3)
+ *
+ *     where P_1 and P_2 are 32-bit numbers (so multiplication by N
+ *     is exact) and P_3 is a 53-bit number. Together, these
+ *     approximate pi well enough for all cases in the restricted
+ *     range.
+ *
+ *     The main reduction sequence is:
+ *
+ *             y = 32/pi * x
+ *             N = integer(y)
+ *     (computed by adding and subtracting off SHIFTER)
+ *
+ *             m_1 = N * P_1
+ *             m_2 = N * P_2
+ *             r_1 = x - m_1
+ *             r = r_1 - m_2
+ *     (this r can be used for most of the calculation)
+ *
+ *             c_1 = r_1 - r
+ *             m_3 = N * P_3
+ *             c_2 = c_1 - m_2
+ *             c = c_2 - m_3
+ *
+ *     2. MAIN ALGORITHM
+ *
+ *     The algorithm uses a table lookup based on B = M * pi / 32
+ *     where M = N mod 64. The stored values are:
+ *       sigma             closest power of 2 to cos(B)
+ *       C_hl              53-bit cos(B) - sigma
+ *       S_hi + S_lo       2 * 53-bit sin(B)
+ *
+ *     The computation is organized as follows:
+ *
+ *          sin(B + r + c) = [sin(B) + sigma * r] +
+ *                           r * (cos(B) - sigma) +
+ *                           sin(B) * [cos(r + c) - 1] +
+ *                           cos(B) * [sin(r + c) - r]
+ *
+ *     which is approximately:
+ *
+ *          [S_hi + sigma * r] +
+ *          C_hl * r +
+ *          S_lo + S_hi * [(cos(r) - 1) - r * c] +
+ *          (C_hl + sigma) * [(sin(r) - r) + c]
+ *
+ *     and this is what is actually computed. We separate this sum
+ *     into four parts:
+ *
+ *          hi + med + pols + corr
+ *
+ *     where
+ *
+ *          hi       = S_hi + sigma r
+ *          med      = C_hl * r
+ *          pols     = S_hi * (cos(r) - 1) + (C_hl + sigma) * (sin(r) - r)
+ *          corr     = S_lo + c * ((C_hl + sigma) - S_hi * r)
+ *
+ *     3. POLYNOMIAL
+ *
+ *     The polynomial S_hi * (cos(r) - 1) + (C_hl + sigma) *
+ *     (sin(r) - r) can be rearranged freely, since it is quite
+ *     small, so we exploit parallelism to the fullest.
+ *
+ *          psc4       =   SC_4 * r_1
+ *          msc4       =   psc4 * r
+ *          r2         =   r * r
+ *          msc2       =   SC_2 * r2
+ *          r4         =   r2 * r2
+ *          psc3       =   SC_3 + msc4
+ *          psc1       =   SC_1 + msc2
+ *          msc3       =   r4 * psc3
+ *          sincospols =   psc1 + msc3
+ *          pols       =   sincospols *
+ *                         
+ *
+ *     4. CORRECTION TERM
+ *
+ *     This is where the "c" component of the range reduction is
+ *     taken into account; recall that just "r" is used for most of
+ *     the calculation.
+ *
+ *          -c   = m_3 - c_2
+ *          -d   = S_hi * r - (C_hl + sigma)
+ *          corr = -c * -d + S_lo
+ *
+ *     5. COMPENSATED SUMMATIONS
+ *
+ *     The two successive compensated summations add up the high
+ *     and medium parts, leaving just the low parts to add up at
+ *     the end.
+ *
+ *          rs        =  sigma * r
+ *          res_int   =  S_hi + rs
+ *          k_0       =  S_hi - res_int
+ *          k_2       =  k_0 + rs
+ *          med       =  C_hl * r
+ *          res_hi    =  res_int + med
+ *          k_1       =  res_int - res_hi
+ *          k_3       =  k_1 + med
+ *
+ *     6. FINAL SUMMATION
+ *
+ *     We now add up all the small parts:
+ *
+ *          res_lo = pols(hi) + pols(lo) + corr + k_1 + k_3
+ *
+ *     Now the overall result is just:
+ *
+ *          res_hi + res_lo
+ *
+ *     7. SMALL ARGUMENTS
+ *
+ *     Inputs with |X| < 2^-252 are treated specially as
+ *     1 - |x|.
+ *
+ * Special cases:
+ *  cos(NaN) = quiet NaN, and raise invalid exception
+ *  cos(INF) = NaN and raise invalid exception
+ *  cos(0) = 1
+ *
+ */ +public final class AMD64MathCosOp extends AMD64MathIntrinsicUnaryOp { + + public static final LIRInstructionClass TYPE = LIRInstructionClass.create(AMD64MathCosOp.class); + + public AMD64MathCosOp() { + super(TYPE, /* GPR */ rax, rcx, rdx, rbx, rsi, rdi, r8, r9, r10, r11, + /* XMM */ xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7); + } + + private ArrayDataPointerConstant onehalf = pointerConstant(16, new int[]{ + // @formatter:off + 0x00000000, 0x3fe00000, 0x00000000, 0x3fe00000 + // @formatter:on + }); + + private ArrayDataPointerConstant p2 = pointerConstant(16, new int[]{ + // @formatter:off + 0x1a600000, 0x3d90b461, 0x1a600000, 0x3d90b461 + // @formatter:on + }); + + private ArrayDataPointerConstant sc4 = pointerConstant(16, new int[]{ + // @formatter:off + 0xa556c734, 0x3ec71de3, 0x1a01a01a, 0x3efa01a0 + // @formatter:on + }); + + private ArrayDataPointerConstant ctable = pointerConstant(16, new int[]{ + // @formatter:off + 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x3ff00000, 0x176d6d31, 0xbf73b92e, + 0xbc29b42c, 0x3fb917a6, 0xe0000000, 0xbc3e2718, 0x00000000, + 0x3ff00000, 0x011469fb, 0xbf93ad06, 0x3c69a60b, 0x3fc8f8b8, + 0xc0000000, 0xbc626d19, 0x00000000, 0x3ff00000, 0x939d225a, + 0xbfa60bea, 0x2ed59f06, 0x3fd29406, 0xa0000000, 0xbc75d28d, + 0x00000000, 0x3ff00000, 0x866b95cf, 0xbfb37ca1, 0xa6aea963, + 0x3fd87de2, 0xe0000000, 0xbc672ced, 0x00000000, 0x3ff00000, + 0x73fa1279, 0xbfbe3a68, 0x3806f63b, 0x3fde2b5d, 0x20000000, + 0x3c5e0d89, 0x00000000, 0x3ff00000, 0x5bc57974, 0xbfc59267, + 0x39ae68c8, 0x3fe1c73b, 0x20000000, 0x3c8b25dd, 0x00000000, + 0x3ff00000, 0x53aba2fd, 0xbfcd0dfe, 0x25091dd6, 0x3fe44cf3, + 0x20000000, 0x3c68076a, 0x00000000, 0x3ff00000, 0x99fcef32, + 0x3fca8279, 0x667f3bcd, 0x3fe6a09e, 0x20000000, 0xbc8bdd34, + 0x00000000, 0x3fe00000, 0x94247758, 0x3fc133cc, 0x6b151741, + 0x3fe8bc80, 0x20000000, 0xbc82c5e1, 0x00000000, 0x3fe00000, + 0x9ae68c87, 0x3fac73b3, 0x290ea1a3, 0x3fea9b66, 0xe0000000, + 0x3c39f630, 0x00000000, 0x3fe00000, 0x7f909c4e, 0xbf9d4a2c, + 0xf180bdb1, 0x3fec38b2, 0x80000000, 0xbc76e0b1, 0x00000000, + 0x3fe00000, 0x65455a75, 0xbfbe0875, 0xcf328d46, 0x3fed906b, + 0x20000000, 0x3c7457e6, 0x00000000, 0x3fe00000, 0x76acf82d, + 0x3fa4a031, 0x56c62dda, 0x3fee9f41, 0xe0000000, 0x3c8760b1, + 0x00000000, 0x3fd00000, 0x0e5967d5, 0xbfac1d1f, 0xcff75cb0, + 0x3fef6297, 0x20000000, 0x3c756217, 0x00000000, 0x3fd00000, + 0x0f592f50, 0xbf9ba165, 0xa3d12526, 0x3fefd88d, 0x40000000, + 0xbc887df6, 0x00000000, 0x3fc00000, 0x00000000, 0x00000000, + 0x00000000, 0x3ff00000, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x0f592f50, 0x3f9ba165, 0xa3d12526, 0x3fefd88d, + 0x40000000, 0xbc887df6, 0x00000000, 0xbfc00000, 0x0e5967d5, + 0x3fac1d1f, 0xcff75cb0, 0x3fef6297, 0x20000000, 0x3c756217, + 0x00000000, 0xbfd00000, 0x76acf82d, 0xbfa4a031, 0x56c62dda, + 0x3fee9f41, 0xe0000000, 0x3c8760b1, 0x00000000, 0xbfd00000, + 0x65455a75, 0x3fbe0875, 0xcf328d46, 0x3fed906b, 0x20000000, + 0x3c7457e6, 0x00000000, 0xbfe00000, 0x7f909c4e, 0x3f9d4a2c, + 0xf180bdb1, 0x3fec38b2, 0x80000000, 0xbc76e0b1, 0x00000000, + 0xbfe00000, 0x9ae68c87, 0xbfac73b3, 0x290ea1a3, 0x3fea9b66, + 0xe0000000, 0x3c39f630, 0x00000000, 0xbfe00000, 0x94247758, + 0xbfc133cc, 0x6b151741, 0x3fe8bc80, 0x20000000, 0xbc82c5e1, + 0x00000000, 0xbfe00000, 0x99fcef32, 0xbfca8279, 0x667f3bcd, + 0x3fe6a09e, 0x20000000, 0xbc8bdd34, 0x00000000, 0xbfe00000, + 0x53aba2fd, 0x3fcd0dfe, 0x25091dd6, 0x3fe44cf3, 0x20000000, + 0x3c68076a, 0x00000000, 0xbff00000, 0x5bc57974, 0x3fc59267, + 0x39ae68c8, 0x3fe1c73b, 0x20000000, 0x3c8b25dd, 0x00000000, + 0xbff00000, 0x73fa1279, 0x3fbe3a68, 0x3806f63b, 0x3fde2b5d, + 0x20000000, 0x3c5e0d89, 0x00000000, 0xbff00000, 0x866b95cf, + 0x3fb37ca1, 0xa6aea963, 0x3fd87de2, 0xe0000000, 0xbc672ced, + 0x00000000, 0xbff00000, 0x939d225a, 0x3fa60bea, 0x2ed59f06, + 0x3fd29406, 0xa0000000, 0xbc75d28d, 0x00000000, 0xbff00000, + 0x011469fb, 0x3f93ad06, 0x3c69a60b, 0x3fc8f8b8, 0xc0000000, + 0xbc626d19, 0x00000000, 0xbff00000, 0x176d6d31, 0x3f73b92e, + 0xbc29b42c, 0x3fb917a6, 0xe0000000, 0xbc3e2718, 0x00000000, + 0xbff00000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0xbff00000, 0x176d6d31, + 0x3f73b92e, 0xbc29b42c, 0xbfb917a6, 0xe0000000, 0x3c3e2718, + 0x00000000, 0xbff00000, 0x011469fb, 0x3f93ad06, 0x3c69a60b, + 0xbfc8f8b8, 0xc0000000, 0x3c626d19, 0x00000000, 0xbff00000, + 0x939d225a, 0x3fa60bea, 0x2ed59f06, 0xbfd29406, 0xa0000000, + 0x3c75d28d, 0x00000000, 0xbff00000, 0x866b95cf, 0x3fb37ca1, + 0xa6aea963, 0xbfd87de2, 0xe0000000, 0x3c672ced, 0x00000000, + 0xbff00000, 0x73fa1279, 0x3fbe3a68, 0x3806f63b, 0xbfde2b5d, + 0x20000000, 0xbc5e0d89, 0x00000000, 0xbff00000, 0x5bc57974, + 0x3fc59267, 0x39ae68c8, 0xbfe1c73b, 0x20000000, 0xbc8b25dd, + 0x00000000, 0xbff00000, 0x53aba2fd, 0x3fcd0dfe, 0x25091dd6, + 0xbfe44cf3, 0x20000000, 0xbc68076a, 0x00000000, 0xbff00000, + 0x99fcef32, 0xbfca8279, 0x667f3bcd, 0xbfe6a09e, 0x20000000, + 0x3c8bdd34, 0x00000000, 0xbfe00000, 0x94247758, 0xbfc133cc, + 0x6b151741, 0xbfe8bc80, 0x20000000, 0x3c82c5e1, 0x00000000, + 0xbfe00000, 0x9ae68c87, 0xbfac73b3, 0x290ea1a3, 0xbfea9b66, + 0xe0000000, 0xbc39f630, 0x00000000, 0xbfe00000, 0x7f909c4e, + 0x3f9d4a2c, 0xf180bdb1, 0xbfec38b2, 0x80000000, 0x3c76e0b1, + 0x00000000, 0xbfe00000, 0x65455a75, 0x3fbe0875, 0xcf328d46, + 0xbfed906b, 0x20000000, 0xbc7457e6, 0x00000000, 0xbfe00000, + 0x76acf82d, 0xbfa4a031, 0x56c62dda, 0xbfee9f41, 0xe0000000, + 0xbc8760b1, 0x00000000, 0xbfd00000, 0x0e5967d5, 0x3fac1d1f, + 0xcff75cb0, 0xbfef6297, 0x20000000, 0xbc756217, 0x00000000, + 0xbfd00000, 0x0f592f50, 0x3f9ba165, 0xa3d12526, 0xbfefd88d, + 0x40000000, 0x3c887df6, 0x00000000, 0xbfc00000, 0x00000000, + 0x00000000, 0x00000000, 0xbff00000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x0f592f50, 0xbf9ba165, 0xa3d12526, + 0xbfefd88d, 0x40000000, 0x3c887df6, 0x00000000, 0x3fc00000, + 0x0e5967d5, 0xbfac1d1f, 0xcff75cb0, 0xbfef6297, 0x20000000, + 0xbc756217, 0x00000000, 0x3fd00000, 0x76acf82d, 0x3fa4a031, + 0x56c62dda, 0xbfee9f41, 0xe0000000, 0xbc8760b1, 0x00000000, + 0x3fd00000, 0x65455a75, 0xbfbe0875, 0xcf328d46, 0xbfed906b, + 0x20000000, 0xbc7457e6, 0x00000000, 0x3fe00000, 0x7f909c4e, + 0xbf9d4a2c, 0xf180bdb1, 0xbfec38b2, 0x80000000, 0x3c76e0b1, + 0x00000000, 0x3fe00000, 0x9ae68c87, 0x3fac73b3, 0x290ea1a3, + 0xbfea9b66, 0xe0000000, 0xbc39f630, 0x00000000, 0x3fe00000, + 0x94247758, 0x3fc133cc, 0x6b151741, 0xbfe8bc80, 0x20000000, + 0x3c82c5e1, 0x00000000, 0x3fe00000, 0x99fcef32, 0x3fca8279, + 0x667f3bcd, 0xbfe6a09e, 0x20000000, 0x3c8bdd34, 0x00000000, + 0x3fe00000, 0x53aba2fd, 0xbfcd0dfe, 0x25091dd6, 0xbfe44cf3, + 0x20000000, 0xbc68076a, 0x00000000, 0x3ff00000, 0x5bc57974, + 0xbfc59267, 0x39ae68c8, 0xbfe1c73b, 0x20000000, 0xbc8b25dd, + 0x00000000, 0x3ff00000, 0x73fa1279, 0xbfbe3a68, 0x3806f63b, + 0xbfde2b5d, 0x20000000, 0xbc5e0d89, 0x00000000, 0x3ff00000, + 0x866b95cf, 0xbfb37ca1, 0xa6aea963, 0xbfd87de2, 0xe0000000, + 0x3c672ced, 0x00000000, 0x3ff00000, 0x939d225a, 0xbfa60bea, + 0x2ed59f06, 0xbfd29406, 0xa0000000, 0x3c75d28d, 0x00000000, + 0x3ff00000, 0x011469fb, 0xbf93ad06, 0x3c69a60b, 0xbfc8f8b8, + 0xc0000000, 0x3c626d19, 0x00000000, 0x3ff00000, 0x176d6d31, + 0xbf73b92e, 0xbc29b42c, 0xbfb917a6, 0xe0000000, 0x3c3e2718, + 0x00000000, 0x3ff00000 + // @formatter:on + }); + + private ArrayDataPointerConstant sc2 = pointerConstant(16, new int[]{ + // @formatter:off + 0x11111111, 0x3f811111, 0x55555555, 0x3fa55555 + // @formatter:on + }); + + private ArrayDataPointerConstant sc3 = pointerConstant(16, new int[]{ + // @formatter:off + 0x1a01a01a, 0xbf2a01a0, 0x16c16c17, 0xbf56c16c + // @formatter:on + }); + + private ArrayDataPointerConstant sc1 = pointerConstant(16, new int[]{ + // @formatter:off + 0x55555555, 0xbfc55555, 0x00000000, 0xbfe00000 + // @formatter:on + }); + + private ArrayDataPointerConstant piInvTable = pointerConstant(16, new int[]{ + // @formatter:off + 0x00000000, 0x00000000, 0xa2f9836e, 0x4e441529, 0xfc2757d1, + 0xf534ddc0, 0xdb629599, 0x3c439041, 0xfe5163ab, 0xdebbc561, + 0xb7246e3a, 0x424dd2e0, 0x06492eea, 0x09d1921c, 0xfe1deb1c, + 0xb129a73e, 0xe88235f5, 0x2ebb4484, 0xe99c7026, 0xb45f7e41, + 0x3991d639, 0x835339f4, 0x9c845f8b, 0xbdf9283b, 0x1ff897ff, + 0xde05980f, 0xef2f118b, 0x5a0a6d1f, 0x6d367ecf, 0x27cb09b7, + 0x4f463f66, 0x9e5fea2d, 0x7527bac7, 0xebe5f17b, 0x3d0739f7, + 0x8a5292ea, 0x6bfb5fb1, 0x1f8d5d08, 0x56033046, 0xfc7b6bab, + 0xf0cfbc21 + // @formatter:on + }); + + private ArrayDataPointerConstant pi4 = pointerConstant(8, new int[]{ + // @formatter:off + 0x40000000, 0x3fe921fb, 0x18469899, 0x3e64442d + // @formatter:on + }); + + private ArrayDataPointerConstant pi48 = pointerConstant(8, new int[]{ + // @formatter:off + 0x18469899, 0x3e64442d + // @formatter:on + }); + + private ArrayDataPointerConstant pi32Inv = pointerConstant(8, new int[]{ + // @formatter:off + 0x6dc9c883, 0x40245f30 + // @formatter:on + }); + + private ArrayDataPointerConstant signMask = pointerConstant(8, new int[]{ + // @formatter:off + 0x00000000, 0x80000000 + // @formatter:on + }); + + private ArrayDataPointerConstant p3 = pointerConstant(8, new int[]{ + // @formatter:off + 0x2e037073, 0x3b63198a + // @formatter:on + }); + + private ArrayDataPointerConstant p1 = pointerConstant(8, new int[]{ + // @formatter:off + 0x54400000, 0x3fb921fb + // @formatter:on + }); + + private ArrayDataPointerConstant negZero = pointerConstant(8, new int[]{ + // @formatter:off + 0x00000000, 0x80000000 + // @formatter:on + }); + + // The 64 bit code is at most SSE2 compliant + private ArrayDataPointerConstant one = pointerConstant(8, new int[]{ + // @formatter:off + 0x00000000, 0x3ff00000 + // @formatter:on + }); + + @Override + public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { + Label block0 = new Label(); + Label block1 = new Label(); + Label block2 = new Label(); + Label block3 = new Label(); + Label block4 = new Label(); + Label block5 = new Label(); + Label block6 = new Label(); + Label block7 = new Label(); + Label block8 = new Label(); + Label block9 = new Label(); + Label block10 = new Label(); + Label block11 = new Label(); + Label block12 = new Label(); + Label block13 = new Label(); + + masm.push(rbx); + masm.subq(rsp, 16); + masm.movsd(new AMD64Address(rsp, 8), xmm0); + + masm.movl(rax, new AMD64Address(rsp, 12)); + masm.movq(xmm1, recordExternalAddress(crb, pi32Inv)); // 0x6dc9c883, 0x40245f30 + masm.andl(rax, 2147418112); + masm.subl(rax, 808452096); + masm.cmpl(rax, 281346048); + masm.jcc(ConditionFlag.Above, block0); + masm.mulsd(xmm1, xmm0); + masm.movdqu(xmm5, recordExternalAddress(crb, onehalf)); // 0x00000000, 0x3fe00000, + // 0x00000000, 0x3fe00000 + masm.movq(xmm4, recordExternalAddress(crb, signMask)); // 0x00000000, 0x80000000 + masm.pand(xmm4, xmm0); + masm.por(xmm5, xmm4); + masm.addpd(xmm1, xmm5); + masm.cvttsd2sil(rdx, xmm1); + masm.cvtsi2sdl(xmm1, rdx); + masm.movdqu(xmm2, recordExternalAddress(crb, p2)); // 0x1a600000, 0x3d90b461, + // 0x1a600000, 0x3d90b461 + masm.movq(xmm3, recordExternalAddress(crb, p1)); // 0x54400000, 0x3fb921fb + masm.mulsd(xmm3, xmm1); + masm.unpcklpd(xmm1, xmm1); + masm.addq(rdx, 1865232); + masm.movdqu(xmm4, xmm0); + masm.andq(rdx, 63); + masm.movdqu(xmm5, recordExternalAddress(crb, sc4)); // 0xa556c734, 0x3ec71de3, + // 0x1a01a01a, 0x3efa01a0 + masm.leaq(rax, recordExternalAddress(crb, ctable)); + masm.shlq(rdx, 5); + masm.addq(rax, rdx); + masm.mulpd(xmm2, xmm1); + masm.subsd(xmm0, xmm3); + masm.mulsd(xmm1, recordExternalAddress(crb, p3)); // 0x2e037073, 0x3b63198a + masm.subsd(xmm4, xmm3); + masm.movq(xmm7, new AMD64Address(rax, 8)); + masm.unpcklpd(xmm0, xmm0); + masm.movdqu(xmm3, xmm4); + masm.subsd(xmm4, xmm2); + masm.mulpd(xmm5, xmm0); + masm.subpd(xmm0, xmm2); + masm.movdqu(xmm6, recordExternalAddress(crb, sc2)); // 0x11111111, 0x3f811111, + // 0x55555555, 0x3fa55555 + masm.mulsd(xmm7, xmm4); + masm.subsd(xmm3, xmm4); + masm.mulpd(xmm5, xmm0); + masm.mulpd(xmm0, xmm0); + masm.subsd(xmm3, xmm2); + masm.movdqu(xmm2, new AMD64Address(rax, 0)); + masm.subsd(xmm1, xmm3); + masm.movq(xmm3, new AMD64Address(rax, 24)); + masm.addsd(xmm2, xmm3); + masm.subsd(xmm7, xmm2); + masm.mulsd(xmm2, xmm4); + masm.mulpd(xmm6, xmm0); + masm.mulsd(xmm3, xmm4); + masm.mulpd(xmm2, xmm0); + masm.mulpd(xmm0, xmm0); + masm.addpd(xmm5, recordExternalAddress(crb, sc3)); // 0x1a01a01a, 0xbf2a01a0, + // 0x16c16c17, 0xbf56c16c + masm.mulsd(xmm4, new AMD64Address(rax, 0)); + masm.addpd(xmm6, recordExternalAddress(crb, sc1)); // 0x55555555, 0xbfc55555, + // 0x00000000, 0xbfe00000 + masm.mulpd(xmm5, xmm0); + masm.movdqu(xmm0, xmm3); + masm.addsd(xmm3, new AMD64Address(rax, 8)); + masm.mulpd(xmm1, xmm7); + masm.movdqu(xmm7, xmm4); + masm.addsd(xmm4, xmm3); + masm.addpd(xmm6, xmm5); + masm.movq(xmm5, new AMD64Address(rax, 8)); + masm.subsd(xmm5, xmm3); + masm.subsd(xmm3, xmm4); + masm.addsd(xmm1, new AMD64Address(rax, 16)); + masm.mulpd(xmm6, xmm2); + masm.addsd(xmm0, xmm5); + masm.addsd(xmm3, xmm7); + masm.addsd(xmm0, xmm1); + masm.addsd(xmm0, xmm3); + masm.addsd(xmm0, xmm6); + masm.unpckhpd(xmm6, xmm6); + masm.addsd(xmm0, xmm6); + masm.addsd(xmm0, xmm4); + masm.jmp(block13); + + masm.bind(block0); + masm.jcc(ConditionFlag.Greater, block1); + masm.pextrw(rax, xmm0, 3); + masm.andl(rax, 32767); + masm.pinsrw(xmm0, rax, 3); + masm.movq(xmm1, recordExternalAddress(crb, one)); // 0x00000000, 0x3ff00000 + masm.subsd(xmm1, xmm0); + masm.movdqu(xmm0, xmm1); + masm.jmp(block13); + + masm.bind(block1); + masm.pextrw(rax, xmm0, 3); + masm.andl(rax, 32752); + masm.cmpl(rax, 32752); + masm.jcc(ConditionFlag.Equal, block2); + masm.pextrw(rcx, xmm0, 3); + masm.andl(rcx, 32752); + masm.subl(rcx, 16224); + masm.shrl(rcx, 7); + masm.andl(rcx, 65532); + masm.leaq(r11, recordExternalAddress(crb, piInvTable)); + masm.addq(rcx, r11); + masm.movdq(rax, xmm0); + masm.movl(r10, new AMD64Address(rcx, 20)); + masm.movl(r8, new AMD64Address(rcx, 24)); + masm.movl(rdx, rax); + masm.shrq(rax, 21); + masm.orl(rax, Integer.MIN_VALUE); + masm.shrl(rax, 11); + masm.movl(r9, r10); + masm.imulq(r10, rdx); + masm.imulq(r9, rax); + masm.imulq(r8, rax); + masm.movl(rsi, new AMD64Address(rcx, 16)); + masm.movl(rdi, new AMD64Address(rcx, 12)); + masm.movl(r11, r10); + masm.shrq(r10, 32); + masm.addq(r9, r10); + masm.addq(r11, r8); + masm.movl(r8, r11); + masm.shrq(r11, 32); + masm.addq(r9, r11); + masm.movl(r10, rsi); + masm.imulq(rsi, rdx); + masm.imulq(r10, rax); + masm.movl(r11, rdi); + masm.imulq(rdi, rdx); + masm.movl(rbx, rsi); + masm.shrq(rsi, 32); + masm.addq(r9, rbx); + masm.movl(rbx, r9); + masm.shrq(r9, 32); + masm.addq(r10, rsi); + masm.addq(r10, r9); + masm.shlq(rbx, 32); + masm.orq(r8, rbx); + masm.imulq(r11, rax); + masm.movl(r9, new AMD64Address(rcx, 8)); + masm.movl(rsi, new AMD64Address(rcx, 4)); + masm.movl(rbx, rdi); + masm.shrq(rdi, 32); + masm.addq(r10, rbx); + masm.movl(rbx, r10); + masm.shrq(r10, 32); + masm.addq(r11, rdi); + masm.addq(r11, r10); + masm.movq(rdi, r9); + masm.imulq(r9, rdx); + masm.imulq(rdi, rax); + masm.movl(r10, r9); + masm.shrq(r9, 32); + masm.addq(r11, r10); + masm.movl(r10, r11); + masm.shrq(r11, 32); + masm.addq(rdi, r9); + masm.addq(rdi, r11); + masm.movq(r9, rsi); + masm.imulq(rsi, rdx); + masm.imulq(r9, rax); + masm.shlq(r10, 32); + masm.orq(r10, rbx); + masm.movl(rax, new AMD64Address(rcx, 0)); + masm.movl(r11, rsi); + masm.shrq(rsi, 32); + masm.addq(rdi, r11); + masm.movl(r11, rdi); + masm.shrq(rdi, 32); + masm.addq(r9, rsi); + masm.addq(r9, rdi); + masm.imulq(rdx, rax); + masm.pextrw(rbx, xmm0, 3); + masm.leaq(rdi, recordExternalAddress(crb, piInvTable)); + masm.subq(rcx, rdi); + masm.addl(rcx, rcx); + masm.addl(rcx, rcx); + masm.addl(rcx, rcx); + masm.addl(rcx, 19); + masm.movl(rsi, 32768); + masm.andl(rsi, rbx); + masm.shrl(rbx, 4); + masm.andl(rbx, 2047); + masm.subl(rbx, 1023); + masm.subl(rcx, rbx); + masm.addq(r9, rdx); + masm.movl(rdx, rcx); + masm.addl(rdx, 32); + masm.cmpl(rcx, 1); + masm.jcc(ConditionFlag.Less, block3); + masm.negl(rcx); + masm.addl(rcx, 29); + masm.shll(r9); + masm.movl(rdi, r9); + masm.andl(r9, 536870911); + masm.testl(r9, 268435456); + masm.jcc(ConditionFlag.NotEqual, block4); + masm.shrl(r9); + masm.movl(rbx, 0); + masm.shlq(r9, 32); + masm.orq(r9, r11); + + masm.bind(block5); + + masm.bind(block6); + masm.cmpq(r9, 0); + masm.jcc(ConditionFlag.Equal, block7); + + masm.bind(block8); + masm.bsrq(r11, r9); + masm.movl(rcx, 29); + masm.subl(rcx, r11); + masm.jcc(ConditionFlag.LessEqual, block9); + masm.shlq(r9); + masm.movq(rax, r10); + masm.shlq(r10); + masm.addl(rdx, rcx); + masm.negl(rcx); + masm.addl(rcx, 64); + masm.shrq(rax); + masm.shrq(r8); + masm.orq(r9, rax); + masm.orq(r10, r8); + + masm.bind(block10); + masm.cvtsi2sdq(xmm0, r9); + masm.shrq(r10, 1); + masm.cvtsi2sdq(xmm3, r10); + masm.xorpd(xmm4, xmm4); + masm.shll(rdx, 4); + masm.negl(rdx); + masm.addl(rdx, 16368); + masm.orl(rdx, rsi); + masm.xorl(rdx, rbx); + masm.pinsrw(xmm4, rdx, 3); + masm.movq(xmm2, recordExternalAddress(crb, pi4)); // 0x40000000, 0x3fe921fb, + // 0x18469899, 0x3e64442d + masm.movq(xmm6, recordExternalAddress(crb, pi48)); // 0x3fe921fb, 0x18469899, + // 0x3e64442d + masm.xorpd(xmm5, xmm5); + masm.subl(rdx, 1008); + masm.pinsrw(xmm5, rdx, 3); + masm.mulsd(xmm0, xmm4); + masm.shll(rsi, 16); + masm.sarl(rsi, 31); + masm.mulsd(xmm3, xmm5); + masm.movdqu(xmm1, xmm0); + masm.mulsd(xmm0, xmm2); + masm.shrl(rdi, 29); + masm.addsd(xmm1, xmm3); + masm.mulsd(xmm3, xmm2); + masm.addl(rdi, rsi); + masm.xorl(rdi, rsi); + masm.mulsd(xmm6, xmm1); + masm.movl(rax, rdi); + masm.addsd(xmm6, xmm3); + masm.movdqu(xmm2, xmm0); + masm.addsd(xmm0, xmm6); + masm.subsd(xmm2, xmm0); + masm.addsd(xmm6, xmm2); + + masm.bind(block11); + masm.movq(xmm1, recordExternalAddress(crb, pi32Inv)); // 0x6dc9c883, 0x40245f30 + masm.mulsd(xmm1, xmm0); + masm.movq(xmm5, recordExternalAddress(crb, onehalf)); // 0x00000000, 0x3fe00000, + // 0x00000000, 0x3fe00000 + masm.movq(xmm4, recordExternalAddress(crb, signMask)); // 0x00000000, 0x80000000 + masm.pand(xmm4, xmm0); + masm.por(xmm5, xmm4); + masm.addpd(xmm1, xmm5); + masm.cvttsd2siq(rdx, xmm1); + masm.cvtsi2sdq(xmm1, rdx); + masm.movq(xmm3, recordExternalAddress(crb, p1)); // 0x54400000, 0x3fb921fb + masm.movdqu(xmm2, recordExternalAddress(crb, p2)); // 0x1a600000, 0x3d90b461, + // 0x1a600000, 0x3d90b461 + masm.mulsd(xmm3, xmm1); + masm.unpcklpd(xmm1, xmm1); + masm.shll(rax, 3); + masm.addl(rdx, 1865232); + masm.movdqu(xmm4, xmm0); + masm.addl(rdx, rax); + masm.andl(rdx, 63); + masm.movdqu(xmm5, recordExternalAddress(crb, sc4)); // 0xa556c734, 0x3ec71de3, + // 0x1a01a01a, 0x3efa01a0 + masm.leaq(rax, recordExternalAddress(crb, ctable)); + masm.shll(rdx, 5); + masm.addq(rax, rdx); + masm.mulpd(xmm2, xmm1); + masm.subsd(xmm0, xmm3); + masm.mulsd(xmm1, recordExternalAddress(crb, p3)); // 0x2e037073, 0x3b63198a + masm.subsd(xmm4, xmm3); + masm.movq(xmm7, new AMD64Address(rax, 8)); + masm.unpcklpd(xmm0, xmm0); + masm.movdqu(xmm3, xmm4); + masm.subsd(xmm4, xmm2); + masm.mulpd(xmm5, xmm0); + masm.subpd(xmm0, xmm2); + masm.mulsd(xmm7, xmm4); + masm.subsd(xmm3, xmm4); + masm.mulpd(xmm5, xmm0); + masm.mulpd(xmm0, xmm0); + masm.subsd(xmm3, xmm2); + masm.movdqu(xmm2, new AMD64Address(rax, 0)); + masm.subsd(xmm1, xmm3); + masm.movq(xmm3, new AMD64Address(rax, 24)); + masm.addsd(xmm2, xmm3); + masm.subsd(xmm7, xmm2); + masm.subsd(xmm1, xmm6); + masm.movdqu(xmm6, recordExternalAddress(crb, sc2)); // 0x11111111, 0x3f811111, + // 0x55555555, 0x3fa55555 + masm.mulsd(xmm2, xmm4); + masm.mulpd(xmm6, xmm0); + masm.mulsd(xmm3, xmm4); + masm.mulpd(xmm2, xmm0); + masm.mulpd(xmm0, xmm0); + masm.addpd(xmm5, recordExternalAddress(crb, sc3)); // 0x1a01a01a, 0xbf2a01a0, + // 0x16c16c17, 0xbf56c16c + masm.mulsd(xmm4, new AMD64Address(rax, 0)); + masm.addpd(xmm6, recordExternalAddress(crb, sc1)); // 0x55555555, 0xbfc55555, + // 0x00000000, 0xbfe00000 + masm.mulpd(xmm5, xmm0); + masm.movdqu(xmm0, xmm3); + masm.addsd(xmm3, new AMD64Address(rax, 8)); + masm.mulpd(xmm1, xmm7); + masm.movdqu(xmm7, xmm4); + masm.addsd(xmm4, xmm3); + masm.addpd(xmm6, xmm5); + masm.movq(xmm5, new AMD64Address(rax, 8)); + masm.subsd(xmm5, xmm3); + masm.subsd(xmm3, xmm4); + masm.addsd(xmm1, new AMD64Address(rax, 16)); + masm.mulpd(xmm6, xmm2); + masm.addsd(xmm5, xmm0); + masm.addsd(xmm3, xmm7); + masm.addsd(xmm1, xmm5); + masm.addsd(xmm1, xmm3); + masm.addsd(xmm1, xmm6); + masm.unpckhpd(xmm6, xmm6); + masm.movdqu(xmm0, xmm4); + masm.addsd(xmm1, xmm6); + masm.addsd(xmm0, xmm1); + masm.jmp(block13); + + masm.bind(block7); + masm.addl(rdx, 64); + masm.movq(r9, r10); + masm.movq(r10, r8); + masm.movl(r8, 0); + masm.cmpq(r9, 0); + masm.jcc(ConditionFlag.NotEqual, block8); + masm.addl(rdx, 64); + masm.movq(r9, r10); + masm.movq(r10, r8); + masm.cmpq(r9, 0); + masm.jcc(ConditionFlag.NotEqual, block8); + masm.xorpd(xmm0, xmm0); + masm.xorpd(xmm6, xmm6); + masm.jmp(block11); + + masm.bind(block9); + masm.jcc(ConditionFlag.Equal, block10); + masm.negl(rcx); + masm.shrq(r10); + masm.movq(rax, r9); + masm.shrq(r9); + masm.subl(rdx, rcx); + masm.negl(rcx); + masm.addl(rcx, 64); + masm.shlq(rax); + masm.orq(r10, rax); + masm.jmp(block10); + masm.bind(block3); + masm.negl(rcx); + masm.shlq(r9, 32); + masm.orq(r9, r11); + masm.shlq(r9); + masm.movq(rdi, r9); + masm.testl(r9, Integer.MIN_VALUE); + masm.jcc(ConditionFlag.NotEqual, block12); + masm.shrl(r9); + masm.movl(rbx, 0); + masm.shrq(rdi, 3); + masm.jmp(block6); + + masm.bind(block4); + masm.shrl(r9); + masm.movl(rbx, 536870912); + masm.shrl(rbx); + masm.shlq(r9, 32); + masm.orq(r9, r11); + masm.shlq(rbx, 32); + masm.addl(rdi, 536870912); + masm.movl(rcx, 0); + masm.movl(r11, 0); + masm.subq(rcx, r8); + masm.sbbq(r11, r10); + masm.sbbq(rbx, r9); + masm.movq(r8, rcx); + masm.movq(r10, r11); + masm.movq(r9, rbx); + masm.movl(rbx, 32768); + masm.jmp(block5); + + masm.bind(block12); + masm.shrl(r9); + masm.movq(rbx, 0x100000000L); + masm.shrq(rbx); + masm.movl(rcx, 0); + masm.movl(r11, 0); + masm.subq(rcx, r8); + masm.sbbq(r11, r10); + masm.sbbq(rbx, r9); + masm.movq(r8, rcx); + masm.movq(r10, r11); + masm.movq(r9, rbx); + masm.movl(rbx, 32768); + masm.shrq(rdi, 3); + masm.addl(rdi, 536870912); + masm.jmp(block6); + + masm.bind(block2); + masm.movsd(xmm0, new AMD64Address(rsp, 8)); + masm.mulsd(xmm0, recordExternalAddress(crb, negZero)); // 0x00000000, 0x80000000 + masm.movq(new AMD64Address(rsp, 0), xmm0); + + masm.bind(block13); + masm.addq(rsp, 16); + masm.pop(rbx); + } + +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathExpOp.java 2019-03-12 08:10:46.320031306 +0100 @@ -0,0 +1,459 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, Intel Corporation. All rights reserved. + * Intel Math Library (LIBM) Source Code + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.lir.amd64; + +import static jdk.vm.ci.amd64.AMD64.r11; +import static jdk.vm.ci.amd64.AMD64.rax; +import static jdk.vm.ci.amd64.AMD64.rcx; +import static jdk.vm.ci.amd64.AMD64.rdx; +import static jdk.vm.ci.amd64.AMD64.rsp; +import static jdk.vm.ci.amd64.AMD64.xmm0; +import static jdk.vm.ci.amd64.AMD64.xmm1; +import static jdk.vm.ci.amd64.AMD64.xmm2; +import static jdk.vm.ci.amd64.AMD64.xmm3; +import static jdk.vm.ci.amd64.AMD64.xmm4; +import static jdk.vm.ci.amd64.AMD64.xmm5; +import static jdk.vm.ci.amd64.AMD64.xmm6; +import static jdk.vm.ci.amd64.AMD64.xmm7; +import static org.graalvm.compiler.lir.amd64.AMD64HotSpotHelper.pointerConstant; +import static org.graalvm.compiler.lir.amd64.AMD64HotSpotHelper.recordExternalAddress; + +import org.graalvm.compiler.asm.Label; +import org.graalvm.compiler.asm.amd64.AMD64Address; +import org.graalvm.compiler.asm.amd64.AMD64Assembler.ConditionFlag; +import org.graalvm.compiler.asm.amd64.AMD64MacroAssembler; +import org.graalvm.compiler.lir.LIRInstructionClass; +import org.graalvm.compiler.lir.asm.ArrayDataPointerConstant; +import org.graalvm.compiler.lir.asm.CompilationResultBuilder; + +/** + * 
+ *                     ALGORITHM DESCRIPTION - EXP()
+ *                     ---------------------
+ *
+ * Description:
+ *  Let K = 64 (table size).
+ *        x    x/log(2)     n
+ *       e  = 2          = 2 * T[j] * (1 + P(y))
+ *  where
+ *       x = m*log(2)/K + y,    y in [-log(2)/K..log(2)/K]
+ *       m = n*K + j,           m,n,j - signed integer, j in [-K/2..K/2]
+ *                  j/K
+ *       values of 2   are tabulated as T[j] = T_hi[j] ( 1 + T_lo[j]).
+ *
+ *       P(y) is a minimax polynomial approximation of exp(x)-1
+ *       on small interval [-log(2)/K..log(2)/K] (were calculated by Maple V).
+ *
+ *  To avoid problems with arithmetic overflow and underflow,
+ *            n                        n1  n2
+ *  value of 2  is safely computed as 2 * 2 where n1 in [-BIAS/2..BIAS/2]
+ *  where BIAS is a value of exponent bias.
+ *
+ * Special cases:
+ *  exp(NaN) = NaN
+ *  exp(+INF) = +INF
+ *  exp(-INF) = 0
+ *  exp(x) = 1 for subnormals
+ *  for finite argument, only exp(0)=1 is exact
+ *  For IEEE double
+ *    if x >  709.782712893383973096 then exp(x) overflow
+ *    if x < -745.133219101941108420 then exp(x) underflow
+ *
+ */ +public final class AMD64MathExpOp extends AMD64MathIntrinsicUnaryOp { + + public static final LIRInstructionClass TYPE = LIRInstructionClass.create(AMD64MathExpOp.class); + + public AMD64MathExpOp() { + super(TYPE, /* GPR */ rax, rcx, rdx, r11, + /* XMM */ xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7); + } + + private ArrayDataPointerConstant cv = pointerConstant(16, new int[]{ + // @formatter:off + 0x652b82fe, 0x40571547, 0x652b82fe, 0x40571547, + }); + private ArrayDataPointerConstant cv16 = pointerConstant(16, new int[]{ + 0xfefa0000, 0x3f862e42, 0xfefa0000, 0x3f862e42, + }); + private ArrayDataPointerConstant cv32 = pointerConstant(16, new int[]{ + 0xbc9e3b3a, 0x3d1cf79a, 0xbc9e3b3a, 0x3d1cf79a, + }); + private ArrayDataPointerConstant cv48 = pointerConstant(16, new int[]{ + 0xfffffffe, 0x3fdfffff, 0xfffffffe, 0x3fdfffff, + }); + private ArrayDataPointerConstant cv64 = pointerConstant(16, new int[]{ + 0xe3289860, 0x3f56c15c, 0x555b9e25, 0x3fa55555, + }); + private ArrayDataPointerConstant cv80 = pointerConstant(16, new int[]{ + 0xc090cf0f, 0x3f811115, 0x55548ba1, 0x3fc55555 + // @formatter:on + }); + + private ArrayDataPointerConstant shifter = pointerConstant(16, new int[]{ + // @formatter:off + 0x00000000, 0x43380000, 0x00000000, 0x43380000 + // @formatter:on + }); + + private ArrayDataPointerConstant mmask = pointerConstant(16, new int[]{ + // @formatter:off + 0xffffffc0, 0x00000000, 0xffffffc0, 0x00000000 + // @formatter:on + }); + + private ArrayDataPointerConstant bias = pointerConstant(16, new int[]{ + // @formatter:off + 0x0000ffc0, 0x00000000, 0x0000ffc0, 0x00000000 + // @formatter:on + }); + + private ArrayDataPointerConstant tblAddr = pointerConstant(16, new int[]{ + // @formatter:off + 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x0e03754d, + 0x3cad7bbf, 0x3e778060, 0x00002c9a, 0x3567f613, 0x3c8cd252, + 0xd3158574, 0x000059b0, 0x61e6c861, 0x3c60f74e, 0x18759bc8, + 0x00008745, 0x5d837b6c, 0x3c979aa6, 0x6cf9890f, 0x0000b558, + 0x702f9cd1, 0x3c3ebe3d, 0x32d3d1a2, 0x0000e3ec, 0x1e63bcd8, + 0x3ca3516e, 0xd0125b50, 0x00011301, 0x26f0387b, 0x3ca4c554, + 0xaea92ddf, 0x0001429a, 0x62523fb6, 0x3ca95153, 0x3c7d517a, + 0x000172b8, 0x3f1353bf, 0x3c8b898c, 0xeb6fcb75, 0x0001a35b, + 0x3e3a2f5f, 0x3c9aecf7, 0x3168b9aa, 0x0001d487, 0x44a6c38d, + 0x3c8a6f41, 0x88628cd6, 0x0002063b, 0xe3a8a894, 0x3c968efd, + 0x6e756238, 0x0002387a, 0x981fe7f2, 0x3c80472b, 0x65e27cdd, + 0x00026b45, 0x6d09ab31, 0x3c82f7e1, 0xf51fdee1, 0x00029e9d, + 0x720c0ab3, 0x3c8b3782, 0xa6e4030b, 0x0002d285, 0x4db0abb6, + 0x3c834d75, 0x0a31b715, 0x000306fe, 0x5dd3f84a, 0x3c8fdd39, + 0xb26416ff, 0x00033c08, 0xcc187d29, 0x3ca12f8c, 0x373aa9ca, + 0x000371a7, 0x738b5e8b, 0x3ca7d229, 0x34e59ff6, 0x0003a7db, + 0xa72a4c6d, 0x3c859f48, 0x4c123422, 0x0003dea6, 0x259d9205, + 0x3ca8b846, 0x21f72e29, 0x0004160a, 0x60c2ac12, 0x3c4363ed, + 0x6061892d, 0x00044e08, 0xdaa10379, 0x3c6ecce1, 0xb5c13cd0, + 0x000486a2, 0xbb7aafb0, 0x3c7690ce, 0xd5362a27, 0x0004bfda, + 0x9b282a09, 0x3ca083cc, 0x769d2ca6, 0x0004f9b2, 0xc1aae707, + 0x3ca509b0, 0x569d4f81, 0x0005342b, 0x18fdd78e, 0x3c933505, + 0x36b527da, 0x00056f47, 0xe21c5409, 0x3c9063e1, 0xdd485429, + 0x0005ab07, 0x2b64c035, 0x3c9432e6, 0x15ad2148, 0x0005e76f, + 0x99f08c0a, 0x3ca01284, 0xb03a5584, 0x0006247e, 0x0073dc06, + 0x3c99f087, 0x82552224, 0x00066238, 0x0da05571, 0x3c998d4d, + 0x667f3bcc, 0x0006a09e, 0x86ce4786, 0x3ca52bb9, 0x3c651a2e, + 0x0006dfb2, 0x206f0dab, 0x3ca32092, 0xe8ec5f73, 0x00071f75, + 0x8e17a7a6, 0x3ca06122, 0x564267c8, 0x00075feb, 0x461e9f86, + 0x3ca244ac, 0x73eb0186, 0x0007a114, 0xabd66c55, 0x3c65ebe1, + 0x36cf4e62, 0x0007e2f3, 0xbbff67d0, 0x3c96fe9f, 0x994cce12, + 0x00082589, 0x14c801df, 0x3c951f14, 0x9b4492ec, 0x000868d9, + 0xc1f0eab4, 0x3c8db72f, 0x422aa0db, 0x0008ace5, 0x59f35f44, + 0x3c7bf683, 0x99157736, 0x0008f1ae, 0x9c06283c, 0x3ca360ba, + 0xb0cdc5e4, 0x00093737, 0x20f962aa, 0x3c95e8d1, 0x9fde4e4f, + 0x00097d82, 0x2b91ce27, 0x3c71affc, 0x82a3f090, 0x0009c491, + 0x589a2ebd, 0x3c9b6d34, 0x7b5de564, 0x000a0c66, 0x9ab89880, + 0x3c95277c, 0xb23e255c, 0x000a5503, 0x6e735ab3, 0x3c846984, + 0x5579fdbf, 0x000a9e6b, 0x92cb3387, 0x3c8c1a77, 0x995ad3ad, + 0x000ae89f, 0xdc2d1d96, 0x3ca22466, 0xb84f15fa, 0x000b33a2, + 0xb19505ae, 0x3ca1112e, 0xf2fb5e46, 0x000b7f76, 0x0a5fddcd, + 0x3c74ffd7, 0x904bc1d2, 0x000bcc1e, 0x30af0cb3, 0x3c736eae, + 0xdd85529c, 0x000c199b, 0xd10959ac, 0x3c84e08f, 0x2e57d14b, + 0x000c67f1, 0x6c921968, 0x3c676b2c, 0xdcef9069, 0x000cb720, + 0x36df99b3, 0x3c937009, 0x4a07897b, 0x000d072d, 0xa63d07a7, + 0x3c74a385, 0xdcfba487, 0x000d5818, 0xd5c192ac, 0x3c8e5a50, + 0x03db3285, 0x000da9e6, 0x1c4a9792, 0x3c98bb73, 0x337b9b5e, + 0x000dfc97, 0x603a88d3, 0x3c74b604, 0xe78b3ff6, 0x000e502e, + 0x92094926, 0x3c916f27, 0xa2a490d9, 0x000ea4af, 0x41aa2008, + 0x3c8ec3bc, 0xee615a27, 0x000efa1b, 0x31d185ee, 0x3c8a64a9, + 0x5b6e4540, 0x000f5076, 0x4d91cd9d, 0x3c77893b, 0x819e90d8, + 0x000fa7c1 + // @formatter:on + }); + + private ArrayDataPointerConstant allones = pointerConstant(16, new int[]{ + // @formatter:off + 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff + // @formatter:on + }); + + private ArrayDataPointerConstant ebias = pointerConstant(16, new int[]{ + // @formatter:off + 0x00000000, 0x3ff00000, 0x00000000, 0x3ff00000 + // @formatter:on + }); + + private ArrayDataPointerConstant xmax = pointerConstant(4, new int[]{ + // @formatter:off + 0xffffffff, 0x7fefffff + // @formatter:on + }); + + private ArrayDataPointerConstant xmin = pointerConstant(4, new int[]{ + // @formatter:off + 0x00000000, 0x00100000 + // @formatter:on + }); + + private ArrayDataPointerConstant inf = pointerConstant(4, new int[]{ + // @formatter:off + 0x00000000, 0x7ff00000 + // @formatter:on + }); + + private ArrayDataPointerConstant zero = pointerConstant(4, new int[]{ + // @formatter:off + 0x00000000, 0x00000000 + // @formatter:on + }); + + private ArrayDataPointerConstant oneVal = pointerConstant(4, new int[]{ + // @formatter:off + 0x00000000, 0x3ff00000 + // @formatter:on + }); + + @Override + public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { + // Registers: + // input: xmm0 + // scratch: xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7 + // rax, rdx, rcx, tmp - r11 + + // Code generated by Intel C compiler for LIBM library + Label block0 = new Label(); + Label block1 = new Label(); + Label block2 = new Label(); + Label block3 = new Label(); + Label block4 = new Label(); + Label block5 = new Label(); + Label block6 = new Label(); + Label block7 = new Label(); + Label block8 = new Label(); + Label block9 = new Label(); + Label block10 = new Label(); + Label block11 = new Label(); + Label block12 = new Label(); + Label block13 = new Label(); + + masm.subq(rsp, 24); + masm.movsd(new AMD64Address(rsp, 8), xmm0); + masm.unpcklpd(xmm0, xmm0); + masm.movdqu(xmm1, recordExternalAddress(crb, cv)); // 0x652b82fe, 0x40571547, + // 0x652b82fe, 0x40571547 + masm.movdqu(xmm6, recordExternalAddress(crb, shifter)); // 0x00000000, 0x43380000, + // 0x00000000, 0x43380000 + masm.movdqu(xmm2, recordExternalAddress(crb, cv16)); // 0xfefa0000, 0x3f862e42, + // 0xfefa0000, 0x3f862e42 + masm.movdqu(xmm3, recordExternalAddress(crb, cv32)); // 0xbc9e3b3a, 0x3d1cf79a, + // 0xbc9e3b3a, 0x3d1cf79a + masm.pextrw(rax, xmm0, 3); + masm.andl(rax, 32767); + masm.movl(rdx, 16527); + masm.subl(rdx, rax); + masm.subl(rax, 15504); + masm.orl(rdx, rax); + masm.cmpl(rdx, Integer.MIN_VALUE); + masm.jcc(ConditionFlag.AboveEqual, block0); + masm.mulpd(xmm1, xmm0); + masm.addpd(xmm1, xmm6); + masm.movapd(xmm7, xmm1); + masm.subpd(xmm1, xmm6); + masm.mulpd(xmm2, xmm1); + masm.movdqu(xmm4, recordExternalAddress(crb, cv64)); // 0xe3289860, 0x3f56c15c, + // 0x555b9e25, 0x3fa55555 + masm.mulpd(xmm3, xmm1); + masm.movdqu(xmm5, recordExternalAddress(crb, cv80)); // 0xc090cf0f, 0x3f811115, + // 0x55548ba1, 0x3fc55555 + masm.subpd(xmm0, xmm2); + masm.movdl(rax, xmm7); + masm.movl(rcx, rax); + masm.andl(rcx, 63); + masm.shll(rcx, 4); + masm.sarl(rax, 6); + masm.movl(rdx, rax); + masm.movdqu(xmm6, recordExternalAddress(crb, mmask)); // 0xffffffc0, 0x00000000, + // 0xffffffc0, 0x00000000 + masm.pand(xmm7, xmm6); + masm.movdqu(xmm6, recordExternalAddress(crb, bias)); // 0x0000ffc0, 0x00000000, + // 0x0000ffc0, 0x00000000 + masm.paddq(xmm7, xmm6); + masm.psllq(xmm7, 46); + masm.subpd(xmm0, xmm3); + masm.leaq(r11, recordExternalAddress(crb, tblAddr)); + masm.movdqu(xmm2, new AMD64Address(rcx, r11, AMD64Address.Scale.Times1)); + masm.mulpd(xmm4, xmm0); + masm.movapd(xmm6, xmm0); + masm.movapd(xmm1, xmm0); + masm.mulpd(xmm6, xmm6); + masm.mulpd(xmm0, xmm6); + masm.addpd(xmm5, xmm4); + masm.mulsd(xmm0, xmm6); + masm.mulpd(xmm6, recordExternalAddress(crb, cv48)); // 0xfffffffe, 0x3fdfffff, + // 0xfffffffe, 0x3fdfffff + masm.addsd(xmm1, xmm2); + masm.unpckhpd(xmm2, xmm2); + masm.mulpd(xmm0, xmm5); + masm.addsd(xmm1, xmm0); + masm.por(xmm2, xmm7); + masm.unpckhpd(xmm0, xmm0); + masm.addsd(xmm0, xmm1); + masm.addsd(xmm0, xmm6); + masm.addl(rdx, 894); + masm.cmpl(rdx, 1916); + masm.jcc(ConditionFlag.Above, block1); + masm.mulsd(xmm0, xmm2); + masm.addsd(xmm0, xmm2); + masm.jmp(block13); + + masm.bind(block1); + masm.xorpd(xmm3, xmm3); + masm.movdqu(xmm4, recordExternalAddress(crb, allones)); // 0xffffffff, 0xffffffff, + // 0xffffffff, 0xffffffff + masm.movl(rdx, -1022); + masm.subl(rdx, rax); + masm.movdl(xmm5, rdx); + masm.psllq(xmm4, xmm5); + masm.movl(rcx, rax); + masm.sarl(rax, 1); + masm.pinsrw(xmm3, rax, 3); + masm.movdqu(xmm6, recordExternalAddress(crb, ebias)); // 0x00000000, 0x3ff00000, + // 0x00000000, 0x3ff00000 + masm.psllq(xmm3, 4); + masm.psubd(xmm2, xmm3); + masm.mulsd(xmm0, xmm2); + masm.cmpl(rdx, 52); + masm.jcc(ConditionFlag.Greater, block2); + masm.pand(xmm4, xmm2); + masm.paddd(xmm3, xmm6); + masm.subsd(xmm2, xmm4); + masm.addsd(xmm0, xmm2); + masm.cmpl(rcx, 1023); + masm.jcc(ConditionFlag.GreaterEqual, block3); + masm.pextrw(rcx, xmm0, 3); + masm.andl(rcx, 32768); + masm.orl(rdx, rcx); + masm.cmpl(rdx, 0); + masm.jcc(ConditionFlag.Equal, block4); + masm.movapd(xmm6, xmm0); + masm.addsd(xmm0, xmm4); + masm.mulsd(xmm0, xmm3); + masm.pextrw(rcx, xmm0, 3); + masm.andl(rcx, 32752); + masm.cmpl(rcx, 0); + masm.jcc(ConditionFlag.Equal, block5); + masm.jmp(block13); + + masm.bind(block5); + masm.mulsd(xmm6, xmm3); + masm.mulsd(xmm4, xmm3); + masm.movdqu(xmm0, xmm6); + masm.pxor(xmm6, xmm4); + masm.psrad(xmm6, 31); + masm.pshufd(xmm6, xmm6, 85); + masm.psllq(xmm0, 1); + masm.psrlq(xmm0, 1); + masm.pxor(xmm0, xmm6); + masm.psrlq(xmm6, 63); + masm.paddq(xmm0, xmm6); + masm.paddq(xmm0, xmm4); + masm.movl(new AMD64Address(rsp, 0), 15); + masm.jmp(block6); + + masm.bind(block4); + masm.addsd(xmm0, xmm4); + masm.mulsd(xmm0, xmm3); + masm.jmp(block13); + + masm.bind(block3); + masm.addsd(xmm0, xmm4); + masm.mulsd(xmm0, xmm3); + masm.pextrw(rcx, xmm0, 3); + masm.andl(rcx, 32752); + masm.cmpl(rcx, 32752); + masm.jcc(ConditionFlag.AboveEqual, block7); + masm.jmp(block13); + + masm.bind(block2); + masm.paddd(xmm3, xmm6); + masm.addpd(xmm0, xmm2); + masm.mulsd(xmm0, xmm3); + masm.movl(new AMD64Address(rsp, 0), 15); + masm.jmp(block6); + + masm.bind(block8); + masm.cmpl(rax, 2146435072); + masm.jcc(ConditionFlag.AboveEqual, block9); + masm.movl(rax, new AMD64Address(rsp, 12)); + masm.cmpl(rax, Integer.MIN_VALUE); + masm.jcc(ConditionFlag.AboveEqual, block10); + masm.movsd(xmm0, recordExternalAddress(crb, xmax)); // 0xffffffff, 0x7fefffff + masm.mulsd(xmm0, xmm0); + + masm.bind(block7); + masm.movl(new AMD64Address(rsp, 0), 14); + masm.jmp(block6); + + masm.bind(block10); + masm.movsd(xmm0, recordExternalAddress(crb, xmin)); // 0x00000000, 0x00100000 + masm.mulsd(xmm0, xmm0); + masm.movl(new AMD64Address(rsp, 0), 15); + masm.jmp(block6); + + masm.bind(block9); + masm.movl(rdx, new AMD64Address(rsp, 8)); + masm.cmpl(rax, 2146435072); + masm.jcc(ConditionFlag.Above, block11); + masm.cmpl(rdx, 0); + masm.jcc(ConditionFlag.NotEqual, block11); + masm.movl(rax, new AMD64Address(rsp, 12)); + masm.cmpl(rax, 2146435072); + masm.jcc(ConditionFlag.NotEqual, block12); + masm.movsd(xmm0, recordExternalAddress(crb, inf)); // 0x00000000, 0x7ff00000 + masm.jmp(block13); + + masm.bind(block12); + masm.movsd(xmm0, recordExternalAddress(crb, zero)); // 0x00000000, 0x00000000 + masm.jmp(block13); + + masm.bind(block11); + masm.movsd(xmm0, new AMD64Address(rsp, 8)); + masm.addsd(xmm0, xmm0); + masm.jmp(block13); + + masm.bind(block0); + masm.movl(rax, new AMD64Address(rsp, 12)); + masm.andl(rax, 2147483647); + masm.cmpl(rax, 1083179008); + masm.jcc(ConditionFlag.AboveEqual, block8); + masm.movsd(new AMD64Address(rsp, 8), xmm0); + masm.addsd(xmm0, recordExternalAddress(crb, oneVal)); // 0x00000000, 0x3ff00000 + masm.jmp(block13); + + masm.bind(block6); + masm.movq(new AMD64Address(rsp, 16), xmm0); + + masm.movq(xmm0, new AMD64Address(rsp, 16)); + + masm.bind(block13); + masm.addq(rsp, 24); + } +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathLog10Op.java 2019-03-12 08:10:46.808034463 +0100 @@ -0,0 +1,424 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, Intel Corporation. All rights reserved. + * Intel Math Library (LIBM) Source Code + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.lir.amd64; + +import static jdk.vm.ci.amd64.AMD64.r11; +import static jdk.vm.ci.amd64.AMD64.r8; +import static jdk.vm.ci.amd64.AMD64.rax; +import static jdk.vm.ci.amd64.AMD64.rcx; +import static jdk.vm.ci.amd64.AMD64.rdx; +import static jdk.vm.ci.amd64.AMD64.rsp; +import static jdk.vm.ci.amd64.AMD64.xmm0; +import static jdk.vm.ci.amd64.AMD64.xmm1; +import static jdk.vm.ci.amd64.AMD64.xmm2; +import static jdk.vm.ci.amd64.AMD64.xmm3; +import static jdk.vm.ci.amd64.AMD64.xmm4; +import static jdk.vm.ci.amd64.AMD64.xmm5; +import static jdk.vm.ci.amd64.AMD64.xmm6; +import static jdk.vm.ci.amd64.AMD64.xmm7; +import static org.graalvm.compiler.lir.amd64.AMD64HotSpotHelper.pointerConstant; +import static org.graalvm.compiler.lir.amd64.AMD64HotSpotHelper.recordExternalAddress; + +import org.graalvm.compiler.asm.Label; +import org.graalvm.compiler.asm.amd64.AMD64Address; +import org.graalvm.compiler.asm.amd64.AMD64Assembler; +import org.graalvm.compiler.asm.amd64.AMD64MacroAssembler; +import org.graalvm.compiler.lir.LIRInstructionClass; +import org.graalvm.compiler.lir.asm.ArrayDataPointerConstant; +import org.graalvm.compiler.lir.asm.CompilationResultBuilder; + +/** + * 
+ *                     ALGORITHM DESCRIPTION - LOG10()
+ *                     ---------------------
+ *
+ *    Let x=2^k * mx, mx in [1,2)
+ *
+ *    Get B~1/mx based on the output of rcpss instruction (B0)
+ *    B = int((B0*LH*2^7+0.5))/2^7
+ *    LH is a short approximation for log10(e)
+ *
+ *    Reduced argument: r=B*mx-LH (computed accurately in high and low parts)
+ *
+ *    Result:  k*log10(2) - log(B) + p(r)
+ *             p(r) is a degree 7 polynomial
+ *             -log(B) read from data table (high, low parts)
+ *             Result is formed from high and low parts.
+ *
+ * Special cases:
+ *  log10(0) = -INF with divide-by-zero exception raised
+ *  log10(1) = +0
+ *  log10(x) = NaN with invalid exception raised if x < -0, including -INF
+ *  log10(+INF) = +INF
+ *
+ */ +public final class AMD64MathLog10Op extends AMD64MathIntrinsicUnaryOp { + + public static final LIRInstructionClass TYPE = LIRInstructionClass.create(AMD64MathLog10Op.class); + + public AMD64MathLog10Op() { + super(TYPE, /* GPR */ rax, rcx, rdx, r8, r11, + /* XMM */ xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7); + } + + private ArrayDataPointerConstant highsigmask = pointerConstant(16, new int[]{ + // @formatter:off + 0xf8000000, 0xffffffff, 0x00000000, 0xffffe000 + // @formatter:on + }); + + private ArrayDataPointerConstant log10E = pointerConstant(8, new int[]{ + // @formatter:off + 0x00000000, 0x3fdbc000, + }); + private ArrayDataPointerConstant log10E8 = pointerConstant(8, new int[]{ + 0xbf2e4108, 0x3f5a7a6c + // @formatter:on + }); + + private ArrayDataPointerConstant lTbl = pointerConstant(16, new int[]{ + // @formatter:off + 0x509f7800, 0x3fd34413, 0x1f12b358, 0x3d1fef31, 0x80333400, + 0x3fd32418, 0xc671d9d0, 0xbcf542bf, 0x51195000, 0x3fd30442, + 0x78a4b0c3, 0x3d18216a, 0x6fc79400, 0x3fd2e490, 0x80fa389d, + 0xbc902869, 0x89d04000, 0x3fd2c502, 0x75c2f564, 0x3d040754, + 0x4ddd1c00, 0x3fd2a598, 0xd219b2c3, 0xbcfa1d84, 0x6baa7c00, + 0x3fd28651, 0xfd9abec1, 0x3d1be6d3, 0x94028800, 0x3fd2672d, + 0xe289a455, 0xbd1ede5e, 0x78b86400, 0x3fd2482c, 0x6734d179, + 0x3d1fe79b, 0xcca3c800, 0x3fd2294d, 0x981a40b8, 0xbced34ea, + 0x439c5000, 0x3fd20a91, 0xcc392737, 0xbd1a9cc3, 0x92752c00, + 0x3fd1ebf6, 0x03c9afe7, 0x3d1e98f8, 0x6ef8dc00, 0x3fd1cd7d, + 0x71dae7f4, 0x3d08a86c, 0x8fe4dc00, 0x3fd1af25, 0xee9185a1, + 0xbcff3412, 0xace59400, 0x3fd190ee, 0xc2cab353, 0x3cf17ed9, + 0x7e925000, 0x3fd172d8, 0x6952c1b2, 0x3cf1521c, 0xbe694400, + 0x3fd154e2, 0xcacb79ca, 0xbd0bdc78, 0x26cbac00, 0x3fd1370d, + 0xf71f4de1, 0xbd01f8be, 0x72fa0800, 0x3fd11957, 0x55bf910b, + 0x3c946e2b, 0x5f106000, 0x3fd0fbc1, 0x39e639c1, 0x3d14a84b, + 0xa802a800, 0x3fd0de4a, 0xd3f31d5d, 0xbd178385, 0x0b992000, + 0x3fd0c0f3, 0x3843106f, 0xbd1f602f, 0x486ce800, 0x3fd0a3ba, + 0x8819497c, 0x3cef987a, 0x1de49400, 0x3fd086a0, 0x1caa0467, + 0x3d0faec7, 0x4c30cc00, 0x3fd069a4, 0xa4424372, 0xbd1618fc, + 0x94490000, 0x3fd04cc6, 0x946517d2, 0xbd18384b, 0xb7e84000, + 0x3fd03006, 0xe0109c37, 0xbd19a6ac, 0x798a0c00, 0x3fd01364, + 0x5121e864, 0xbd164cf7, 0x38ce8000, 0x3fcfedbf, 0x46214d1a, + 0xbcbbc402, 0xc8e62000, 0x3fcfb4ef, 0xdab93203, 0x3d1e0176, + 0x2cb02800, 0x3fcf7c5a, 0x2a2ea8e4, 0xbcfec86a, 0xeeeaa000, + 0x3fcf43fd, 0xc18e49a4, 0x3cf110a8, 0x9bb6e800, 0x3fcf0bda, + 0x923cc9c0, 0xbd15ce99, 0xc093f000, 0x3fced3ef, 0x4d4b51e9, + 0x3d1a04c7, 0xec58f800, 0x3fce9c3c, 0x163cad59, 0x3cac8260, + 0x9a907000, 0x3fce2d7d, 0x3fa93646, 0x3ce4a1c0, 0x37311000, + 0x3fcdbf99, 0x32abd1fd, 0x3d07ea9d, 0x6744b800, 0x3fcd528c, + 0x4dcbdfd4, 0xbd1b08e2, 0xe36de800, 0x3fcce653, 0x0b7b7f7f, + 0xbd1b8f03, 0x77506800, 0x3fcc7aec, 0xa821c9fb, 0x3d13c163, + 0x00ff8800, 0x3fcc1053, 0x536bca76, 0xbd074ee5, 0x70719800, + 0x3fcba684, 0xd7da9b6b, 0xbd1fbf16, 0xc6f8d800, 0x3fcb3d7d, + 0xe2220bb3, 0x3d1a295d, 0x16c15800, 0x3fcad53c, 0xe724911e, + 0xbcf55822, 0x82533800, 0x3fca6dbc, 0x6d982371, 0x3cac567c, + 0x3c19e800, 0x3fca06fc, 0x84d17d80, 0x3d1da204, 0x85ef8000, + 0x3fc9a0f8, 0x54466a6a, 0xbd002204, 0xb0ac2000, 0x3fc93bae, + 0xd601fd65, 0x3d18840c, 0x1bb9b000, 0x3fc8d71c, 0x7bf58766, + 0xbd14f897, 0x34aae800, 0x3fc8733e, 0x3af6ac24, 0xbd0f5c45, + 0x76d68000, 0x3fc81012, 0x4303e1a1, 0xbd1f9a80, 0x6af57800, + 0x3fc7ad96, 0x43fbcb46, 0x3cf4c33e, 0xa6c51000, 0x3fc74bc7, + 0x70f0eac5, 0xbd192e3b, 0xccab9800, 0x3fc6eaa3, 0xc0093dfe, + 0xbd0faf15, 0x8b60b800, 0x3fc68a28, 0xde78d5fd, 0xbc9ea4ee, + 0x9d987000, 0x3fc62a53, 0x962bea6e, 0xbd194084, 0xc9b0e800, + 0x3fc5cb22, 0x888dd999, 0x3d1fe201, 0xe1634800, 0x3fc56c93, + 0x16ada7ad, 0x3d1b1188, 0xc176c000, 0x3fc50ea4, 0x4159b5b5, + 0xbcf09c08, 0x51766000, 0x3fc4b153, 0x84393d23, 0xbcf6a89c, + 0x83695000, 0x3fc4549d, 0x9f0b8bbb, 0x3d1c4b8c, 0x538d5800, + 0x3fc3f881, 0xf49df747, 0x3cf89b99, 0xc8138000, 0x3fc39cfc, + 0xd503b834, 0xbd13b99f, 0xf0df0800, 0x3fc3420d, 0xf011b386, + 0xbd05d8be, 0xe7466800, 0x3fc2e7b2, 0xf39c7bc2, 0xbd1bb94e, + 0xcdd62800, 0x3fc28de9, 0x05e6d69b, 0xbd10ed05, 0xd015d800, + 0x3fc234b0, 0xe29b6c9d, 0xbd1ff967, 0x224ea800, 0x3fc1dc06, + 0x727711fc, 0xbcffb30d, 0x01540000, 0x3fc183e8, 0x39786c5a, + 0x3cc23f57, 0xb24d9800, 0x3fc12c54, 0xc905a342, 0x3d003a1d, + 0x82835800, 0x3fc0d54a, 0x9b9920c0, 0x3d03b25a, 0xc72ac000, + 0x3fc07ec7, 0x46f26a24, 0x3cf0fa41, 0xdd35d800, 0x3fc028ca, + 0x41d9d6dc, 0x3d034a65, 0x52474000, 0x3fbfa6a4, 0x44f66449, + 0x3d19cad3, 0x2da3d000, 0x3fbefcb8, 0x67832999, 0x3d18400f, + 0x32a10000, 0x3fbe53ce, 0x9c0e3b1a, 0xbcff62fd, 0x556b7000, + 0x3fbdabe3, 0x02976913, 0xbcf8243b, 0x97e88000, 0x3fbd04f4, + 0xec793797, 0x3d1c0578, 0x09647000, 0x3fbc5eff, 0x05fc0565, + 0xbd1d799e, 0xc6426000, 0x3fbbb9ff, 0x4625f5ed, 0x3d1f5723, + 0xf7afd000, 0x3fbb15f3, 0xdd5aae61, 0xbd1a7e1e, 0xd358b000, + 0x3fba72d8, 0x3314e4d3, 0x3d17bc91, 0x9b1f5000, 0x3fb9d0ab, + 0x9a4d514b, 0x3cf18c9b, 0x9cd4e000, 0x3fb92f69, 0x7e4496ab, + 0x3cf1f96d, 0x31f4f000, 0x3fb88f10, 0xf56479e7, 0x3d165818, + 0xbf628000, 0x3fb7ef9c, 0x26bf486d, 0xbd1113a6, 0xb526b000, + 0x3fb7510c, 0x1a1c3384, 0x3ca9898d, 0x8e31e000, 0x3fb6b35d, + 0xb3875361, 0xbd0661ac, 0xd01de000, 0x3fb6168c, 0x2a7cacfa, + 0xbd1bdf10, 0x0af23000, 0x3fb57a98, 0xff868816, 0x3cf046d0, + 0xd8ea0000, 0x3fb4df7c, 0x1515fbe7, 0xbd1fd529, 0xde3b2000, + 0x3fb44538, 0x6e59a132, 0x3d1faeee, 0xc8df9000, 0x3fb3abc9, + 0xf1322361, 0xbd198807, 0x505f1000, 0x3fb3132d, 0x0888e6ab, + 0x3d1e5380, 0x359bd000, 0x3fb27b61, 0xdfbcbb22, 0xbcfe2724, + 0x429ee000, 0x3fb1e463, 0x6eb4c58c, 0xbcfe4dd6, 0x4a673000, + 0x3fb14e31, 0x4ce1ac9b, 0x3d1ba691, 0x28b96000, 0x3fb0b8c9, + 0x8c7813b8, 0xbd0b3872, 0xc1f08000, 0x3fb02428, 0xc2bc8c2c, + 0x3cb5ea6b, 0x05a1a000, 0x3faf209c, 0x72e8f18e, 0xbce8df84, + 0xc0b5e000, 0x3fadfa6d, 0x9fdef436, 0x3d087364, 0xaf416000, + 0x3facd5c2, 0x1068c3a9, 0x3d0827e7, 0xdb356000, 0x3fabb296, + 0x120a34d3, 0x3d101a9f, 0x5dfea000, 0x3faa90e6, 0xdaded264, + 0xbd14c392, 0x6034c000, 0x3fa970ad, 0x1c9d06a9, 0xbd1b705e, + 0x194c6000, 0x3fa851e8, 0x83996ad9, 0xbd0117bc, 0xcf4ac000, + 0x3fa73492, 0xb1a94a62, 0xbca5ea42, 0xd67b4000, 0x3fa618a9, + 0x75aed8ca, 0xbd07119b, 0x9126c000, 0x3fa4fe29, 0x5291d533, + 0x3d12658f, 0x6f4d4000, 0x3fa3e50e, 0xcd2c5cd9, 0x3d1d5c70, + 0xee608000, 0x3fa2cd54, 0xd1008489, 0x3d1a4802, 0x9900e000, + 0x3fa1b6f9, 0x54fb5598, 0xbd16593f, 0x06bb6000, 0x3fa0a1f9, + 0x64ef57b4, 0xbd17636b, 0xb7940000, 0x3f9f1c9f, 0xee6a4737, + 0x3cb5d479, 0x91aa0000, 0x3f9cf7f5, 0x3a16373c, 0x3d087114, + 0x156b8000, 0x3f9ad5ed, 0x836c554a, 0x3c6900b0, 0xd4764000, + 0x3f98b67f, 0xed12f17b, 0xbcffc974, 0x77dec000, 0x3f9699a7, + 0x232ce7ea, 0x3d1e35bb, 0xbfbf4000, 0x3f947f5d, 0xd84ffa6e, + 0x3d0e0a49, 0x82c7c000, 0x3f92679c, 0x8d170e90, 0xbd14d9f2, + 0xadd20000, 0x3f90525d, 0x86d9f88e, 0x3cdeb986, 0x86f10000, + 0x3f8c7f36, 0xb9e0a517, 0x3ce29faa, 0xb75c8000, 0x3f885e9e, + 0x542568cb, 0xbd1f7bdb, 0x46b30000, 0x3f8442e8, 0xb954e7d9, + 0x3d1e5287, 0xb7e60000, 0x3f802c07, 0x22da0b17, 0xbd19fb27, + 0x6c8b0000, 0x3f7833e3, 0x821271ef, 0xbd190f96, 0x29910000, + 0x3f701936, 0xbc3491a5, 0xbd1bcf45, 0x354a0000, 0x3f600fe3, + 0xc0ff520a, 0xbd19d71c, 0x00000000, 0x00000000, 0x00000000, + 0x00000000 + // @formatter:on + }); + + private ArrayDataPointerConstant log2 = pointerConstant(8, new int[]{ + // @formatter:off + 0x509f7800, 0x3f934413, + }); + private ArrayDataPointerConstant log28 = pointerConstant(8, new int[]{ + 0x1f12b358, 0x3cdfef31 + // @formatter:on + }); + + private ArrayDataPointerConstant coeff = pointerConstant(16, new int[]{ + // @formatter:off + 0xc1a5f12e, 0x40358874, 0x64d4ef0d, 0xc0089309, + }); + private ArrayDataPointerConstant coeff16 = pointerConstant(16, new int[]{ + 0x385593b1, 0xc025c917, 0xdc963467, 0x3ffc6a02, + }); + private ArrayDataPointerConstant coeff32 = pointerConstant(16, new int[]{ + 0x7f9d3aa1, 0x4016ab9f, 0xdc77b115, 0xbff27af2 + // @formatter:on + }); + + @Override + public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { + // Registers: + // input: xmm0 + // scratch: xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7 + // rax, rdx, rcx, tmp - r11 + // Code generated by Intel C compiler for LIBM library + + Label block0 = new Label(); + Label block1 = new Label(); + Label block2 = new Label(); + Label block3 = new Label(); + Label block4 = new Label(); + Label block5 = new Label(); + Label block6 = new Label(); + Label block7 = new Label(); + Label block8 = new Label(); + Label block9 = new Label(); + + masm.subq(rsp, 24); + masm.movsd(new AMD64Address(rsp, 0), xmm0); + + masm.xorpd(xmm2, xmm2); + masm.movl(rax, 16368); + masm.pinsrw(xmm2, rax, 3); + masm.movl(rcx, 1054736384); + masm.movdl(xmm7, rcx); + masm.xorpd(xmm3, xmm3); + masm.movl(rdx, 30704); + masm.pinsrw(xmm3, rdx, 3); + masm.movdqu(xmm1, xmm0); + masm.movl(rdx, 32768); + masm.movdl(xmm4, rdx); + masm.movdqu(xmm5, recordExternalAddress(crb, highsigmask)); // 0xf8000000, 0xffffffff, + // 0x00000000, 0xffffe000 + masm.pextrw(rax, xmm0, 3); + masm.por(xmm0, xmm2); + masm.movl(rcx, 16352); + masm.psrlq(xmm0, 27); + masm.movdqu(xmm2, recordExternalAddress(crb, log10E)); // 0x00000000, 0x3fdbc000, + // 0xbf2e4108, 0x3f5a7a6c + masm.psrld(xmm0, 2); + masm.rcpps(xmm0, xmm0); + masm.psllq(xmm1, 12); + masm.pshufd(xmm6, xmm5, 78); + masm.psrlq(xmm1, 12); + masm.subl(rax, 16); + masm.cmpl(rax, 32736); + masm.jcc(AMD64Assembler.ConditionFlag.AboveEqual, block0); + + masm.bind(block1); + masm.mulss(xmm0, xmm7); + masm.por(xmm1, xmm3); + masm.leaq(r11, recordExternalAddress(crb, lTbl)); + masm.andpd(xmm5, xmm1); + masm.paddd(xmm0, xmm4); + masm.subsd(xmm1, xmm5); + masm.movdl(rdx, xmm0); + masm.psllq(xmm0, 29); + masm.andpd(xmm0, xmm6); + masm.andl(rax, 32752); + masm.subl(rax, rcx); + masm.cvtsi2sdl(xmm7, rax); + masm.mulpd(xmm5, xmm0); + masm.mulsd(xmm1, xmm0); + masm.movq(xmm6, recordExternalAddress(crb, log2)); // 0x509f7800, 0x3f934413, + // 0x1f12b358, 0x3cdfef31 + masm.movdqu(xmm3, recordExternalAddress(crb, coeff)); // 0xc1a5f12e, 0x40358874, + // 0x64d4ef0d, 0xc0089309 + masm.subsd(xmm5, xmm2); + masm.andl(rdx, 16711680); + masm.shrl(rdx, 12); + masm.movdqu(xmm0, new AMD64Address(r11, rdx, AMD64Address.Scale.Times1, -1504)); + masm.movdqu(xmm4, recordExternalAddress(crb, coeff16)); // 0x385593b1, 0xc025c917, + // 0xdc963467, 0x3ffc6a02 + masm.addsd(xmm1, xmm5); + masm.movdqu(xmm2, recordExternalAddress(crb, coeff32)); // 0x7f9d3aa1, 0x4016ab9f, + // 0xdc77b115, 0xbff27af2 + masm.mulsd(xmm6, xmm7); + masm.pshufd(xmm5, xmm1, 68); + masm.mulsd(xmm7, recordExternalAddress(crb, log28)); // 0x1f12b358, 0x3cdfef31 + masm.mulsd(xmm3, xmm1); + masm.addsd(xmm0, xmm6); + masm.mulpd(xmm4, xmm5); + masm.movq(xmm6, recordExternalAddress(crb, log10E8)); // 0xbf2e4108, 0x3f5a7a6c + masm.mulpd(xmm5, xmm5); + masm.addpd(xmm4, xmm2); + masm.mulpd(xmm3, xmm5); + masm.pshufd(xmm2, xmm0, 228); + masm.addsd(xmm0, xmm1); + masm.mulsd(xmm4, xmm1); + masm.subsd(xmm2, xmm0); + masm.mulsd(xmm6, xmm1); + masm.addsd(xmm1, xmm2); + masm.pshufd(xmm2, xmm0, 238); + masm.mulsd(xmm5, xmm5); + masm.addsd(xmm7, xmm2); + masm.addsd(xmm1, xmm6); + masm.addpd(xmm4, xmm3); + masm.addsd(xmm1, xmm7); + masm.mulpd(xmm4, xmm5); + masm.addsd(xmm1, xmm4); + masm.pshufd(xmm5, xmm4, 238); + masm.addsd(xmm1, xmm5); + masm.addsd(xmm0, xmm1); + masm.jmp(block9); + + masm.bind(block0); + masm.movq(xmm0, new AMD64Address(rsp, 0)); + masm.movq(xmm1, new AMD64Address(rsp, 0)); + masm.addl(rax, 16); + masm.cmpl(rax, 32768); + masm.jcc(AMD64Assembler.ConditionFlag.AboveEqual, block2); + masm.cmpl(rax, 16); + masm.jcc(AMD64Assembler.ConditionFlag.Below, block3); + + masm.bind(block4); + masm.addsd(xmm0, xmm0); + masm.jmp(block9); + + masm.bind(block5); + masm.jcc(AMD64Assembler.ConditionFlag.Above, block4); + masm.cmpl(rdx, 0); + masm.jcc(AMD64Assembler.ConditionFlag.Above, block4); + masm.jmp(block6); + + masm.bind(block3); + masm.xorpd(xmm1, xmm1); + masm.addsd(xmm1, xmm0); + masm.movdl(rdx, xmm1); + masm.psrlq(xmm1, 32); + masm.movdl(rcx, xmm1); + masm.orl(rdx, rcx); + masm.cmpl(rdx, 0); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block7); + masm.xorpd(xmm1, xmm1); + masm.movl(rax, 18416); + masm.pinsrw(xmm1, rax, 3); + masm.mulsd(xmm0, xmm1); + masm.xorpd(xmm2, xmm2); + masm.movl(rax, 16368); + masm.pinsrw(xmm2, rax, 3); + masm.movdqu(xmm1, xmm0); + masm.pextrw(rax, xmm0, 3); + masm.por(xmm0, xmm2); + masm.movl(rcx, 18416); + masm.psrlq(xmm0, 27); + masm.movdqu(xmm2, recordExternalAddress(crb, log10E)); // 0x00000000, 0x3fdbc000, + // 0xbf2e4108, 0x3f5a7a6c + masm.psrld(xmm0, 2); + masm.rcpps(xmm0, xmm0); + masm.psllq(xmm1, 12); + masm.pshufd(xmm6, xmm5, 78); + masm.psrlq(xmm1, 12); + masm.jmp(block1); + + masm.bind(block2); + masm.movdl(rdx, xmm1); + masm.psrlq(xmm1, 32); + masm.movdl(rcx, xmm1); + masm.addl(rcx, rcx); + masm.cmpl(rcx, -2097152); + masm.jcc(AMD64Assembler.ConditionFlag.AboveEqual, block5); + masm.orl(rdx, rcx); + masm.cmpl(rdx, 0); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block7); + + masm.bind(block6); + masm.xorpd(xmm1, xmm1); + masm.xorpd(xmm0, xmm0); + masm.movl(rax, 32752); + masm.pinsrw(xmm1, rax, 3); + masm.mulsd(xmm0, xmm1); + masm.movl(new AMD64Address(rsp, 16), 9); + masm.jmp(block8); + + masm.bind(block7); + masm.xorpd(xmm1, xmm1); + masm.xorpd(xmm0, xmm0); + masm.movl(rax, 49136); + masm.pinsrw(xmm0, rax, 3); + masm.divsd(xmm0, xmm1); + masm.movl(new AMD64Address(rsp, 16), 8); + + masm.bind(block8); + masm.movq(new AMD64Address(rsp, 8), xmm0); + masm.movq(xmm0, new AMD64Address(rsp, 8)); + + masm.bind(block9); + masm.addq(rsp, 24); + } +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathLogOp.java 2019-03-12 08:10:47.296037618 +0100 @@ -0,0 +1,402 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, Intel Corporation. All rights reserved. + * Intel Math Library (LIBM) Source Code + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.lir.amd64; + +import static jdk.vm.ci.amd64.AMD64.r11; +import static jdk.vm.ci.amd64.AMD64.r8; +import static jdk.vm.ci.amd64.AMD64.rax; +import static jdk.vm.ci.amd64.AMD64.rcx; +import static jdk.vm.ci.amd64.AMD64.rdx; +import static jdk.vm.ci.amd64.AMD64.rsp; +import static jdk.vm.ci.amd64.AMD64.xmm0; +import static jdk.vm.ci.amd64.AMD64.xmm1; +import static jdk.vm.ci.amd64.AMD64.xmm2; +import static jdk.vm.ci.amd64.AMD64.xmm3; +import static jdk.vm.ci.amd64.AMD64.xmm4; +import static jdk.vm.ci.amd64.AMD64.xmm5; +import static jdk.vm.ci.amd64.AMD64.xmm6; +import static jdk.vm.ci.amd64.AMD64.xmm7; +import static org.graalvm.compiler.lir.amd64.AMD64HotSpotHelper.pointerConstant; +import static org.graalvm.compiler.lir.amd64.AMD64HotSpotHelper.recordExternalAddress; + +import org.graalvm.compiler.asm.Label; +import org.graalvm.compiler.asm.amd64.AMD64Address; +import org.graalvm.compiler.asm.amd64.AMD64Assembler; +import org.graalvm.compiler.asm.amd64.AMD64MacroAssembler; +import org.graalvm.compiler.lir.LIRInstructionClass; +import org.graalvm.compiler.lir.asm.ArrayDataPointerConstant; +import org.graalvm.compiler.lir.asm.CompilationResultBuilder; + +import jdk.vm.ci.amd64.AMD64; + +/** + * 
+ *                     ALGORITHM DESCRIPTION - LOG()
+ *                     ---------------------
+ *
+ *    x=2^k * mx, mx in [1,2)
+ *
+ *    Get B~1/mx based on the output of rcpss instruction (B0)
+ *    B = int((B0*2^7+0.5))/2^7
+ *
+ *    Reduced argument: r=B*mx-1.0 (computed accurately in high and low parts)
+ *
+ *    Result:  k*log(2) - log(B) + p(r) if |x-1| >= small value (2^-6)  and
+ *             p(r) is a degree 7 polynomial
+ *             -log(B) read from data table (high, low parts)
+ *             Result is formed from high and low parts.
+ *
+ * Special cases:
+ *  log(NaN) = quiet NaN, and raise invalid exception
+ *  log(+INF) = that INF
+ *  log(0) = -INF with divide-by-zero exception raised
+ *  log(1) = +0
+ *  log(x) = NaN with invalid exception raised if x < -0, including -INF
+ *
+ */ +public final class AMD64MathLogOp extends AMD64MathIntrinsicUnaryOp { + + public static final LIRInstructionClass TYPE = LIRInstructionClass.create(AMD64MathLogOp.class); + + public AMD64MathLogOp() { + super(TYPE, /* GPR */ rax, rcx, rdx, r8, r11, + /* XMM */ xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7); + } + + private ArrayDataPointerConstant lTbl = pointerConstant(16, new int[]{ + // @formatter:off + 0xfefa3800, 0x3fe62e42, 0x93c76730, 0x3d2ef357, 0xaa241800, + 0x3fe5ee82, 0x0cda46be, 0x3d220238, 0x5c364800, 0x3fe5af40, + 0xac10c9fb, 0x3d2dfa63, 0x26bb8c00, 0x3fe5707a, 0xff3303dd, + 0x3d09980b, 0x26867800, 0x3fe5322e, 0x5d257531, 0x3d05ccc4, + 0x835a5000, 0x3fe4f45a, 0x6d93b8fb, 0xbd2e6c51, 0x6f970c00, + 0x3fe4b6fd, 0xed4c541c, 0x3cef7115, 0x27e8a400, 0x3fe47a15, + 0xf94d60aa, 0xbd22cb6a, 0xf2f92400, 0x3fe43d9f, 0x481051f7, + 0xbcfd984f, 0x2125cc00, 0x3fe4019c, 0x30f0c74c, 0xbd26ce79, + 0x0c36c000, 0x3fe3c608, 0x7cfe13c2, 0xbd02b736, 0x17197800, + 0x3fe38ae2, 0xbb5569a4, 0xbd218b7a, 0xad9d8c00, 0x3fe35028, + 0x9527e6ac, 0x3d10b83f, 0x44340800, 0x3fe315da, 0xc5a0ed9c, + 0xbd274e93, 0x57b0e000, 0x3fe2dbf5, 0x07b9dc11, 0xbd17a6e5, + 0x6d0ec000, 0x3fe2a278, 0xe797882d, 0x3d206d2b, 0x1134dc00, + 0x3fe26962, 0x05226250, 0xbd0b61f1, 0xd8bebc00, 0x3fe230b0, + 0x6e48667b, 0x3d12fc06, 0x5fc61800, 0x3fe1f863, 0xc9fe81d3, + 0xbd2a7242, 0x49ae6000, 0x3fe1c078, 0xed70e667, 0x3cccacde, + 0x40f23c00, 0x3fe188ee, 0xf8ab4650, 0x3d14cc4e, 0xf6f29800, + 0x3fe151c3, 0xa293ae49, 0xbd2edd97, 0x23c75c00, 0x3fe11af8, + 0xbb9ddcb2, 0xbd258647, 0x8611cc00, 0x3fe0e489, 0x07801742, + 0x3d1c2998, 0xe2d05400, 0x3fe0ae76, 0x887e7e27, 0x3d1f486b, + 0x0533c400, 0x3fe078bf, 0x41edf5fd, 0x3d268122, 0xbe760400, + 0x3fe04360, 0xe79539e0, 0xbd04c45f, 0xe5b20800, 0x3fe00e5a, + 0xb1727b1c, 0xbd053ba3, 0xaf7a4800, 0x3fdfb358, 0x3c164935, + 0x3d0085fa, 0xee031800, 0x3fdf4aa7, 0x6f014a8b, 0x3d12cde5, + 0x56b41000, 0x3fdee2a1, 0x5a470251, 0x3d2f27f4, 0xc3ddb000, + 0x3fde7b42, 0x5372bd08, 0xbd246550, 0x1a272800, 0x3fde148a, + 0x07322938, 0xbd1326b2, 0x484c9800, 0x3fddae75, 0x60dc616a, + 0xbd1ea42d, 0x46def800, 0x3fdd4902, 0xe9a767a8, 0x3d235baf, + 0x18064800, 0x3fdce42f, 0x3ec7a6b0, 0xbd0797c3, 0xc7455800, + 0x3fdc7ff9, 0xc15249ae, 0xbd29b6dd, 0x693fa000, 0x3fdc1c60, + 0x7fe8e180, 0x3d2cec80, 0x1b80e000, 0x3fdbb961, 0xf40a666d, + 0x3d27d85b, 0x04462800, 0x3fdb56fa, 0x2d841995, 0x3d109525, + 0x5248d000, 0x3fdaf529, 0x52774458, 0xbd217cc5, 0x3c8ad800, + 0x3fda93ed, 0xbea77a5d, 0x3d1e36f2, 0x0224f800, 0x3fda3344, + 0x7f9d79f5, 0x3d23c645, 0xea15f000, 0x3fd9d32b, 0x10d0c0b0, + 0xbd26279e, 0x43135800, 0x3fd973a3, 0xa502d9f0, 0xbd152313, + 0x635bf800, 0x3fd914a8, 0x2ee6307d, 0xbd1766b5, 0xa88b3000, + 0x3fd8b639, 0xe5e70470, 0xbd205ae1, 0x776dc800, 0x3fd85855, + 0x3333778a, 0x3d2fd56f, 0x3bd81800, 0x3fd7fafa, 0xc812566a, + 0xbd272090, 0x687cf800, 0x3fd79e26, 0x2efd1778, 0x3d29ec7d, + 0x76c67800, 0x3fd741d8, 0x49dc60b3, 0x3d2d8b09, 0xe6af1800, + 0x3fd6e60e, 0x7c222d87, 0x3d172165, 0x3e9c6800, 0x3fd68ac8, + 0x2756eba0, 0x3d20a0d3, 0x0b3ab000, 0x3fd63003, 0xe731ae00, + 0xbd2db623, 0xdf596000, 0x3fd5d5bd, 0x08a465dc, 0xbd0a0b2a, + 0x53c8d000, 0x3fd57bf7, 0xee5d40ef, 0x3d1faded, 0x0738a000, + 0x3fd522ae, 0x8164c759, 0x3d2ebe70, 0x9e173000, 0x3fd4c9e0, + 0x1b0ad8a4, 0xbd2e2089, 0xc271c800, 0x3fd4718d, 0x0967d675, + 0xbd2f27ce, 0x23d5e800, 0x3fd419b4, 0xec90e09d, 0x3d08e436, + 0x77333000, 0x3fd3c252, 0xb606bd5c, 0x3d183b54, 0x76be1000, + 0x3fd36b67, 0xb0f177c8, 0x3d116ecd, 0xe1d36000, 0x3fd314f1, + 0xd3213cb8, 0xbd28e27a, 0x7cdc9000, 0x3fd2bef0, 0x4a5004f4, + 0x3d2a9cfa, 0x1134d800, 0x3fd26962, 0xdf5bb3b6, 0x3d2c93c1, + 0x6d0eb800, 0x3fd21445, 0xba46baea, 0x3d0a87de, 0x635a6800, + 0x3fd1bf99, 0x5147bdb7, 0x3d2ca6ed, 0xcbacf800, 0x3fd16b5c, + 0xf7a51681, 0x3d2b9acd, 0x8227e800, 0x3fd1178e, 0x63a5f01c, + 0xbd2c210e, 0x67616000, 0x3fd0c42d, 0x163ceae9, 0x3d27188b, + 0x604d5800, 0x3fd07138, 0x16ed4e91, 0x3cf89cdb, 0x5626c800, + 0x3fd01eae, 0x1485e94a, 0xbd16f08c, 0x6cb3b000, 0x3fcf991c, + 0xca0cdf30, 0x3d1bcbec, 0xe4dd0000, 0x3fcef5ad, 0x65bb8e11, + 0xbcca2115, 0xffe71000, 0x3fce530e, 0x6041f430, 0x3cc21227, + 0xb0d49000, 0x3fcdb13d, 0xf715b035, 0xbd2aff2a, 0xf2656000, + 0x3fcd1037, 0x75b6f6e4, 0xbd084a7e, 0xc6f01000, 0x3fcc6ffb, + 0xc5962bd2, 0xbcf1ec72, 0x383be000, 0x3fcbd087, 0x595412b6, + 0xbd2d4bc4, 0x575bd000, 0x3fcb31d8, 0x4eace1aa, 0xbd0c358d, + 0x3c8ae000, 0x3fca93ed, 0x50562169, 0xbd287243, 0x07089000, + 0x3fc9f6c4, 0x6865817a, 0x3d29904d, 0xdcf70000, 0x3fc95a5a, + 0x58a0ff6f, 0x3d07f228, 0xeb390000, 0x3fc8beaf, 0xaae92cd1, + 0xbd073d54, 0x6551a000, 0x3fc823c1, 0x9a631e83, 0x3d1e0ddb, + 0x85445000, 0x3fc7898d, 0x70914305, 0xbd1c6610, 0x8b757000, + 0x3fc6f012, 0xe59c21e1, 0xbd25118d, 0xbe8c1000, 0x3fc6574e, + 0x2c3c2e78, 0x3d19cf8b, 0x6b544000, 0x3fc5bf40, 0xeb68981c, + 0xbd127023, 0xe4a1b000, 0x3fc527e5, 0xe5697dc7, 0x3d2633e8, + 0x8333b000, 0x3fc4913d, 0x54fdb678, 0x3d258379, 0xa5993000, + 0x3fc3fb45, 0x7e6a354d, 0xbd2cd1d8, 0xb0159000, 0x3fc365fc, + 0x234b7289, 0x3cc62fa8, 0x0c868000, 0x3fc2d161, 0xcb81b4a1, + 0x3d039d6c, 0x2a49c000, 0x3fc23d71, 0x8fd3df5c, 0x3d100d23, + 0x7e23f000, 0x3fc1aa2b, 0x44389934, 0x3d2ca78e, 0x8227e000, + 0x3fc1178e, 0xce2d07f2, 0x3d21ef78, 0xb59e4000, 0x3fc08598, + 0x7009902c, 0xbd27e5dd, 0x39dbe000, 0x3fbfe891, 0x4fa10afd, + 0xbd2534d6, 0x830a2000, 0x3fbec739, 0xafe645e0, 0xbd2dc068, + 0x63844000, 0x3fbda727, 0x1fa71733, 0x3d1a8940, 0x01bc4000, + 0x3fbc8858, 0xc65aacd3, 0x3d2646d1, 0x8dad6000, 0x3fbb6ac8, + 0x2bf768e5, 0xbd139080, 0x40b1c000, 0x3fba4e76, 0xb94407c8, + 0xbd0e42b6, 0x5d594000, 0x3fb9335e, 0x3abd47da, 0x3d23115c, + 0x2f40e000, 0x3fb8197e, 0xf96ffdf7, 0x3d0f80dc, 0x0aeac000, + 0x3fb700d3, 0xa99ded32, 0x3cec1e8d, 0x4d97a000, 0x3fb5e95a, + 0x3c5d1d1e, 0xbd2c6906, 0x5d208000, 0x3fb4d311, 0x82f4e1ef, + 0xbcf53a25, 0xa7d1e000, 0x3fb3bdf5, 0xa5db4ed7, 0x3d2cc85e, + 0xa4472000, 0x3fb2aa04, 0xae9c697d, 0xbd20b6e8, 0xd1466000, + 0x3fb1973b, 0x560d9e9b, 0xbd25325d, 0xb59e4000, 0x3fb08598, + 0x7009902c, 0xbd17e5dd, 0xc006c000, 0x3faeea31, 0x4fc93b7b, + 0xbd0e113e, 0xcdddc000, 0x3faccb73, 0x47d82807, 0xbd1a68f2, + 0xd0fb0000, 0x3faaaef2, 0x353bb42e, 0x3d20fc1a, 0x149fc000, + 0x3fa894aa, 0xd05a267d, 0xbd197995, 0xf2d4c000, 0x3fa67c94, + 0xec19afa2, 0xbd029efb, 0xd42e0000, 0x3fa466ae, 0x75bdfd28, + 0xbd2c1673, 0x2f8d0000, 0x3fa252f3, 0xe021b67b, 0x3d283e9a, + 0x89e74000, 0x3fa0415d, 0x5cf1d753, 0x3d0111c0, 0xec148000, + 0x3f9c63d2, 0x3f9eb2f3, 0x3d2578c6, 0x28c90000, 0x3f984925, + 0x325a0c34, 0xbd2aa0ba, 0x25980000, 0x3f9432a9, 0x928637fe, + 0x3d098139, 0x58938000, 0x3f902056, 0x06e2f7d2, 0xbd23dc5b, + 0xa3890000, 0x3f882448, 0xda74f640, 0xbd275577, 0x75890000, + 0x3f801015, 0x999d2be8, 0xbd10c76b, 0x59580000, 0x3f700805, + 0xcb31c67b, 0x3d2166af, 0x00000000, 0x00000000, 0x00000000, + 0x80000000 + // @formatter:on + }); + + private ArrayDataPointerConstant log2 = pointerConstant(8, new int[]{ + // @formatter:off + 0xfefa3800, 0x3fa62e42, + }); + private ArrayDataPointerConstant log28 = pointerConstant(8, new int[]{ + 0x93c76730, 0x3ceef357 + // @formatter:on + }); + + private ArrayDataPointerConstant coeff = pointerConstant(16, new int[]{ + // @formatter:off + 0x92492492, 0x3fc24924, 0x00000000, 0xbfd00000, + }); + private ArrayDataPointerConstant coeff16 = pointerConstant(16, new int[]{ + 0x3d6fb175, 0xbfc5555e, 0x55555555, 0x3fd55555, + }); + private ArrayDataPointerConstant coeff32 = pointerConstant(16, new int[]{ + 0x9999999a, 0x3fc99999, 0x00000000, 0xbfe00000 + // @formatter:on + }); + + @Override + public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { + // registers, + // input: xmm0 + // scratch: xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7 + // rax, rdx, rcx, r8, r11 + Label block0 = new Label(); + Label block1 = new Label(); + Label block2 = new Label(); + Label block3 = new Label(); + Label block4 = new Label(); + Label block5 = new Label(); + Label block6 = new Label(); + Label block7 = new Label(); + Label block8 = new Label(); + Label block9 = new Label(); + + masm.subq(rsp, 24); + masm.movsd(new AMD64Address(rsp, 0), xmm0); + masm.movq(rax, 0x3ff0000000000000L); + masm.movdq(xmm2, rax); + masm.movq(rdx, 0x77f0000000000000L); + masm.movdq(xmm3, rdx); + masm.movl(rcx, 32768); + masm.movdl(xmm4, rcx); + masm.movq(r8, 0xffffe00000000000L); + masm.movdq(xmm5, r8); + masm.movdqu(xmm1, xmm0); + masm.pextrw(rax, xmm0, 3); + masm.por(xmm0, xmm2); + masm.movl(rcx, 16352); + masm.psrlq(xmm0, 27); + masm.leaq(r11, recordExternalAddress(crb, lTbl)); + masm.psrld(xmm0, 2); + masm.rcpps(xmm0, xmm0); + masm.psllq(xmm1, 12); + masm.pshufd(xmm6, xmm5, 228); + masm.psrlq(xmm1, 12); + masm.subl(rax, 16); + masm.cmpl(rax, 32736); + masm.jcc(AMD64Assembler.ConditionFlag.AboveEqual, block0); + + masm.bind(block1); + masm.paddd(xmm0, xmm4); + masm.por(xmm1, xmm3); + masm.movdl(rdx, xmm0); + masm.psllq(xmm0, 29); + masm.pand(xmm5, xmm1); + masm.pand(xmm0, xmm6); + masm.subsd(xmm1, xmm5); + masm.mulpd(xmm5, xmm0); + masm.andl(rax, 32752); + masm.subl(rax, rcx); + masm.cvtsi2sdl(xmm7, rax); + masm.mulsd(xmm1, xmm0); + masm.movq(xmm6, recordExternalAddress(crb, log2)); // 0xfefa3800, 0x3fa62e42 + masm.movdqu(xmm3, recordExternalAddress(crb, coeff)); // 0x92492492, 0x3fc24924, + // 0x00000000, 0xbfd00000 + masm.subsd(xmm5, xmm2); + masm.andl(rdx, 16711680); + masm.shrl(rdx, 12); + masm.movdqu(xmm0, new AMD64Address(r11, rdx, AMD64Address.Scale.Times1)); + masm.movdqu(xmm4, recordExternalAddress(crb, coeff16)); // 0x3d6fb175, 0xbfc5555e, + // 0x55555555, 0x3fd55555 + masm.addsd(xmm1, xmm5); + masm.movdqu(xmm2, recordExternalAddress(crb, coeff32)); // 0x9999999a, 0x3fc99999, + // 0x00000000, 0xbfe00000 + masm.mulsd(xmm6, xmm7); + if (masm.supports(AMD64.CPUFeature.SSE3)) { + masm.movddup(xmm5, xmm1); + } else { + masm.movdqu(xmm5, xmm1); + masm.movlhps(xmm5, xmm5); + } + masm.mulsd(xmm7, recordExternalAddress(crb, log28)); // 0x93c76730, 0x3ceef357 + masm.mulsd(xmm3, xmm1); + masm.addsd(xmm0, xmm6); + masm.mulpd(xmm4, xmm5); + masm.mulpd(xmm5, xmm5); + if (masm.supports(AMD64.CPUFeature.SSE3)) { + masm.movddup(xmm6, xmm0); + } else { + masm.movdqu(xmm6, xmm0); + masm.movlhps(xmm6, xmm6); + } + masm.addsd(xmm0, xmm1); + masm.addpd(xmm4, xmm2); + masm.mulpd(xmm3, xmm5); + masm.subsd(xmm6, xmm0); + masm.mulsd(xmm4, xmm1); + masm.pshufd(xmm2, xmm0, 238); + masm.addsd(xmm1, xmm6); + masm.mulsd(xmm5, xmm5); + masm.addsd(xmm7, xmm2); + masm.addpd(xmm4, xmm3); + masm.addsd(xmm1, xmm7); + masm.mulpd(xmm4, xmm5); + masm.addsd(xmm1, xmm4); + masm.pshufd(xmm5, xmm4, 238); + masm.addsd(xmm1, xmm5); + masm.addsd(xmm0, xmm1); + masm.jmp(block9); + + masm.bind(block0); + masm.movq(xmm0, new AMD64Address(rsp, 0)); + masm.movq(xmm1, new AMD64Address(rsp, 0)); + masm.addl(rax, 16); + masm.cmpl(rax, 32768); + masm.jcc(AMD64Assembler.ConditionFlag.AboveEqual, block2); + masm.cmpl(rax, 16); + masm.jcc(AMD64Assembler.ConditionFlag.Below, block3); + + masm.bind(block4); + masm.addsd(xmm0, xmm0); + masm.jmp(block9); + + masm.bind(block5); + masm.jcc(AMD64Assembler.ConditionFlag.Above, block4); + masm.cmpl(rdx, 0); + masm.jcc(AMD64Assembler.ConditionFlag.Above, block4); + masm.jmp(block6); + + masm.bind(block3); + masm.xorpd(xmm1, xmm1); + masm.addsd(xmm1, xmm0); + masm.movdl(rdx, xmm1); + masm.psrlq(xmm1, 32); + masm.movdl(rcx, xmm1); + masm.orl(rdx, rcx); + masm.cmpl(rdx, 0); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block7); + masm.xorpd(xmm1, xmm1); + masm.movl(rax, 18416); + masm.pinsrw(xmm1, rax, 3); + masm.mulsd(xmm0, xmm1); + masm.movdqu(xmm1, xmm0); + masm.pextrw(rax, xmm0, 3); + masm.por(xmm0, xmm2); + masm.psrlq(xmm0, 27); + masm.movl(rcx, 18416); + masm.psrld(xmm0, 2); + masm.rcpps(xmm0, xmm0); + masm.psllq(xmm1, 12); + masm.pshufd(xmm6, xmm5, 228); + masm.psrlq(xmm1, 12); + masm.jmp(block1); + + masm.bind(block2); + masm.movdl(rdx, xmm1); + masm.psrlq(xmm1, 32); + masm.movdl(rcx, xmm1); + masm.addl(rcx, rcx); + masm.cmpl(rcx, -2097152); + masm.jcc(AMD64Assembler.ConditionFlag.AboveEqual, block5); + masm.orl(rdx, rcx); + masm.cmpl(rdx, 0); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block7); + + masm.bind(block6); + masm.xorpd(xmm1, xmm1); + masm.xorpd(xmm0, xmm0); + masm.movl(rax, 32752); + masm.pinsrw(xmm1, rax, 3); + masm.mulsd(xmm0, xmm1); + masm.movl(new AMD64Address(rsp, 16), 3); + masm.jmp(block8); + masm.bind(block7); + masm.xorpd(xmm1, xmm1); + masm.xorpd(xmm0, xmm0); + masm.movl(rax, 49136); + masm.pinsrw(xmm0, rax, 3); + masm.divsd(xmm0, xmm1); + masm.movl(new AMD64Address(rsp, 16), 2); + + masm.bind(block8); + masm.movq(new AMD64Address(rsp, 8), xmm0); + + masm.movq(xmm0, new AMD64Address(rsp, 8)); + + masm.bind(block9); + masm.addq(rsp, 24); + } +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathPowOp.java 2019-03-12 08:10:47.784040773 +0100 @@ -0,0 +1,1979 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, Intel Corporation. All rights reserved. + * Intel Math Library (LIBM) Source Code + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.lir.amd64; + +import static jdk.vm.ci.amd64.AMD64.r10; +import static jdk.vm.ci.amd64.AMD64.r11; +import static jdk.vm.ci.amd64.AMD64.r8; +import static jdk.vm.ci.amd64.AMD64.r9; +import static jdk.vm.ci.amd64.AMD64.rax; +import static jdk.vm.ci.amd64.AMD64.rcx; +import static jdk.vm.ci.amd64.AMD64.rdx; +import static jdk.vm.ci.amd64.AMD64.rsp; +import static jdk.vm.ci.amd64.AMD64.xmm0; +import static jdk.vm.ci.amd64.AMD64.xmm1; +import static jdk.vm.ci.amd64.AMD64.xmm2; +import static jdk.vm.ci.amd64.AMD64.xmm3; +import static jdk.vm.ci.amd64.AMD64.xmm4; +import static jdk.vm.ci.amd64.AMD64.xmm5; +import static jdk.vm.ci.amd64.AMD64.xmm6; +import static jdk.vm.ci.amd64.AMD64.xmm7; +import static org.graalvm.compiler.lir.amd64.AMD64HotSpotHelper.pointerConstant; +import static org.graalvm.compiler.lir.amd64.AMD64HotSpotHelper.recordExternalAddress; + +import org.graalvm.compiler.asm.Label; +import org.graalvm.compiler.asm.amd64.AMD64Address; +import org.graalvm.compiler.asm.amd64.AMD64Assembler; +import org.graalvm.compiler.asm.amd64.AMD64MacroAssembler; +import org.graalvm.compiler.lir.LIRInstructionClass; +import org.graalvm.compiler.lir.asm.ArrayDataPointerConstant; +import org.graalvm.compiler.lir.asm.CompilationResultBuilder; + +import jdk.vm.ci.code.Register; + +/** + * 
+ *                     ALGORITHM DESCRIPTION  - POW()
+ *                     ---------------------
+ *
+ *    Let x=2^k * mx, mx in [1,2)
+ *
+ *    log2(x) calculation:
+ *
+ *    Get B~1/mx based on the output of rcpps instruction (B0)
+ *    B = int((B0*LH*2^9+0.5))/2^9
+ *    LH is a short approximation for log2(e)
+ *
+ *    Reduced argument, scaled by LH:
+ *                r=B*mx-LH (computed accurately in high and low parts)
+ *
+ *    log2(x) result:  k - log2(B) + p(r)
+ *             p(r) is a degree 8 polynomial
+ *             -log2(B) read from data table (high, low parts)
+ *             log2(x) is formed from high and low parts
+ *    For |x| in [1-1/32, 1+1/16), a slower but more accurate computation
+ *    based om the same table design is performed.
+ *
+ *   Main path is taken if | floor(log2(|log2(|x|)|) + floor(log2|y|) | < 8,
+ *   to filter out all potential OF/UF cases.
+ *   exp2(y*log2(x)) is computed using an 8-bit index table and a degree 5
+ *   polynomial
+ *
+ * Special cases:
+ *  pow(-0,y) = -INF and raises the divide-by-zero exception for y an odd
+ *  integer < 0.
+ *  pow(-0,y) = +INF and raises the divide-by-zero exception for y < 0 and
+ *  not an odd integer.
+ *  pow(-0,y) = -0 for y an odd integer > 0.
+ *  pow(-0,y) = +0 for y > 0 and not an odd integer.
+ *  pow(-1,-INF) = NaN.
+ *  pow(+1,y) = NaN for any y, even a NaN.
+ *  pow(x,-0) = 1 for any x, even a NaN.
+ *  pow(x,y) = a NaN and raises the invalid exception for finite x < 0 and
+ *  finite non-integer y.
+ *  pow(x,-INF) = +INF for |x|<1.
+ *  pow(x,-INF) = +0 for |x|>1.
+ *  pow(x,+INF) = +0 for |x|<1.
+ *  pow(x,+INF) = +INF for |x|>1.
+ *  pow(-INF,y) = -0 for y an odd integer < 0.
+ *  pow(-INF,y) = +0 for y < 0 and not an odd integer.
+ *  pow(-INF,y) = -INF for y an odd integer > 0.
+ *  pow(-INF,y) = +INF for y > 0 and not an odd integer.
+ *  pow(+INF,y) = +0 for y <0.
+ *  pow(+INF,y) = +INF for y >0.
+ *
+ */ +public final class AMD64MathPowOp extends AMD64MathIntrinsicBinaryOp { + + public static final LIRInstructionClass TYPE = LIRInstructionClass.create(AMD64MathPowOp.class); + + public AMD64MathPowOp() { + super(TYPE, /* GPR */ rax, rcx, rdx, r8, r9, r10, r11, + /* XMM */ xmm2, xmm3, xmm4, xmm5, xmm6, xmm7); + } + + private ArrayDataPointerConstant highsigmask = pointerConstant(16, new int[]{ + //@formatter:off + 0x00000000, 0xfffff800, 0x00000000, 0xfffff800 + //@formatter:on + }); + + private ArrayDataPointerConstant log2E = pointerConstant(16, new int[]{ + //@formatter:off + 0x00000000, 0x3ff72000, 0x161bb241, 0xbf5dabe1 + //@formatter:on + }); + + private ArrayDataPointerConstant highmaskY = pointerConstant(8, new int[]{ + //@formatter:off + 0x00000000, 0xfffffff8, + }); + private ArrayDataPointerConstant highmaskY8 = pointerConstant(8, new int[]{ + 0x00000000, 0xffffffff + //@formatter:on + }); + + private ArrayDataPointerConstant tExp = pointerConstant(16, new int[]{ + //@formatter:off + 0x00000000, 0x3ff00000, 0x00000000, 0x3b700000, 0xfa5abcbf, + 0x3ff00b1a, 0xa7609f71, 0xbc84f6b2, 0xa9fb3335, 0x3ff0163d, + 0x9ab8cdb7, 0x3c9b6129, 0x143b0281, 0x3ff02168, 0x0fc54eb6, + 0xbc82bf31, 0x3e778061, 0x3ff02c9a, 0x535b085d, 0xbc719083, + 0x2e11bbcc, 0x3ff037d4, 0xeeade11a, 0x3c656811, 0xe86e7f85, + 0x3ff04315, 0x1977c96e, 0xbc90a31c, 0x72f654b1, 0x3ff04e5f, + 0x3aa0d08c, 0x3c84c379, 0xd3158574, 0x3ff059b0, 0xa475b465, + 0x3c8d73e2, 0x0e3c1f89, 0x3ff0650a, 0x5799c397, 0xbc95cb7b, + 0x29ddf6de, 0x3ff0706b, 0xe2b13c27, 0xbc8c91df, 0x2b72a836, + 0x3ff07bd4, 0x54458700, 0x3c832334, 0x18759bc8, 0x3ff08745, + 0x4bb284ff, 0x3c6186be, 0xf66607e0, 0x3ff092bd, 0x800a3fd1, + 0xbc968063, 0xcac6f383, 0x3ff09e3e, 0x18316136, 0x3c914878, + 0x9b1f3919, 0x3ff0a9c7, 0x873d1d38, 0x3c85d16c, 0x6cf9890f, + 0x3ff0b558, 0x4adc610b, 0x3c98a62e, 0x45e46c85, 0x3ff0c0f1, + 0x06d21cef, 0x3c94f989, 0x2b7247f7, 0x3ff0cc92, 0x16e24f71, + 0x3c901edc, 0x23395dec, 0x3ff0d83b, 0xe43f316a, 0xbc9bc14d, + 0x32d3d1a2, 0x3ff0e3ec, 0x27c57b52, 0x3c403a17, 0x5fdfa9c5, + 0x3ff0efa5, 0xbc54021b, 0xbc949db9, 0xaffed31b, 0x3ff0fb66, + 0xc44ebd7b, 0xbc6b9bed, 0x28d7233e, 0x3ff10730, 0x1692fdd5, + 0x3c8d46eb, 0xd0125b51, 0x3ff11301, 0x39449b3a, 0xbc96c510, + 0xab5e2ab6, 0x3ff11edb, 0xf703fb72, 0xbc9ca454, 0xc06c31cc, + 0x3ff12abd, 0xb36ca5c7, 0xbc51b514, 0x14f204ab, 0x3ff136a8, + 0xba48dcf0, 0xbc67108f, 0xaea92de0, 0x3ff1429a, 0x9af1369e, + 0xbc932fbf, 0x934f312e, 0x3ff14e95, 0x39bf44ab, 0xbc8b91e8, + 0xc8a58e51, 0x3ff15a98, 0xb9eeab0a, 0x3c82406a, 0x5471c3c2, + 0x3ff166a4, 0x82ea1a32, 0x3c58f23b, 0x3c7d517b, 0x3ff172b8, + 0xb9d78a76, 0xbc819041, 0x8695bbc0, 0x3ff17ed4, 0xe2ac5a64, + 0x3c709e3f, 0x388c8dea, 0x3ff18af9, 0xd1970f6c, 0xbc911023, + 0x58375d2f, 0x3ff19726, 0x85f17e08, 0x3c94aadd, 0xeb6fcb75, + 0x3ff1a35b, 0x7b4968e4, 0x3c8e5b4c, 0xf8138a1c, 0x3ff1af99, + 0xa4b69280, 0x3c97bf85, 0x84045cd4, 0x3ff1bbe0, 0x352ef607, + 0xbc995386, 0x95281c6b, 0x3ff1c82f, 0x8010f8c9, 0x3c900977, + 0x3168b9aa, 0x3ff1d487, 0x00a2643c, 0x3c9e016e, 0x5eb44027, + 0x3ff1e0e7, 0x088cb6de, 0xbc96fdd8, 0x22fcd91d, 0x3ff1ed50, + 0x027bb78c, 0xbc91df98, 0x8438ce4d, 0x3ff1f9c1, 0xa097af5c, + 0xbc9bf524, 0x88628cd6, 0x3ff2063b, 0x814a8495, 0x3c8dc775, + 0x3578a819, 0x3ff212be, 0x2cfcaac9, 0x3c93592d, 0x917ddc96, + 0x3ff21f49, 0x9494a5ee, 0x3c82a97e, 0xa27912d1, 0x3ff22bdd, + 0x5577d69f, 0x3c8d34fb, 0x6e756238, 0x3ff2387a, 0xb6c70573, + 0x3c99b07e, 0xfb82140a, 0x3ff2451f, 0x911ca996, 0x3c8acfcc, + 0x4fb2a63f, 0x3ff251ce, 0xbef4f4a4, 0x3c8ac155, 0x711ece75, + 0x3ff25e85, 0x4ac31b2c, 0x3c93e1a2, 0x65e27cdd, 0x3ff26b45, + 0x9940e9d9, 0x3c82bd33, 0x341ddf29, 0x3ff2780e, 0x05f9e76c, + 0x3c9e067c, 0xe1f56381, 0x3ff284df, 0x8c3f0d7e, 0xbc9a4c3a, + 0x7591bb70, 0x3ff291ba, 0x28401cbd, 0xbc82cc72, 0xf51fdee1, + 0x3ff29e9d, 0xafad1255, 0x3c8612e8, 0x66d10f13, 0x3ff2ab8a, + 0x191690a7, 0xbc995743, 0xd0dad990, 0x3ff2b87f, 0xd6381aa4, + 0xbc410adc, 0x39771b2f, 0x3ff2c57e, 0xa6eb5124, 0xbc950145, + 0xa6e4030b, 0x3ff2d285, 0x54db41d5, 0x3c900247, 0x1f641589, + 0x3ff2df96, 0xfbbce198, 0x3c9d16cf, 0xa93e2f56, 0x3ff2ecaf, + 0x45d52383, 0x3c71ca0f, 0x4abd886b, 0x3ff2f9d2, 0x532bda93, + 0xbc653c55, 0x0a31b715, 0x3ff306fe, 0xd23182e4, 0x3c86f46a, + 0xedeeb2fd, 0x3ff31432, 0xf3f3fcd1, 0x3c8959a3, 0xfc4cd831, + 0x3ff32170, 0x8e18047c, 0x3c8a9ce7, 0x3ba8ea32, 0x3ff32eb8, + 0x3cb4f318, 0xbc9c45e8, 0xb26416ff, 0x3ff33c08, 0x843659a6, + 0x3c932721, 0x66e3fa2d, 0x3ff34962, 0x930881a4, 0xbc835a75, + 0x5f929ff1, 0x3ff356c5, 0x5c4e4628, 0xbc8b5cee, 0xa2de883b, + 0x3ff36431, 0xa06cb85e, 0xbc8c3144, 0x373aa9cb, 0x3ff371a7, + 0xbf42eae2, 0xbc963aea, 0x231e754a, 0x3ff37f26, 0x9eceb23c, + 0xbc99f5ca, 0x6d05d866, 0x3ff38cae, 0x3c9904bd, 0xbc9e958d, + 0x1b7140ef, 0x3ff39a40, 0xfc8e2934, 0xbc99a9a5, 0x34e59ff7, + 0x3ff3a7db, 0xd661f5e3, 0xbc75e436, 0xbfec6cf4, 0x3ff3b57f, + 0xe26fff18, 0x3c954c66, 0xc313a8e5, 0x3ff3c32d, 0x375d29c3, + 0xbc9efff8, 0x44ede173, 0x3ff3d0e5, 0x8c284c71, 0x3c7fe8d0, + 0x4c123422, 0x3ff3dea6, 0x11f09ebc, 0x3c8ada09, 0xdf1c5175, + 0x3ff3ec70, 0x7b8c9bca, 0xbc8af663, 0x04ac801c, 0x3ff3fa45, + 0xf956f9f3, 0xbc97d023, 0xc367a024, 0x3ff40822, 0xb6f4d048, + 0x3c8bddf8, 0x21f72e2a, 0x3ff4160a, 0x1c309278, 0xbc5ef369, + 0x2709468a, 0x3ff423fb, 0xc0b314dd, 0xbc98462d, 0xd950a897, + 0x3ff431f5, 0xe35f7999, 0xbc81c7dd, 0x3f84b9d4, 0x3ff43ffa, + 0x9704c003, 0x3c8880be, 0x6061892d, 0x3ff44e08, 0x04ef80d0, + 0x3c489b7a, 0x42a7d232, 0x3ff45c20, 0x82fb1f8e, 0xbc686419, + 0xed1d0057, 0x3ff46a41, 0xd1648a76, 0x3c9c944b, 0x668b3237, + 0x3ff4786d, 0xed445733, 0xbc9c20f0, 0xb5c13cd0, 0x3ff486a2, + 0xb69062f0, 0x3c73c1a3, 0xe192aed2, 0x3ff494e1, 0x5e499ea0, + 0xbc83b289, 0xf0d7d3de, 0x3ff4a32a, 0xf3d1be56, 0x3c99cb62, + 0xea6db7d7, 0x3ff4b17d, 0x7f2897f0, 0xbc8125b8, 0xd5362a27, + 0x3ff4bfda, 0xafec42e2, 0x3c7d4397, 0xb817c114, 0x3ff4ce41, + 0x690abd5d, 0x3c905e29, 0x99fddd0d, 0x3ff4dcb2, 0xbc6a7833, + 0x3c98ecdb, 0x81d8abff, 0x3ff4eb2d, 0x2e5d7a52, 0xbc95257d, + 0x769d2ca7, 0x3ff4f9b2, 0xd25957e3, 0xbc94b309, 0x7f4531ee, + 0x3ff50841, 0x49b7465f, 0x3c7a249b, 0xa2cf6642, 0x3ff516da, + 0x69bd93ef, 0xbc8f7685, 0xe83f4eef, 0x3ff5257d, 0x43efef71, + 0xbc7c998d, 0x569d4f82, 0x3ff5342b, 0x1db13cad, 0xbc807abe, + 0xf4f6ad27, 0x3ff542e2, 0x192d5f7e, 0x3c87926d, 0xca5d920f, + 0x3ff551a4, 0xefede59b, 0xbc8d689c, 0xdde910d2, 0x3ff56070, + 0x168eebf0, 0xbc90fb6e, 0x36b527da, 0x3ff56f47, 0x011d93ad, + 0x3c99bb2c, 0xdbe2c4cf, 0x3ff57e27, 0x8a57b9c4, 0xbc90b98c, + 0xd497c7fd, 0x3ff58d12, 0x5b9a1de8, 0x3c8295e1, 0x27ff07cc, + 0x3ff59c08, 0xe467e60f, 0xbc97e2ce, 0xdd485429, 0x3ff5ab07, + 0x054647ad, 0x3c96324c, 0xfba87a03, 0x3ff5ba11, 0x4c233e1a, + 0xbc9b77a1, 0x8a5946b7, 0x3ff5c926, 0x816986a2, 0x3c3c4b1b, + 0x90998b93, 0x3ff5d845, 0xa8b45643, 0xbc9cd6a7, 0x15ad2148, + 0x3ff5e76f, 0x3080e65e, 0x3c9ba6f9, 0x20dceb71, 0x3ff5f6a3, + 0xe3cdcf92, 0xbc89eadd, 0xb976dc09, 0x3ff605e1, 0x9b56de47, + 0xbc93e242, 0xe6cdf6f4, 0x3ff6152a, 0x4ab84c27, 0x3c9e4b3e, + 0xb03a5585, 0x3ff6247e, 0x7e40b497, 0xbc9383c1, 0x1d1929fd, + 0x3ff633dd, 0xbeb964e5, 0x3c984710, 0x34ccc320, 0x3ff64346, + 0x759d8933, 0xbc8c483c, 0xfebc8fb7, 0x3ff652b9, 0xc9a73e09, + 0xbc9ae3d5, 0x82552225, 0x3ff66238, 0x87591c34, 0xbc9bb609, + 0xc70833f6, 0x3ff671c1, 0x586c6134, 0xbc8e8732, 0xd44ca973, + 0x3ff68155, 0x44f73e65, 0x3c6038ae, 0xb19e9538, 0x3ff690f4, + 0x9aeb445d, 0x3c8804bd, 0x667f3bcd, 0x3ff6a09e, 0x13b26456, + 0xbc9bdd34, 0xfa75173e, 0x3ff6b052, 0x2c9a9d0e, 0x3c7a38f5, + 0x750bdabf, 0x3ff6c012, 0x67ff0b0d, 0xbc728956, 0xddd47645, + 0x3ff6cfdc, 0xb6f17309, 0x3c9c7aa9, 0x3c651a2f, 0x3ff6dfb2, + 0x683c88ab, 0xbc6bbe3a, 0x98593ae5, 0x3ff6ef92, 0x9e1ac8b2, + 0xbc90b974, 0xf9519484, 0x3ff6ff7d, 0x25860ef6, 0xbc883c0f, + 0x66f42e87, 0x3ff70f74, 0xd45aa65f, 0x3c59d644, 0xe8ec5f74, + 0x3ff71f75, 0x86887a99, 0xbc816e47, 0x86ead08a, 0x3ff72f82, + 0x2cd62c72, 0xbc920aa0, 0x48a58174, 0x3ff73f9a, 0x6c65d53c, + 0xbc90a8d9, 0x35d7cbfd, 0x3ff74fbd, 0x618a6e1c, 0x3c9047fd, + 0x564267c9, 0x3ff75feb, 0x57316dd3, 0xbc902459, 0xb1ab6e09, + 0x3ff77024, 0x169147f8, 0x3c9b7877, 0x4fde5d3f, 0x3ff78069, + 0x0a02162d, 0x3c9866b8, 0x38ac1cf6, 0x3ff790b9, 0x62aadd3e, + 0x3c9349a8, 0x73eb0187, 0x3ff7a114, 0xee04992f, 0xbc841577, + 0x0976cfdb, 0x3ff7b17b, 0x8468dc88, 0xbc9bebb5, 0x0130c132, + 0x3ff7c1ed, 0xd1164dd6, 0x3c9f124c, 0x62ff86f0, 0x3ff7d26a, + 0xfb72b8b4, 0x3c91bddb, 0x36cf4e62, 0x3ff7e2f3, 0xba15797e, + 0x3c705d02, 0x8491c491, 0x3ff7f387, 0xcf9311ae, 0xbc807f11, + 0x543e1a12, 0x3ff80427, 0x626d972b, 0xbc927c86, 0xadd106d9, + 0x3ff814d2, 0x0d151d4d, 0x3c946437, 0x994cce13, 0x3ff82589, + 0xd41532d8, 0xbc9d4c1d, 0x1eb941f7, 0x3ff8364c, 0x31df2bd5, + 0x3c999b9a, 0x4623c7ad, 0x3ff8471a, 0xa341cdfb, 0xbc88d684, + 0x179f5b21, 0x3ff857f4, 0xf8b216d0, 0xbc5ba748, 0x9b4492ed, + 0x3ff868d9, 0x9bd4f6ba, 0xbc9fc6f8, 0xd931a436, 0x3ff879ca, + 0xd2db47bd, 0x3c85d2d7, 0xd98a6699, 0x3ff88ac7, 0xf37cb53a, + 0x3c9994c2, 0xa478580f, 0x3ff89bd0, 0x4475202a, 0x3c9d5395, + 0x422aa0db, 0x3ff8ace5, 0x56864b27, 0x3c96e9f1, 0xbad61778, + 0x3ff8be05, 0xfc43446e, 0x3c9ecb5e, 0x16b5448c, 0x3ff8cf32, + 0x32e9e3aa, 0xbc70d55e, 0x5e0866d9, 0x3ff8e06a, 0x6fc9b2e6, + 0xbc97114a, 0x99157736, 0x3ff8f1ae, 0xa2e3976c, 0x3c85cc13, + 0xd0282c8a, 0x3ff902fe, 0x85fe3fd2, 0x3c9592ca, 0x0b91ffc6, + 0x3ff9145b, 0x2e582524, 0xbc9dd679, 0x53aa2fe2, 0x3ff925c3, + 0xa639db7f, 0xbc83455f, 0xb0cdc5e5, 0x3ff93737, 0x81b57ebc, + 0xbc675fc7, 0x2b5f98e5, 0x3ff948b8, 0x797d2d99, 0xbc8dc3d6, + 0xcbc8520f, 0x3ff95a44, 0x96a5f039, 0xbc764b7c, 0x9a7670b3, + 0x3ff96bdd, 0x7f19c896, 0xbc5ba596, 0x9fde4e50, 0x3ff97d82, + 0x7c1b85d1, 0xbc9d185b, 0xe47a22a2, 0x3ff98f33, 0xa24c78ec, + 0x3c7cabda, 0x70ca07ba, 0x3ff9a0f1, 0x91cee632, 0xbc9173bd, + 0x4d53fe0d, 0x3ff9b2bb, 0x4df6d518, 0xbc9dd84e, 0x82a3f090, + 0x3ff9c491, 0xb071f2be, 0x3c7c7c46, 0x194bb8d5, 0x3ff9d674, + 0xa3dd8233, 0xbc9516be, 0x19e32323, 0x3ff9e863, 0x78e64c6e, + 0x3c7824ca, 0x8d07f29e, 0x3ff9fa5e, 0xaaf1face, 0xbc84a9ce, + 0x7b5de565, 0x3ffa0c66, 0x5d1cd533, 0xbc935949, 0xed8eb8bb, + 0x3ffa1e7a, 0xee8be70e, 0x3c9c6618, 0xec4a2d33, 0x3ffa309b, + 0x7ddc36ab, 0x3c96305c, 0x80460ad8, 0x3ffa42c9, 0x589fb120, + 0xbc9aa780, 0xb23e255d, 0x3ffa5503, 0xdb8d41e1, 0xbc9d2f6e, + 0x8af46052, 0x3ffa674a, 0x30670366, 0x3c650f56, 0x1330b358, + 0x3ffa799e, 0xcac563c7, 0x3c9bcb7e, 0x53c12e59, 0x3ffa8bfe, + 0xb2ba15a9, 0xbc94f867, 0x5579fdbf, 0x3ffa9e6b, 0x0ef7fd31, + 0x3c90fac9, 0x21356eba, 0x3ffab0e5, 0xdae94545, 0x3c889c31, + 0xbfd3f37a, 0x3ffac36b, 0xcae76cd0, 0xbc8f9234, 0x3a3c2774, + 0x3ffad5ff, 0xb6b1b8e5, 0x3c97ef3b, 0x995ad3ad, 0x3ffae89f, + 0x345dcc81, 0x3c97a1cd, 0xe622f2ff, 0x3ffafb4c, 0x0f315ecd, + 0xbc94b2fc, 0x298db666, 0x3ffb0e07, 0x4c80e425, 0xbc9bdef5, + 0x6c9a8952, 0x3ffb20ce, 0x4a0756cc, 0x3c94dd02, 0xb84f15fb, + 0x3ffb33a2, 0x3084d708, 0xbc62805e, 0x15b749b1, 0x3ffb4684, + 0xe9df7c90, 0xbc7f763d, 0x8de5593a, 0x3ffb5972, 0xbbba6de3, + 0xbc9c71df, 0x29f1c52a, 0x3ffb6c6e, 0x52883f6e, 0x3c92a8f3, + 0xf2fb5e47, 0x3ffb7f76, 0x7e54ac3b, 0xbc75584f, 0xf22749e4, + 0x3ffb928c, 0x54cb65c6, 0xbc9b7216, 0x30a1064a, 0x3ffba5b0, + 0x0e54292e, 0xbc9efcd3, 0xb79a6f1f, 0x3ffbb8e0, 0xc9696205, + 0xbc3f52d1, 0x904bc1d2, 0x3ffbcc1e, 0x7a2d9e84, 0x3c823dd0, + 0xc3f3a207, 0x3ffbdf69, 0x60ea5b53, 0xbc3c2623, 0x5bd71e09, + 0x3ffbf2c2, 0x3f6b9c73, 0xbc9efdca, 0x6141b33d, 0x3ffc0628, + 0xa1fbca34, 0xbc8d8a5a, 0xdd85529c, 0x3ffc199b, 0x895048dd, + 0x3c811065, 0xd9fa652c, 0x3ffc2d1c, 0x17c8a5d7, 0xbc96e516, + 0x5fffd07a, 0x3ffc40ab, 0xe083c60a, 0x3c9b4537, 0x78fafb22, + 0x3ffc5447, 0x2493b5af, 0x3c912f07, 0x2e57d14b, 0x3ffc67f1, + 0xff483cad, 0x3c92884d, 0x8988c933, 0x3ffc7ba8, 0xbe255559, + 0xbc8e76bb, 0x9406e7b5, 0x3ffc8f6d, 0x48805c44, 0x3c71acbc, + 0x5751c4db, 0x3ffca340, 0xd10d08f5, 0xbc87f2be, 0xdcef9069, + 0x3ffcb720, 0xd1e949db, 0x3c7503cb, 0x2e6d1675, 0x3ffccb0f, + 0x86009092, 0xbc7d220f, 0x555dc3fa, 0x3ffcdf0b, 0x53829d72, + 0xbc8dd83b, 0x5b5bab74, 0x3ffcf315, 0xb86dff57, 0xbc9a08e9, + 0x4a07897c, 0x3ffd072d, 0x43797a9c, 0xbc9cbc37, 0x2b08c968, + 0x3ffd1b53, 0x219a36ee, 0x3c955636, 0x080d89f2, 0x3ffd2f87, + 0x719d8578, 0xbc9d487b, 0xeacaa1d6, 0x3ffd43c8, 0xbf5a1614, + 0x3c93db53, 0xdcfba487, 0x3ffd5818, 0xd75b3707, 0x3c82ed02, + 0xe862e6d3, 0x3ffd6c76, 0x4a8165a0, 0x3c5fe87a, 0x16c98398, + 0x3ffd80e3, 0x8beddfe8, 0xbc911ec1, 0x71ff6075, 0x3ffd955d, + 0xbb9af6be, 0x3c9a052d, 0x03db3285, 0x3ffda9e6, 0x696db532, + 0x3c9c2300, 0xd63a8315, 0x3ffdbe7c, 0x926b8be4, 0xbc9b76f1, + 0xf301b460, 0x3ffdd321, 0x78f018c3, 0x3c92da57, 0x641c0658, + 0x3ffde7d5, 0x8e79ba8f, 0xbc9ca552, 0x337b9b5f, 0x3ffdfc97, + 0x4f184b5c, 0xbc91a5cd, 0x6b197d17, 0x3ffe1167, 0xbd5c7f44, + 0xbc72b529, 0x14f5a129, 0x3ffe2646, 0x817a1496, 0xbc97b627, + 0x3b16ee12, 0x3ffe3b33, 0x31fdc68b, 0xbc99f4a4, 0xe78b3ff6, + 0x3ffe502e, 0x80a9cc8f, 0x3c839e89, 0x24676d76, 0x3ffe6539, + 0x7522b735, 0xbc863ff8, 0xfbc74c83, 0x3ffe7a51, 0xca0c8de2, + 0x3c92d522, 0x77cdb740, 0x3ffe8f79, 0x80b054b1, 0xbc910894, + 0xa2a490da, 0x3ffea4af, 0x179c2893, 0xbc9e9c23, 0x867cca6e, + 0x3ffeb9f4, 0x2293e4f2, 0x3c94832f, 0x2d8e67f1, 0x3ffecf48, + 0xb411ad8c, 0xbc9c93f3, 0xa2188510, 0x3ffee4aa, 0xa487568d, + 0x3c91c68d, 0xee615a27, 0x3ffefa1b, 0x86a4b6b0, 0x3c9dc7f4, + 0x1cb6412a, 0x3fff0f9c, 0x65181d45, 0xbc932200, 0x376bba97, + 0x3fff252b, 0xbf0d8e43, 0x3c93a1a5, 0x48dd7274, 0x3fff3ac9, + 0x3ed837de, 0xbc795a5a, 0x5b6e4540, 0x3fff5076, 0x2dd8a18b, + 0x3c99d3e1, 0x798844f8, 0x3fff6632, 0x3539343e, 0x3c9fa37b, + 0xad9cbe14, 0x3fff7bfd, 0xd006350a, 0xbc9dbb12, 0x02243c89, + 0x3fff91d8, 0xa779f689, 0xbc612ea8, 0x819e90d8, 0x3fffa7c1, + 0xf3a5931e, 0x3c874853, 0x3692d514, 0x3fffbdba, 0x15098eb6, + 0xbc796773, 0x2b8f71f1, 0x3fffd3c2, 0x966579e7, 0x3c62eb74, + 0x6b2a23d9, 0x3fffe9d9, 0x7442fde3, 0x3c74a603 + //@formatter:on + }); + + private ArrayDataPointerConstant eCoeff = pointerConstant(16, new int[]{ + //@formatter:off + 0xe78a6731, 0x3f55d87f, 0xd704a0c0, 0x3fac6b08, + }); + private ArrayDataPointerConstant eCoeff16 = pointerConstant(16, new int[]{ + 0x6fba4e77, 0x3f83b2ab, 0xff82c58f, 0x3fcebfbd, + }); + private ArrayDataPointerConstant eCoeff32 = pointerConstant(16, new int[]{ + 0xfefa39ef, 0x3fe62e42, 0x00000000, 0x00000000 + //@formatter:on + }); + + private ArrayDataPointerConstant coeffH = pointerConstant(8, new int[]{ + //@formatter:off + 0x00000000, 0xbfd61a00, + }); + private ArrayDataPointerConstant coeffH8 = pointerConstant(8, new int[]{ + 0x00000000, 0xbf5dabe1 + //@formatter:on + }); + + private ArrayDataPointerConstant highmaskLogX = pointerConstant(16, new int[]{ + //@formatter:off + 0xf8000000, 0xffffffff, 0x00000000, 0xfffff800 + //@formatter:on + }); + + private ArrayDataPointerConstant halfmask = pointerConstant(8, new int[]{ + //@formatter:off + 0xf8000000, 0xffffffff, 0xf8000000, 0xffffffff + //@formatter:on + }); + + private ArrayDataPointerConstant coeff = pointerConstant(16, new int[]{ + //@formatter:off + 0x6dc96112, 0xbf836578, 0xee241472, 0xbf9b0301, + }); + private ArrayDataPointerConstant coeff16 = pointerConstant(16, new int[]{ + 0x9f95985a, 0xbfb528db, 0xb3841d2a, 0xbfd619b6, + }); + private ArrayDataPointerConstant coeff32 = pointerConstant(16, new int[]{ + 0x518775e3, 0x3f9004f2, 0xac8349bb, 0x3fa76c9b, + }); + private ArrayDataPointerConstant coeff48 = pointerConstant(16, new int[]{ + 0x486ececc, 0x3fc4635e, 0x161bb241, 0xbf5dabe1, + }); + private ArrayDataPointerConstant coeff64 = pointerConstant(16, new int[]{ + 0x9f95985a, 0xbfb528db, 0xf8b5787d, 0x3ef2531e, + }); + private ArrayDataPointerConstant coeff80 = pointerConstant(16, new int[]{ + 0x486ececb, 0x3fc4635e, 0x412055cc, 0xbdd61bb2 + //@formatter:on + }); + + private ArrayDataPointerConstant lTbl = pointerConstant(16, new int[]{ + //@formatter:off + 0x00000000, 0x3ff00000, 0x00000000, 0x00000000, 0x20000000, + 0x3feff00a, 0x96621f95, 0x3e5b1856, 0xe0000000, 0x3fefe019, + 0xe5916f9e, 0xbe325278, 0x00000000, 0x3fefd02f, 0x859a1062, + 0x3e595fb7, 0xc0000000, 0x3fefc049, 0xb245f18f, 0xbe529c38, + 0xe0000000, 0x3fefb069, 0xad2880a7, 0xbe501230, 0x60000000, + 0x3fefa08f, 0xc8e72420, 0x3e597bd1, 0x80000000, 0x3fef90ba, + 0xc30c4500, 0xbe5d6c75, 0xe0000000, 0x3fef80ea, 0x02c63f43, + 0x3e2e1318, 0xc0000000, 0x3fef7120, 0xb3d4cccc, 0xbe44c52a, + 0x00000000, 0x3fef615c, 0xdbd91397, 0xbe4e7d6c, 0xa0000000, + 0x3fef519c, 0x65c5cd68, 0xbe522dc8, 0xa0000000, 0x3fef41e2, + 0x46d1306c, 0xbe5a840e, 0xe0000000, 0x3fef322d, 0xd2980e94, + 0x3e5071af, 0xa0000000, 0x3fef227e, 0x773abade, 0xbe5891e5, + 0xa0000000, 0x3fef12d4, 0xdc6bf46b, 0xbe5cccbe, 0xe0000000, + 0x3fef032f, 0xbc7247fa, 0xbe2bab83, 0x80000000, 0x3feef390, + 0xbcaa1e46, 0xbe53bb3b, 0x60000000, 0x3feee3f6, 0x5f6c682d, + 0xbe54c619, 0x80000000, 0x3feed461, 0x5141e368, 0xbe4b6d86, + 0xe0000000, 0x3feec4d1, 0xec678f76, 0xbe369af6, 0x80000000, + 0x3feeb547, 0x41301f55, 0xbe2d4312, 0x60000000, 0x3feea5c2, + 0x676da6bd, 0xbe4d8dd0, 0x60000000, 0x3fee9642, 0x57a891c4, + 0x3e51f991, 0xa0000000, 0x3fee86c7, 0xe4eb491e, 0x3e579bf9, + 0x20000000, 0x3fee7752, 0xfddc4a2c, 0xbe3356e6, 0xc0000000, + 0x3fee67e1, 0xd75b5bf1, 0xbe449531, 0x80000000, 0x3fee5876, + 0xbd423b8e, 0x3df54fe4, 0x60000000, 0x3fee4910, 0x330e51b9, + 0x3e54289c, 0x80000000, 0x3fee39af, 0x8651a95f, 0xbe55aad6, + 0xa0000000, 0x3fee2a53, 0x5e98c708, 0xbe2fc4a9, 0xe0000000, + 0x3fee1afc, 0x0989328d, 0x3e23958c, 0x40000000, 0x3fee0bab, + 0xee642abd, 0xbe425dd8, 0xa0000000, 0x3fedfc5e, 0xc394d236, + 0x3e526362, 0x20000000, 0x3feded17, 0xe104aa8e, 0x3e4ce247, + 0xc0000000, 0x3fedddd4, 0x265a9be4, 0xbe5bb77a, 0x40000000, + 0x3fedce97, 0x0ecac52f, 0x3e4a7cb1, 0xe0000000, 0x3fedbf5e, + 0x124cb3b8, 0x3e257024, 0x80000000, 0x3fedb02b, 0xe6d4febe, + 0xbe2033ee, 0x20000000, 0x3feda0fd, 0x39cca00e, 0xbe3ddabc, + 0xc0000000, 0x3fed91d3, 0xef8a552a, 0xbe543390, 0x40000000, + 0x3fed82af, 0xb8e85204, 0x3e513850, 0xe0000000, 0x3fed738f, + 0x3d59fe08, 0xbe5db728, 0x40000000, 0x3fed6475, 0x3aa7ead1, + 0x3e58804b, 0xc0000000, 0x3fed555f, 0xf8a35ba9, 0xbe5298b0, + 0x00000000, 0x3fed464f, 0x9a88dd15, 0x3e5a8cdb, 0x40000000, + 0x3fed3743, 0xb0b0a190, 0x3e598635, 0x80000000, 0x3fed283c, + 0xe2113295, 0xbe5c1119, 0x80000000, 0x3fed193a, 0xafbf1728, + 0xbe492e9c, 0x60000000, 0x3fed0a3d, 0xe4a4ccf3, 0x3e19b90e, + 0x20000000, 0x3fecfb45, 0xba3cbeb8, 0x3e406b50, 0xc0000000, + 0x3fecec51, 0x110f7ddd, 0x3e0d6806, 0x40000000, 0x3fecdd63, + 0x7dd7d508, 0xbe5a8943, 0x80000000, 0x3fecce79, 0x9b60f271, + 0xbe50676a, 0x80000000, 0x3fecbf94, 0x0b9ad660, 0x3e59174f, + 0x60000000, 0x3fecb0b4, 0x00823d9c, 0x3e5bbf72, 0x20000000, + 0x3feca1d9, 0x38a6ec89, 0xbe4d38f9, 0x80000000, 0x3fec9302, + 0x3a0b7d8e, 0x3e53dbfd, 0xc0000000, 0x3fec8430, 0xc6826b34, + 0xbe27c5c9, 0xc0000000, 0x3fec7563, 0x0c706381, 0xbe593653, + 0x60000000, 0x3fec669b, 0x7df34ec7, 0x3e461ab5, 0xe0000000, + 0x3fec57d7, 0x40e5e7e8, 0xbe5c3dae, 0x00000000, 0x3fec4919, + 0x5602770f, 0xbe55219d, 0xc0000000, 0x3fec3a5e, 0xec7911eb, + 0x3e5a5d25, 0x60000000, 0x3fec2ba9, 0xb39ea225, 0xbe53c00b, + 0x80000000, 0x3fec1cf8, 0x967a212e, 0x3e5a8ddf, 0x60000000, + 0x3fec0e4c, 0x580798bd, 0x3e5f53ab, 0x00000000, 0x3febffa5, + 0xb8282df6, 0xbe46b874, 0x20000000, 0x3febf102, 0xe33a6729, + 0x3e54963f, 0x00000000, 0x3febe264, 0x3b53e88a, 0xbe3adce1, + 0x60000000, 0x3febd3ca, 0xc2585084, 0x3e5cde9f, 0x80000000, + 0x3febc535, 0xa335c5ee, 0xbe39fd9c, 0x20000000, 0x3febb6a5, + 0x7325b04d, 0x3e42ba15, 0x60000000, 0x3feba819, 0x1564540f, + 0x3e3a9f35, 0x40000000, 0x3feb9992, 0x83fff592, 0xbe5465ce, + 0xa0000000, 0x3feb8b0f, 0xb9da63d3, 0xbe4b1a0a, 0x80000000, + 0x3feb7c91, 0x6d6f1ea4, 0x3e557657, 0x00000000, 0x3feb6e18, + 0x5e80a1bf, 0x3e4ddbb6, 0x00000000, 0x3feb5fa3, 0x1c9eacb5, + 0x3e592877, 0xa0000000, 0x3feb5132, 0x6d40beb3, 0xbe51858c, + 0xa0000000, 0x3feb42c6, 0xd740c67b, 0x3e427ad2, 0x40000000, + 0x3feb345f, 0xa3e0ccee, 0xbe5c2fc4, 0x40000000, 0x3feb25fc, + 0x8e752b50, 0xbe3da3c2, 0xc0000000, 0x3feb179d, 0xa892e7de, + 0x3e1fb481, 0xc0000000, 0x3feb0943, 0x21ed71e9, 0xbe365206, + 0x20000000, 0x3feafaee, 0x0e1380a3, 0x3e5c5b7b, 0x20000000, + 0x3feaec9d, 0x3c3d640e, 0xbe5dbbd0, 0x60000000, 0x3feade50, + 0x8f97a715, 0x3e3a8ec5, 0x20000000, 0x3fead008, 0x23ab2839, + 0x3e2fe98a, 0x40000000, 0x3feac1c4, 0xf4bbd50f, 0x3e54d8f6, + 0xe0000000, 0x3feab384, 0x14757c4d, 0xbe48774c, 0xc0000000, + 0x3feaa549, 0x7c7b0eea, 0x3e5b51bb, 0x20000000, 0x3fea9713, + 0xf56f7013, 0x3e386200, 0xe0000000, 0x3fea88e0, 0xbe428ebe, + 0xbe514af5, 0xe0000000, 0x3fea7ab2, 0x8d0e4496, 0x3e4f9165, + 0x60000000, 0x3fea6c89, 0xdbacc5d5, 0xbe5c063b, 0x20000000, + 0x3fea5e64, 0x3f19d970, 0xbe5a0c8c, 0x20000000, 0x3fea5043, + 0x09ea3e6b, 0x3e5065dc, 0x80000000, 0x3fea4226, 0x78df246c, + 0x3e5e05f6, 0x40000000, 0x3fea340e, 0x4057d4a0, 0x3e431b2b, + 0x40000000, 0x3fea25fa, 0x82867bb5, 0x3e4b76be, 0xa0000000, + 0x3fea17ea, 0x9436f40a, 0xbe5aad39, 0x20000000, 0x3fea09df, + 0x4b5253b3, 0x3e46380b, 0x00000000, 0x3fe9fbd8, 0x8fc52466, + 0xbe386f9b, 0x20000000, 0x3fe9edd5, 0x22d3f344, 0xbe538347, + 0x60000000, 0x3fe9dfd6, 0x1ac33522, 0x3e5dbc53, 0x00000000, + 0x3fe9d1dc, 0xeabdff1d, 0x3e40fc0c, 0xe0000000, 0x3fe9c3e5, + 0xafd30e73, 0xbe585e63, 0xe0000000, 0x3fe9b5f3, 0xa52f226a, + 0xbe43e8f9, 0x20000000, 0x3fe9a806, 0xecb8698d, 0xbe515b36, + 0x80000000, 0x3fe99a1c, 0xf2b4e89d, 0x3e48b62b, 0x20000000, + 0x3fe98c37, 0x7c9a88fb, 0x3e44414c, 0x00000000, 0x3fe97e56, + 0xda015741, 0xbe5d13ba, 0xe0000000, 0x3fe97078, 0x5fdace06, + 0x3e51b947, 0x00000000, 0x3fe962a0, 0x956ca094, 0x3e518785, + 0x40000000, 0x3fe954cb, 0x01164c1d, 0x3e5d5b57, 0xc0000000, + 0x3fe946fa, 0xe63b3767, 0xbe4f84e7, 0x40000000, 0x3fe9392e, + 0xe57cc2a9, 0x3e34eda3, 0xe0000000, 0x3fe92b65, 0x8c75b544, + 0x3e5766a0, 0xc0000000, 0x3fe91da1, 0x37d1d087, 0xbe5e2ab1, + 0x80000000, 0x3fe90fe1, 0xa953dc20, 0x3e5fa1f3, 0x80000000, + 0x3fe90225, 0xdbd3f369, 0x3e47d6db, 0xa0000000, 0x3fe8f46d, + 0x1c9be989, 0xbe5e2b0a, 0xa0000000, 0x3fe8e6b9, 0x3c93d76a, + 0x3e5c8618, 0xe0000000, 0x3fe8d909, 0x2182fc9a, 0xbe41aa9e, + 0x20000000, 0x3fe8cb5e, 0xe6b3539d, 0xbe530d19, 0x60000000, + 0x3fe8bdb6, 0x49e58cc3, 0xbe3bb374, 0xa0000000, 0x3fe8b012, + 0xa7cfeb8f, 0x3e56c412, 0x00000000, 0x3fe8a273, 0x8d52bc19, + 0x3e1429b8, 0x60000000, 0x3fe894d7, 0x4dc32c6c, 0xbe48604c, + 0xc0000000, 0x3fe8873f, 0x0c868e56, 0xbe564ee5, 0x00000000, + 0x3fe879ac, 0x56aee828, 0x3e5e2fd8, 0x60000000, 0x3fe86c1c, + 0x7ceab8ec, 0x3e493365, 0xc0000000, 0x3fe85e90, 0x78d4dadc, + 0xbe4f7f25, 0x00000000, 0x3fe85109, 0x0ccd8280, 0x3e31e7a2, + 0x40000000, 0x3fe84385, 0x34ba4e15, 0x3e328077, 0x80000000, + 0x3fe83605, 0xa670975a, 0xbe53eee5, 0xa0000000, 0x3fe82889, + 0xf61b77b2, 0xbe43a20a, 0xa0000000, 0x3fe81b11, 0x13e6643b, + 0x3e5e5fe5, 0xc0000000, 0x3fe80d9d, 0x82cc94e8, 0xbe5ff1f9, + 0xa0000000, 0x3fe8002d, 0x8a0c9c5d, 0xbe42b0e7, 0x60000000, + 0x3fe7f2c1, 0x22a16f01, 0x3e5d9ea0, 0x20000000, 0x3fe7e559, + 0xc38cd451, 0x3e506963, 0xc0000000, 0x3fe7d7f4, 0x9902bc71, + 0x3e4503d7, 0x40000000, 0x3fe7ca94, 0xdef2a3c0, 0x3e3d98ed, + 0xa0000000, 0x3fe7bd37, 0xed49abb0, 0x3e24c1ff, 0xe0000000, + 0x3fe7afde, 0xe3b0be70, 0xbe40c467, 0x00000000, 0x3fe7a28a, + 0xaf9f193c, 0xbe5dff6c, 0xe0000000, 0x3fe79538, 0xb74cf6b6, + 0xbe258ed0, 0xa0000000, 0x3fe787eb, 0x1d9127c7, 0x3e345fb0, + 0x40000000, 0x3fe77aa2, 0x1028c21d, 0xbe4619bd, 0xa0000000, + 0x3fe76d5c, 0x7cb0b5e4, 0x3e40f1a2, 0xe0000000, 0x3fe7601a, + 0x2b1bc4ad, 0xbe32e8bb, 0xe0000000, 0x3fe752dc, 0x6839f64e, + 0x3e41f57b, 0xc0000000, 0x3fe745a2, 0xc4121f7e, 0xbe52c40a, + 0x60000000, 0x3fe7386c, 0xd6852d72, 0xbe5c4e6b, 0xc0000000, + 0x3fe72b39, 0x91d690f7, 0xbe57f88f, 0xe0000000, 0x3fe71e0a, + 0x627a2159, 0xbe4425d5, 0xc0000000, 0x3fe710df, 0x50a54033, + 0x3e422b7e, 0x60000000, 0x3fe703b8, 0x3b0b5f91, 0x3e5d3857, + 0xe0000000, 0x3fe6f694, 0x84d628a2, 0xbe51f090, 0x00000000, + 0x3fe6e975, 0x306d8894, 0xbe414d83, 0xe0000000, 0x3fe6dc58, + 0x30bf24aa, 0xbe4650ca, 0x80000000, 0x3fe6cf40, 0xd4628d69, + 0xbe5db007, 0xc0000000, 0x3fe6c22b, 0xa2aae57b, 0xbe31d279, + 0xc0000000, 0x3fe6b51a, 0x860edf7e, 0xbe2d4c4a, 0x80000000, + 0x3fe6a80d, 0xf3559341, 0xbe5f7e98, 0xe0000000, 0x3fe69b03, + 0xa885899e, 0xbe5c2011, 0xe0000000, 0x3fe68dfd, 0x2bdc6d37, + 0x3e224a82, 0xa0000000, 0x3fe680fb, 0xc12ad1b9, 0xbe40cf56, + 0x00000000, 0x3fe673fd, 0x1bcdf659, 0xbdf52f2d, 0x00000000, + 0x3fe66702, 0x5df10408, 0x3e5663e0, 0xc0000000, 0x3fe65a0a, + 0xa4070568, 0xbe40b12f, 0x00000000, 0x3fe64d17, 0x71c54c47, + 0x3e5f5e8b, 0x00000000, 0x3fe64027, 0xbd4b7e83, 0x3e42ead6, + 0xa0000000, 0x3fe6333a, 0x61598bd2, 0xbe4c48d4, 0xc0000000, + 0x3fe62651, 0x6f538d61, 0x3e548401, 0xa0000000, 0x3fe6196c, + 0x14344120, 0xbe529af6, 0x00000000, 0x3fe60c8b, 0x5982c587, + 0xbe3e1e4f, 0x00000000, 0x3fe5ffad, 0xfe51d4ea, 0xbe4c897a, + 0x80000000, 0x3fe5f2d2, 0xfd46ebe1, 0x3e552e00, 0xa0000000, + 0x3fe5e5fb, 0xa4695699, 0x3e5ed471, 0x60000000, 0x3fe5d928, + 0x80d118ae, 0x3e456b61, 0xa0000000, 0x3fe5cc58, 0x304c330b, + 0x3e54dc29, 0x80000000, 0x3fe5bf8c, 0x0af2dedf, 0xbe3aa9bd, + 0xe0000000, 0x3fe5b2c3, 0x15fc9258, 0xbe479a37, 0xc0000000, + 0x3fe5a5fe, 0x9292c7ea, 0x3e188650, 0x20000000, 0x3fe5993d, + 0x33b4d380, 0x3e5d6d93, 0x20000000, 0x3fe58c7f, 0x02fd16c7, + 0x3e2fe961, 0xa0000000, 0x3fe57fc4, 0x4a05edb6, 0xbe4d55b4, + 0xa0000000, 0x3fe5730d, 0x3d443abb, 0xbe5e6954, 0x00000000, + 0x3fe5665a, 0x024acfea, 0x3e50e61b, 0x00000000, 0x3fe559aa, + 0xcc9edd09, 0xbe325403, 0x60000000, 0x3fe54cfd, 0x1fe26950, + 0x3e5d500e, 0x60000000, 0x3fe54054, 0x6c5ae164, 0xbe4a79b4, + 0xc0000000, 0x3fe533ae, 0x154b0287, 0xbe401571, 0xa0000000, + 0x3fe5270c, 0x0673f401, 0xbe56e56b, 0xe0000000, 0x3fe51a6d, + 0x751b639c, 0x3e235269, 0xa0000000, 0x3fe50dd2, 0x7c7b2bed, + 0x3ddec887, 0xc0000000, 0x3fe5013a, 0xafab4e17, 0x3e5e7575, + 0x60000000, 0x3fe4f4a6, 0x2e308668, 0x3e59aed6, 0x80000000, + 0x3fe4e815, 0xf33e2a76, 0xbe51f184, 0xe0000000, 0x3fe4db87, + 0x839f3e3e, 0x3e57db01, 0xc0000000, 0x3fe4cefd, 0xa9eda7bb, + 0x3e535e0f, 0x00000000, 0x3fe4c277, 0x2a8f66a5, 0x3e5ce451, + 0xc0000000, 0x3fe4b5f3, 0x05192456, 0xbe4e8518, 0xc0000000, + 0x3fe4a973, 0x4aa7cd1d, 0x3e46784a, 0x40000000, 0x3fe49cf7, + 0x8e23025e, 0xbe5749f2, 0x00000000, 0x3fe4907e, 0x18d30215, + 0x3e360f39, 0x20000000, 0x3fe48408, 0x63dcf2f3, 0x3e5e00fe, + 0xc0000000, 0x3fe47795, 0x46182d09, 0xbe5173d9, 0xa0000000, + 0x3fe46b26, 0x8f0e62aa, 0xbe48f281, 0xe0000000, 0x3fe45eba, + 0x5775c40c, 0xbe56aad4, 0x60000000, 0x3fe45252, 0x0fe25f69, + 0x3e48bd71, 0x40000000, 0x3fe445ed, 0xe9989ec5, 0x3e590d97, + 0x80000000, 0x3fe4398b, 0xb3d9ffe3, 0x3e479dbc, 0x20000000, + 0x3fe42d2d, 0x388e4d2e, 0xbe5eed80, 0xe0000000, 0x3fe420d1, + 0x6f797c18, 0x3e554b4c, 0x20000000, 0x3fe4147a, 0x31048bb4, + 0xbe5b1112, 0x80000000, 0x3fe40825, 0x2efba4f9, 0x3e48ebc7, + 0x40000000, 0x3fe3fbd4, 0x50201119, 0x3e40b701, 0x40000000, + 0x3fe3ef86, 0x0a4db32c, 0x3e551de8, 0xa0000000, 0x3fe3e33b, + 0x0c9c148b, 0xbe50c1f6, 0x20000000, 0x3fe3d6f4, 0xc9129447, + 0x3e533fa0, 0x00000000, 0x3fe3cab0, 0xaae5b5a0, 0xbe22b68e, + 0x20000000, 0x3fe3be6f, 0x02305e8a, 0xbe54fc08, 0x60000000, + 0x3fe3b231, 0x7f908258, 0x3e57dc05, 0x00000000, 0x3fe3a5f7, + 0x1a09af78, 0x3e08038b, 0xe0000000, 0x3fe399bf, 0x490643c1, + 0xbe5dbe42, 0xe0000000, 0x3fe38d8b, 0x5e8ad724, 0xbe3c2b72, + 0x20000000, 0x3fe3815b, 0xc67196b6, 0x3e1713cf, 0xa0000000, + 0x3fe3752d, 0x6182e429, 0xbe3ec14c, 0x40000000, 0x3fe36903, + 0xab6eb1ae, 0x3e5a2cc5, 0x40000000, 0x3fe35cdc, 0xfe5dc064, + 0xbe5c5878, 0x40000000, 0x3fe350b8, 0x0ba6b9e4, 0x3e51619b, + 0x80000000, 0x3fe34497, 0x857761aa, 0x3e5fff53, 0x00000000, + 0x3fe3387a, 0xf872d68c, 0x3e484f4d, 0xa0000000, 0x3fe32c5f, + 0x087e97c2, 0x3e52842e, 0x80000000, 0x3fe32048, 0x73d6d0c0, + 0xbe503edf, 0x80000000, 0x3fe31434, 0x0c1456a1, 0xbe5f72ad, + 0xa0000000, 0x3fe30823, 0x83a1a4d5, 0xbe5e65cc, 0xe0000000, + 0x3fe2fc15, 0x855a7390, 0xbe506438, 0x40000000, 0x3fe2f00b, + 0xa2898287, 0x3e3d22a2, 0xe0000000, 0x3fe2e403, 0x8b56f66f, + 0xbe5aa5fd, 0x80000000, 0x3fe2d7ff, 0x52db119a, 0x3e3a2e3d, + 0x60000000, 0x3fe2cbfe, 0xe2ddd4c0, 0xbe586469, 0x40000000, + 0x3fe2c000, 0x6b01bf10, 0x3e352b9d, 0x40000000, 0x3fe2b405, + 0xb07a1cdf, 0x3e5c5cda, 0x80000000, 0x3fe2a80d, 0xc7b5f868, + 0xbe5668b3, 0xc0000000, 0x3fe29c18, 0x185edf62, 0xbe563d66, + 0x00000000, 0x3fe29027, 0xf729e1cc, 0x3e59a9a0, 0x80000000, + 0x3fe28438, 0x6433c727, 0xbe43cc89, 0x00000000, 0x3fe2784d, + 0x41782631, 0xbe30750c, 0xa0000000, 0x3fe26c64, 0x914911b7, + 0xbe58290e, 0x40000000, 0x3fe2607f, 0x3dcc73e1, 0xbe4269cd, + 0x00000000, 0x3fe2549d, 0x2751bf70, 0xbe5a6998, 0xc0000000, + 0x3fe248bd, 0x4248b9fb, 0xbe4ddb00, 0x80000000, 0x3fe23ce1, + 0xf35cf82f, 0x3e561b71, 0x60000000, 0x3fe23108, 0x8e481a2d, + 0x3e518fb9, 0x60000000, 0x3fe22532, 0x5ab96edc, 0xbe5fafc5, + 0x40000000, 0x3fe2195f, 0x80943911, 0xbe07f819, 0x40000000, + 0x3fe20d8f, 0x386f2d6c, 0xbe54ba8b, 0x40000000, 0x3fe201c2, + 0xf29664ac, 0xbe5eb815, 0x20000000, 0x3fe1f5f8, 0x64f03390, + 0x3e5e320c, 0x20000000, 0x3fe1ea31, 0x747ff696, 0x3e5ef0a5, + 0x40000000, 0x3fe1de6d, 0x3e9ceb51, 0xbe5f8d27, 0x20000000, + 0x3fe1d2ac, 0x4ae0b55e, 0x3e5faa21, 0x20000000, 0x3fe1c6ee, + 0x28569a5e, 0x3e598a4f, 0x20000000, 0x3fe1bb33, 0x54b33e07, + 0x3e46130a, 0x20000000, 0x3fe1af7b, 0x024f1078, 0xbe4dbf93, + 0x00000000, 0x3fe1a3c6, 0xb0783bfa, 0x3e419248, 0xe0000000, + 0x3fe19813, 0x2f02b836, 0x3e4e02b7, 0xc0000000, 0x3fe18c64, + 0x28dec9d4, 0x3e09064f, 0x80000000, 0x3fe180b8, 0x45cbf406, + 0x3e5b1f46, 0x40000000, 0x3fe1750f, 0x03d9964c, 0x3e5b0a79, + 0x00000000, 0x3fe16969, 0x8b5b882b, 0xbe238086, 0xa0000000, + 0x3fe15dc5, 0x73bad6f8, 0xbdf1fca4, 0x20000000, 0x3fe15225, + 0x5385769c, 0x3e5e8d76, 0xa0000000, 0x3fe14687, 0x1676dc6b, + 0x3e571d08, 0x20000000, 0x3fe13aed, 0xa8c41c7f, 0xbe598a25, + 0x60000000, 0x3fe12f55, 0xc4e1aaf0, 0x3e435277, 0xa0000000, + 0x3fe123c0, 0x403638e1, 0xbe21aa7c, 0xc0000000, 0x3fe1182e, + 0x557a092b, 0xbdd0116b, 0xc0000000, 0x3fe10c9f, 0x7d779f66, + 0x3e4a61ba, 0xc0000000, 0x3fe10113, 0x2b09c645, 0xbe5d586e, + 0x20000000, 0x3fe0ea04, 0xea2cad46, 0x3e5aa97c, 0x20000000, + 0x3fe0d300, 0x23190e54, 0x3e50f1a7, 0xa0000000, 0x3fe0bc07, + 0x1379a5a6, 0xbe51619d, 0x60000000, 0x3fe0a51a, 0x926a3d4a, + 0x3e5cf019, 0xa0000000, 0x3fe08e38, 0xa8c24358, 0x3e35241e, + 0x20000000, 0x3fe07762, 0x24317e7a, 0x3e512cfa, 0x00000000, + 0x3fe06097, 0xfd9cf274, 0xbe55bef3, 0x00000000, 0x3fe049d7, + 0x3689b49d, 0xbe36d26d, 0x40000000, 0x3fe03322, 0xf72ef6c4, + 0xbe54cd08, 0xa0000000, 0x3fe01c78, 0x23702d2d, 0xbe5900bf, + 0x00000000, 0x3fe005da, 0x3f59c14c, 0x3e57d80b, 0x40000000, + 0x3fdfde8d, 0xad67766d, 0xbe57fad4, 0x40000000, 0x3fdfb17c, + 0x644f4ae7, 0x3e1ee43b, 0x40000000, 0x3fdf8481, 0x903234d2, + 0x3e501a86, 0x40000000, 0x3fdf579c, 0xafe9e509, 0xbe267c3e, + 0x00000000, 0x3fdf2acd, 0xb7dfda0b, 0xbe48149b, 0x40000000, + 0x3fdefe13, 0x3b94305e, 0x3e5f4ea7, 0x80000000, 0x3fded16f, + 0x5d95da61, 0xbe55c198, 0x00000000, 0x3fdea4e1, 0x406960c9, + 0xbdd99a19, 0x00000000, 0x3fde7868, 0xd22f3539, 0x3e470c78, + 0x80000000, 0x3fde4c04, 0x83eec535, 0xbe3e1232, 0x40000000, + 0x3fde1fb6, 0x3dfbffcb, 0xbe4b7d71, 0x40000000, 0x3fddf37d, + 0x7e1be4e0, 0xbe5b8f8f, 0x40000000, 0x3fddc759, 0x46dae887, + 0xbe350458, 0x80000000, 0x3fdd9b4a, 0xed6ecc49, 0xbe5f0045, + 0x80000000, 0x3fdd6f50, 0x2e9e883c, 0x3e2915da, 0x80000000, + 0x3fdd436b, 0xf0bccb32, 0x3e4a68c9, 0x80000000, 0x3fdd179b, + 0x9bbfc779, 0xbe54a26a, 0x00000000, 0x3fdcebe0, 0x7cea33ab, + 0x3e43c6b7, 0x40000000, 0x3fdcc039, 0xe740fd06, 0x3e5526c2, + 0x40000000, 0x3fdc94a7, 0x9eadeb1a, 0xbe396d8d, 0xc0000000, + 0x3fdc6929, 0xf0a8f95a, 0xbe5c0ab2, 0x80000000, 0x3fdc3dc0, + 0x6ee2693b, 0x3e0992e6, 0xc0000000, 0x3fdc126b, 0x5ac6b581, + 0xbe2834b6, 0x40000000, 0x3fdbe72b, 0x8cc226ff, 0x3e3596a6, + 0x00000000, 0x3fdbbbff, 0xf92a74bb, 0x3e3c5813, 0x00000000, + 0x3fdb90e7, 0x479664c0, 0xbe50d644, 0x00000000, 0x3fdb65e3, + 0x5004975b, 0xbe55258f, 0x00000000, 0x3fdb3af3, 0xe4b23194, + 0xbe588407, 0xc0000000, 0x3fdb1016, 0xe65d4d0a, 0x3e527c26, + 0x80000000, 0x3fdae54e, 0x814fddd6, 0x3e5962a2, 0x40000000, + 0x3fdaba9a, 0xe19d0913, 0xbe562f4e, 0x80000000, 0x3fda8ff9, + 0x43cfd006, 0xbe4cfdeb, 0x40000000, 0x3fda656c, 0x686f0a4e, + 0x3e5e47a8, 0xc0000000, 0x3fda3af2, 0x7200d410, 0x3e5e1199, + 0xc0000000, 0x3fda108c, 0xabd2266e, 0x3e5ee4d1, 0x40000000, + 0x3fd9e63a, 0x396f8f2c, 0x3e4dbffb, 0x00000000, 0x3fd9bbfb, + 0xe32b25dd, 0x3e5c3a54, 0x40000000, 0x3fd991cf, 0x431e4035, + 0xbe457925, 0x80000000, 0x3fd967b6, 0x7bed3dd3, 0x3e40c61d, + 0x00000000, 0x3fd93db1, 0xd7449365, 0x3e306419, 0x80000000, + 0x3fd913be, 0x1746e791, 0x3e56fcfc, 0x40000000, 0x3fd8e9df, + 0xf3a9028b, 0xbe5041b9, 0xc0000000, 0x3fd8c012, 0x56840c50, + 0xbe26e20a, 0x40000000, 0x3fd89659, 0x19763102, 0xbe51f466, + 0x80000000, 0x3fd86cb2, 0x7032de7c, 0xbe4d298a, 0x80000000, + 0x3fd8431e, 0xdeb39fab, 0xbe4361eb, 0x40000000, 0x3fd8199d, + 0x5d01cbe0, 0xbe5425b3, 0x80000000, 0x3fd7f02e, 0x3ce99aa9, + 0x3e146fa8, 0x80000000, 0x3fd7c6d2, 0xd1a262b9, 0xbe5a1a69, + 0xc0000000, 0x3fd79d88, 0x8606c236, 0x3e423a08, 0x80000000, + 0x3fd77451, 0x8fd1e1b7, 0x3e5a6a63, 0xc0000000, 0x3fd74b2c, + 0xe491456a, 0x3e42c1ca, 0x40000000, 0x3fd7221a, 0x4499a6d7, + 0x3e36a69a, 0x00000000, 0x3fd6f91a, 0x5237df94, 0xbe0f8f02, + 0x00000000, 0x3fd6d02c, 0xb6482c6e, 0xbe5abcf7, 0x00000000, + 0x3fd6a750, 0x1919fd61, 0xbe57ade2, 0x00000000, 0x3fd67e86, + 0xaa7a994d, 0xbe3f3fbd, 0x00000000, 0x3fd655ce, 0x67db014c, + 0x3e33c550, 0x00000000, 0x3fd62d28, 0xa82856b7, 0xbe1409d1, + 0xc0000000, 0x3fd60493, 0x1e6a300d, 0x3e55d899, 0x80000000, + 0x3fd5dc11, 0x1222bd5c, 0xbe35bfc0, 0xc0000000, 0x3fd5b3a0, + 0x6e8dc2d3, 0x3e5d4d79, 0x00000000, 0x3fd58b42, 0xe0e4ace6, + 0xbe517303, 0x80000000, 0x3fd562f4, 0xb306e0a8, 0x3e5edf0f, + 0xc0000000, 0x3fd53ab8, 0x6574bc54, 0x3e5ee859, 0x80000000, + 0x3fd5128e, 0xea902207, 0x3e5f6188, 0xc0000000, 0x3fd4ea75, + 0x9f911d79, 0x3e511735, 0x80000000, 0x3fd4c26e, 0xf9c77397, + 0xbe5b1643, 0x40000000, 0x3fd49a78, 0x15fc9258, 0x3e479a37, + 0x80000000, 0x3fd47293, 0xd5a04dd9, 0xbe426e56, 0xc0000000, + 0x3fd44abf, 0xe04042f5, 0x3e56f7c6, 0x40000000, 0x3fd422fd, + 0x1d8bf2c8, 0x3e5d8810, 0x00000000, 0x3fd3fb4c, 0x88a8ddee, + 0xbe311454, 0xc0000000, 0x3fd3d3ab, 0x3e3b5e47, 0xbe5d1b72, + 0x40000000, 0x3fd3ac1c, 0xc2ab5d59, 0x3e31b02b, 0xc0000000, + 0x3fd3849d, 0xd4e34b9e, 0x3e51cb2f, 0x40000000, 0x3fd35d30, + 0x177204fb, 0xbe2b8cd7, 0x80000000, 0x3fd335d3, 0xfcd38c82, + 0xbe4356e1, 0x80000000, 0x3fd30e87, 0x64f54acc, 0xbe4e6224, + 0x00000000, 0x3fd2e74c, 0xaa7975d9, 0x3e5dc0fe, 0x80000000, + 0x3fd2c021, 0x516dab3f, 0xbe50ffa3, 0x40000000, 0x3fd29907, + 0x2bfb7313, 0x3e5674a2, 0xc0000000, 0x3fd271fd, 0x0549fc99, + 0x3e385d29, 0xc0000000, 0x3fd24b04, 0x55b63073, 0xbe500c6d, + 0x00000000, 0x3fd2241c, 0x3f91953a, 0x3e389977, 0xc0000000, + 0x3fd1fd43, 0xa1543f71, 0xbe3487ab, 0xc0000000, 0x3fd1d67b, + 0x4ec8867c, 0x3df6a2dc, 0x00000000, 0x3fd1afc4, 0x4328e3bb, + 0x3e41d9c0, 0x80000000, 0x3fd1891c, 0x2e1cda84, 0x3e3bdd87, + 0x40000000, 0x3fd16285, 0x4b5331ae, 0xbe53128e, 0x00000000, + 0x3fd13bfe, 0xb9aec164, 0xbe52ac98, 0xc0000000, 0x3fd11586, + 0xd91e1316, 0xbe350630, 0x80000000, 0x3fd0ef1f, 0x7cacc12c, + 0x3e3f5219, 0x40000000, 0x3fd0c8c8, 0xbce277b7, 0x3e3d30c0, + 0x00000000, 0x3fd0a281, 0x2a63447d, 0xbe541377, 0x80000000, + 0x3fd07c49, 0xfac483b5, 0xbe5772ec, 0xc0000000, 0x3fd05621, + 0x36b8a570, 0xbe4fd4bd, 0xc0000000, 0x3fd03009, 0xbae505f7, + 0xbe450388, 0x80000000, 0x3fd00a01, 0x3e35aead, 0xbe5430fc, + 0x80000000, 0x3fcfc811, 0x707475ac, 0x3e38806e, 0x80000000, + 0x3fcf7c3f, 0xc91817fc, 0xbe40ccea, 0x80000000, 0x3fcf308c, + 0xae05d5e9, 0xbe4919b8, 0x80000000, 0x3fcee4f8, 0xae6cc9e6, + 0xbe530b94, 0x00000000, 0x3fce9983, 0x1efe3e8e, 0x3e57747e, + 0x00000000, 0x3fce4e2d, 0xda78d9bf, 0xbe59a608, 0x00000000, + 0x3fce02f5, 0x8abe2c2e, 0x3e4a35ad, 0x00000000, 0x3fcdb7dc, + 0x1495450d, 0xbe0872cc, 0x80000000, 0x3fcd6ce1, 0x86ee0ba0, + 0xbe4f59a0, 0x00000000, 0x3fcd2205, 0xe81ca888, 0x3e5402c3, + 0x00000000, 0x3fccd747, 0x3b4424b9, 0x3e5dfdc3, 0x80000000, + 0x3fcc8ca7, 0xd305b56c, 0x3e202da6, 0x00000000, 0x3fcc4226, + 0x399a6910, 0xbe482a1c, 0x80000000, 0x3fcbf7c2, 0x747f7938, + 0xbe587372, 0x80000000, 0x3fcbad7c, 0x6fc246a0, 0x3e50d83d, + 0x00000000, 0x3fcb6355, 0xee9e9be5, 0xbe5c35bd, 0x80000000, + 0x3fcb194a, 0x8416c0bc, 0x3e546d4f, 0x00000000, 0x3fcacf5e, + 0x49f7f08f, 0x3e56da76, 0x00000000, 0x3fca858f, 0x5dc30de2, + 0x3e5f390c, 0x00000000, 0x3fca3bde, 0x950583b6, 0xbe5e4169, + 0x80000000, 0x3fc9f249, 0x33631553, 0x3e52aeb1, 0x00000000, + 0x3fc9a8d3, 0xde8795a6, 0xbe59a504, 0x00000000, 0x3fc95f79, + 0x076bf41e, 0x3e5122fe, 0x80000000, 0x3fc9163c, 0x2914c8e7, + 0x3e3dd064, 0x00000000, 0x3fc8cd1d, 0x3a30eca3, 0xbe21b4aa, + 0x80000000, 0x3fc8841a, 0xb2a96650, 0xbe575444, 0x80000000, + 0x3fc83b34, 0x2376c0cb, 0xbe2a74c7, 0x80000000, 0x3fc7f26b, + 0xd8a0b653, 0xbe5181b6, 0x00000000, 0x3fc7a9bf, 0x32257882, + 0xbe4a78b4, 0x00000000, 0x3fc7612f, 0x1eee8bd9, 0xbe1bfe9d, + 0x80000000, 0x3fc718bb, 0x0c603cc4, 0x3e36fdc9, 0x80000000, + 0x3fc6d064, 0x3728b8cf, 0xbe1e542e, 0x80000000, 0x3fc68829, + 0xc79a4067, 0x3e5c380f, 0x00000000, 0x3fc6400b, 0xf69eac69, + 0x3e550a84, 0x80000000, 0x3fc5f808, 0xb7a780a4, 0x3e5d9224, + 0x80000000, 0x3fc5b022, 0xad9dfb1e, 0xbe55242f, 0x00000000, + 0x3fc56858, 0x659b18be, 0xbe4bfda3, 0x80000000, 0x3fc520a9, + 0x66ee3631, 0xbe57d769, 0x80000000, 0x3fc4d916, 0x1ec62819, + 0x3e2427f7, 0x80000000, 0x3fc4919f, 0xdec25369, 0xbe435431, + 0x00000000, 0x3fc44a44, 0xa8acfc4b, 0xbe3c62e8, 0x00000000, + 0x3fc40304, 0xcf1d3eab, 0xbdfba29f, 0x80000000, 0x3fc3bbdf, + 0x79aba3ea, 0xbdf1b7c8, 0x80000000, 0x3fc374d6, 0xb8d186da, + 0xbe5130cf, 0x80000000, 0x3fc32de8, 0x9d74f152, 0x3e2285b6, + 0x00000000, 0x3fc2e716, 0x50ae7ca9, 0xbe503920, 0x80000000, + 0x3fc2a05e, 0x6caed92e, 0xbe533924, 0x00000000, 0x3fc259c2, + 0x9cb5034e, 0xbe510e31, 0x80000000, 0x3fc21340, 0x12c4d378, + 0xbe540b43, 0x80000000, 0x3fc1ccd9, 0xcc418706, 0x3e59887a, + 0x00000000, 0x3fc1868e, 0x921f4106, 0xbe528e67, 0x80000000, + 0x3fc1405c, 0x3969441e, 0x3e5d8051, 0x00000000, 0x3fc0fa46, + 0xd941ef5b, 0x3e5f9079, 0x80000000, 0x3fc0b44a, 0x5a3e81b2, + 0xbe567691, 0x00000000, 0x3fc06e69, 0x9d66afe7, 0xbe4d43fb, + 0x00000000, 0x3fc028a2, 0x0a92a162, 0xbe52f394, 0x00000000, + 0x3fbfc5ea, 0x209897e5, 0x3e529e37, 0x00000000, 0x3fbf3ac5, + 0x8458bd7b, 0x3e582831, 0x00000000, 0x3fbeafd5, 0xb8d8b4b8, + 0xbe486b4a, 0x00000000, 0x3fbe2518, 0xe0a3b7b6, 0x3e5bafd2, + 0x00000000, 0x3fbd9a90, 0x2bf2710e, 0x3e383b2b, 0x00000000, + 0x3fbd103c, 0x73eb6ab7, 0xbe56d78d, 0x00000000, 0x3fbc861b, + 0x32ceaff5, 0xbe32dc5a, 0x00000000, 0x3fbbfc2e, 0xbee04cb7, + 0xbe4a71a4, 0x00000000, 0x3fbb7274, 0x35ae9577, 0x3e38142f, + 0x00000000, 0x3fbae8ee, 0xcbaddab4, 0xbe5490f0, 0x00000000, + 0x3fba5f9a, 0x95ce1114, 0x3e597c71, 0x00000000, 0x3fb9d67a, + 0x6d7c0f78, 0x3e3abc2d, 0x00000000, 0x3fb94d8d, 0x2841a782, + 0xbe566cbc, 0x00000000, 0x3fb8c4d2, 0x6ed429c6, 0xbe3cfff9, + 0x00000000, 0x3fb83c4a, 0xe4a49fbb, 0xbe552964, 0x00000000, + 0x3fb7b3f4, 0x2193d81e, 0xbe42fa72, 0x00000000, 0x3fb72bd0, + 0xdd70c122, 0x3e527a8c, 0x00000000, 0x3fb6a3df, 0x03108a54, + 0xbe450393, 0x00000000, 0x3fb61c1f, 0x30ff7954, 0x3e565840, + 0x00000000, 0x3fb59492, 0xdedd460c, 0xbe5422b5, 0x00000000, + 0x3fb50d36, 0x950f9f45, 0xbe5313f6, 0x00000000, 0x3fb4860b, + 0x582cdcb1, 0x3e506d39, 0x00000000, 0x3fb3ff12, 0x7216d3a6, + 0x3e4aa719, 0x00000000, 0x3fb3784a, 0x57a423fd, 0x3e5a9b9f, + 0x00000000, 0x3fb2f1b4, 0x7a138b41, 0xbe50b418, 0x00000000, + 0x3fb26b4e, 0x2fbfd7ea, 0x3e23a53e, 0x00000000, 0x3fb1e519, + 0x18913ccb, 0x3e465fc1, 0x00000000, 0x3fb15f15, 0x7ea24e21, + 0x3e042843, 0x00000000, 0x3fb0d941, 0x7c6d9c77, 0x3e59f61e, + 0x00000000, 0x3fb0539e, 0x114efd44, 0x3e4ccab7, 0x00000000, + 0x3faf9c56, 0x1777f657, 0x3e552f65, 0x00000000, 0x3fae91d2, + 0xc317b86a, 0xbe5a61e0, 0x00000000, 0x3fad87ac, 0xb7664efb, + 0xbe41f64e, 0x00000000, 0x3fac7de6, 0x5d3d03a9, 0x3e0807a0, + 0x00000000, 0x3fab7480, 0x743c38eb, 0xbe3726e1, 0x00000000, + 0x3faa6b78, 0x06a253f1, 0x3e5ad636, 0x00000000, 0x3fa962d0, + 0xa35f541b, 0x3e5a187a, 0x00000000, 0x3fa85a88, 0x4b86e446, + 0xbe508150, 0x00000000, 0x3fa7529c, 0x2589cacf, 0x3e52938a, + 0x00000000, 0x3fa64b10, 0xaf6b11f2, 0xbe3454cd, 0x00000000, + 0x3fa543e2, 0x97506fef, 0xbe5fdec5, 0x00000000, 0x3fa43d10, + 0xe75f7dd9, 0xbe388dd3, 0x00000000, 0x3fa3369c, 0xa4139632, + 0xbdea5177, 0x00000000, 0x3fa23086, 0x352d6f1e, 0xbe565ad6, + 0x00000000, 0x3fa12acc, 0x77449eb7, 0xbe50d5c7, 0x00000000, + 0x3fa0256e, 0x7478da78, 0x3e404724, 0x00000000, 0x3f9e40dc, + 0xf59cef7f, 0xbe539d0a, 0x00000000, 0x3f9c3790, 0x1511d43c, + 0x3e53c2c8, 0x00000000, 0x3f9a2f00, 0x9b8bff3c, 0xbe43b3e1, + 0x00000000, 0x3f982724, 0xad1e22a5, 0x3e46f0bd, 0x00000000, + 0x3f962000, 0x130d9356, 0x3e475ba0, 0x00000000, 0x3f941994, + 0x8f86f883, 0xbe513d0b, 0x00000000, 0x3f9213dc, 0x914d0dc8, + 0xbe534335, 0x00000000, 0x3f900ed8, 0x2d73e5e7, 0xbe22ba75, + 0x00000000, 0x3f8c1510, 0xc5b7d70e, 0x3e599c5d, 0x00000000, + 0x3f880de0, 0x8a27857e, 0xbe3d28c8, 0x00000000, 0x3f840810, + 0xda767328, 0x3e531b3d, 0x00000000, 0x3f8003b0, 0x77bacaf3, + 0xbe5f04e3, 0x00000000, 0x3f780150, 0xdf4b0720, 0x3e5a8bff, + 0x00000000, 0x3f6ffc40, 0x34c48e71, 0xbe3fcd99, 0x00000000, + 0x3f5ff6c0, 0x1ad218af, 0xbe4c78a7, 0x00000000, 0x00000000, + 0x00000000, 0x80000000 + //@formatter:on + }); + + private ArrayDataPointerConstant log2 = pointerConstant(8, new int[]{ + //@formatter:off + 0xfefa39ef, 0x3fe62e42, 0xfefa39ef, 0xbfe62e42 + //@formatter:on + }); + + private ArrayDataPointerConstant double2 = pointerConstant(8, new int[]{ + //@formatter:off + 0x00000000, 0x40000000 + //@formatter:on + }); + + @Override + public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { + // registers, + // input: xmm0, xmm1 + // scratch: xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7 + // rax, rdx, rcx, r8, r11 + + // Code generated by Intel C compiler for LIBM library + Label block0 = new Label(); + Label block1 = new Label(); + Label block2 = new Label(); + Label block3 = new Label(); + Label block4 = new Label(); + Label block5 = new Label(); + Label block6 = new Label(); + Label block7 = new Label(); + Label block8 = new Label(); + Label block9 = new Label(); + Label block10 = new Label(); + Label block11 = new Label(); + Label block12 = new Label(); + Label block13 = new Label(); + Label block14 = new Label(); + Label block15 = new Label(); + Label block16 = new Label(); + Label block17 = new Label(); + Label block18 = new Label(); + Label block19 = new Label(); + Label block20 = new Label(); + Label block21 = new Label(); + Label block22 = new Label(); + Label block23 = new Label(); + Label block24 = new Label(); + Label block25 = new Label(); + Label block26 = new Label(); + Label block27 = new Label(); + Label block28 = new Label(); + Label block29 = new Label(); + Label block30 = new Label(); + Label block31 = new Label(); + Label block32 = new Label(); + Label block33 = new Label(); + Label block34 = new Label(); + Label block35 = new Label(); + Label block36 = new Label(); + Label block37 = new Label(); + Label block38 = new Label(); + Label block39 = new Label(); + Label block40 = new Label(); + Label block41 = new Label(); + Label block42 = new Label(); + Label block43 = new Label(); + Label block44 = new Label(); + Label block45 = new Label(); + Label block46 = new Label(); + Label block47 = new Label(); + Label block48 = new Label(); + Label block49 = new Label(); + Label block50 = new Label(); + Label block51 = new Label(); + Label block52 = new Label(); + Label block53 = new Label(); + Label block54 = new Label(); + Label block55 = new Label(); + Label block56 = new Label(); + Label block57 = new Label(); + + Register tmp1 = r8; + Register tmp2 = r9; + Register tmp3 = r10; + Register tmp4 = r11; + + masm.subq(rsp, 40); + masm.movsd(new AMD64Address(rsp, 8), xmm0); + masm.movsd(new AMD64Address(rsp, 16), xmm1); + + // Special case: pow(x, 2.0) => x * x + masm.movdq(tmp1, xmm1); + masm.cmpq(tmp1, recordExternalAddress(crb, double2)); + masm.jccb(AMD64Assembler.ConditionFlag.NotEqual, block57); + masm.mulsd(xmm0, xmm0); + masm.jmp(block56); + + masm.bind(block57); + masm.pextrw(rax, xmm0, 3); + masm.xorpd(xmm2, xmm2); + masm.movq(tmp2, 0x3ff0000000000000L); + masm.movdq(xmm2, tmp2); + masm.movl(tmp1, 1069088768); + masm.movdq(xmm7, tmp1); + masm.xorpd(xmm1, xmm1); + masm.movq(tmp3, 0x77f0000000000000L); + masm.movdq(xmm1, tmp3); + masm.movdqu(xmm3, xmm0); + masm.movl(rdx, 32752); + masm.andl(rdx, rax); + masm.subl(rdx, 16368); + masm.movl(rcx, rdx); + masm.sarl(rdx, 31); + masm.addl(rcx, rdx); + masm.xorl(rcx, rdx); + masm.por(xmm0, xmm2); + masm.movdqu(xmm6, recordExternalAddress(crb, highsigmask)); // 0x00000000, 0xfffff800, + // 0x00000000, 0xfffff800 + masm.psrlq(xmm0, 27); + masm.movq(xmm2, recordExternalAddress(crb, log2E)); // 0x00000000, 0x3ff72000, + // 0x161bb241, 0xbf5dabe1 + masm.psrld(xmm0, 2); + masm.addl(rcx, 16); + masm.bsrl(rcx, rcx); + masm.rcpps(xmm0, xmm0); + masm.psllq(xmm3, 12); + masm.movl(tmp4, 8192); + masm.movdq(xmm4, tmp4); + masm.psrlq(xmm3, 12); + masm.subl(rax, 16); + masm.cmpl(rax, 32736); + masm.jcc(AMD64Assembler.ConditionFlag.AboveEqual, block0); + masm.movl(tmp1, 0); + + masm.bind(block1); + masm.mulss(xmm0, xmm7); + masm.movl(rdx, -1); + masm.subl(rcx, 4); + masm.shll(rdx); + masm.shlq(rdx, 32); + masm.movdq(xmm5, rdx); + masm.por(xmm3, xmm1); + masm.subl(rax, 16351); + masm.cmpl(rax, 1); + masm.jcc(AMD64Assembler.ConditionFlag.BelowEqual, block2); + masm.paddd(xmm0, xmm4); + masm.pand(xmm5, xmm3); + masm.movdl(rdx, xmm0); + masm.psllq(xmm0, 29); + + masm.bind(block3); + masm.subsd(xmm3, xmm5); + masm.pand(xmm0, xmm6); + masm.subl(rax, 1); + masm.sarl(rax, 4); + masm.cvtsi2sdl(xmm7, rax); + masm.mulpd(xmm5, xmm0); + + masm.bind(block4); + masm.mulsd(xmm3, xmm0); + masm.movdqu(xmm1, recordExternalAddress(crb, coeff)); // 0x6dc96112, 0xbf836578, + // 0xee241472, 0xbf9b0301 + masm.leaq(tmp4, recordExternalAddress(crb, lTbl)); + masm.subsd(xmm5, xmm2); + masm.movdqu(xmm4, recordExternalAddress(crb, coeff16)); // 0x9f95985a, 0xbfb528db, + // 0xb3841d2a, 0xbfd619b6 + masm.movl(rcx, rax); + masm.sarl(rax, 31); + masm.addl(rcx, rax); + masm.xorl(rax, rcx); + masm.addl(rax, 1); + masm.bsrl(rax, rax); + masm.unpcklpd(xmm5, xmm3); + masm.movdqu(xmm6, recordExternalAddress(crb, coeff32)); // 0x518775e3, 0x3f9004f2, + // 0xac8349bb, 0x3fa76c9b + masm.addsd(xmm3, xmm5); + masm.andl(rdx, 16760832); + masm.shrl(rdx, 10); + masm.addpd(xmm5, new AMD64Address(tmp4, rdx, AMD64Address.Scale.Times1, -3648)); + masm.movdqu(xmm0, recordExternalAddress(crb, coeff48)); // 0x486ececc, 0x3fc4635e, + // 0x161bb241, 0xbf5dabe1 + masm.pshufd(xmm2, xmm3, 68); + masm.mulsd(xmm3, xmm3); + masm.mulpd(xmm1, xmm2); + masm.mulpd(xmm4, xmm2); + masm.addsd(xmm5, xmm7); + masm.mulsd(xmm2, xmm3); + masm.addpd(xmm6, xmm1); + masm.mulsd(xmm3, xmm3); + masm.addpd(xmm0, xmm4); + masm.movq(xmm1, new AMD64Address(rsp, 16)); + masm.movw(rcx, new AMD64Address(rsp, 22)); + masm.pshufd(xmm7, xmm5, 238); + masm.movq(xmm4, recordExternalAddress(crb, highmaskY)); // 0x00000000, 0xfffffff8, + // 0x00000000, 0xffffffff + masm.mulpd(xmm6, xmm2); + masm.pshufd(xmm3, xmm3, 68); + masm.mulpd(xmm0, xmm2); + masm.shll(rax, 4); + masm.subl(rax, 15872); + masm.andl(rcx, 32752); + masm.addl(rax, rcx); + masm.mulpd(xmm3, xmm6); + masm.cmpl(rax, 624); + masm.jcc(AMD64Assembler.ConditionFlag.AboveEqual, block5); + masm.xorpd(xmm6, xmm6); + masm.movl(rdx, 17080); + masm.pinsrw(xmm6, rdx, 3); + masm.movdqu(xmm2, xmm1); + masm.pand(xmm4, xmm1); + masm.subsd(xmm1, xmm4); + masm.mulsd(xmm4, xmm5); + masm.addsd(xmm0, xmm7); + masm.mulsd(xmm1, xmm5); + masm.movdqu(xmm7, xmm6); + masm.addsd(xmm6, xmm4); + masm.leaq(tmp4, recordExternalAddress(crb, tExp)); + masm.addpd(xmm3, xmm0); + masm.movdl(rdx, xmm6); + masm.subsd(xmm6, xmm7); + masm.pshufd(xmm0, xmm3, 238); + masm.subsd(xmm4, xmm6); + masm.addsd(xmm0, xmm3); + masm.movl(rcx, rdx); + masm.andl(rdx, 255); + masm.addl(rdx, rdx); + masm.movdqu(xmm5, new AMD64Address(tmp4, rdx, AMD64Address.Scale.Times8, 0)); + masm.addsd(xmm4, xmm1); + masm.mulsd(xmm2, xmm0); + masm.movdqu(xmm7, recordExternalAddress(crb, eCoeff)); // 0xe78a6731, 0x3f55d87f, + // 0xd704a0c0, 0x3fac6b08 + masm.movdqu(xmm3, recordExternalAddress(crb, eCoeff16)); // 0x6fba4e77, 0x3f83b2ab, + // 0xff82c58f, 0x3fcebfbd + masm.shll(rcx, 12); + masm.xorl(rcx, tmp1); + masm.andl(rcx, -1048576); + masm.movdq(xmm6, rcx); + masm.addsd(xmm2, xmm4); + masm.movq(tmp2, 0x3fe62e42fefa39efL); + masm.movdq(xmm1, tmp2); + masm.pshufd(xmm0, xmm2, 68); + masm.pshufd(xmm4, xmm2, 68); + masm.mulsd(xmm1, xmm2); + masm.pshufd(xmm6, xmm6, 17); + masm.mulpd(xmm0, xmm0); + masm.mulpd(xmm7, xmm4); + masm.paddd(xmm5, xmm6); + masm.mulsd(xmm1, xmm5); + masm.pshufd(xmm6, xmm5, 238); + masm.mulsd(xmm0, xmm0); + masm.addpd(xmm3, xmm7); + masm.addsd(xmm1, xmm6); + masm.mulpd(xmm0, xmm3); + masm.pshufd(xmm3, xmm0, 238); + masm.mulsd(xmm0, xmm5); + masm.mulsd(xmm3, xmm5); + masm.addsd(xmm0, xmm1); + masm.addsd(xmm0, xmm3); + masm.addsd(xmm0, xmm5); + masm.jmp(block56); + + masm.bind(block0); + masm.addl(rax, 16); + masm.movl(rdx, 32752); + masm.andl(rdx, rax); + masm.cmpl(rdx, 32752); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block6); + masm.testl(rax, 32768); + masm.jcc(AMD64Assembler.ConditionFlag.NotEqual, block7); + + masm.bind(block8); + masm.movq(xmm0, new AMD64Address(rsp, 8)); + masm.movq(xmm3, new AMD64Address(rsp, 8)); + masm.movdl(rdx, xmm3); + masm.psrlq(xmm3, 32); + masm.movdl(rcx, xmm3); + masm.orl(rdx, rcx); + masm.cmpl(rdx, 0); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block9); + masm.xorpd(xmm3, xmm3); + masm.movl(rax, 18416); + masm.pinsrw(xmm3, rax, 3); + masm.mulsd(xmm0, xmm3); + masm.xorpd(xmm2, xmm2); + masm.movl(rax, 16368); + masm.pinsrw(xmm2, rax, 3); + masm.movdqu(xmm3, xmm0); + masm.pextrw(rax, xmm0, 3); + masm.por(xmm0, xmm2); + masm.movl(rcx, 18416); + masm.psrlq(xmm0, 27); + masm.movq(xmm2, recordExternalAddress(crb, log2E)); // 0x00000000, 0x3ff72000, + // 0x161bb241, 0xbf5dabe1 + masm.psrld(xmm0, 2); + masm.rcpps(xmm0, xmm0); + masm.psllq(xmm3, 12); + masm.movdqu(xmm6, recordExternalAddress(crb, highsigmask)); // 0x00000000, 0xfffff800, + // 0x00000000, 0xfffff800 + masm.psrlq(xmm3, 12); + masm.mulss(xmm0, xmm7); + masm.movl(rdx, -1024); + masm.movdl(xmm5, rdx); + masm.por(xmm3, xmm1); + masm.paddd(xmm0, xmm4); + masm.psllq(xmm5, 32); + masm.movdl(rdx, xmm0); + masm.psllq(xmm0, 29); + masm.pand(xmm5, xmm3); + masm.movl(tmp1, 0); + masm.pand(xmm0, xmm6); + masm.subsd(xmm3, xmm5); + masm.andl(rax, 32752); + masm.subl(rax, 18416); + masm.sarl(rax, 4); + masm.cvtsi2sdl(xmm7, rax); + masm.mulpd(xmm5, xmm0); + masm.jmp(block4); + + masm.bind(block10); + masm.movq(xmm0, new AMD64Address(rsp, 8)); + masm.movq(xmm3, new AMD64Address(rsp, 8)); + masm.movdl(rdx, xmm3); + masm.psrlq(xmm3, 32); + masm.movdl(rcx, xmm3); + masm.orl(rdx, rcx); + masm.cmpl(rdx, 0); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block9); + masm.xorpd(xmm3, xmm3); + masm.movl(rax, 18416); + masm.pinsrw(xmm3, rax, 3); + masm.mulsd(xmm0, xmm3); + masm.xorpd(xmm2, xmm2); + masm.movl(rax, 16368); + masm.pinsrw(xmm2, rax, 3); + masm.movdqu(xmm3, xmm0); + masm.pextrw(rax, xmm0, 3); + masm.por(xmm0, xmm2); + masm.movl(rcx, 18416); + masm.psrlq(xmm0, 27); + masm.movq(xmm2, recordExternalAddress(crb, log2E)); // 0x00000000, 0x3ff72000, + // 0x161bb241, 0xbf5dabe1 + masm.psrld(xmm0, 2); + masm.rcpps(xmm0, xmm0); + masm.psllq(xmm3, 12); + masm.movdqu(xmm6, recordExternalAddress(crb, highsigmask)); // 0x00000000, 0xfffff800, + // 0x00000000, 0xfffff800 + masm.psrlq(xmm3, 12); + masm.mulss(xmm0, xmm7); + masm.movl(rdx, -1024); + masm.movdl(xmm5, rdx); + masm.por(xmm3, xmm1); + masm.paddd(xmm0, xmm4); + masm.psllq(xmm5, 32); + masm.movdl(rdx, xmm0); + masm.psllq(xmm0, 29); + masm.pand(xmm5, xmm3); + masm.movl(tmp1, Integer.MIN_VALUE); + masm.pand(xmm0, xmm6); + masm.subsd(xmm3, xmm5); + masm.andl(rax, 32752); + masm.subl(rax, 18416); + masm.sarl(rax, 4); + masm.cvtsi2sdl(xmm7, rax); + masm.mulpd(xmm5, xmm0); + masm.jmp(block4); + + masm.bind(block5); + masm.cmpl(rax, 0); + masm.jcc(AMD64Assembler.ConditionFlag.Less, block11); + masm.cmpl(rax, 752); + masm.jcc(AMD64Assembler.ConditionFlag.AboveEqual, block12); + masm.addsd(xmm0, xmm7); + masm.movq(xmm2, recordExternalAddress(crb, halfmask)); // 0xf8000000, 0xffffffff, + // 0xf8000000, 0xffffffff + masm.addpd(xmm3, xmm0); + masm.xorpd(xmm6, xmm6); + masm.movl(rax, 17080); + masm.pinsrw(xmm6, rax, 3); + masm.pshufd(xmm0, xmm3, 238); + masm.addsd(xmm0, xmm3); + masm.movdqu(xmm3, xmm5); + masm.addsd(xmm5, xmm0); + masm.movdqu(xmm4, xmm2); + masm.subsd(xmm3, xmm5); + masm.movdqu(xmm7, xmm5); + masm.pand(xmm5, xmm2); + masm.movdqu(xmm2, xmm1); + masm.pand(xmm4, xmm1); + masm.subsd(xmm7, xmm5); + masm.addsd(xmm0, xmm3); + masm.subsd(xmm1, xmm4); + masm.mulsd(xmm4, xmm5); + masm.addsd(xmm0, xmm7); + masm.mulsd(xmm2, xmm0); + masm.movdqu(xmm7, xmm6); + masm.mulsd(xmm1, xmm5); + masm.addsd(xmm6, xmm4); + masm.movdl(rax, xmm6); + masm.subsd(xmm6, xmm7); + masm.leaq(tmp4, recordExternalAddress(crb, tExp)); + masm.addsd(xmm2, xmm1); + masm.movdqu(xmm7, recordExternalAddress(crb, eCoeff)); // 0xe78a6731, 0x3f55d87f, + // 0xd704a0c0, 0x3fac6b08 + masm.movdqu(xmm3, recordExternalAddress(crb, eCoeff16)); // 0x6fba4e77, 0x3f83b2ab, + // 0xff82c58f, 0x3fcebfbd + masm.subsd(xmm4, xmm6); + masm.pextrw(rdx, xmm6, 3); + masm.movl(rcx, rax); + masm.andl(rax, 255); + masm.addl(rax, rax); + masm.movdqu(xmm5, new AMD64Address(tmp4, rax, AMD64Address.Scale.Times8, 0)); + masm.addsd(xmm2, xmm4); + masm.sarl(rcx, 8); + masm.movl(rax, rcx); + masm.sarl(rcx, 1); + masm.subl(rax, rcx); + masm.shll(rcx, 20); + masm.xorl(rcx, tmp1); + masm.movdl(xmm6, rcx); + masm.movq(xmm1, recordExternalAddress(crb, eCoeff32)); // 0xfefa39ef, 0x3fe62e42, + // 0x00000000, 0x00000000 + masm.andl(rdx, 32767); + masm.cmpl(rdx, 16529); + masm.jcc(AMD64Assembler.ConditionFlag.Above, block12); + masm.pshufd(xmm0, xmm2, 68); + masm.pshufd(xmm4, xmm2, 68); + masm.mulpd(xmm0, xmm0); + masm.mulpd(xmm7, xmm4); + masm.pshufd(xmm6, xmm6, 17); + masm.mulsd(xmm1, xmm2); + masm.mulsd(xmm0, xmm0); + masm.paddd(xmm5, xmm6); + masm.addpd(xmm3, xmm7); + masm.mulsd(xmm1, xmm5); + masm.pshufd(xmm6, xmm5, 238); + masm.mulpd(xmm0, xmm3); + masm.addsd(xmm1, xmm6); + masm.pshufd(xmm3, xmm0, 238); + masm.mulsd(xmm0, xmm5); + masm.mulsd(xmm3, xmm5); + masm.shll(rax, 4); + masm.xorpd(xmm4, xmm4); + masm.addl(rax, 16368); + masm.pinsrw(xmm4, rax, 3); + masm.addsd(xmm0, xmm1); + masm.addsd(xmm0, xmm3); + masm.movdqu(xmm1, xmm0); + masm.addsd(xmm0, xmm5); + masm.mulsd(xmm0, xmm4); + masm.pextrw(rax, xmm0, 3); + masm.andl(rax, 32752); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block13); + masm.cmpl(rax, 32752); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block14); + masm.jmp(block56); + + masm.bind(block6); + masm.movq(xmm1, new AMD64Address(rsp, 16)); + masm.movq(xmm0, new AMD64Address(rsp, 8)); + masm.movdqu(xmm2, xmm0); + masm.movdl(rax, xmm2); + masm.psrlq(xmm2, 20); + masm.movdl(rdx, xmm2); + masm.orl(rax, rdx); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block15); + masm.movdl(rax, xmm1); + masm.psrlq(xmm1, 32); + masm.movdl(rdx, xmm1); + masm.movl(rcx, rdx); + masm.addl(rdx, rdx); + masm.orl(rax, rdx); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block16); + masm.addsd(xmm0, xmm0); + masm.jmp(block56); + + masm.bind(block16); + masm.xorpd(xmm0, xmm0); + masm.movl(rax, 16368); + masm.pinsrw(xmm0, rax, 3); + masm.movl(new AMD64Address(rsp, 0), 29); + masm.jmp(block17); + + masm.bind(block18); + masm.movq(xmm0, new AMD64Address(rsp, 16)); + masm.addpd(xmm0, xmm0); + masm.jmp(block56); + + masm.bind(block15); + masm.movdl(rax, xmm1); + masm.movdqu(xmm2, xmm1); + masm.psrlq(xmm1, 32); + masm.movdl(rdx, xmm1); + masm.movl(rcx, rdx); + masm.addl(rdx, rdx); + masm.orl(rax, rdx); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block19); + masm.pextrw(rax, xmm2, 3); + masm.andl(rax, 32752); + masm.cmpl(rax, 32752); + masm.jcc(AMD64Assembler.ConditionFlag.NotEqual, block20); + masm.movdl(rax, xmm2); + masm.psrlq(xmm2, 20); + masm.movdl(rdx, xmm2); + masm.orl(rax, rdx); + masm.jcc(AMD64Assembler.ConditionFlag.NotEqual, block18); + + masm.bind(block20); + masm.pextrw(rax, xmm0, 3); + masm.testl(rax, 32768); + masm.jcc(AMD64Assembler.ConditionFlag.NotEqual, block21); + masm.testl(rcx, Integer.MIN_VALUE); + masm.jcc(AMD64Assembler.ConditionFlag.NotEqual, block22); + masm.jmp(block56); + + masm.bind(block23); + masm.movq(xmm1, new AMD64Address(rsp, 16)); + masm.movdl(rax, xmm1); + masm.testl(rax, 1); + masm.jcc(AMD64Assembler.ConditionFlag.NotEqual, block24); + masm.testl(rax, 2); + masm.jcc(AMD64Assembler.ConditionFlag.NotEqual, block25); + masm.jmp(block24); + + masm.bind(block21); + masm.shrl(rcx, 20); + masm.andl(rcx, 2047); + masm.cmpl(rcx, 1075); + masm.jcc(AMD64Assembler.ConditionFlag.Above, block24); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block26); + masm.cmpl(rcx, 1074); + masm.jcc(AMD64Assembler.ConditionFlag.Above, block23); + masm.cmpl(rcx, 1023); + masm.jcc(AMD64Assembler.ConditionFlag.Below, block24); + masm.movq(xmm1, new AMD64Address(rsp, 16)); + masm.movl(rax, 17208); + masm.xorpd(xmm3, xmm3); + masm.pinsrw(xmm3, rax, 3); + masm.movdqu(xmm4, xmm3); + masm.addsd(xmm3, xmm1); + masm.subsd(xmm4, xmm3); + masm.addsd(xmm1, xmm4); + masm.pextrw(rax, xmm1, 3); + masm.andl(rax, 32752); + masm.jcc(AMD64Assembler.ConditionFlag.NotEqual, block24); + masm.movdl(rax, xmm3); + masm.andl(rax, 1); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block24); + + masm.bind(block25); + masm.movq(xmm1, new AMD64Address(rsp, 16)); + masm.pextrw(rax, xmm1, 3); + masm.andl(rax, 32768); + masm.jcc(AMD64Assembler.ConditionFlag.NotEqual, block27); + masm.jmp(block56); + + masm.bind(block27); + masm.xorpd(xmm0, xmm0); + masm.movl(rax, 32768); + masm.pinsrw(xmm0, rax, 3); + masm.jmp(block56); + + masm.bind(block24); + masm.movq(xmm1, new AMD64Address(rsp, 16)); + masm.pextrw(rax, xmm1, 3); + masm.andl(rax, 32768); + masm.jcc(AMD64Assembler.ConditionFlag.NotEqual, block22); + masm.xorpd(xmm0, xmm0); + masm.movl(rax, 32752); + masm.pinsrw(xmm0, rax, 3); + masm.jmp(block56); + + masm.bind(block26); + masm.movq(xmm1, new AMD64Address(rsp, 16)); + masm.movdl(rax, xmm1); + masm.andl(rax, 1); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block24); + masm.jmp(block25); + + masm.bind(block28); + masm.movdl(rax, xmm1); + masm.psrlq(xmm1, 20); + masm.movdl(rdx, xmm1); + masm.orl(rax, rdx); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block29); + masm.movq(xmm0, new AMD64Address(rsp, 16)); + masm.addsd(xmm0, xmm0); + masm.jmp(block56); + + masm.bind(block29); + masm.movq(xmm0, new AMD64Address(rsp, 8)); + masm.pextrw(rax, xmm0, 3); + masm.cmpl(rax, 49136); + masm.jcc(AMD64Assembler.ConditionFlag.NotEqual, block30); + masm.movdl(rcx, xmm0); + masm.psrlq(xmm0, 20); + masm.movdl(rdx, xmm0); + masm.orl(rcx, rdx); + masm.jcc(AMD64Assembler.ConditionFlag.NotEqual, block30); + masm.xorpd(xmm0, xmm0); + masm.movl(rax, 32760); + masm.pinsrw(xmm0, rax, 3); + masm.jmp(block56); + + masm.bind(block30); + masm.movq(xmm1, new AMD64Address(rsp, 16)); + masm.andl(rax, 32752); + masm.subl(rax, 16368); + masm.pextrw(rdx, xmm1, 3); + masm.xorpd(xmm0, xmm0); + masm.xorl(rax, rdx); + masm.andl(rax, 32768); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block31); + masm.jmp(block56); + + masm.bind(block31); + masm.movl(rcx, 32752); + masm.pinsrw(xmm0, rcx, 3); + masm.jmp(block56); + + masm.bind(block32); + masm.movdl(rax, xmm1); + masm.cmpl(rdx, 17184); + masm.jcc(AMD64Assembler.ConditionFlag.Above, block33); + masm.testl(rax, 1); + masm.jcc(AMD64Assembler.ConditionFlag.NotEqual, block34); + masm.testl(rax, 2); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block35); + masm.jmp(block36); + + masm.bind(block33); + masm.testl(rax, 1); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block35); + masm.jmp(block36); + + masm.bind(block7); + masm.movq(xmm2, new AMD64Address(rsp, 8)); + masm.movdl(rax, xmm2); + masm.psrlq(xmm2, 31); + masm.movdl(rcx, xmm2); + masm.orl(rax, rcx); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block9); + masm.movq(xmm1, new AMD64Address(rsp, 16)); + masm.pextrw(rdx, xmm1, 3); + masm.movdl(rax, xmm1); + masm.movdqu(xmm2, xmm1); + masm.psrlq(xmm2, 32); + masm.movdl(rcx, xmm2); + masm.addl(rcx, rcx); + masm.orl(rcx, rax); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block37); + masm.andl(rdx, 32752); + masm.cmpl(rdx, 32752); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block28); + masm.cmpl(rdx, 17200); + masm.jcc(AMD64Assembler.ConditionFlag.Above, block35); + masm.cmpl(rdx, 17184); + masm.jcc(AMD64Assembler.ConditionFlag.AboveEqual, block32); + masm.cmpl(rdx, 16368); + masm.jcc(AMD64Assembler.ConditionFlag.Below, block34); + masm.movl(rax, 17208); + masm.xorpd(xmm2, xmm2); + masm.pinsrw(xmm2, rax, 3); + masm.movdqu(xmm4, xmm2); + masm.addsd(xmm2, xmm1); + masm.subsd(xmm4, xmm2); + masm.addsd(xmm1, xmm4); + masm.pextrw(rax, xmm1, 3); + masm.andl(rax, 32767); + masm.jcc(AMD64Assembler.ConditionFlag.NotEqual, block34); + masm.movdl(rax, xmm2); + masm.andl(rax, 1); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block35); + + masm.bind(block36); + masm.xorpd(xmm1, xmm1); + masm.movl(rdx, 30704); + masm.pinsrw(xmm1, rdx, 3); + masm.movq(xmm2, recordExternalAddress(crb, log2E)); // 0x00000000, 0x3ff72000, + // 0x161bb241, 0xbf5dabe1 + masm.movq(xmm4, new AMD64Address(rsp, 8)); + masm.pextrw(rax, xmm4, 3); + masm.movl(rdx, 8192); + masm.movdl(xmm4, rdx); + masm.andl(rax, 32767); + masm.subl(rax, 16); + masm.jcc(AMD64Assembler.ConditionFlag.Less, block10); + masm.movl(rdx, rax); + masm.andl(rdx, 32752); + masm.subl(rdx, 16368); + masm.movl(rcx, rdx); + masm.sarl(rdx, 31); + masm.addl(rcx, rdx); + masm.xorl(rcx, rdx); + masm.addl(rcx, 16); + masm.bsrl(rcx, rcx); + masm.movl(tmp1, Integer.MIN_VALUE); + masm.jmp(block1); + + masm.bind(block34); + masm.xorpd(xmm1, xmm1); + masm.movl(rax, 32752); + masm.pinsrw(xmm1, rax, 3); + masm.xorpd(xmm0, xmm0); + masm.mulsd(xmm0, xmm1); + masm.movl(new AMD64Address(rsp, 0), 28); + masm.jmp(block17); + + masm.bind(block35); + masm.xorpd(xmm1, xmm1); + masm.movl(rdx, 30704); + masm.pinsrw(xmm1, rdx, 3); + masm.movq(xmm2, recordExternalAddress(crb, log2E)); // 0x00000000, 0x3ff72000, + // 0x161bb241, 0xbf5dabe1 + masm.movq(xmm4, new AMD64Address(rsp, 8)); + masm.pextrw(rax, xmm4, 3); + masm.movl(rdx, 8192); + masm.movdl(xmm4, rdx); + masm.andl(rax, 32767); + masm.subl(rax, 16); + masm.jcc(AMD64Assembler.ConditionFlag.Less, block8); + masm.movl(rdx, rax); + masm.andl(rdx, 32752); + masm.subl(rdx, 16368); + masm.movl(rcx, rdx); + masm.sarl(rdx, 31); + masm.addl(rcx, rdx); + masm.xorl(rcx, rdx); + masm.addl(rcx, 16); + masm.bsrl(rcx, rcx); + masm.movl(tmp1, 0); + masm.jmp(block1); + + masm.bind(block19); + masm.xorpd(xmm0, xmm0); + masm.movl(rax, 16368); + masm.pinsrw(xmm0, rax, 3); + masm.jmp(block56); + + masm.bind(block22); + masm.xorpd(xmm0, xmm0); + masm.jmp(block56); + + masm.bind(block11); + masm.addl(rax, 384); + masm.cmpl(rax, 0); + masm.jcc(AMD64Assembler.ConditionFlag.Less, block38); + masm.mulsd(xmm5, xmm1); + masm.addsd(xmm0, xmm7); + masm.shrl(tmp1, 31); + masm.addpd(xmm3, xmm0); + masm.pshufd(xmm0, xmm3, 238); + masm.addsd(xmm3, xmm0); + masm.leaq(tmp4, recordExternalAddress(crb, log2)); // 0xfefa39ef, 0x3fe62e42, + // 0xfefa39ef, 0xbfe62e42 + masm.movq(xmm4, new AMD64Address(tmp4, tmp1, AMD64Address.Scale.Times8, 0)); + masm.mulsd(xmm1, xmm3); + masm.xorpd(xmm0, xmm0); + masm.movl(rax, 16368); + masm.shll(tmp1, 15); + masm.orl(rax, tmp1); + masm.pinsrw(xmm0, rax, 3); + masm.addsd(xmm5, xmm1); + masm.mulsd(xmm5, xmm4); + masm.addsd(xmm0, xmm5); + masm.jmp(block56); + + masm.bind(block38); + + masm.bind(block37); + masm.xorpd(xmm0, xmm0); + masm.movl(rax, 16368); + masm.pinsrw(xmm0, rax, 3); + masm.jmp(block56); + + masm.bind(block39); + masm.xorpd(xmm0, xmm0); + masm.movl(rax, 16368); + masm.pinsrw(xmm0, rax, 3); + masm.movl(new AMD64Address(rsp, 0), 26); + masm.jmp(block17); + + masm.bind(block9); + masm.movq(xmm1, new AMD64Address(rsp, 16)); + masm.movdqu(xmm2, xmm1); + masm.pextrw(rax, xmm1, 3); + masm.andl(rax, 32752); + masm.cmpl(rax, 32752); + masm.jcc(AMD64Assembler.ConditionFlag.NotEqual, block40); + masm.movdl(rax, xmm2); + masm.psrlq(xmm2, 20); + masm.movdl(rdx, xmm2); + masm.orl(rax, rdx); + masm.jcc(AMD64Assembler.ConditionFlag.NotEqual, block18); + + masm.bind(block40); + masm.movdl(rax, xmm1); + masm.psrlq(xmm1, 32); + masm.movdl(rdx, xmm1); + masm.movl(rcx, rdx); + masm.addl(rdx, rdx); + masm.orl(rax, rdx); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block39); + masm.shrl(rdx, 21); + masm.cmpl(rdx, 1075); + masm.jcc(AMD64Assembler.ConditionFlag.Above, block41); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block42); + masm.cmpl(rdx, 1023); + masm.jcc(AMD64Assembler.ConditionFlag.Below, block41); + masm.movq(xmm1, new AMD64Address(rsp, 16)); + masm.movl(rax, 17208); + masm.xorpd(xmm3, xmm3); + masm.pinsrw(xmm3, rax, 3); + masm.movdqu(xmm4, xmm3); + masm.addsd(xmm3, xmm1); + masm.subsd(xmm4, xmm3); + masm.addsd(xmm1, xmm4); + masm.pextrw(rax, xmm1, 3); + masm.andl(rax, 32752); + masm.jcc(AMD64Assembler.ConditionFlag.NotEqual, block41); + masm.movdl(rax, xmm3); + masm.andl(rax, 1); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block41); + + masm.bind(block43); + masm.movq(xmm0, new AMD64Address(rsp, 8)); + masm.testl(rcx, Integer.MIN_VALUE); + masm.jcc(AMD64Assembler.ConditionFlag.NotEqual, block44); + masm.jmp(block56); + + masm.bind(block42); + masm.movq(xmm1, new AMD64Address(rsp, 16)); + masm.movdl(rax, xmm1); + masm.testl(rax, 1); + masm.jcc(AMD64Assembler.ConditionFlag.NotEqual, block43); + + masm.bind(block41); + masm.testl(rcx, Integer.MIN_VALUE); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block22); + masm.xorpd(xmm0, xmm0); + + masm.bind(block44); + masm.movl(rax, 16368); + masm.xorpd(xmm1, xmm1); + masm.pinsrw(xmm1, rax, 3); + masm.divsd(xmm1, xmm0); + masm.movdqu(xmm0, xmm1); + masm.movl(new AMD64Address(rsp, 0), 27); + masm.jmp(block17); + + masm.bind(block12); + masm.movq(xmm2, new AMD64Address(rsp, 8)); + masm.movq(xmm6, new AMD64Address(rsp, 16)); + masm.pextrw(rax, xmm2, 3); + masm.pextrw(rdx, xmm6, 3); + masm.movl(rcx, 32752); + masm.andl(rcx, rdx); + masm.cmpl(rcx, 32752); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block45); + masm.andl(rax, 32752); + masm.subl(rax, 16368); + masm.xorl(rdx, rax); + masm.testl(rdx, 32768); + masm.jcc(AMD64Assembler.ConditionFlag.NotEqual, block46); + + masm.bind(block47); + masm.movl(rax, 32736); + masm.pinsrw(xmm0, rax, 3); + masm.shrl(tmp1, 16); + masm.orl(rax, tmp1); + masm.pinsrw(xmm1, rax, 3); + masm.mulsd(xmm0, xmm1); + + masm.bind(block14); + masm.movl(new AMD64Address(rsp, 0), 24); + masm.jmp(block17); + + masm.bind(block46); + masm.movl(rax, 16); + masm.pinsrw(xmm0, rax, 3); + masm.mulsd(xmm0, xmm0); + masm.testl(tmp1, Integer.MIN_VALUE); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block48); + masm.movq(tmp2, 0x8000000000000000L); + masm.movdq(xmm2, tmp2); + masm.xorpd(xmm0, xmm2); + + masm.bind(block48); + masm.movl(new AMD64Address(rsp, 0), 25); + masm.jmp(block17); + + masm.bind(block13); + masm.pextrw(rcx, xmm5, 3); + masm.pextrw(rdx, xmm4, 3); + masm.movl(rax, -1); + masm.andl(rcx, 32752); + masm.subl(rcx, 16368); + masm.andl(rdx, 32752); + masm.addl(rdx, rcx); + masm.movl(rcx, -31); + masm.sarl(rdx, 4); + masm.subl(rcx, rdx); + masm.jcc(AMD64Assembler.ConditionFlag.LessEqual, block49); + masm.cmpl(rcx, 20); + masm.jcc(AMD64Assembler.ConditionFlag.Above, block50); + masm.shll(rax); + + masm.bind(block49); + masm.movdl(xmm0, rax); + masm.psllq(xmm0, 32); + masm.pand(xmm0, xmm5); + masm.subsd(xmm5, xmm0); + masm.addsd(xmm5, xmm1); + masm.mulsd(xmm0, xmm4); + masm.mulsd(xmm5, xmm4); + masm.addsd(xmm0, xmm5); + + masm.bind(block50); + masm.jmp(block48); + + masm.bind(block2); + masm.movw(rcx, new AMD64Address(rsp, 22)); + masm.movl(rdx, Integer.MIN_VALUE); + masm.movdl(xmm1, rdx); + masm.xorpd(xmm7, xmm7); + masm.paddd(xmm0, xmm4); + masm.movdl(rdx, xmm0); + masm.psllq(xmm0, 29); + masm.paddq(xmm1, xmm3); + masm.pand(xmm5, xmm1); + masm.andl(rcx, 32752); + masm.cmpl(rcx, 16560); + masm.jcc(AMD64Assembler.ConditionFlag.Less, block3); + masm.pand(xmm0, xmm6); + masm.subsd(xmm3, xmm5); + masm.addl(rax, 16351); + masm.shrl(rax, 4); + masm.subl(rax, 1022); + masm.cvtsi2sdl(xmm7, rax); + masm.mulpd(xmm5, xmm0); + masm.leaq(r11, recordExternalAddress(crb, lTbl)); + masm.movq(xmm4, recordExternalAddress(crb, coeffH)); // 0x00000000, 0xbfd61a00, + // 0x00000000, 0xbf5dabe1 + masm.mulsd(xmm3, xmm0); + masm.movq(xmm6, recordExternalAddress(crb, coeffH)); // 0x00000000, 0xbfd61a00, + // 0x00000000, 0xbf5dabe1 + masm.subsd(xmm5, xmm2); + masm.movq(xmm1, recordExternalAddress(crb, coeffH8)); // 0x00000000, 0xbf5dabe1 + masm.pshufd(xmm2, xmm3, 68); + masm.unpcklpd(xmm5, xmm3); + masm.addsd(xmm3, xmm5); + masm.movq(xmm0, recordExternalAddress(crb, coeffH8)); // 0x00000000, 0xbf5dabe1 + masm.andl(rdx, 16760832); + masm.shrl(rdx, 10); + masm.addpd(xmm7, new AMD64Address(tmp4, rdx, AMD64Address.Scale.Times1, -3648)); + masm.mulsd(xmm4, xmm5); + masm.mulsd(xmm0, xmm5); + masm.mulsd(xmm6, xmm2); + masm.mulsd(xmm1, xmm2); + masm.movdqu(xmm2, xmm5); + masm.mulsd(xmm4, xmm5); + masm.addsd(xmm5, xmm0); + masm.movdqu(xmm0, xmm7); + masm.addsd(xmm2, xmm3); + masm.addsd(xmm7, xmm5); + masm.mulsd(xmm6, xmm2); + masm.subsd(xmm0, xmm7); + masm.movdqu(xmm2, xmm7); + masm.addsd(xmm7, xmm4); + masm.addsd(xmm0, xmm5); + masm.subsd(xmm2, xmm7); + masm.addsd(xmm4, xmm2); + masm.pshufd(xmm2, xmm5, 238); + masm.movdqu(xmm5, xmm7); + masm.addsd(xmm7, xmm2); + masm.addsd(xmm4, xmm0); + masm.movdqu(xmm0, recordExternalAddress(crb, coeff)); // 0x6dc96112, 0xbf836578, + // 0xee241472, 0xbf9b0301 + masm.subsd(xmm5, xmm7); + masm.addsd(xmm6, xmm4); + masm.movdqu(xmm4, xmm7); + masm.addsd(xmm5, xmm2); + masm.addsd(xmm7, xmm1); + masm.movdqu(xmm2, recordExternalAddress(crb, coeff64)); // 0x486ececc, 0x3fc4635e, + // 0x161bb241, 0xbf5dabe1 + masm.subsd(xmm4, xmm7); + masm.addsd(xmm6, xmm5); + masm.addsd(xmm4, xmm1); + masm.pshufd(xmm5, xmm7, 238); + masm.movapd(xmm1, xmm7); + masm.addsd(xmm7, xmm5); + masm.subsd(xmm1, xmm7); + masm.addsd(xmm1, xmm5); + masm.movdqu(xmm5, recordExternalAddress(crb, coeff80)); // 0x9f95985a, 0xbfb528db, + // 0xf8b5787d, 0x3ef2531e + masm.pshufd(xmm3, xmm3, 68); + masm.addsd(xmm6, xmm4); + masm.addsd(xmm6, xmm1); + masm.movdqu(xmm1, recordExternalAddress(crb, coeff32)); // 0x9f95985a, 0xbfb528db, + // 0xb3841d2a, 0xbfd619b6 + masm.mulpd(xmm0, xmm3); + masm.mulpd(xmm2, xmm3); + masm.pshufd(xmm4, xmm3, 68); + masm.mulpd(xmm3, xmm3); + masm.addpd(xmm0, xmm1); + masm.addpd(xmm5, xmm2); + masm.mulsd(xmm4, xmm3); + masm.movq(xmm2, recordExternalAddress(crb, highmaskLogX)); // 0xf8000000, 0xffffffff, + // 0x00000000, 0xfffff800 + masm.mulpd(xmm3, xmm3); + masm.movq(xmm1, new AMD64Address(rsp, 16)); + masm.movw(rcx, new AMD64Address(rsp, 22)); + masm.mulpd(xmm0, xmm4); + masm.pextrw(rax, xmm7, 3); + masm.mulpd(xmm5, xmm4); + masm.mulpd(xmm0, xmm3); + masm.movq(xmm4, recordExternalAddress(crb, highmaskY8)); // 0x00000000, 0xffffffff + masm.pand(xmm2, xmm7); + masm.addsd(xmm5, xmm6); + masm.subsd(xmm7, xmm2); + masm.addpd(xmm5, xmm0); + masm.andl(rax, 32752); + masm.subl(rax, 16368); + masm.andl(rcx, 32752); + masm.cmpl(rcx, 32752); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block45); + masm.addl(rcx, rax); + masm.cmpl(rcx, 16576); + masm.jcc(AMD64Assembler.ConditionFlag.AboveEqual, block51); + masm.pshufd(xmm0, xmm5, 238); + masm.pand(xmm4, xmm1); + masm.movdqu(xmm3, xmm1); + masm.addsd(xmm5, xmm0); + masm.subsd(xmm1, xmm4); + masm.xorpd(xmm6, xmm6); + masm.movl(rdx, 17080); + masm.pinsrw(xmm6, rdx, 3); + masm.addsd(xmm7, xmm5); + masm.mulsd(xmm4, xmm2); + masm.mulsd(xmm1, xmm2); + masm.movdqu(xmm5, xmm6); + masm.mulsd(xmm3, xmm7); + masm.addsd(xmm6, xmm4); + masm.addsd(xmm1, xmm3); + masm.movdqu(xmm7, recordExternalAddress(crb, eCoeff)); // 0xe78a6731, 0x3f55d87f, + // 0xd704a0c0, 0x3fac6b08 + masm.movdl(rdx, xmm6); + masm.subsd(xmm6, xmm5); + masm.leaq(tmp4, recordExternalAddress(crb, tExp)); + masm.movdqu(xmm3, recordExternalAddress(crb, eCoeff16)); // 0x6fba4e77, 0x3f83b2ab, + // 0xff82c58f, 0x3fcebfbd + masm.movq(xmm2, recordExternalAddress(crb, eCoeff32)); // 0xfefa39ef, 0x3fe62e42, + // 0x00000000, 0x00000000 + masm.subsd(xmm4, xmm6); + masm.movl(rcx, rdx); + masm.andl(rdx, 255); + masm.addl(rdx, rdx); + masm.movdqu(xmm5, new AMD64Address(tmp4, rdx, AMD64Address.Scale.Times8, 0)); + masm.addsd(xmm4, xmm1); + masm.pextrw(rdx, xmm6, 3); + masm.shrl(rcx, 8); + masm.movl(rax, rcx); + masm.shrl(rcx, 1); + masm.subl(rax, rcx); + masm.shll(rcx, 20); + masm.movdl(xmm6, rcx); + masm.pshufd(xmm0, xmm4, 68); + masm.pshufd(xmm1, xmm4, 68); + masm.mulpd(xmm0, xmm0); + masm.mulpd(xmm7, xmm1); + masm.pshufd(xmm6, xmm6, 17); + masm.mulsd(xmm2, xmm4); + masm.andl(rdx, 32767); + masm.cmpl(rdx, 16529); + masm.jcc(AMD64Assembler.ConditionFlag.Above, block12); + masm.mulsd(xmm0, xmm0); + masm.paddd(xmm5, xmm6); + masm.addpd(xmm3, xmm7); + masm.mulsd(xmm2, xmm5); + masm.pshufd(xmm6, xmm5, 238); + masm.mulpd(xmm0, xmm3); + masm.addsd(xmm2, xmm6); + masm.pshufd(xmm3, xmm0, 238); + masm.addl(rax, 1023); + masm.shll(rax, 20); + masm.orl(rax, tmp1); + masm.movdl(xmm4, rax); + masm.mulsd(xmm0, xmm5); + masm.mulsd(xmm3, xmm5); + masm.addsd(xmm0, xmm2); + masm.psllq(xmm4, 32); + masm.addsd(xmm0, xmm3); + masm.movdqu(xmm1, xmm0); + masm.addsd(xmm0, xmm5); + masm.mulsd(xmm0, xmm4); + masm.pextrw(rax, xmm0, 3); + masm.andl(rax, 32752); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block13); + masm.cmpl(rax, 32752); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block14); + + masm.bind(block52); + masm.jmp(block56); + + masm.bind(block45); + masm.movq(xmm0, new AMD64Address(rsp, 8)); + masm.xorpd(xmm2, xmm2); + masm.movl(rax, 49136); + masm.pinsrw(xmm2, rax, 3); + masm.addsd(xmm2, xmm0); + masm.pextrw(rax, xmm2, 3); + masm.cmpl(rax, 0); + masm.jcc(AMD64Assembler.ConditionFlag.NotEqual, block53); + masm.xorpd(xmm0, xmm0); + masm.movl(rax, 32760); + masm.pinsrw(xmm0, rax, 3); + masm.jmp(block56); + + masm.bind(block53); + masm.movq(xmm1, new AMD64Address(rsp, 16)); + masm.movdl(rdx, xmm1); + masm.movdqu(xmm3, xmm1); + masm.psrlq(xmm3, 20); + masm.movdl(rcx, xmm3); + masm.orl(rcx, rdx); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block54); + masm.addsd(xmm1, xmm1); + masm.movdqu(xmm0, xmm1); + masm.jmp(block56); + + masm.bind(block51); + masm.pextrw(rax, xmm1, 3); + masm.pextrw(rcx, xmm2, 3); + masm.xorl(rax, rcx); + masm.testl(rax, 32768); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block47); + masm.jmp(block46); + + masm.bind(block54); + masm.pextrw(rax, xmm0, 3); + masm.andl(rax, 32752); + masm.pextrw(rdx, xmm1, 3); + masm.xorpd(xmm0, xmm0); + masm.subl(rax, 16368); + masm.xorl(rax, rdx); + masm.testl(rax, 32768); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block55); + masm.jmp(block56); + + masm.bind(block55); + masm.movl(rdx, 32752); + masm.pinsrw(xmm0, rdx, 3); + masm.jmp(block56); + + masm.bind(block17); + masm.movq(new AMD64Address(rsp, 24), xmm0); + + masm.movq(xmm0, new AMD64Address(rsp, 24)); + + masm.bind(block56); + masm.addq(rsp, 40); + } +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathSinOp.java 2019-03-12 08:10:48.284044004 +0100 @@ -0,0 +1,902 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, Intel Corporation. All rights reserved. + * Intel Math Library (LIBM) Source Code + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.lir.amd64; + +import static jdk.vm.ci.amd64.AMD64.r10; +import static jdk.vm.ci.amd64.AMD64.r11; +import static jdk.vm.ci.amd64.AMD64.r8; +import static jdk.vm.ci.amd64.AMD64.r9; +import static jdk.vm.ci.amd64.AMD64.rax; +import static jdk.vm.ci.amd64.AMD64.rbx; +import static jdk.vm.ci.amd64.AMD64.rcx; +import static jdk.vm.ci.amd64.AMD64.rdi; +import static jdk.vm.ci.amd64.AMD64.rdx; +import static jdk.vm.ci.amd64.AMD64.rsi; +import static jdk.vm.ci.amd64.AMD64.rsp; +import static jdk.vm.ci.amd64.AMD64.xmm0; +import static jdk.vm.ci.amd64.AMD64.xmm1; +import static jdk.vm.ci.amd64.AMD64.xmm2; +import static jdk.vm.ci.amd64.AMD64.xmm3; +import static jdk.vm.ci.amd64.AMD64.xmm4; +import static jdk.vm.ci.amd64.AMD64.xmm5; +import static jdk.vm.ci.amd64.AMD64.xmm6; +import static jdk.vm.ci.amd64.AMD64.xmm7; +import static org.graalvm.compiler.lir.amd64.AMD64HotSpotHelper.pointerConstant; +import static org.graalvm.compiler.lir.amd64.AMD64HotSpotHelper.recordExternalAddress; + +import org.graalvm.compiler.asm.Label; +import org.graalvm.compiler.asm.amd64.AMD64Address; +import org.graalvm.compiler.asm.amd64.AMD64Assembler.ConditionFlag; +import org.graalvm.compiler.asm.amd64.AMD64MacroAssembler; +import org.graalvm.compiler.lir.LIRInstructionClass; +import org.graalvm.compiler.lir.asm.ArrayDataPointerConstant; +import org.graalvm.compiler.lir.asm.CompilationResultBuilder; + +import jdk.vm.ci.amd64.AMD64; + +/** + * 
+ *                     ALGORITHM DESCRIPTION - SIN()
+ *                     ---------------------
+ *
+ *     1. RANGE REDUCTION
+ *
+ *     We perform an initial range reduction from X to r with
+ *
+ *          X =~= N * pi/32 + r
+ *
+ *     so that |r| <= pi/64 + epsilon. We restrict inputs to those
+ *     where |N| <= 932560. Beyond this, the range reduction is
+ *     insufficiently accurate. For extremely small inputs,
+ *     denormalization can occur internally, impacting performance.
+ *     This means that the main path is actually only taken for
+ *     2^-252 <= |X| < 90112.
+ *
+ *     To avoid branches, we perform the range reduction to full
+ *     accuracy each time.
+ *
+ *          X - N * (P_1 + P_2 + P_3)
+ *
+ *     where P_1 and P_2 are 32-bit numbers (so multiplication by N
+ *     is exact) and P_3 is a 53-bit number. Together, these
+ *     approximate pi well enough for all cases in the restricted
+ *     range.
+ *
+ *     The main reduction sequence is:
+ *
+ *             y = 32/pi * x
+ *             N = integer(y)
+ *     (computed by adding and subtracting off SHIFTER)
+ *
+ *             m_1 = N * P_1
+ *             m_2 = N * P_2
+ *             r_1 = x - m_1
+ *             r = r_1 - m_2
+ *     (this r can be used for most of the calculation)
+ *
+ *             c_1 = r_1 - r
+ *             m_3 = N * P_3
+ *             c_2 = c_1 - m_2
+ *             c = c_2 - m_3
+ *
+ *     2. MAIN ALGORITHM
+ *
+ *     The algorithm uses a table lookup based on B = M * pi / 32
+ *     where M = N mod 64. The stored values are:
+ *       sigma             closest power of 2 to cos(B)
+ *       C_hl              53-bit cos(B) - sigma
+ *       S_hi + S_lo       2 * 53-bit sin(B)
+ *
+ *     The computation is organized as follows:
+ *
+ *          sin(B + r + c) = [sin(B) + sigma * r] +
+ *                           r * (cos(B) - sigma) +
+ *                           sin(B) * [cos(r + c) - 1] +
+ *                           cos(B) * [sin(r + c) - r]
+ *
+ *     which is approximately:
+ *
+ *          [S_hi + sigma * r] +
+ *          C_hl * r +
+ *          S_lo + S_hi * [(cos(r) - 1) - r * c] +
+ *          (C_hl + sigma) * [(sin(r) - r) + c]
+ *
+ *     and this is what is actually computed. We separate this sum
+ *     into four parts:
+ *
+ *          hi + med + pols + corr
+ *
+ *     where
+ *
+ *          hi       = S_hi + sigma r
+ *          med      = C_hl * r
+ *          pols     = S_hi * (cos(r) - 1) + (C_hl + sigma) * (sin(r) - r)
+ *          corr     = S_lo + c * ((C_hl + sigma) - S_hi * r)
+ *
+ *     3. POLYNOMIAL
+ *
+ *     The polynomial S_hi * (cos(r) - 1) + (C_hl + sigma) *
+ *     (sin(r) - r) can be rearranged freely, since it is quite
+ *     small, so we exploit parallelism to the fullest.
+ *
+ *          psc4       =   SC_4 * r_1
+ *          msc4       =   psc4 * r
+ *          r2         =   r * r
+ *          msc2       =   SC_2 * r2
+ *          r4         =   r2 * r2
+ *          psc3       =   SC_3 + msc4
+ *          psc1       =   SC_1 + msc2
+ *          msc3       =   r4 * psc3
+ *          sincospols =   psc1 + msc3
+ *          pols       =   sincospols *
+ *                         
+ *
+ *     4. CORRECTION TERM
+ *
+ *     This is where the "c" component of the range reduction is
+ *     taken into account; recall that just "r" is used for most of
+ *     the calculation.
+ *
+ *          -c   = m_3 - c_2
+ *          -d   = S_hi * r - (C_hl + sigma)
+ *          corr = -c * -d + S_lo
+ *
+ *     5. COMPENSATED SUMMATIONS
+ *
+ *     The two successive compensated summations add up the high
+ *     and medium parts, leaving just the low parts to add up at
+ *     the end.
+ *
+ *          rs        =  sigma * r
+ *          res_int   =  S_hi + rs
+ *          k_0       =  S_hi - res_int
+ *          k_2       =  k_0 + rs
+ *          med       =  C_hl * r
+ *          res_hi    =  res_int + med
+ *          k_1       =  res_int - res_hi
+ *          k_3       =  k_1 + med
+ *
+ *     6. FINAL SUMMATION
+ *
+ *     We now add up all the small parts:
+ *
+ *          res_lo = pols(hi) + pols(lo) + corr + k_1 + k_3
+ *
+ *     Now the overall result is just:
+ *
+ *          res_hi + res_lo
+ *
+ *     7. SMALL ARGUMENTS
+ *
+ *     If |x| < SNN (SNN meaning the smallest normal number), we
+ *     simply perform 0.1111111 cdots 1111 * x. For SNN <= |x|, we
+ *     do 2^-55 * (2^55 * x - x).
+ *
+ * Special cases:
+ *  sin(NaN) = quiet NaN, and raise invalid exception
+ *  sin(INF) = NaN and raise invalid exception
+ *  sin(+/-0) = +/-0
+ *
+ */ +public final class AMD64MathSinOp extends AMD64MathIntrinsicUnaryOp { + + public static final LIRInstructionClass TYPE = LIRInstructionClass.create(AMD64MathSinOp.class); + + public AMD64MathSinOp() { + super(TYPE, /* GPR */ rax, rcx, rdx, rbx, rsi, rdi, r8, r9, r10, r11, + /* XMM */ xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7); + } + + private ArrayDataPointerConstant onehalf = pointerConstant(16, new int[]{ + // @formatter:off + 0x00000000, 0x3fe00000, 0x00000000, 0x3fe00000 + // @formatter:on + }); + + private ArrayDataPointerConstant p2 = pointerConstant(16, new int[]{ + // @formatter:off + 0x1a600000, 0x3d90b461, 0x1a600000, 0x3d90b461 + // @formatter:on + }); + + private ArrayDataPointerConstant sc4 = pointerConstant(16, new int[]{ + // @formatter:off + 0xa556c734, 0x3ec71de3, 0x1a01a01a, 0x3efa01a0 + // @formatter:on + }); + + private ArrayDataPointerConstant ctable = pointerConstant(16, new int[]{ + // @formatter:off + 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x3ff00000, 0x176d6d31, 0xbf73b92e, + 0xbc29b42c, 0x3fb917a6, 0xe0000000, 0xbc3e2718, 0x00000000, + 0x3ff00000, 0x011469fb, 0xbf93ad06, 0x3c69a60b, 0x3fc8f8b8, + 0xc0000000, 0xbc626d19, 0x00000000, 0x3ff00000, 0x939d225a, + 0xbfa60bea, 0x2ed59f06, 0x3fd29406, 0xa0000000, 0xbc75d28d, + 0x00000000, 0x3ff00000, 0x866b95cf, 0xbfb37ca1, 0xa6aea963, + 0x3fd87de2, 0xe0000000, 0xbc672ced, 0x00000000, 0x3ff00000, + 0x73fa1279, 0xbfbe3a68, 0x3806f63b, 0x3fde2b5d, 0x20000000, + 0x3c5e0d89, 0x00000000, 0x3ff00000, 0x5bc57974, 0xbfc59267, + 0x39ae68c8, 0x3fe1c73b, 0x20000000, 0x3c8b25dd, 0x00000000, + 0x3ff00000, 0x53aba2fd, 0xbfcd0dfe, 0x25091dd6, 0x3fe44cf3, + 0x20000000, 0x3c68076a, 0x00000000, 0x3ff00000, 0x99fcef32, + 0x3fca8279, 0x667f3bcd, 0x3fe6a09e, 0x20000000, 0xbc8bdd34, + 0x00000000, 0x3fe00000, 0x94247758, 0x3fc133cc, 0x6b151741, + 0x3fe8bc80, 0x20000000, 0xbc82c5e1, 0x00000000, 0x3fe00000, + 0x9ae68c87, 0x3fac73b3, 0x290ea1a3, 0x3fea9b66, 0xe0000000, + 0x3c39f630, 0x00000000, 0x3fe00000, 0x7f909c4e, 0xbf9d4a2c, + 0xf180bdb1, 0x3fec38b2, 0x80000000, 0xbc76e0b1, 0x00000000, + 0x3fe00000, 0x65455a75, 0xbfbe0875, 0xcf328d46, 0x3fed906b, + 0x20000000, 0x3c7457e6, 0x00000000, 0x3fe00000, 0x76acf82d, + 0x3fa4a031, 0x56c62dda, 0x3fee9f41, 0xe0000000, 0x3c8760b1, + 0x00000000, 0x3fd00000, 0x0e5967d5, 0xbfac1d1f, 0xcff75cb0, + 0x3fef6297, 0x20000000, 0x3c756217, 0x00000000, 0x3fd00000, + 0x0f592f50, 0xbf9ba165, 0xa3d12526, 0x3fefd88d, 0x40000000, + 0xbc887df6, 0x00000000, 0x3fc00000, 0x00000000, 0x00000000, + 0x00000000, 0x3ff00000, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x0f592f50, 0x3f9ba165, 0xa3d12526, 0x3fefd88d, + 0x40000000, 0xbc887df6, 0x00000000, 0xbfc00000, 0x0e5967d5, + 0x3fac1d1f, 0xcff75cb0, 0x3fef6297, 0x20000000, 0x3c756217, + 0x00000000, 0xbfd00000, 0x76acf82d, 0xbfa4a031, 0x56c62dda, + 0x3fee9f41, 0xe0000000, 0x3c8760b1, 0x00000000, 0xbfd00000, + 0x65455a75, 0x3fbe0875, 0xcf328d46, 0x3fed906b, 0x20000000, + 0x3c7457e6, 0x00000000, 0xbfe00000, 0x7f909c4e, 0x3f9d4a2c, + 0xf180bdb1, 0x3fec38b2, 0x80000000, 0xbc76e0b1, 0x00000000, + 0xbfe00000, 0x9ae68c87, 0xbfac73b3, 0x290ea1a3, 0x3fea9b66, + 0xe0000000, 0x3c39f630, 0x00000000, 0xbfe00000, 0x94247758, + 0xbfc133cc, 0x6b151741, 0x3fe8bc80, 0x20000000, 0xbc82c5e1, + 0x00000000, 0xbfe00000, 0x99fcef32, 0xbfca8279, 0x667f3bcd, + 0x3fe6a09e, 0x20000000, 0xbc8bdd34, 0x00000000, 0xbfe00000, + 0x53aba2fd, 0x3fcd0dfe, 0x25091dd6, 0x3fe44cf3, 0x20000000, + 0x3c68076a, 0x00000000, 0xbff00000, 0x5bc57974, 0x3fc59267, + 0x39ae68c8, 0x3fe1c73b, 0x20000000, 0x3c8b25dd, 0x00000000, + 0xbff00000, 0x73fa1279, 0x3fbe3a68, 0x3806f63b, 0x3fde2b5d, + 0x20000000, 0x3c5e0d89, 0x00000000, 0xbff00000, 0x866b95cf, + 0x3fb37ca1, 0xa6aea963, 0x3fd87de2, 0xe0000000, 0xbc672ced, + 0x00000000, 0xbff00000, 0x939d225a, 0x3fa60bea, 0x2ed59f06, + 0x3fd29406, 0xa0000000, 0xbc75d28d, 0x00000000, 0xbff00000, + 0x011469fb, 0x3f93ad06, 0x3c69a60b, 0x3fc8f8b8, 0xc0000000, + 0xbc626d19, 0x00000000, 0xbff00000, 0x176d6d31, 0x3f73b92e, + 0xbc29b42c, 0x3fb917a6, 0xe0000000, 0xbc3e2718, 0x00000000, + 0xbff00000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0xbff00000, 0x176d6d31, + 0x3f73b92e, 0xbc29b42c, 0xbfb917a6, 0xe0000000, 0x3c3e2718, + 0x00000000, 0xbff00000, 0x011469fb, 0x3f93ad06, 0x3c69a60b, + 0xbfc8f8b8, 0xc0000000, 0x3c626d19, 0x00000000, 0xbff00000, + 0x939d225a, 0x3fa60bea, 0x2ed59f06, 0xbfd29406, 0xa0000000, + 0x3c75d28d, 0x00000000, 0xbff00000, 0x866b95cf, 0x3fb37ca1, + 0xa6aea963, 0xbfd87de2, 0xe0000000, 0x3c672ced, 0x00000000, + 0xbff00000, 0x73fa1279, 0x3fbe3a68, 0x3806f63b, 0xbfde2b5d, + 0x20000000, 0xbc5e0d89, 0x00000000, 0xbff00000, 0x5bc57974, + 0x3fc59267, 0x39ae68c8, 0xbfe1c73b, 0x20000000, 0xbc8b25dd, + 0x00000000, 0xbff00000, 0x53aba2fd, 0x3fcd0dfe, 0x25091dd6, + 0xbfe44cf3, 0x20000000, 0xbc68076a, 0x00000000, 0xbff00000, + 0x99fcef32, 0xbfca8279, 0x667f3bcd, 0xbfe6a09e, 0x20000000, + 0x3c8bdd34, 0x00000000, 0xbfe00000, 0x94247758, 0xbfc133cc, + 0x6b151741, 0xbfe8bc80, 0x20000000, 0x3c82c5e1, 0x00000000, + 0xbfe00000, 0x9ae68c87, 0xbfac73b3, 0x290ea1a3, 0xbfea9b66, + 0xe0000000, 0xbc39f630, 0x00000000, 0xbfe00000, 0x7f909c4e, + 0x3f9d4a2c, 0xf180bdb1, 0xbfec38b2, 0x80000000, 0x3c76e0b1, + 0x00000000, 0xbfe00000, 0x65455a75, 0x3fbe0875, 0xcf328d46, + 0xbfed906b, 0x20000000, 0xbc7457e6, 0x00000000, 0xbfe00000, + 0x76acf82d, 0xbfa4a031, 0x56c62dda, 0xbfee9f41, 0xe0000000, + 0xbc8760b1, 0x00000000, 0xbfd00000, 0x0e5967d5, 0x3fac1d1f, + 0xcff75cb0, 0xbfef6297, 0x20000000, 0xbc756217, 0x00000000, + 0xbfd00000, 0x0f592f50, 0x3f9ba165, 0xa3d12526, 0xbfefd88d, + 0x40000000, 0x3c887df6, 0x00000000, 0xbfc00000, 0x00000000, + 0x00000000, 0x00000000, 0xbff00000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x0f592f50, 0xbf9ba165, 0xa3d12526, + 0xbfefd88d, 0x40000000, 0x3c887df6, 0x00000000, 0x3fc00000, + 0x0e5967d5, 0xbfac1d1f, 0xcff75cb0, 0xbfef6297, 0x20000000, + 0xbc756217, 0x00000000, 0x3fd00000, 0x76acf82d, 0x3fa4a031, + 0x56c62dda, 0xbfee9f41, 0xe0000000, 0xbc8760b1, 0x00000000, + 0x3fd00000, 0x65455a75, 0xbfbe0875, 0xcf328d46, 0xbfed906b, + 0x20000000, 0xbc7457e6, 0x00000000, 0x3fe00000, 0x7f909c4e, + 0xbf9d4a2c, 0xf180bdb1, 0xbfec38b2, 0x80000000, 0x3c76e0b1, + 0x00000000, 0x3fe00000, 0x9ae68c87, 0x3fac73b3, 0x290ea1a3, + 0xbfea9b66, 0xe0000000, 0xbc39f630, 0x00000000, 0x3fe00000, + 0x94247758, 0x3fc133cc, 0x6b151741, 0xbfe8bc80, 0x20000000, + 0x3c82c5e1, 0x00000000, 0x3fe00000, 0x99fcef32, 0x3fca8279, + 0x667f3bcd, 0xbfe6a09e, 0x20000000, 0x3c8bdd34, 0x00000000, + 0x3fe00000, 0x53aba2fd, 0xbfcd0dfe, 0x25091dd6, 0xbfe44cf3, + 0x20000000, 0xbc68076a, 0x00000000, 0x3ff00000, 0x5bc57974, + 0xbfc59267, 0x39ae68c8, 0xbfe1c73b, 0x20000000, 0xbc8b25dd, + 0x00000000, 0x3ff00000, 0x73fa1279, 0xbfbe3a68, 0x3806f63b, + 0xbfde2b5d, 0x20000000, 0xbc5e0d89, 0x00000000, 0x3ff00000, + 0x866b95cf, 0xbfb37ca1, 0xa6aea963, 0xbfd87de2, 0xe0000000, + 0x3c672ced, 0x00000000, 0x3ff00000, 0x939d225a, 0xbfa60bea, + 0x2ed59f06, 0xbfd29406, 0xa0000000, 0x3c75d28d, 0x00000000, + 0x3ff00000, 0x011469fb, 0xbf93ad06, 0x3c69a60b, 0xbfc8f8b8, + 0xc0000000, 0x3c626d19, 0x00000000, 0x3ff00000, 0x176d6d31, + 0xbf73b92e, 0xbc29b42c, 0xbfb917a6, 0xe0000000, 0x3c3e2718, + 0x00000000, 0x3ff00000 + // @formatter:on + }); + + private ArrayDataPointerConstant sc2 = pointerConstant(16, new int[]{ + // @formatter:off + 0x11111111, 0x3f811111, 0x55555555, 0x3fa55555 + // @formatter:on + }); + + private ArrayDataPointerConstant sc3 = pointerConstant(16, new int[]{ + // @formatter:off + 0x1a01a01a, 0xbf2a01a0, 0x16c16c17, 0xbf56c16c + // @formatter:on + }); + + private ArrayDataPointerConstant sc1 = pointerConstant(16, new int[]{ + // @formatter:off + 0x55555555, 0xbfc55555, 0x00000000, 0xbfe00000 + // @formatter:on + }); + + private ArrayDataPointerConstant piInvTable = pointerConstant(16, new int[]{ + // @formatter:off + 0x00000000, 0x00000000, 0xa2f9836e, 0x4e441529, 0xfc2757d1, + 0xf534ddc0, 0xdb629599, 0x3c439041, 0xfe5163ab, 0xdebbc561, + 0xb7246e3a, 0x424dd2e0, 0x06492eea, 0x09d1921c, 0xfe1deb1c, + 0xb129a73e, 0xe88235f5, 0x2ebb4484, 0xe99c7026, 0xb45f7e41, + 0x3991d639, 0x835339f4, 0x9c845f8b, 0xbdf9283b, 0x1ff897ff, + 0xde05980f, 0xef2f118b, 0x5a0a6d1f, 0x6d367ecf, 0x27cb09b7, + 0x4f463f66, 0x9e5fea2d, 0x7527bac7, 0xebe5f17b, 0x3d0739f7, + 0x8a5292ea, 0x6bfb5fb1, 0x1f8d5d08, 0x56033046, 0xfc7b6bab, + 0xf0cfbc21 + // @formatter:on + }); + + private ArrayDataPointerConstant pi4 = pointerConstant(8, new int[]{ + // @formatter:off + 0x40000000, 0x3fe921fb, + }); + private ArrayDataPointerConstant pi48 = pointerConstant(8, new int[]{ + 0x18469899, 0x3e64442d + // @formatter:on + }); + + private ArrayDataPointerConstant pi32Inv = pointerConstant(8, new int[]{ + // @formatter:off + 0x6dc9c883, 0x40245f30 + // @formatter:on + }); + + private ArrayDataPointerConstant shifter = pointerConstant(8, new int[]{ + // @formatter:off + 0x00000000, 0x43380000 + // @formatter:on + }); + + private ArrayDataPointerConstant signMask = pointerConstant(8, new int[]{ + // @formatter:off + 0x00000000, 0x80000000 + // @formatter:on + }); + + private ArrayDataPointerConstant p3 = pointerConstant(8, new int[]{ + // @formatter:off + 0x2e037073, 0x3b63198a + // @formatter:on + }); + + private ArrayDataPointerConstant allOnes = pointerConstant(8, new int[]{ + // @formatter:off + 0xffffffff, 0x3fefffff + // @formatter:on + }); + + private ArrayDataPointerConstant twoPow55 = pointerConstant(8, new int[]{ + // @formatter:off + 0x00000000, 0x43600000 + // @formatter:on + }); + + private ArrayDataPointerConstant twoPowM55 = pointerConstant(8, new int[]{ + // @formatter:off + 0x00000000, 0x3c800000 + // @formatter:on + }); + + private ArrayDataPointerConstant p1 = pointerConstant(8, new int[]{ + // @formatter:off + 0x54400000, 0x3fb921fb + // @formatter:on + }); + + private ArrayDataPointerConstant negZero = pointerConstant(8, new int[]{ + // @formatter:off + 0x00000000, 0x80000000 + // @formatter:on + }); + + @Override + public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { + Label block0 = new Label(); + Label block1 = new Label(); + Label block2 = new Label(); + Label block3 = new Label(); + Label block4 = new Label(); + Label block5 = new Label(); + Label block6 = new Label(); + Label block7 = new Label(); + Label block8 = new Label(); + Label block9 = new Label(); + Label block10 = new Label(); + Label block11 = new Label(); + Label block12 = new Label(); + Label block13 = new Label(); + Label block14 = new Label(); + + masm.push(AMD64.rbx); + masm.subq(rsp, 16); + masm.movsd(new AMD64Address(rsp, 8), xmm0); + masm.movl(rax, new AMD64Address(rsp, 12)); + masm.movq(xmm1, recordExternalAddress(crb, pi32Inv)); // 0x6dc9c883, 0x40245f30 + masm.movq(xmm2, recordExternalAddress(crb, shifter)); // 0x00000000, 0x43380000 + masm.andl(rax, 2147418112); + masm.subl(rax, 808452096); + masm.cmpl(rax, 281346048); + masm.jcc(ConditionFlag.Above, block0); + masm.mulsd(xmm1, xmm0); + masm.movdqu(xmm5, recordExternalAddress(crb, onehalf)); // 0x00000000, 0x3fe00000, + // 0x00000000, 0x3fe00000 + masm.movq(xmm4, recordExternalAddress(crb, signMask)); // 0x00000000, 0x80000000 + masm.pand(xmm4, xmm0); + masm.por(xmm5, xmm4); + masm.addpd(xmm1, xmm5); + masm.cvttsd2sil(rdx, xmm1); + masm.cvtsi2sdl(xmm1, rdx); + masm.movdqu(xmm6, recordExternalAddress(crb, p2)); // 0x1a600000, 0x3d90b461, + // 0x1a600000, 0x3d90b461 + masm.movq(r8, 0x3fb921fb54400000L); + masm.movdq(xmm3, r8); + masm.movdqu(xmm5, recordExternalAddress(crb, sc4)); // 0xa556c734, 0x3ec71de3, + // 0x1a01a01a, 0x3efa01a0 + masm.pshufd(xmm4, xmm0, 68); + masm.mulsd(xmm3, xmm1); + if (masm.supports(AMD64.CPUFeature.SSE3)) { + masm.movddup(xmm1, xmm1); + } else { + masm.movlhps(xmm1, xmm1); + } + masm.andl(rdx, 63); + masm.shll(rdx, 5); + masm.leaq(AMD64.rax, recordExternalAddress(crb, ctable)); + masm.addq(AMD64.rax, AMD64.rdx); + masm.mulpd(xmm6, xmm1); + masm.mulsd(xmm1, recordExternalAddress(crb, p3)); // 0x2e037073, 0x3b63198a + masm.subsd(xmm4, xmm3); + masm.movq(xmm7, new AMD64Address(AMD64.rax, 8)); + masm.subsd(xmm0, xmm3); + if (masm.supports(AMD64.CPUFeature.SSE3)) { + masm.movddup(xmm3, xmm4); + } else { + masm.movdqu(xmm3, xmm4); + masm.movlhps(xmm3, xmm3); + } + masm.subsd(xmm4, xmm6); + masm.pshufd(xmm0, xmm0, 68); + masm.movdqu(xmm2, new AMD64Address(AMD64.rax, 0)); + masm.mulpd(xmm5, xmm0); + masm.subpd(xmm0, xmm6); + masm.mulsd(xmm7, xmm4); + masm.subsd(xmm3, xmm4); + masm.mulpd(xmm5, xmm0); + masm.mulpd(xmm0, xmm0); + masm.subsd(xmm3, xmm6); + masm.movdqu(xmm6, recordExternalAddress(crb, sc2)); // 0x11111111, 0x3f811111, + // 0x55555555, 0x3fa55555 + masm.subsd(xmm1, xmm3); + masm.movq(xmm3, new AMD64Address(AMD64.rax, 24)); + masm.addsd(xmm2, xmm3); + masm.subsd(xmm7, xmm2); + masm.mulsd(xmm2, xmm4); + masm.mulpd(xmm6, xmm0); + masm.mulsd(xmm3, xmm4); + masm.mulpd(xmm2, xmm0); + masm.mulpd(xmm0, xmm0); + masm.addpd(xmm5, recordExternalAddress(crb, sc3)); // 0x1a01a01a, 0xbf2a01a0, + // 0x16c16c17, 0xbf56c16c + masm.mulsd(xmm4, new AMD64Address(AMD64.rax, 0)); + masm.addpd(xmm6, recordExternalAddress(crb, sc1)); // 0x55555555, 0xbfc55555, + // 0x00000000, 0xbfe00000 + masm.mulpd(xmm5, xmm0); + masm.movdqu(xmm0, xmm3); + masm.addsd(xmm3, new AMD64Address(AMD64.rax, 8)); + masm.mulpd(xmm1, xmm7); + masm.movdqu(xmm7, xmm4); + masm.addsd(xmm4, xmm3); + masm.addpd(xmm6, xmm5); + masm.movq(xmm5, new AMD64Address(AMD64.rax, 8)); + masm.subsd(xmm5, xmm3); + masm.subsd(xmm3, xmm4); + masm.addsd(xmm1, new AMD64Address(AMD64.rax, 16)); + masm.mulpd(xmm6, xmm2); + masm.addsd(xmm5, xmm0); + masm.addsd(xmm3, xmm7); + masm.addsd(xmm1, xmm5); + masm.addsd(xmm1, xmm3); + masm.addsd(xmm1, xmm6); + masm.unpckhpd(xmm6, xmm6); + masm.movdqu(xmm0, xmm4); + masm.addsd(xmm1, xmm6); + masm.addsd(xmm0, xmm1); + masm.jmp(block14); + + masm.bind(block0); + masm.jcc(ConditionFlag.Greater, block1); + masm.shrl(rax, 20); + masm.cmpl(rax, 3325); + masm.jcc(ConditionFlag.NotEqual, block2); + masm.mulsd(xmm0, recordExternalAddress(crb, allOnes)); // 0xffffffff, 0x3fefffff + masm.jmp(block14); + + masm.bind(block2); + masm.movq(xmm3, recordExternalAddress(crb, twoPow55)); // 0x00000000, 0x43600000 + masm.mulsd(xmm3, xmm0); + masm.subsd(xmm3, xmm0); + masm.mulsd(xmm3, recordExternalAddress(crb, twoPowM55)); // 0x00000000, 0x3c800000 + masm.jmp(block14); + + masm.bind(block1); + masm.pextrw(rax, xmm0, 3); + masm.andl(rax, 32752); + masm.cmpl(rax, 32752); + masm.jcc(ConditionFlag.Equal, block3); + masm.pextrw(rcx, xmm0, 3); + masm.andl(rcx, 32752); + masm.subl(rcx, 16224); + masm.shrl(rcx, 7); + masm.andl(rcx, 65532); + masm.leaq(r11, recordExternalAddress(crb, piInvTable)); + masm.addq(AMD64.rcx, r11); + masm.movdq(AMD64.rax, xmm0); + masm.movl(r10, new AMD64Address(AMD64.rcx, 20)); + masm.movl(r8, new AMD64Address(AMD64.rcx, 24)); + masm.movl(rdx, rax); + masm.shrq(AMD64.rax, 21); + masm.orl(rax, Integer.MIN_VALUE); + masm.shrl(rax, 11); + masm.movl(r9, r10); + masm.imulq(r10, AMD64.rdx); + masm.imulq(r9, AMD64.rax); + masm.imulq(r8, AMD64.rax); + masm.movl(rsi, new AMD64Address(AMD64.rcx, 16)); + masm.movl(rdi, new AMD64Address(AMD64.rcx, 12)); + masm.movl(r11, r10); + masm.shrq(r10, 32); + masm.addq(r9, r10); + masm.addq(r11, r8); + masm.movl(r8, r11); + masm.shrq(r11, 32); + masm.addq(r9, r11); + masm.movl(r10, rsi); + masm.imulq(rsi, AMD64.rdx); + masm.imulq(r10, AMD64.rax); + masm.movl(r11, rdi); + masm.imulq(rdi, AMD64.rdx); + masm.movl(rbx, rsi); + masm.shrq(rsi, 32); + masm.addq(r9, AMD64.rbx); + masm.movl(rbx, r9); + masm.shrq(r9, 32); + masm.addq(r10, rsi); + masm.addq(r10, r9); + masm.shlq(AMD64.rbx, 32); + masm.orq(r8, AMD64.rbx); + masm.imulq(r11, AMD64.rax); + masm.movl(r9, new AMD64Address(AMD64.rcx, 8)); + masm.movl(rsi, new AMD64Address(AMD64.rcx, 4)); + masm.movl(rbx, rdi); + masm.shrq(rdi, 32); + masm.addq(r10, AMD64.rbx); + masm.movl(rbx, r10); + masm.shrq(r10, 32); + masm.addq(r11, rdi); + masm.addq(r11, r10); + masm.movq(rdi, r9); + masm.imulq(r9, AMD64.rdx); + masm.imulq(rdi, AMD64.rax); + masm.movl(r10, r9); + masm.shrq(r9, 32); + masm.addq(r11, r10); + masm.movl(r10, r11); + masm.shrq(r11, 32); + masm.addq(rdi, r9); + masm.addq(rdi, r11); + masm.movq(r9, rsi); + masm.imulq(rsi, AMD64.rdx); + masm.imulq(r9, AMD64.rax); + masm.shlq(r10, 32); + masm.orq(r10, AMD64.rbx); + masm.movl(rax, new AMD64Address(AMD64.rcx, 0)); + masm.movl(r11, rsi); + masm.shrq(rsi, 32); + masm.addq(rdi, r11); + masm.movl(r11, rdi); + masm.shrq(rdi, 32); + masm.addq(r9, rsi); + masm.addq(r9, rdi); + masm.imulq(AMD64.rdx, AMD64.rax); + masm.pextrw(rbx, xmm0, 3); + masm.leaq(rdi, recordExternalAddress(crb, piInvTable)); + masm.subq(AMD64.rcx, rdi); + masm.addl(rcx, rcx); + masm.addl(rcx, rcx); + masm.addl(rcx, rcx); + masm.addl(rcx, 19); + masm.movl(rsi, 32768); + masm.andl(rsi, rbx); + masm.shrl(rbx, 4); + masm.andl(rbx, 2047); + masm.subl(rbx, 1023); + masm.subl(rcx, rbx); + masm.addq(r9, AMD64.rdx); + masm.movl(rdx, rcx); + masm.addl(rdx, 32); + masm.cmpl(rcx, 1); + masm.jcc(ConditionFlag.Less, block4); + masm.negl(rcx); + masm.addl(rcx, 29); + masm.shll(r9); + masm.movl(rdi, r9); + masm.andl(r9, 536870911); + masm.testl(r9, 268435456); + masm.jcc(ConditionFlag.NotEqual, block5); + masm.shrl(r9); + masm.movl(rbx, 0); + masm.shlq(r9, 32); + masm.orq(r9, r11); + + masm.bind(block6); + + masm.bind(block7); + + masm.cmpq(r9, 0); + masm.jcc(ConditionFlag.Equal, block8); + + masm.bind(block9); + masm.bsrq(r11, r9); + masm.movl(rcx, 29); + masm.subl(rcx, r11); + masm.jcc(ConditionFlag.LessEqual, block10); + masm.shlq(r9); + masm.movq(AMD64.rax, r10); + masm.shlq(r10); + masm.addl(rdx, rcx); + masm.negl(rcx); + masm.addl(rcx, 64); + masm.shrq(AMD64.rax); + masm.shrq(r8); + masm.orq(r9, AMD64.rax); + masm.orq(r10, r8); + + masm.bind(block11); + masm.cvtsi2sdq(xmm0, r9); + masm.shrq(r10, 1); + masm.cvtsi2sdq(xmm3, r10); + masm.xorpd(xmm4, xmm4); + masm.shll(rdx, 4); + masm.negl(rdx); + masm.addl(rdx, 16368); + masm.orl(rdx, rsi); + masm.xorl(rdx, rbx); + masm.pinsrw(xmm4, rdx, 3); + masm.movq(xmm2, recordExternalAddress(crb, pi4)); // 0x40000000, 0x3fe921fb, + // 0x18469899, 0x3e64442d + masm.movq(xmm6, recordExternalAddress(crb, pi48)); // 0x3fe921fb, 0x18469899, + // 0x3e64442d + masm.xorpd(xmm5, xmm5); + masm.subl(rdx, 1008); + masm.pinsrw(xmm5, rdx, 3); + masm.mulsd(xmm0, xmm4); + masm.shll(rsi, 16); + masm.sarl(rsi, 31); + masm.mulsd(xmm3, xmm5); + masm.movdqu(xmm1, xmm0); + masm.mulsd(xmm0, xmm2); + masm.shrl(rdi, 29); + masm.addsd(xmm1, xmm3); + masm.mulsd(xmm3, xmm2); + masm.addl(rdi, rsi); + masm.xorl(rdi, rsi); + masm.mulsd(xmm6, xmm1); + masm.movl(rax, rdi); + masm.addsd(xmm6, xmm3); + masm.movdqu(xmm2, xmm0); + masm.addsd(xmm0, xmm6); + masm.subsd(xmm2, xmm0); + masm.addsd(xmm6, xmm2); + + masm.bind(block12); + masm.movq(xmm1, recordExternalAddress(crb, pi32Inv)); // 0x6dc9c883, 0x40245f30 + masm.mulsd(xmm1, xmm0); + masm.movq(xmm5, recordExternalAddress(crb, onehalf)); // 0x00000000, 0x3fe00000, + // 0x00000000, 0x3fe00000 + masm.movq(xmm4, recordExternalAddress(crb, signMask)); // 0x00000000, 0x80000000 + masm.pand(xmm4, xmm0); + masm.por(xmm5, xmm4); + masm.addpd(xmm1, xmm5); + masm.cvttsd2sil(rdx, xmm1); + masm.cvtsi2sdl(xmm1, rdx); + masm.movq(xmm3, recordExternalAddress(crb, p1)); // 0x54400000, 0x3fb921fb + masm.movdqu(xmm2, recordExternalAddress(crb, p2)); // 0x1a600000, 0x3d90b461, + // 0x1a600000, 0x3d90b461 + masm.mulsd(xmm3, xmm1); + masm.unpcklpd(xmm1, xmm1); + masm.shll(rax, 3); + masm.addl(rdx, 1865216); + masm.movdqu(xmm4, xmm0); + masm.addl(rdx, rax); + masm.andl(rdx, 63); + masm.movdqu(xmm5, recordExternalAddress(crb, sc4)); // 0x54400000, 0x3fb921fb + masm.leaq(AMD64.rax, recordExternalAddress(crb, ctable)); + masm.shll(rdx, 5); + masm.addq(AMD64.rax, AMD64.rdx); + masm.mulpd(xmm2, xmm1); + masm.subsd(xmm0, xmm3); + masm.mulsd(xmm1, recordExternalAddress(crb, p3)); // 0x2e037073, 0x3b63198a + masm.subsd(xmm4, xmm3); + masm.movq(xmm7, new AMD64Address(AMD64.rax, 8)); + masm.unpcklpd(xmm0, xmm0); + masm.movdqu(xmm3, xmm4); + masm.subsd(xmm4, xmm2); + masm.mulpd(xmm5, xmm0); + masm.subpd(xmm0, xmm2); + masm.mulsd(xmm7, xmm4); + masm.subsd(xmm3, xmm4); + masm.mulpd(xmm5, xmm0); + masm.mulpd(xmm0, xmm0); + masm.subsd(xmm3, xmm2); + masm.movdqu(xmm2, new AMD64Address(AMD64.rax, 0)); + masm.subsd(xmm1, xmm3); + masm.movq(xmm3, new AMD64Address(AMD64.rax, 24)); + masm.addsd(xmm2, xmm3); + masm.subsd(xmm7, xmm2); + masm.subsd(xmm1, xmm6); + masm.movdqu(xmm6, recordExternalAddress(crb, sc2)); // 0x11111111, 0x3f811111, + // 0x55555555, 0x3fa55555 + masm.mulsd(xmm2, xmm4); + masm.mulpd(xmm6, xmm0); + masm.mulsd(xmm3, xmm4); + masm.mulpd(xmm2, xmm0); + masm.mulpd(xmm0, xmm0); + masm.addpd(xmm5, recordExternalAddress(crb, sc3)); // 0x1a01a01a, 0xbf2a01a0, + // 0x16c16c17, 0xbf56c16c + masm.mulsd(xmm4, new AMD64Address(AMD64.rax, 0)); + masm.addpd(xmm6, recordExternalAddress(crb, sc1)); // 0x55555555, 0xbfc55555, + // 0x00000000, 0xbfe00000 + masm.mulpd(xmm5, xmm0); + masm.movdqu(xmm0, xmm3); + masm.addsd(xmm3, new AMD64Address(AMD64.rax, 8)); + masm.mulpd(xmm1, xmm7); + masm.movdqu(xmm7, xmm4); + masm.addsd(xmm4, xmm3); + masm.addpd(xmm6, xmm5); + masm.movq(xmm5, new AMD64Address(AMD64.rax, 8)); + masm.subsd(xmm5, xmm3); + masm.subsd(xmm3, xmm4); + masm.addsd(xmm1, new AMD64Address(AMD64.rax, 16)); + masm.mulpd(xmm6, xmm2); + masm.addsd(xmm5, xmm0); + masm.addsd(xmm3, xmm7); + masm.addsd(xmm1, xmm5); + masm.addsd(xmm1, xmm3); + masm.addsd(xmm1, xmm6); + masm.unpckhpd(xmm6, xmm6); + masm.movdqu(xmm0, xmm4); + masm.addsd(xmm1, xmm6); + masm.addsd(xmm0, xmm1); + masm.jmp(block14); + + masm.bind(block8); + masm.addl(rdx, 64); + masm.movq(r9, r10); + masm.movq(r10, r8); + masm.movl(r8, 0); + masm.cmpq(r9, 0); + masm.jcc(ConditionFlag.NotEqual, block9); + masm.addl(rdx, 64); + masm.movq(r9, r10); + masm.movq(r10, r8); + masm.cmpq(r9, 0); + masm.jcc(ConditionFlag.NotEqual, block9); + masm.xorpd(xmm0, xmm0); + masm.xorpd(xmm6, xmm6); + masm.jmp(block12); + + masm.bind(block10); + masm.jcc(ConditionFlag.Equal, block11); + masm.negl(rcx); + masm.shrq(r10); + masm.movq(AMD64.rax, r9); + masm.shrq(r9); + masm.subl(rdx, rcx); + masm.negl(rcx); + masm.addl(rcx, 64); + masm.shlq(AMD64.rax); + masm.orq(r10, AMD64.rax); + masm.jmp(block11); + + masm.bind(block4); + masm.negl(rcx); + masm.shlq(r9, 32); + masm.orq(r9, r11); + masm.shlq(r9); + masm.movq(rdi, r9); + masm.testl(r9, Integer.MIN_VALUE); + masm.jcc(ConditionFlag.NotEqual, block13); + masm.shrl(r9); + masm.movl(rbx, 0); + masm.shrq(rdi, 3); + masm.jmp(block7); + + masm.bind(block5); + masm.shrl(r9); + masm.movl(rbx, 536870912); + masm.shrl(rbx); + masm.shlq(r9, 32); + masm.orq(r9, r11); + masm.shlq(AMD64.rbx, 32); + masm.addl(rdi, 536870912); + masm.movl(AMD64.rcx, 0); + masm.movl(r11, 0); + masm.subq(AMD64.rcx, r8); + masm.sbbq(r11, r10); + masm.sbbq(AMD64.rbx, r9); + masm.movq(r8, AMD64.rcx); + masm.movq(r10, r11); + masm.movq(r9, AMD64.rbx); + masm.movl(rbx, 32768); + masm.jmp(block6); + + masm.bind(block13); + masm.shrl(r9); + masm.movq(AMD64.rbx, 0x100000000L); + masm.shrq(AMD64.rbx); + masm.movl(AMD64.rcx, 0); + masm.movl(r11, 0); + masm.subq(AMD64.rcx, r8); + masm.sbbq(r11, r10); + masm.sbbq(AMD64.rbx, r9); + masm.movq(r8, AMD64.rcx); + masm.movq(r10, r11); + masm.movq(r9, AMD64.rbx); + masm.movl(rbx, 32768); + masm.shrq(rdi, 3); + masm.addl(rdi, 536870912); + masm.jmp(block7); + + masm.bind(block3); + masm.movq(xmm0, new AMD64Address(rsp, 8)); + masm.mulsd(xmm0, recordExternalAddress(crb, negZero)); // 0x00000000, 0x80000000 + masm.movq(new AMD64Address(rsp, 0), xmm0); + + masm.bind(block14); + masm.addq(rsp, 16); + masm.pop(AMD64.rbx); + } +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathTanOp.java 2019-03-12 08:10:48.780047212 +0100 @@ -0,0 +1,1119 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, Intel Corporation. All rights reserved. + * Intel Math Library (LIBM) Source Code + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.lir.amd64; + +import static jdk.vm.ci.amd64.AMD64.r10; +import static jdk.vm.ci.amd64.AMD64.r11; +import static jdk.vm.ci.amd64.AMD64.r8; +import static jdk.vm.ci.amd64.AMD64.r9; +import static jdk.vm.ci.amd64.AMD64.rax; +import static jdk.vm.ci.amd64.AMD64.rbx; +import static jdk.vm.ci.amd64.AMD64.rcx; +import static jdk.vm.ci.amd64.AMD64.rdi; +import static jdk.vm.ci.amd64.AMD64.rdx; +import static jdk.vm.ci.amd64.AMD64.rsi; +import static jdk.vm.ci.amd64.AMD64.rsp; +import static jdk.vm.ci.amd64.AMD64.xmm0; +import static jdk.vm.ci.amd64.AMD64.xmm1; +import static jdk.vm.ci.amd64.AMD64.xmm2; +import static jdk.vm.ci.amd64.AMD64.xmm3; +import static jdk.vm.ci.amd64.AMD64.xmm4; +import static jdk.vm.ci.amd64.AMD64.xmm5; +import static jdk.vm.ci.amd64.AMD64.xmm6; +import static jdk.vm.ci.amd64.AMD64.xmm7; +import static org.graalvm.compiler.lir.amd64.AMD64HotSpotHelper.pointerConstant; +import static org.graalvm.compiler.lir.amd64.AMD64HotSpotHelper.recordExternalAddress; + +import org.graalvm.compiler.asm.Label; +import org.graalvm.compiler.asm.amd64.AMD64Address; +import org.graalvm.compiler.asm.amd64.AMD64Assembler; +import org.graalvm.compiler.asm.amd64.AMD64MacroAssembler; +import org.graalvm.compiler.lir.LIRInstructionClass; +import org.graalvm.compiler.lir.asm.ArrayDataPointerConstant; +import org.graalvm.compiler.lir.asm.CompilationResultBuilder; + +import jdk.vm.ci.amd64.AMD64; + +/** + * 
+ *                     ALGORITHM DESCRIPTION - TAN()
+ *                     ---------------------
+ *
+ * Polynomials coefficients and other constants.
+ *
+ * Note that in this algorithm, there is a different polynomial for
+ * each breakpoint, so there are 32 sets of polynomial coefficients
+ * as well as 32 instances of the other constants.
+ *
+ * The polynomial coefficients and constants are offset from the start
+ * of the main block as follows:
+ *
+ *   0:  c8 | c0
+ *  16:  c9 | c1
+ *  32: c10 | c2
+ *  48: c11 | c3
+ *  64: c12 | c4
+ *  80: c13 | c5
+ *  96: c14 | c6
+ * 112: c15 | c7
+ * 128: T_hi
+ * 136: T_lo
+ * 144: Sigma
+ * 152: T_hl
+ * 160: Tau
+ * 168: Mask
+ * 176: (end of block)
+ *
+ * The total table size is therefore 5632 bytes.
+ *
+ * Note that c0 and c1 are always zero. We could try storing
+ * other constants here, and just loading the low part of the
+ * SIMD register in these cases, after ensuring the high part
+ * is zero.
+ *
+ * The higher terms of the polynomial are computed in the *low*
+ * part of the SIMD register. This is so we can overlap the
+ * multiplication by r^8 and the unpacking of the other part.
+ *
+ * The constants are:
+ * T_hi + T_lo = accurate constant term in power series
+ * Sigma + T_hl = accurate coefficient of r in power series (Sigma=1 bit)
+ * Tau = multiplier for the reciprocal, always -1 or 0
+ *
+ * The basic reconstruction formula using these constants is:
+ *
+ * High = tau * recip_hi + t_hi
+ * Med = (sgn * r + t_hl * r)_hi
+ * Low = (sgn * r + t_hl * r)_lo +
+ *       tau * recip_lo + T_lo + (T_hl + sigma) * c + pol
+ *
+ * where pol = c0 + c1 * r + c2 * r^2 + ... + c15 * r^15
+ *
+ * (c0 = c1 = 0, but using them keeps SIMD regularity)
+ *
+ * We then do a compensated sum High + Med, add the low parts together
+ * and then do the final sum.
+ *
+ * Here recip_hi + recip_lo is an accurate reciprocal of the remainder
+ * modulo pi/2
+ *
+ * Special cases:
+ *  tan(NaN) = quiet NaN, and raise invalid exception
+ *  tan(INF) = NaN and raise invalid exception
+ *  tan(+/-0) = +/-0
+ *
+ */ +public final class AMD64MathTanOp extends AMD64MathIntrinsicUnaryOp { + + public static final LIRInstructionClass TYPE = LIRInstructionClass.create(AMD64MathTanOp.class); + + public AMD64MathTanOp() { + super(TYPE, /* GPR */ rax, rcx, rdx, rbx, rsi, rdi, r8, r9, r10, r11, + /* XMM */ xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7); + } + + private ArrayDataPointerConstant onehalf = pointerConstant(16, new int[]{ + // @formatter:off + 0x00000000, 0x3fe00000, 0x00000000, 0x3fe00000 + // @formatter:on + }); + + private ArrayDataPointerConstant mul16 = pointerConstant(16, new int[]{ + // @formatter:off + 0x00000000, 0x40300000, 0x00000000, 0x3ff00000 + // @formatter:on + }); + + private ArrayDataPointerConstant signMask = pointerConstant(16, new int[]{ + // @formatter:off + 0x00000000, 0x80000000, 0x00000000, 0x80000000 + // @formatter:on + }); + + private ArrayDataPointerConstant pi32Inv = pointerConstant(16, new int[]{ + // @formatter:off + 0x6dc9c883, 0x3fe45f30, 0x6dc9c883, 0x40245f30 + // @formatter:on + }); + + private ArrayDataPointerConstant p1 = pointerConstant(16, new int[]{ + // @formatter:off + 0x54444000, 0x3fb921fb, 0x54440000, 0x3fb921fb + // @formatter:on + }); + + private ArrayDataPointerConstant p2 = pointerConstant(16, new int[]{ + // @formatter:off + 0x67674000, 0xbd32e7b9, 0x4c4c0000, 0x3d468c23 + // @formatter:on + }); + + private ArrayDataPointerConstant p3 = pointerConstant(16, new int[]{ + // @formatter:off + 0x3707344a, 0x3aa8a2e0, 0x03707345, 0x3ae98a2e + // @formatter:on + }); + + private ArrayDataPointerConstant ctable = pointerConstant(16, new int[]{ + // @formatter:off + 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x882c10fa, + 0x3f9664f4, 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x55e6c23d, 0x3f8226e3, 0x55555555, + 0x3fd55555, 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x0e157de0, 0x3f6d6d3d, 0x11111111, 0x3fc11111, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x452b75e3, 0x3f57da36, + 0x1ba1ba1c, 0x3faba1ba, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x3ff00000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x4e435f9b, + 0x3f953f83, 0x00000000, 0x00000000, 0x3c6e8e46, 0x3f9b74ea, + 0x00000000, 0x00000000, 0xda5b7511, 0x3f85ad63, 0xdc230b9b, + 0x3fb97558, 0x26cb3788, 0x3f881308, 0x76fc4985, 0x3fd62ac9, + 0x77bb08ba, 0x3f757c85, 0xb6247521, 0x3fb1381e, 0x5922170c, + 0x3f754e95, 0x8746482d, 0x3fc27f83, 0x11055b30, 0x3f64e391, + 0x3e666320, 0x3fa3e609, 0x0de9dae3, 0x3f6301df, 0x1f1dca06, + 0x3fafa8ae, 0x8c5b2da2, 0x3fb936bb, 0x4e88f7a5, 0x3c587d05, + 0x00000000, 0x3ff00000, 0xa8935dd9, 0x3f83dde2, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x5a279ea3, 0x3faa3407, + 0x00000000, 0x00000000, 0x432d65fa, 0x3fa70153, 0x00000000, + 0x00000000, 0x891a4602, 0x3f9d03ef, 0xd62ca5f8, 0x3fca77d9, + 0xb35f4628, 0x3f97a265, 0x433258fa, 0x3fd8cf51, 0xb58fd909, + 0x3f8f88e3, 0x01771cea, 0x3fc2b154, 0xf3562f8e, 0x3f888f57, + 0xc028a723, 0x3fc7370f, 0x20b7f9f0, 0x3f80f44c, 0x214368e9, + 0x3fb6dfaa, 0x28891863, 0x3f79b4b6, 0x172dbbf0, 0x3fb6cb8e, + 0xe0553158, 0x3fc975f5, 0x593fe814, 0x3c2ef5d3, 0x00000000, + 0x3ff00000, 0x03dec550, 0x3fa44203, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x9314533e, 0x3fbb8ec5, 0x00000000, + 0x00000000, 0x09aa36d0, 0x3fb6d3f4, 0x00000000, 0x00000000, + 0xdcb427fd, 0x3fb13950, 0xd87ab0bb, 0x3fd5335e, 0xce0ae8a5, + 0x3fabb382, 0x79143126, 0x3fddba41, 0x5f2b28d4, 0x3fa552f1, + 0x59f21a6d, 0x3fd015ab, 0x22c27d95, 0x3fa0e984, 0xe19fc6aa, + 0x3fd0576c, 0x8f2c2950, 0x3f9a4898, 0xc0b3f22c, 0x3fc59462, + 0x1883a4b8, 0x3f94b61c, 0x3f838640, 0x3fc30eb8, 0x355c63dc, + 0x3fd36a08, 0x1dce993d, 0xbc6d704d, 0x00000000, 0x3ff00000, + 0x2b82ab63, 0x3fb78e92, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x56f37042, 0x3fccfc56, 0x00000000, 0x00000000, + 0xaa563951, 0x3fc90125, 0x00000000, 0x00000000, 0x3d0e7c5d, + 0x3fc50533, 0x9bed9b2e, 0x3fdf0ed9, 0x5fe7c47c, 0x3fc1f250, + 0x96c125e5, 0x3fe2edd9, 0x5a02bbd8, 0x3fbe5c71, 0x86362c20, + 0x3fda08b7, 0x4b4435ed, 0x3fb9d342, 0x4b494091, 0x3fd911bd, + 0xb56658be, 0x3fb5e4c7, 0x93a2fd76, 0x3fd3c092, 0xda271794, + 0x3fb29910, 0x3303df2b, 0x3fd189be, 0x99fcef32, 0x3fda8279, + 0xb68c1467, 0x3c708b2f, 0x00000000, 0x3ff00000, 0x980c4337, + 0x3fc5f619, 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0xcc03e501, 0x3fdff10f, 0x00000000, 0x00000000, 0x44a4e845, + 0x3fddb63b, 0x00000000, 0x00000000, 0x3768ad9f, 0x3fdb72a4, + 0x3dd01cca, 0x3fe5fdb9, 0xa61d2811, 0x3fd972b2, 0x5645ad0b, + 0x3fe977f9, 0xd013b3ab, 0x3fd78ca3, 0xbf0bf914, 0x3fe4f192, + 0x4d53e730, 0x3fd5d060, 0x3f8b9000, 0x3fe49933, 0xe2b82f08, + 0x3fd4322a, 0x5936a835, 0x3fe27ae1, 0xb1c61c9b, 0x3fd2b3fb, + 0xef478605, 0x3fe1659e, 0x190834ec, 0x3fe11ab7, 0xcdb625ea, + 0xbc8e564b, 0x00000000, 0x3ff00000, 0xb07217e3, 0x3fd248f1, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x2b2c49d0, + 0x3ff2de9c, 0x00000000, 0x00000000, 0x2655bc98, 0x3ff33e58, + 0x00000000, 0x00000000, 0xff691fa2, 0x3ff3972e, 0xe93463bd, + 0x3feeed87, 0x070e10a0, 0x3ff3f5b2, 0xf4d790a4, 0x3ff20c10, + 0xa04e8ea3, 0x3ff4541a, 0x386accd3, 0x3ff1369e, 0x222a66dd, + 0x3ff4b521, 0x22a9777e, 0x3ff20817, 0x52a04a6e, 0x3ff5178f, + 0xddaa0031, 0x3ff22137, 0x4447d47c, 0x3ff57c01, 0x1e9c7f1d, + 0x3ff29311, 0x2ab7f990, 0x3fe561b8, 0x209c7df1, 0x3c87a8c5, + 0x00000000, 0x3ff00000, 0x4170bcc6, 0x3fdc92d8, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0xc7ab4d5a, 0x40085e24, + 0x00000000, 0x00000000, 0xe93ea75d, 0x400b963d, 0x00000000, + 0x00000000, 0x94a7f25a, 0x400f37e2, 0x4b6261cb, 0x3ff5f984, + 0x5a9dd812, 0x4011aab0, 0x74c30018, 0x3ffaf5a5, 0x7f2ce8e3, + 0x4013fe8b, 0xfe8e54fa, 0x3ffd7334, 0x670d618d, 0x4016a10c, + 0x4db97058, 0x4000e012, 0x24df44dd, 0x40199c5f, 0x697d6ece, + 0x4003006e, 0x83298b82, 0x401cfc4d, 0x19d490d6, 0x40058c19, + 0x2ae42850, 0x3fea4300, 0x118e20e6, 0xbc7a6db8, 0x00000000, + 0x40000000, 0xe33345b8, 0xbfd4e526, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x65965966, 0x40219659, 0x00000000, + 0x00000000, 0x882c10fa, 0x402664f4, 0x00000000, 0x00000000, + 0x83cd3723, 0x402c8342, 0x00000000, 0x40000000, 0x55e6c23d, + 0x403226e3, 0x55555555, 0x40055555, 0x34451939, 0x40371c96, + 0xaaaaaaab, 0x400aaaaa, 0x0e157de0, 0x403d6d3d, 0x11111111, + 0x40111111, 0xa738201f, 0x4042bbce, 0x05b05b06, 0x4015b05b, + 0x452b75e3, 0x4047da36, 0x1ba1ba1c, 0x401ba1ba, 0x00000000, + 0x3ff00000, 0x00000000, 0x00000000, 0x00000000, 0x40000000, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x4f48b8d3, 0xbf33eaf9, 0x00000000, 0x00000000, + 0x0cf7586f, 0x3f20b8ea, 0x00000000, 0x00000000, 0xd0258911, + 0xbf0abaf3, 0x23e49fe9, 0xbfab5a8c, 0x2d53222e, 0x3ef60d15, + 0x21169451, 0x3fa172b2, 0xbb254dbc, 0xbee1d3b5, 0xdbf93b8e, + 0xbf84c7db, 0x05b4630b, 0x3ecd3364, 0xee9aada7, 0x3f743924, + 0x794a8297, 0xbeb7b7b9, 0xe015f797, 0xbf5d41f5, 0xe41a4a56, + 0x3ea35dfb, 0xe4c2a251, 0x3f49a2ab, 0x5af9e000, 0xbfce49ce, + 0x8c743719, 0x3d1eb860, 0x00000000, 0x00000000, 0x1b4863cf, + 0x3fd78294, 0x00000000, 0x3ff00000, 0x00000000, 0xfffffff8, + 0x535ad890, 0xbf2b9320, 0x00000000, 0x00000000, 0x018fdf1f, + 0x3f16d61d, 0x00000000, 0x00000000, 0x0359f1be, 0xbf0139e4, + 0xa4317c6d, 0xbfa67e17, 0x82672d0f, 0x3eebb405, 0x2f1b621e, + 0x3f9f455b, 0x51ccf238, 0xbed55317, 0xf437b9ac, 0xbf804bee, + 0xc791a2b5, 0x3ec0e993, 0x919a1db2, 0x3f7080c2, 0x336a5b0e, + 0xbeaa48a2, 0x0a268358, 0xbf55a443, 0xdfd978e4, 0x3e94b61f, + 0xd7767a58, 0x3f431806, 0x2aea0000, 0xbfc9bbe8, 0x7723ea61, + 0xbd3a2369, 0x00000000, 0x00000000, 0xdf7796ff, 0x3fd6e642, + 0x00000000, 0x3ff00000, 0x00000000, 0xfffffff8, 0xb9ff07ce, + 0xbf231c78, 0x00000000, 0x00000000, 0xa5517182, 0x3f0ff0e0, + 0x00000000, 0x00000000, 0x790b4cbc, 0xbef66191, 0x848a46c6, + 0xbfa21ac0, 0xb16435fa, 0x3ee1d3ec, 0x2a1aa832, 0x3f9c71ea, + 0xfdd299ef, 0xbec9dd1a, 0x3f8dbaaf, 0xbf793363, 0x309fc6ea, + 0x3eb415d6, 0xbee60471, 0x3f6b83ba, 0x94a0a697, 0xbe9dae11, + 0x3e5c67b3, 0xbf4fd07b, 0x9a8f3e3e, 0x3e86bd75, 0xa4beb7a4, + 0x3f3d1eb1, 0x29cfc000, 0xbfc549ce, 0xbf159358, 0xbd397b33, + 0x00000000, 0x00000000, 0x871fee6c, 0x3fd666f0, 0x00000000, + 0x3ff00000, 0x00000000, 0xfffffff8, 0x7d98a556, 0xbf1a3958, + 0x00000000, 0x00000000, 0x9d88dc01, 0x3f0704c2, 0x00000000, + 0x00000000, 0x73742a2b, 0xbeed054a, 0x58844587, 0xbf9c2a13, + 0x55688a79, 0x3ed7a326, 0xee33f1d6, 0x3f9a48f4, 0xa8dc9888, + 0xbebf8939, 0xaad4b5b8, 0xbf72f746, 0x9102efa1, 0x3ea88f82, + 0xdabc29cf, 0x3f678228, 0x9289afb8, 0xbe90f456, 0x741fb4ed, + 0xbf46f3a3, 0xa97f6663, 0x3e79b4bf, 0xca89ff3f, 0x3f36db70, + 0xa8a2a000, 0xbfc0ee13, 0x3da24be1, 0xbd338b9f, 0x00000000, + 0x00000000, 0x11cd6c69, 0x3fd601fd, 0x00000000, 0x3ff00000, + 0x00000000, 0xfffffff8, 0x1a154b97, 0xbf116b01, 0x00000000, + 0x00000000, 0x2d427630, 0x3f0147bf, 0x00000000, 0x00000000, + 0xb93820c8, 0xbee264d4, 0xbb6cbb18, 0xbf94ab8c, 0x888d4d92, + 0x3ed0568b, 0x60730f7c, 0x3f98b19b, 0xe4b1fb11, 0xbeb2f950, + 0x22cf9f74, 0xbf6b21cd, 0x4a3ff0a6, 0x3e9f499e, 0xfd2b83ce, + 0x3f64aad7, 0x637b73af, 0xbe83487c, 0xe522591a, 0xbf3fc092, + 0xa158e8bc, 0x3e6e3aae, 0xe5e82ffa, 0x3f329d2f, 0xd636a000, + 0xbfb9477f, 0xc2c2d2bc, 0xbd135ef9, 0x00000000, 0x00000000, + 0xf2fdb123, 0x3fd5b566, 0x00000000, 0x3ff00000, 0x00000000, + 0xfffffff8, 0xc41acb64, 0xbf05448d, 0x00000000, 0x00000000, + 0xdbb03d6f, 0x3efb7ad2, 0x00000000, 0x00000000, 0x9e42962d, + 0xbed5aea5, 0x2579f8ef, 0xbf8b2398, 0x288a1ed9, 0x3ec81441, + 0xb0198dc5, 0x3f979a3a, 0x2fdfe253, 0xbea57cd3, 0x5766336f, + 0xbf617caa, 0x600944c3, 0x3e954ed6, 0xa4e0aaf8, 0x3f62c646, + 0x6b8fb29c, 0xbe74e3a3, 0xdc4c0409, 0xbf33f952, 0x9bffe365, + 0x3e6301ec, 0xb8869e44, 0x3f2fc566, 0xe1e04000, 0xbfb0cc62, + 0x016b907f, 0xbd119cbc, 0x00000000, 0x00000000, 0xe6b9d8fa, + 0x3fd57fb3, 0x00000000, 0x3ff00000, 0x00000000, 0xfffffff8, + 0x5daf22a6, 0xbef429d7, 0x00000000, 0x00000000, 0x06bca545, + 0x3ef7a27d, 0x00000000, 0x00000000, 0x7211c19a, 0xbec41c3e, + 0x956ed53e, 0xbf7ae3f4, 0xee750e72, 0x3ec3901b, 0x91d443f5, + 0x3f96f713, 0x36661e6c, 0xbe936e09, 0x506f9381, 0xbf5122e8, + 0xcb6dd43f, 0x3e9041b9, 0x6698b2ff, 0x3f61b0c7, 0x576bf12b, + 0xbe625a8a, 0xe5a0e9dc, 0xbf23499d, 0x110384dd, 0x3e5b1c2c, + 0x68d43db6, 0x3f2cb899, 0x6ecac000, 0xbfa0c414, 0xcd7dd58c, + 0x3d13500f, 0x00000000, 0x00000000, 0x85a2c8fb, 0x3fd55fe0, + 0x00000000, 0x3ff00000, 0x00000000, 0xfffffff8, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x2bf70ebe, 0x3ef66a8f, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0xd644267f, 0x3ec22805, 0x16c16c17, 0x3f96c16c, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0xc4e09162, + 0x3e8d6db2, 0xbc011567, 0x3f61566a, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x1f79955c, 0x3e57da4e, 0x9334ef0b, + 0x3f2bbd77, 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x55555555, 0x3fd55555, 0x00000000, + 0x3ff00000, 0x00000000, 0xfffffff8, 0x5daf22a6, 0x3ef429d7, + 0x00000000, 0x00000000, 0x06bca545, 0x3ef7a27d, 0x00000000, + 0x00000000, 0x7211c19a, 0x3ec41c3e, 0x956ed53e, 0x3f7ae3f4, + 0xee750e72, 0x3ec3901b, 0x91d443f5, 0x3f96f713, 0x36661e6c, + 0x3e936e09, 0x506f9381, 0x3f5122e8, 0xcb6dd43f, 0x3e9041b9, + 0x6698b2ff, 0x3f61b0c7, 0x576bf12b, 0x3e625a8a, 0xe5a0e9dc, + 0x3f23499d, 0x110384dd, 0x3e5b1c2c, 0x68d43db6, 0x3f2cb899, + 0x6ecac000, 0x3fa0c414, 0xcd7dd58c, 0xbd13500f, 0x00000000, + 0x00000000, 0x85a2c8fb, 0x3fd55fe0, 0x00000000, 0x3ff00000, + 0x00000000, 0xfffffff8, 0xc41acb64, 0x3f05448d, 0x00000000, + 0x00000000, 0xdbb03d6f, 0x3efb7ad2, 0x00000000, 0x00000000, + 0x9e42962d, 0x3ed5aea5, 0x2579f8ef, 0x3f8b2398, 0x288a1ed9, + 0x3ec81441, 0xb0198dc5, 0x3f979a3a, 0x2fdfe253, 0x3ea57cd3, + 0x5766336f, 0x3f617caa, 0x600944c3, 0x3e954ed6, 0xa4e0aaf8, + 0x3f62c646, 0x6b8fb29c, 0x3e74e3a3, 0xdc4c0409, 0x3f33f952, + 0x9bffe365, 0x3e6301ec, 0xb8869e44, 0x3f2fc566, 0xe1e04000, + 0x3fb0cc62, 0x016b907f, 0x3d119cbc, 0x00000000, 0x00000000, + 0xe6b9d8fa, 0x3fd57fb3, 0x00000000, 0x3ff00000, 0x00000000, + 0xfffffff8, 0x1a154b97, 0x3f116b01, 0x00000000, 0x00000000, + 0x2d427630, 0x3f0147bf, 0x00000000, 0x00000000, 0xb93820c8, + 0x3ee264d4, 0xbb6cbb18, 0x3f94ab8c, 0x888d4d92, 0x3ed0568b, + 0x60730f7c, 0x3f98b19b, 0xe4b1fb11, 0x3eb2f950, 0x22cf9f74, + 0x3f6b21cd, 0x4a3ff0a6, 0x3e9f499e, 0xfd2b83ce, 0x3f64aad7, + 0x637b73af, 0x3e83487c, 0xe522591a, 0x3f3fc092, 0xa158e8bc, + 0x3e6e3aae, 0xe5e82ffa, 0x3f329d2f, 0xd636a000, 0x3fb9477f, + 0xc2c2d2bc, 0x3d135ef9, 0x00000000, 0x00000000, 0xf2fdb123, + 0x3fd5b566, 0x00000000, 0x3ff00000, 0x00000000, 0xfffffff8, + 0x7d98a556, 0x3f1a3958, 0x00000000, 0x00000000, 0x9d88dc01, + 0x3f0704c2, 0x00000000, 0x00000000, 0x73742a2b, 0x3eed054a, + 0x58844587, 0x3f9c2a13, 0x55688a79, 0x3ed7a326, 0xee33f1d6, + 0x3f9a48f4, 0xa8dc9888, 0x3ebf8939, 0xaad4b5b8, 0x3f72f746, + 0x9102efa1, 0x3ea88f82, 0xdabc29cf, 0x3f678228, 0x9289afb8, + 0x3e90f456, 0x741fb4ed, 0x3f46f3a3, 0xa97f6663, 0x3e79b4bf, + 0xca89ff3f, 0x3f36db70, 0xa8a2a000, 0x3fc0ee13, 0x3da24be1, + 0x3d338b9f, 0x00000000, 0x00000000, 0x11cd6c69, 0x3fd601fd, + 0x00000000, 0x3ff00000, 0x00000000, 0xfffffff8, 0xb9ff07ce, + 0x3f231c78, 0x00000000, 0x00000000, 0xa5517182, 0x3f0ff0e0, + 0x00000000, 0x00000000, 0x790b4cbc, 0x3ef66191, 0x848a46c6, + 0x3fa21ac0, 0xb16435fa, 0x3ee1d3ec, 0x2a1aa832, 0x3f9c71ea, + 0xfdd299ef, 0x3ec9dd1a, 0x3f8dbaaf, 0x3f793363, 0x309fc6ea, + 0x3eb415d6, 0xbee60471, 0x3f6b83ba, 0x94a0a697, 0x3e9dae11, + 0x3e5c67b3, 0x3f4fd07b, 0x9a8f3e3e, 0x3e86bd75, 0xa4beb7a4, + 0x3f3d1eb1, 0x29cfc000, 0x3fc549ce, 0xbf159358, 0x3d397b33, + 0x00000000, 0x00000000, 0x871fee6c, 0x3fd666f0, 0x00000000, + 0x3ff00000, 0x00000000, 0xfffffff8, 0x535ad890, 0x3f2b9320, + 0x00000000, 0x00000000, 0x018fdf1f, 0x3f16d61d, 0x00000000, + 0x00000000, 0x0359f1be, 0x3f0139e4, 0xa4317c6d, 0x3fa67e17, + 0x82672d0f, 0x3eebb405, 0x2f1b621e, 0x3f9f455b, 0x51ccf238, + 0x3ed55317, 0xf437b9ac, 0x3f804bee, 0xc791a2b5, 0x3ec0e993, + 0x919a1db2, 0x3f7080c2, 0x336a5b0e, 0x3eaa48a2, 0x0a268358, + 0x3f55a443, 0xdfd978e4, 0x3e94b61f, 0xd7767a58, 0x3f431806, + 0x2aea0000, 0x3fc9bbe8, 0x7723ea61, 0x3d3a2369, 0x00000000, + 0x00000000, 0xdf7796ff, 0x3fd6e642, 0x00000000, 0x3ff00000, + 0x00000000, 0xfffffff8, 0x4f48b8d3, 0x3f33eaf9, 0x00000000, + 0x00000000, 0x0cf7586f, 0x3f20b8ea, 0x00000000, 0x00000000, + 0xd0258911, 0x3f0abaf3, 0x23e49fe9, 0x3fab5a8c, 0x2d53222e, + 0x3ef60d15, 0x21169451, 0x3fa172b2, 0xbb254dbc, 0x3ee1d3b5, + 0xdbf93b8e, 0x3f84c7db, 0x05b4630b, 0x3ecd3364, 0xee9aada7, + 0x3f743924, 0x794a8297, 0x3eb7b7b9, 0xe015f797, 0x3f5d41f5, + 0xe41a4a56, 0x3ea35dfb, 0xe4c2a251, 0x3f49a2ab, 0x5af9e000, + 0x3fce49ce, 0x8c743719, 0xbd1eb860, 0x00000000, 0x00000000, + 0x1b4863cf, 0x3fd78294, 0x00000000, 0x3ff00000, 0x00000000, + 0xfffffff8, 0x65965966, 0xc0219659, 0x00000000, 0x00000000, + 0x882c10fa, 0x402664f4, 0x00000000, 0x00000000, 0x83cd3723, + 0xc02c8342, 0x00000000, 0xc0000000, 0x55e6c23d, 0x403226e3, + 0x55555555, 0x40055555, 0x34451939, 0xc0371c96, 0xaaaaaaab, + 0xc00aaaaa, 0x0e157de0, 0x403d6d3d, 0x11111111, 0x40111111, + 0xa738201f, 0xc042bbce, 0x05b05b06, 0xc015b05b, 0x452b75e3, + 0x4047da36, 0x1ba1ba1c, 0x401ba1ba, 0x00000000, 0xbff00000, + 0x00000000, 0x00000000, 0x00000000, 0x40000000, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0xc7ab4d5a, 0xc0085e24, 0x00000000, 0x00000000, 0xe93ea75d, + 0x400b963d, 0x00000000, 0x00000000, 0x94a7f25a, 0xc00f37e2, + 0x4b6261cb, 0xbff5f984, 0x5a9dd812, 0x4011aab0, 0x74c30018, + 0x3ffaf5a5, 0x7f2ce8e3, 0xc013fe8b, 0xfe8e54fa, 0xbffd7334, + 0x670d618d, 0x4016a10c, 0x4db97058, 0x4000e012, 0x24df44dd, + 0xc0199c5f, 0x697d6ece, 0xc003006e, 0x83298b82, 0x401cfc4d, + 0x19d490d6, 0x40058c19, 0x2ae42850, 0xbfea4300, 0x118e20e6, + 0x3c7a6db8, 0x00000000, 0x40000000, 0xe33345b8, 0xbfd4e526, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x2b2c49d0, + 0xbff2de9c, 0x00000000, 0x00000000, 0x2655bc98, 0x3ff33e58, + 0x00000000, 0x00000000, 0xff691fa2, 0xbff3972e, 0xe93463bd, + 0xbfeeed87, 0x070e10a0, 0x3ff3f5b2, 0xf4d790a4, 0x3ff20c10, + 0xa04e8ea3, 0xbff4541a, 0x386accd3, 0xbff1369e, 0x222a66dd, + 0x3ff4b521, 0x22a9777e, 0x3ff20817, 0x52a04a6e, 0xbff5178f, + 0xddaa0031, 0xbff22137, 0x4447d47c, 0x3ff57c01, 0x1e9c7f1d, + 0x3ff29311, 0x2ab7f990, 0xbfe561b8, 0x209c7df1, 0xbc87a8c5, + 0x00000000, 0x3ff00000, 0x4170bcc6, 0x3fdc92d8, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0xcc03e501, 0xbfdff10f, + 0x00000000, 0x00000000, 0x44a4e845, 0x3fddb63b, 0x00000000, + 0x00000000, 0x3768ad9f, 0xbfdb72a4, 0x3dd01cca, 0xbfe5fdb9, + 0xa61d2811, 0x3fd972b2, 0x5645ad0b, 0x3fe977f9, 0xd013b3ab, + 0xbfd78ca3, 0xbf0bf914, 0xbfe4f192, 0x4d53e730, 0x3fd5d060, + 0x3f8b9000, 0x3fe49933, 0xe2b82f08, 0xbfd4322a, 0x5936a835, + 0xbfe27ae1, 0xb1c61c9b, 0x3fd2b3fb, 0xef478605, 0x3fe1659e, + 0x190834ec, 0xbfe11ab7, 0xcdb625ea, 0x3c8e564b, 0x00000000, + 0x3ff00000, 0xb07217e3, 0x3fd248f1, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x56f37042, 0xbfccfc56, 0x00000000, + 0x00000000, 0xaa563951, 0x3fc90125, 0x00000000, 0x00000000, + 0x3d0e7c5d, 0xbfc50533, 0x9bed9b2e, 0xbfdf0ed9, 0x5fe7c47c, + 0x3fc1f250, 0x96c125e5, 0x3fe2edd9, 0x5a02bbd8, 0xbfbe5c71, + 0x86362c20, 0xbfda08b7, 0x4b4435ed, 0x3fb9d342, 0x4b494091, + 0x3fd911bd, 0xb56658be, 0xbfb5e4c7, 0x93a2fd76, 0xbfd3c092, + 0xda271794, 0x3fb29910, 0x3303df2b, 0x3fd189be, 0x99fcef32, + 0xbfda8279, 0xb68c1467, 0xbc708b2f, 0x00000000, 0x3ff00000, + 0x980c4337, 0x3fc5f619, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x9314533e, 0xbfbb8ec5, 0x00000000, 0x00000000, + 0x09aa36d0, 0x3fb6d3f4, 0x00000000, 0x00000000, 0xdcb427fd, + 0xbfb13950, 0xd87ab0bb, 0xbfd5335e, 0xce0ae8a5, 0x3fabb382, + 0x79143126, 0x3fddba41, 0x5f2b28d4, 0xbfa552f1, 0x59f21a6d, + 0xbfd015ab, 0x22c27d95, 0x3fa0e984, 0xe19fc6aa, 0x3fd0576c, + 0x8f2c2950, 0xbf9a4898, 0xc0b3f22c, 0xbfc59462, 0x1883a4b8, + 0x3f94b61c, 0x3f838640, 0x3fc30eb8, 0x355c63dc, 0xbfd36a08, + 0x1dce993d, 0x3c6d704d, 0x00000000, 0x3ff00000, 0x2b82ab63, + 0x3fb78e92, 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x5a279ea3, 0xbfaa3407, 0x00000000, 0x00000000, 0x432d65fa, + 0x3fa70153, 0x00000000, 0x00000000, 0x891a4602, 0xbf9d03ef, + 0xd62ca5f8, 0xbfca77d9, 0xb35f4628, 0x3f97a265, 0x433258fa, + 0x3fd8cf51, 0xb58fd909, 0xbf8f88e3, 0x01771cea, 0xbfc2b154, + 0xf3562f8e, 0x3f888f57, 0xc028a723, 0x3fc7370f, 0x20b7f9f0, + 0xbf80f44c, 0x214368e9, 0xbfb6dfaa, 0x28891863, 0x3f79b4b6, + 0x172dbbf0, 0x3fb6cb8e, 0xe0553158, 0xbfc975f5, 0x593fe814, + 0xbc2ef5d3, 0x00000000, 0x3ff00000, 0x03dec550, 0x3fa44203, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x4e435f9b, + 0xbf953f83, 0x00000000, 0x00000000, 0x3c6e8e46, 0x3f9b74ea, + 0x00000000, 0x00000000, 0xda5b7511, 0xbf85ad63, 0xdc230b9b, + 0xbfb97558, 0x26cb3788, 0x3f881308, 0x76fc4985, 0x3fd62ac9, + 0x77bb08ba, 0xbf757c85, 0xb6247521, 0xbfb1381e, 0x5922170c, + 0x3f754e95, 0x8746482d, 0x3fc27f83, 0x11055b30, 0xbf64e391, + 0x3e666320, 0xbfa3e609, 0x0de9dae3, 0x3f6301df, 0x1f1dca06, + 0x3fafa8ae, 0x8c5b2da2, 0xbfb936bb, 0x4e88f7a5, 0xbc587d05, + 0x00000000, 0x3ff00000, 0xa8935dd9, 0x3f83dde2, 0x00000000, + 0x00000000, 0x00000000, 0x00000000 + // @formatter:on + }); + + private ArrayDataPointerConstant mask35 = pointerConstant(16, new int[]{ + // @formatter:off + 0xfffc0000, 0xffffffff, 0x00000000, 0x00000000 + // @formatter:on + }); + + private ArrayDataPointerConstant q11 = pointerConstant(16, new int[]{ + // @formatter:off + 0xb8fe4d77, 0x3f82609a + // @formatter:on + }); + + private ArrayDataPointerConstant q9 = pointerConstant(16, new int[]{ + // @formatter:off + 0xbf847a43, 0x3f9664a0 + // @formatter:on + }); + + private ArrayDataPointerConstant q7 = pointerConstant(16, new int[]{ + // @formatter:off + 0x52c4c8ab, 0x3faba1ba + // @formatter:on + }); + + private ArrayDataPointerConstant q5 = pointerConstant(16, new int[]{ + // @formatter:off + 0x11092746, 0x3fc11111 + // @formatter:on + }); + + private ArrayDataPointerConstant q3 = pointerConstant(16, new int[]{ + // @formatter:off + 0x55555612, 0x3fd55555 + // @formatter:on + }); + + private ArrayDataPointerConstant piInvTable = pointerConstant(16, new int[]{ + // @formatter:off + 0x00000000, 0x00000000, 0xa2f9836e, 0x4e441529, 0xfc2757d1, + 0xf534ddc0, 0xdb629599, 0x3c439041, 0xfe5163ab, 0xdebbc561, + 0xb7246e3a, 0x424dd2e0, 0x06492eea, 0x09d1921c, 0xfe1deb1c, + 0xb129a73e, 0xe88235f5, 0x2ebb4484, 0xe99c7026, 0xb45f7e41, + 0x3991d639, 0x835339f4, 0x9c845f8b, 0xbdf9283b, 0x1ff897ff, + 0xde05980f, 0xef2f118b, 0x5a0a6d1f, 0x6d367ecf, 0x27cb09b7, + 0x4f463f66, 0x9e5fea2d, 0x7527bac7, 0xebe5f17b, 0x3d0739f7, + 0x8a5292ea, 0x6bfb5fb1, 0x1f8d5d08, 0x56033046, 0xfc7b6bab, + 0xf0cfbc21 + // @formatter:on + }); + + private ArrayDataPointerConstant pi4 = pointerConstant(8, new int[]{ + // @formatter:off + 0x00000000, 0x3fe921fb, + }); + private ArrayDataPointerConstant pi48 = pointerConstant(8, new int[]{ + 0x4611a626, 0x3e85110b + // @formatter:on + }); + + private ArrayDataPointerConstant qq2 = pointerConstant(8, new int[]{ + // @formatter:off + 0x676733af, 0x3d32e7b9 + // @formatter:on + }); + + private ArrayDataPointerConstant one = pointerConstant(8, new int[]{ + // @formatter:off + 0x00000000, 0x3ff00000 + // @formatter:on + }); + + private ArrayDataPointerConstant twoPow55 = pointerConstant(8, new int[]{ + // @formatter:off + 0x00000000, 0x43600000 + // @formatter:on + }); + + private ArrayDataPointerConstant twoPowM55 = pointerConstant(4, new int[]{ + // @formatter:off + 0x00000000, 0x3c800000 + // @formatter:on + }); + + private ArrayDataPointerConstant negZero = pointerConstant(4, new int[]{ + // @formatter:off + 0x00000000, 0x80000000 + // @formatter:on + }); + + @Override + public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { + Label block0 = new Label(); + Label block1 = new Label(); + Label block2 = new Label(); + Label block3 = new Label(); + Label block4 = new Label(); + Label block5 = new Label(); + Label block6 = new Label(); + Label block7 = new Label(); + Label block8 = new Label(); + Label block9 = new Label(); + Label block10 = new Label(); + Label block11 = new Label(); + Label block12 = new Label(); + Label block13 = new Label(); + Label block14 = new Label(); + + masm.push(rbx); + masm.subq(rsp, 16); + masm.movsd(new AMD64Address(rsp, 8), xmm0); + + masm.pextrw(rax, xmm0, 3); + masm.andl(rax, 32767); + masm.subl(rax, 16314); + masm.cmpl(rax, 270); + masm.jcc(AMD64Assembler.ConditionFlag.Above, block0); + masm.movdqu(xmm5, recordExternalAddress(crb, onehalf)); // 0x00000000, 0x3fe00000, + // 0x00000000, 0x3fe00000 + masm.movdqu(xmm6, recordExternalAddress(crb, mul16)); // 0x00000000, 0x40300000, + // 0x00000000, 0x3ff00000 + masm.unpcklpd(xmm0, xmm0); + masm.movdqu(xmm4, recordExternalAddress(crb, signMask)); // 0x00000000, 0x80000000, + // 0x00000000, 0x80000000 + masm.andpd(xmm4, xmm0); + masm.movdqu(xmm1, recordExternalAddress(crb, pi32Inv)); // 0x6dc9c883, 0x3fe45f30, + // 0x6dc9c883, 0x40245f30 + masm.mulpd(xmm1, xmm0); + masm.por(xmm5, xmm4); + masm.addpd(xmm1, xmm5); + masm.movdqu(xmm7, xmm1); + masm.unpckhpd(xmm7, xmm7); + masm.cvttsd2sil(rdx, xmm7); + masm.cvttpd2dq(xmm1, xmm1); + masm.cvtdq2pd(xmm1, xmm1); + masm.mulpd(xmm1, xmm6); + masm.movdqu(xmm3, recordExternalAddress(crb, p1)); // 0x54444000, 0x3fb921fb, + // 0x54440000, 0x3fb921fb + masm.movq(xmm5, recordExternalAddress(crb, qq2)); // 0x676733af, 0x3d32e7b9 + masm.addq(rdx, 469248); + masm.movdqu(xmm4, recordExternalAddress(crb, p2)); // 0x67674000, 0xbd32e7b9, + // 0x4c4c0000, 0x3d468c23 + masm.mulpd(xmm3, xmm1); + masm.andq(rdx, 31); + masm.mulsd(xmm5, xmm1); + masm.movq(rcx, rdx); + masm.mulpd(xmm4, xmm1); + masm.shlq(rcx, 1); + masm.subpd(xmm0, xmm3); + masm.mulpd(xmm1, recordExternalAddress(crb, p3)); // 0x3707344a, 0x3aa8a2e0, + // 0x03707345, 0x3ae98a2e + masm.addq(rdx, rcx); + masm.shlq(rcx, 2); + masm.addq(rdx, rcx); + masm.addsd(xmm5, xmm0); + masm.movdqu(xmm2, xmm0); + masm.subpd(xmm0, xmm4); + masm.movq(xmm6, recordExternalAddress(crb, one)); // 0x00000000, 0x3ff00000 + masm.shlq(rdx, 4); + masm.leaq(rax, recordExternalAddress(crb, ctable)); + masm.andpd(xmm5, recordExternalAddress(crb, mask35)); // 0xfffc0000, 0xffffffff, + // 0x00000000, 0x00000000 + masm.movdqu(xmm3, xmm0); + masm.addq(rax, rdx); + masm.subpd(xmm2, xmm0); + masm.unpckhpd(xmm0, xmm0); + masm.divsd(xmm6, xmm5); + masm.subpd(xmm2, xmm4); + masm.movdqu(xmm7, new AMD64Address(rax, 16)); + masm.subsd(xmm3, xmm5); + masm.mulpd(xmm7, xmm0); + masm.subpd(xmm2, xmm1); + masm.movdqu(xmm1, new AMD64Address(rax, 48)); + masm.mulpd(xmm1, xmm0); + masm.movdqu(xmm4, new AMD64Address(rax, 96)); + masm.mulpd(xmm4, xmm0); + masm.addsd(xmm2, xmm3); + masm.movdqu(xmm3, xmm0); + masm.mulpd(xmm0, xmm0); + masm.addpd(xmm7, new AMD64Address(rax, 0)); + masm.addpd(xmm1, new AMD64Address(rax, 32)); + masm.mulpd(xmm1, xmm0); + masm.addpd(xmm4, new AMD64Address(rax, 80)); + masm.addpd(xmm7, xmm1); + masm.movdqu(xmm1, new AMD64Address(rax, 112)); + masm.mulpd(xmm1, xmm0); + masm.mulpd(xmm0, xmm0); + masm.addpd(xmm4, xmm1); + masm.movdqu(xmm1, new AMD64Address(rax, 64)); + masm.mulpd(xmm1, xmm0); + masm.addpd(xmm7, xmm1); + masm.movdqu(xmm1, xmm3); + masm.mulpd(xmm3, xmm0); + masm.mulsd(xmm0, xmm0); + masm.mulpd(xmm1, new AMD64Address(rax, 144)); + masm.mulpd(xmm4, xmm3); + masm.movdqu(xmm3, xmm1); + masm.addpd(xmm7, xmm4); + masm.movdqu(xmm4, xmm1); + masm.mulsd(xmm0, xmm7); + masm.unpckhpd(xmm7, xmm7); + masm.addsd(xmm0, xmm7); + masm.unpckhpd(xmm1, xmm1); + masm.addsd(xmm3, xmm1); + masm.subsd(xmm4, xmm3); + masm.addsd(xmm1, xmm4); + masm.movdqu(xmm4, xmm2); + masm.movq(xmm7, new AMD64Address(rax, 144)); + masm.unpckhpd(xmm2, xmm2); + masm.addsd(xmm7, new AMD64Address(rax, 152)); + masm.mulsd(xmm7, xmm2); + masm.addsd(xmm7, new AMD64Address(rax, 136)); + masm.addsd(xmm7, xmm1); + masm.addsd(xmm0, xmm7); + masm.movq(xmm7, recordExternalAddress(crb, one)); // 0x00000000, 0x3ff00000 + masm.mulsd(xmm4, xmm6); + masm.movq(xmm2, new AMD64Address(rax, 168)); + masm.andpd(xmm2, xmm6); + masm.mulsd(xmm5, xmm2); + masm.mulsd(xmm6, new AMD64Address(rax, 160)); + masm.subsd(xmm7, xmm5); + masm.subsd(xmm2, new AMD64Address(rax, 128)); + masm.subsd(xmm7, xmm4); + masm.mulsd(xmm7, xmm6); + masm.movdqu(xmm4, xmm3); + masm.subsd(xmm3, xmm2); + masm.addsd(xmm2, xmm3); + masm.subsd(xmm4, xmm2); + masm.addsd(xmm0, xmm4); + masm.subsd(xmm0, xmm7); + masm.addsd(xmm0, xmm3); + masm.jmp(block14); + + masm.bind(block0); + masm.jcc(AMD64Assembler.ConditionFlag.Greater, block1); + masm.pextrw(rax, xmm0, 3); + masm.movl(rdx, rax); + masm.andl(rax, 32752); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block2); + masm.andl(rdx, 32767); + masm.cmpl(rdx, 15904); + masm.jcc(AMD64Assembler.ConditionFlag.Below, block3); + masm.movdqu(xmm2, xmm0); + masm.movdqu(xmm3, xmm0); + masm.movq(xmm1, recordExternalAddress(crb, q11)); // 0xb8fe4d77, 0x3f82609a + masm.mulsd(xmm2, xmm0); + masm.mulsd(xmm3, xmm2); + masm.mulsd(xmm1, xmm2); + masm.addsd(xmm1, recordExternalAddress(crb, q9)); // 0xbf847a43, 0x3f9664a0 + masm.mulsd(xmm1, xmm2); + masm.addsd(xmm1, recordExternalAddress(crb, q7)); // 0x52c4c8ab, 0x3faba1ba + masm.mulsd(xmm1, xmm2); + masm.addsd(xmm1, recordExternalAddress(crb, q5)); // 0x11092746, 0x3fc11111 + masm.mulsd(xmm1, xmm2); + masm.addsd(xmm1, recordExternalAddress(crb, q3)); // 0x55555612, 0x3fd55555 + masm.mulsd(xmm1, xmm3); + masm.addsd(xmm0, xmm1); + masm.jmp(block14); + + masm.bind(block3); + masm.movq(xmm3, recordExternalAddress(crb, twoPow55)); // 0x00000000, 0x43600000 + masm.mulsd(xmm3, xmm0); + masm.addsd(xmm0, xmm3); + masm.mulsd(xmm0, recordExternalAddress(crb, twoPowM55)); // 0x00000000, 0x3c800000 + masm.jmp(block14); + + masm.bind(block2); + masm.movdqu(xmm1, xmm0); + masm.mulsd(xmm1, xmm1); + masm.jmp(block14); + + masm.bind(block1); + masm.pextrw(rax, xmm0, 3); + masm.andl(rax, 32752); + masm.cmpl(rax, 32752); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block4); + masm.pextrw(rcx, xmm0, 3); + masm.andl(rcx, 32752); + masm.subl(rcx, 16224); + masm.shrl(rcx, 7); + masm.andl(rcx, 65532); + masm.leaq(r11, recordExternalAddress(crb, piInvTable)); + masm.addq(rcx, r11); + masm.movdq(rax, xmm0); + masm.movl(r10, new AMD64Address(rcx, 20)); + masm.movl(r8, new AMD64Address(rcx, 24)); + masm.movl(rdx, rax); + masm.shrq(rax, 21); + masm.orl(rax, Integer.MIN_VALUE); + masm.shrl(rax, 11); + masm.movl(r9, r10); + masm.imulq(r10, rdx); + masm.imulq(r9, rax); + masm.imulq(r8, rax); + masm.movl(rsi, new AMD64Address(rcx, 16)); + masm.movl(rdi, new AMD64Address(rcx, 12)); + masm.movl(r11, r10); + masm.shrq(r10, 32); + masm.addq(r9, r10); + masm.addq(r11, r8); + masm.movl(r8, r11); + masm.shrq(r11, 32); + masm.addq(r9, r11); + masm.movl(r10, rsi); + masm.imulq(rsi, rdx); + masm.imulq(r10, rax); + masm.movl(r11, rdi); + masm.imulq(rdi, rdx); + masm.movl(rbx, rsi); + masm.shrq(rsi, 32); + masm.addq(r9, rbx); + masm.movl(rbx, r9); + masm.shrq(r9, 32); + masm.addq(r10, rsi); + masm.addq(r10, r9); + masm.shlq(rbx, 32); + masm.orq(r8, rbx); + masm.imulq(r11, rax); + masm.movl(r9, new AMD64Address(rcx, 8)); + masm.movl(rsi, new AMD64Address(rcx, 4)); + masm.movl(rbx, rdi); + masm.shrq(rdi, 32); + masm.addq(r10, rbx); + masm.movl(rbx, r10); + masm.shrq(r10, 32); + masm.addq(r11, rdi); + masm.addq(r11, r10); + masm.movq(rdi, r9); + masm.imulq(r9, rdx); + masm.imulq(rdi, rax); + masm.movl(r10, r9); + masm.shrq(r9, 32); + masm.addq(r11, r10); + masm.movl(r10, r11); + masm.shrq(r11, 32); + masm.addq(rdi, r9); + masm.addq(rdi, r11); + masm.movq(r9, rsi); + masm.imulq(rsi, rdx); + masm.imulq(r9, rax); + masm.shlq(r10, 32); + masm.orq(r10, rbx); + masm.movl(rax, new AMD64Address(rcx, 0)); + masm.movl(r11, rsi); + masm.shrq(rsi, 32); + masm.addq(rdi, r11); + masm.movl(r11, rdi); + masm.shrq(rdi, 32); + masm.addq(r9, rsi); + masm.addq(r9, rdi); + masm.imulq(rdx, rax); + masm.pextrw(rbx, xmm0, 3); + masm.leaq(rdi, recordExternalAddress(crb, piInvTable)); + masm.subq(rcx, rdi); + masm.addl(rcx, rcx); + masm.addl(rcx, rcx); + masm.addl(rcx, rcx); + masm.addl(rcx, 19); + masm.movl(rsi, 32768); + masm.andl(rsi, rbx); + masm.shrl(rbx, 4); + masm.andl(rbx, 2047); + masm.subl(rbx, 1023); + masm.subl(rcx, rbx); + masm.addq(r9, rdx); + masm.movl(rdx, rcx); + masm.addl(rdx, 32); + masm.cmpl(rcx, 0); + masm.jcc(AMD64Assembler.ConditionFlag.Less, block5); + masm.negl(rcx); + masm.addl(rcx, 29); + masm.shll(r9); + masm.movl(rdi, r9); + masm.andl(r9, 1073741823); + masm.testl(r9, 536870912); + masm.jcc(AMD64Assembler.ConditionFlag.NotEqual, block6); + masm.shrl(r9); + masm.movl(rbx, 0); + masm.shlq(r9, 32); + masm.orq(r9, r11); + + masm.bind(block7); + + masm.bind(block8); + masm.cmpq(r9, 0); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block9); + + masm.bind(block10); + masm.bsrq(r11, r9); + masm.movl(rcx, 29); + masm.subl(rcx, r11); + masm.jcc(AMD64Assembler.ConditionFlag.LessEqual, block11); + masm.shlq(r9); + masm.movq(rax, r10); + masm.shlq(r10); + masm.addl(rdx, rcx); + masm.negl(rcx); + masm.addl(rcx, 64); + masm.shrq(rax); + masm.shrq(r8); + masm.orq(r9, rax); + masm.orq(r10, r8); + + masm.bind(block12); + masm.cvtsi2sdq(xmm0, r9); + masm.shrq(r10, 1); + masm.cvtsi2sdq(xmm3, r10); + masm.xorpd(xmm4, xmm4); + masm.shll(rdx, 4); + masm.negl(rdx); + masm.addl(rdx, 16368); + masm.orl(rdx, rsi); + masm.xorl(rdx, rbx); + masm.pinsrw(xmm4, rdx, 3); + masm.movq(xmm2, recordExternalAddress(crb, pi4)); // 0x00000000, 0x3fe921fb, + // 0x4611a626, 0x3e85110b + masm.movq(xmm7, recordExternalAddress(crb, pi48)); // 0x3fe921fb, 0x4611a626, + // 0x3e85110b + masm.xorpd(xmm5, xmm5); + masm.subl(rdx, 1008); + masm.pinsrw(xmm5, rdx, 3); + masm.mulsd(xmm0, xmm4); + masm.shll(rsi, 16); + masm.sarl(rsi, 31); + masm.mulsd(xmm3, xmm5); + masm.movdqu(xmm1, xmm0); + masm.mulsd(xmm0, xmm2); + masm.shrl(rdi, 30); + masm.addsd(xmm1, xmm3); + masm.mulsd(xmm3, xmm2); + masm.addl(rdi, rsi); + masm.xorl(rdi, rsi); + masm.mulsd(xmm7, xmm1); + masm.movl(rax, rdi); + masm.addsd(xmm7, xmm3); + masm.movdqu(xmm2, xmm0); + masm.addsd(xmm0, xmm7); + masm.subsd(xmm2, xmm0); + masm.addsd(xmm7, xmm2); + masm.movdqu(xmm1, recordExternalAddress(crb, pi32Inv)); // 0x6dc9c883, 0x3fe45f30, + // 0x6dc9c883, 0x40245f30 + if (masm.supports(AMD64.CPUFeature.SSE3)) { + masm.movddup(xmm0, xmm0); + } else { + masm.movlhps(xmm0, xmm0); + } + masm.movdqu(xmm4, recordExternalAddress(crb, signMask)); // 0x00000000, 0x80000000, + // 0x00000000, 0x80000000 + masm.andpd(xmm4, xmm0); + masm.mulpd(xmm1, xmm0); + if (masm.supports(AMD64.CPUFeature.SSE3)) { + masm.movddup(xmm7, xmm7); + } else { + masm.movlhps(xmm7, xmm7); + } + masm.movdqu(xmm5, recordExternalAddress(crb, onehalf)); // 0x00000000, 0x3fe00000, + // 0x00000000, 0x3fe00000 + masm.movdqu(xmm6, recordExternalAddress(crb, mul16)); // 0x00000000, 0x40300000, + // 0x00000000, 0x3ff00000 + masm.por(xmm5, xmm4); + masm.addpd(xmm1, xmm5); + masm.movdqu(xmm5, xmm1); + masm.unpckhpd(xmm5, xmm5); + masm.cvttsd2sil(rdx, xmm5); + masm.cvttpd2dq(xmm1, xmm1); + masm.cvtdq2pd(xmm1, xmm1); + masm.mulpd(xmm1, xmm6); + masm.movdqu(xmm3, recordExternalAddress(crb, p1)); // 0x54444000, 0x3fb921fb, + // 0x54440000, 0x3fb921fb + masm.movq(xmm5, recordExternalAddress(crb, qq2)); // 0x676733af, 0x3d32e7b9 + masm.shll(rax, 4); + masm.addl(rdx, 469248); + masm.movdqu(xmm4, recordExternalAddress(crb, p2)); // 0x67674000, 0xbd32e7b9, + // 0x4c4c0000, 0x3d468c23 + masm.mulpd(xmm3, xmm1); + masm.addl(rdx, rax); + masm.andl(rdx, 31); + masm.mulsd(xmm5, xmm1); + masm.movl(rcx, rdx); + masm.mulpd(xmm4, xmm1); + masm.shll(rcx, 1); + masm.subpd(xmm0, xmm3); + masm.mulpd(xmm1, recordExternalAddress(crb, p3)); // 0x3707344a, 0x3aa8a2e0, + // 0x03707345, 0x3ae98a2e + masm.addl(rdx, rcx); + masm.shll(rcx, 2); + masm.addl(rdx, rcx); + masm.addsd(xmm5, xmm0); + masm.movdqu(xmm2, xmm0); + masm.subpd(xmm0, xmm4); + masm.movq(xmm6, recordExternalAddress(crb, one)); // 0x00000000, 0x3ff00000 + masm.shll(rdx, 4); + masm.leaq(rax, recordExternalAddress(crb, ctable)); + masm.andpd(xmm5, recordExternalAddress(crb, mask35)); // 0xfffc0000, 0xffffffff, + // 0x00000000, 0x00000000 + masm.movdqu(xmm3, xmm0); + masm.addq(rax, rdx); + masm.subpd(xmm2, xmm0); + masm.unpckhpd(xmm0, xmm0); + masm.divsd(xmm6, xmm5); + masm.subpd(xmm2, xmm4); + masm.subsd(xmm3, xmm5); + masm.subpd(xmm2, xmm1); + masm.movdqu(xmm1, new AMD64Address(rax, 48)); + masm.addpd(xmm2, xmm7); + masm.movdqu(xmm7, new AMD64Address(rax, 16)); + masm.mulpd(xmm7, xmm0); + masm.movdqu(xmm4, new AMD64Address(rax, 96)); + masm.mulpd(xmm1, xmm0); + masm.mulpd(xmm4, xmm0); + masm.addsd(xmm2, xmm3); + masm.movdqu(xmm3, xmm0); + masm.mulpd(xmm0, xmm0); + masm.addpd(xmm7, new AMD64Address(rax, 0)); + masm.addpd(xmm1, new AMD64Address(rax, 32)); + masm.mulpd(xmm1, xmm0); + masm.addpd(xmm4, new AMD64Address(rax, 80)); + masm.addpd(xmm7, xmm1); + masm.movdqu(xmm1, new AMD64Address(rax, 112)); + masm.mulpd(xmm1, xmm0); + masm.mulpd(xmm0, xmm0); + masm.addpd(xmm4, xmm1); + masm.movdqu(xmm1, new AMD64Address(rax, 64)); + masm.mulpd(xmm1, xmm0); + masm.addpd(xmm7, xmm1); + masm.movdqu(xmm1, xmm3); + masm.mulpd(xmm3, xmm0); + masm.mulsd(xmm0, xmm0); + masm.mulpd(xmm1, new AMD64Address(rax, 144)); + masm.mulpd(xmm4, xmm3); + masm.movdqu(xmm3, xmm1); + masm.addpd(xmm7, xmm4); + masm.movdqu(xmm4, xmm1); + masm.mulsd(xmm0, xmm7); + masm.unpckhpd(xmm7, xmm7); + masm.addsd(xmm0, xmm7); + masm.unpckhpd(xmm1, xmm1); + masm.addsd(xmm3, xmm1); + masm.subsd(xmm4, xmm3); + masm.addsd(xmm1, xmm4); + masm.movdqu(xmm4, xmm2); + masm.movq(xmm7, new AMD64Address(rax, 144)); + masm.unpckhpd(xmm2, xmm2); + masm.addsd(xmm7, new AMD64Address(rax, 152)); + masm.mulsd(xmm7, xmm2); + masm.addsd(xmm7, new AMD64Address(rax, 136)); + masm.addsd(xmm7, xmm1); + masm.addsd(xmm0, xmm7); + masm.movq(xmm7, recordExternalAddress(crb, one)); // 0x00000000, 0x3ff00000 + masm.mulsd(xmm4, xmm6); + masm.movq(xmm2, new AMD64Address(rax, 168)); + masm.andpd(xmm2, xmm6); + masm.mulsd(xmm5, xmm2); + masm.mulsd(xmm6, new AMD64Address(rax, 160)); + masm.subsd(xmm7, xmm5); + masm.subsd(xmm2, new AMD64Address(rax, 128)); + masm.subsd(xmm7, xmm4); + masm.mulsd(xmm7, xmm6); + masm.movdqu(xmm4, xmm3); + masm.subsd(xmm3, xmm2); + masm.addsd(xmm2, xmm3); + masm.subsd(xmm4, xmm2); + masm.addsd(xmm0, xmm4); + masm.subsd(xmm0, xmm7); + masm.addsd(xmm0, xmm3); + masm.jmp(block14); + + masm.bind(block9); + masm.addl(rdx, 64); + masm.movq(r9, r10); + masm.movq(r10, r8); + masm.movl(r8, 0); + masm.cmpq(r9, 0); + masm.jcc(AMD64Assembler.ConditionFlag.NotEqual, block10); + masm.addl(rdx, 64); + masm.movq(r9, r10); + masm.movq(r10, r8); + masm.cmpq(r9, 0); + masm.jcc(AMD64Assembler.ConditionFlag.NotEqual, block10); + masm.jmp(block12); + + masm.bind(block11); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block12); + masm.negl(rcx); + masm.shrq(r10); + masm.movq(rax, r9); + masm.shrq(r9); + masm.subl(rdx, rcx); + masm.negl(rcx); + masm.addl(rcx, 64); + masm.shlq(rax); + masm.orq(r10, rax); + masm.jmp(block12); + + masm.bind(block5); + masm.notl(rcx); + masm.shlq(r9, 32); + masm.orq(r9, r11); + masm.shlq(r9); + masm.movq(rdi, r9); + masm.testl(r9, Integer.MIN_VALUE); + masm.jcc(AMD64Assembler.ConditionFlag.NotEqual, block13); + masm.shrl(r9); + masm.movl(rbx, 0); + masm.shrq(rdi, 2); + masm.jmp(block8); + + masm.bind(block6); + masm.shrl(r9); + masm.movl(rbx, 1073741824); + masm.shrl(rbx); + masm.shlq(r9, 32); + masm.orq(r9, r11); + masm.shlq(rbx, 32); + masm.addl(rdi, 1073741824); + masm.movl(rcx, 0); + masm.movl(r11, 0); + masm.subq(rcx, r8); + masm.sbbq(r11, r10); + masm.sbbq(rbx, r9); + masm.movq(r8, rcx); + masm.movq(r10, r11); + masm.movq(r9, rbx); + masm.movl(rbx, 32768); + masm.jmp(block7); + + masm.bind(block13); + masm.shrl(r9); + masm.movq(rbx, 0x100000000L); + masm.shrq(rbx); + masm.movl(rcx, 0); + masm.movl(r11, 0); + masm.subq(rcx, r8); + masm.sbbq(r11, r10); + masm.sbbq(rbx, r9); + masm.movq(r8, rcx); + masm.movq(r10, r11); + masm.movq(r9, rbx); + masm.movl(rbx, 32768); + masm.shrq(rdi, 2); + masm.addl(rdi, 1073741824); + masm.jmp(block8); + + masm.bind(block4); + masm.movq(xmm0, new AMD64Address(rsp, 8)); + masm.mulsd(xmm0, recordExternalAddress(crb, negZero)); // 0x00000000, 0x80000000 + masm.movq(new AMD64Address(rsp, 0), xmm0); + + masm.bind(block14); + masm.addq(rsp, 16); + masm.pop(rbx); + } +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/hashing/HashFunction.java 2019-03-12 08:10:49.276050419 +0100 @@ -0,0 +1,156 @@ +/* + * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.lir.hashing; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.function.BiFunction; +import java.util.function.Function; + +import org.graalvm.compiler.lir.gen.ArithmeticLIRGenerator; + +import jdk.vm.ci.meta.JavaConstant; +import jdk.vm.ci.meta.Value; + +/** + * This class provides a set of cheap imperfect hash functions based on the paper "Improving Switch + * Statement Performance with Hashing Optimized at Compile Time". + * (http://programming.sirrida.de/hashsuper.pdf) + */ +public abstract class HashFunction { + + /** + * Applies the hash function. + * + * @param value the value to be hashed + * @param min {@code value} is guaranteed to be greater or equal to this minimum + * @return the hash value within int range + */ + public abstract int apply(long value, long min); + + /** + * Generates LIR that implements the hash function in terms of value and min. + * + * @param value the value to be hashed + * @param min the lowest key + * @param gen the lir generator + * @return new lir value with the hash function applied + */ + public abstract Value gen(Value value, Value min, ArithmeticLIRGenerator gen); + + /** + * Returns an estimate of number of CPU cycles necessary to apply the hash function. + */ + public abstract int effort(); + + /** + * @return a list of all available hash functions + */ + public static final List instances() { + return Collections.unmodifiableList(instances); + } + + private static List instances = new ArrayList<>(); + + private static int[] mersennePrimes = {3, 7, 31, 127, 8191, 131071, 524287, 2147483647}; + + static { + //@formatter:off + + add("val", 0, + (val, min) -> val, + gen -> (val, min) -> val); + + add("val - min", 1, + (val, min) -> val - min, + gen -> (val, min) -> gen.emitSub(val, min, false)); + + add("val >> min", 1, + (val, min) -> val >> min, + gen -> (val, min) -> gen.emitShr(val, min)); + + add("val >> (val & min)", 2, + (val, min) -> val >> (val & min), + gen -> (val, min) -> gen.emitShr(val, gen.emitAnd(val, min))); + + add("(val >> min) ^ val", 2, + (val, min) -> (val >> min) ^ val, + gen -> (val, min) -> gen.emitXor(gen.emitShr(val, min), val)); + + add("(val >> min) * val", 3, + (val, min) -> (val >> min) * val, + gen -> (val, min) -> gen.emitMul(gen.emitShr(val, min), val, false)); + + addWithPrimes("(val * prime) >> min", 3, + prime -> (val, min) -> (val * prime) >> min, + (gen, prime) -> (val, min) -> gen.emitShr(gen.emitMul(val, prime, false), min)); + + addWithPrimes("rotateRight(val, prime)", 3, + prime -> (val, min) -> Long.rotateRight(val, prime), + (gen, prime) -> (val, min) -> gen.emitRor(val, prime)); + + addWithPrimes("rotateRight(val, prime) + val", 4, + prime -> (val, min) -> Long.rotateRight(val, prime) + val, + (gen, prime) -> (val, min) -> gen.emitAdd(gen.emitRor(val, prime), val, false)); + + addWithPrimes("rotateRight(val, prime) ^ val", 4, + prime -> (val, min) -> Long.rotateRight(val, prime) ^ val, + (gen, prime) -> (val, min) -> gen.emitXor(gen.emitRor(val, prime), val)); + //@formatter:on + } + + private static void add(String toString, int effort, BiFunction f, Function> gen) { + instances.add(new HashFunction() { + + @Override + public int apply(long value, long min) { + return f.apply(value, min).intValue(); + } + + @Override + public int effort() { + return effort; + } + + @Override + public String toString() { + return toString; + } + + @Override + public Value gen(Value val, Value min, ArithmeticLIRGenerator t) { + return gen.apply(t).apply(t.emitNarrow(val, 32), t.emitNarrow(min, 32)); + } + }); + } + + private static void addWithPrimes(String toString, int effort, Function> f, + BiFunction> gen) { + for (int p : mersennePrimes) { + add(toString, effort, f.apply(p), g -> gen.apply(g, g.getLIRGen().emitJavaConstant(JavaConstant.forInt(p)))); + } + } +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/hashing/Hasher.java 2019-03-12 08:10:49.768053599 +0100 @@ -0,0 +1,161 @@ +/* + * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.lir.hashing; + +import java.util.Comparator; +import java.util.HashSet; +import java.util.Optional; +import java.util.Set; +import java.util.TreeSet; + +import org.graalvm.compiler.lir.gen.ArithmeticLIRGenerator; + +import jdk.vm.ci.meta.JavaConstant; +import jdk.vm.ci.meta.Value; + +/** + * This class holds a hash function at a specific cardinality and min value (lowest key). The + * cardinality is the required size of the hash table to make the hasher injective for the provided + * keys. + */ +public final class Hasher { + + /** + * Tries to find a hash function without conflicts for the provided keys. + * + * @param keys the keys + * @param minDensity the minimum density of the switch table. Used to determine the maximum + * cardinality of the hash function + * @return an optional hasher + */ + public static Optional forKeys(JavaConstant[] keys, double minDensity) { + if (keys.length <= 2) { + return Optional.empty(); + } else { + int maxCardinality = (int) Math.round(keys.length / minDensity); + assertSorted(keys); + TreeSet candidates = new TreeSet<>(new Comparator() { + @Override + public int compare(Hasher o1, Hasher o2) { + int d = o1.cardinality - o2.cardinality; + if (d != 0) { + return d; + } else { + return o1.effort() - o2.effort(); + } + } + }); + long min = keys[0].asLong(); + for (HashFunction f : HashFunction.instances()) { + for (int cardinality = keys.length; cardinality < maxCardinality; cardinality++) { + if (isValid(keys, min, f, cardinality)) { + candidates.add(new Hasher(f, cardinality, min)); + break; + } + } + } + if (candidates.isEmpty()) { + return Optional.empty(); + } else { + return Optional.of(candidates.first()); + } + } + } + + private static void assertSorted(JavaConstant[] keys) { + for (int i = 1; i < keys.length; i++) { + assert keys[i - 1].asLong() < keys[i].asLong(); + } + } + + private static boolean isValid(JavaConstant[] keys, long min, HashFunction function, int cardinality) { + Set seen = new HashSet<>(keys.length); + for (JavaConstant key : keys) { + int hash = function.apply(key.asLong(), min) & (cardinality - 1); + if (!seen.add(hash)) { + return false; + } + } + return true; + } + + private final HashFunction function; + private final int cardinality; + private final long min; + + private Hasher(HashFunction function, int cardinality, long min) { + this.function = function; + this.cardinality = cardinality; + this.min = min; + } + + /** + * Applies the hash function. + * + * @param value the value to be hashed + * @return the hash value + */ + public int hash(long value) { + return function.apply(value, min) & (cardinality - 1); + } + + /** + * Applies the hash function to a lir value. + * + * @param value the value to be hashed + * @param gen the lir generator + * @return the hashed lir value + */ + public Value hash(Value value, ArithmeticLIRGenerator gen) { + Value h = function.gen(value, gen.getLIRGen().emitJavaConstant(JavaConstant.forLong(min)), gen); + return gen.emitAnd(h, gen.getLIRGen().emitJavaConstant(JavaConstant.forInt(cardinality - 1))); + } + + /** + * @return the hashing effort + */ + public int effort() { + return function.effort() + 1; + } + + /** + * @return the cardinality of the hash function that should match the size of the table switch. + */ + public int cardinality() { + return cardinality; + } + + /** + * @return the hash function + */ + public HashFunction function() { + return function; + } + + @Override + public String toString() { + return "Hasher[function=" + function + ", effort=" + effort() + ", cardinality=" + cardinality + "]"; + } +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop.phases/src/org/graalvm/compiler/loop/phases/ConvertDeoptimizeToGuardPhase.java 2019-03-12 08:10:50.256056753 +0100 @@ -0,0 +1,272 @@ +/* + * Copyright (c) 2011, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.loop.phases; + +import static org.graalvm.compiler.phases.common.DeadCodeEliminationPhase.Optionality.Optional; + +import java.util.List; + +import org.graalvm.compiler.core.common.GraalOptions; +import org.graalvm.compiler.core.common.cfg.Loop; +import org.graalvm.compiler.debug.DebugCloseable; +import org.graalvm.compiler.graph.Node; +import org.graalvm.compiler.graph.NodeSourcePosition; +import org.graalvm.compiler.graph.spi.SimplifierTool; +import org.graalvm.compiler.loop.LoopEx; +import org.graalvm.compiler.loop.LoopsData; +import org.graalvm.compiler.nodeinfo.InputType; +import org.graalvm.compiler.nodes.AbstractBeginNode; +import org.graalvm.compiler.nodes.AbstractEndNode; +import org.graalvm.compiler.nodes.AbstractMergeNode; +import org.graalvm.compiler.nodes.ConstantNode; +import org.graalvm.compiler.nodes.ControlSplitNode; +import org.graalvm.compiler.nodes.DeoptimizeNode; +import org.graalvm.compiler.nodes.EndNode; +import org.graalvm.compiler.nodes.FixedGuardNode; +import org.graalvm.compiler.nodes.FixedNode; +import org.graalvm.compiler.nodes.FixedWithNextNode; +import org.graalvm.compiler.nodes.GuardNode; +import org.graalvm.compiler.nodes.IfNode; +import org.graalvm.compiler.nodes.LogicNode; +import org.graalvm.compiler.nodes.LoopExitNode; +import org.graalvm.compiler.nodes.ProxyNode; +import org.graalvm.compiler.nodes.StartNode; +import org.graalvm.compiler.nodes.StaticDeoptimizingNode; +import org.graalvm.compiler.nodes.StructuredGraph; +import org.graalvm.compiler.nodes.ValueNode; +import org.graalvm.compiler.nodes.ValuePhiNode; +import org.graalvm.compiler.nodes.calc.CompareNode; +import org.graalvm.compiler.nodes.cfg.Block; +import org.graalvm.compiler.nodes.spi.LoweringProvider; +import org.graalvm.compiler.nodes.util.GraphUtil; +import org.graalvm.compiler.phases.BasePhase; +import org.graalvm.compiler.phases.common.DeadCodeEliminationPhase; +import org.graalvm.compiler.phases.common.LazyValue; +import org.graalvm.compiler.phases.tiers.PhaseContext; + +import jdk.vm.ci.meta.Constant; +import jdk.vm.ci.meta.DeoptimizationAction; + +/** + * This phase will find branches which always end with a {@link DeoptimizeNode} and replace their + * {@link ControlSplitNode ControlSplitNodes} with {@link FixedGuardNode FixedGuardNodes}. + * + * This is useful because {@link FixedGuardNode FixedGuardNodes} will be lowered to {@link GuardNode + * GuardNodes} which can later be optimized more aggressively than control-flow constructs. + * + * This is currently only done for branches that start from a {@link IfNode}. If it encounters a + * branch starting at an other kind of {@link ControlSplitNode}, it will only bring the + * {@link DeoptimizeNode} as close to the {@link ControlSplitNode} as possible. + * + */ +public class ConvertDeoptimizeToGuardPhase extends BasePhase { + @Override + @SuppressWarnings("try") + protected void run(final StructuredGraph graph, PhaseContext context) { + assert graph.hasValueProxies() : "ConvertDeoptimizeToGuardPhase always creates proxies"; + assert !graph.getGuardsStage().areFrameStatesAtDeopts() : graph.getGuardsStage(); + LazyValue lazyLoops = new LazyValue<>(() -> new LoopsData(graph)); + + for (DeoptimizeNode d : graph.getNodes(DeoptimizeNode.TYPE)) { + assert d.isAlive(); + if (d.getAction() == DeoptimizationAction.None) { + continue; + } + try (DebugCloseable closable = d.withNodeSourcePosition()) { + propagateFixed(d, d, context != null ? context.getLowerer() : null, lazyLoops); + } + } + + if (context != null) { + for (FixedGuardNode fixedGuard : graph.getNodes(FixedGuardNode.TYPE)) { + try (DebugCloseable closable = fixedGuard.withNodeSourcePosition()) { + trySplitFixedGuard(fixedGuard, context, lazyLoops); + } + } + } + + new DeadCodeEliminationPhase(Optional).apply(graph); + } + + private static void trySplitFixedGuard(FixedGuardNode fixedGuard, PhaseContext context, LazyValue lazyLoops) { + LogicNode condition = fixedGuard.condition(); + if (condition instanceof CompareNode) { + CompareNode compare = (CompareNode) condition; + ValueNode x = compare.getX(); + ValuePhiNode xPhi = (x instanceof ValuePhiNode) ? (ValuePhiNode) x : null; + if (x instanceof ConstantNode || xPhi != null) { + ValueNode y = compare.getY(); + ValuePhiNode yPhi = (y instanceof ValuePhiNode) ? (ValuePhiNode) y : null; + if (y instanceof ConstantNode || yPhi != null) { + processFixedGuardAndPhis(fixedGuard, context, compare, x, xPhi, y, yPhi, lazyLoops); + } + } + } + } + + private static void processFixedGuardAndPhis(FixedGuardNode fixedGuard, PhaseContext context, CompareNode compare, ValueNode x, ValuePhiNode xPhi, ValueNode y, ValuePhiNode yPhi, + LazyValue lazyLoops) { + AbstractBeginNode pred = AbstractBeginNode.prevBegin(fixedGuard); + if (pred instanceof AbstractMergeNode) { + AbstractMergeNode merge = (AbstractMergeNode) pred; + if (xPhi != null && xPhi.merge() != merge) { + return; + } + if (yPhi != null && yPhi.merge() != merge) { + return; + } + + processFixedGuardAndMerge(fixedGuard, context, compare, x, xPhi, y, yPhi, merge, lazyLoops); + } + } + + @SuppressWarnings("try") + private static void processFixedGuardAndMerge(FixedGuardNode fixedGuard, PhaseContext context, CompareNode compare, ValueNode x, ValuePhiNode xPhi, ValueNode y, ValuePhiNode yPhi, + AbstractMergeNode merge, LazyValue lazyLoops) { + List mergePredecessors = merge.cfgPredecessors().snapshot(); + for (AbstractEndNode mergePredecessor : mergePredecessors) { + if (!mergePredecessor.isAlive()) { + break; + } + Constant xs; + if (xPhi == null) { + xs = x.asConstant(); + } else { + xs = xPhi.valueAt(mergePredecessor).asConstant(); + } + Constant ys; + if (yPhi == null) { + ys = y.asConstant(); + } else { + ys = yPhi.valueAt(mergePredecessor).asConstant(); + } + if (xs != null && ys != null && compare.condition().foldCondition(xs, ys, context.getConstantReflection(), compare.unorderedIsTrue()) == fixedGuard.isNegated()) { + try (DebugCloseable position = fixedGuard.withNodeSourcePosition()) { + propagateFixed(mergePredecessor, fixedGuard, context.getLowerer(), lazyLoops); + } + } + } + } + + @SuppressWarnings("try") + private static void propagateFixed(FixedNode from, StaticDeoptimizingNode deopt, LoweringProvider loweringProvider, LazyValue lazyLoops) { + Node current = from; + while (current != null) { + if (GraalOptions.GuardPriorities.getValue(from.getOptions()) && current instanceof FixedGuardNode) { + FixedGuardNode otherGuard = (FixedGuardNode) current; + if (otherGuard.computePriority().isHigherPriorityThan(deopt.computePriority())) { + moveAsDeoptAfter(otherGuard, deopt); + return; + } + } else if (current instanceof AbstractBeginNode) { + if (current instanceof AbstractMergeNode) { + AbstractMergeNode mergeNode = (AbstractMergeNode) current; + FixedNode next = mergeNode.next(); + while (mergeNode.isAlive()) { + AbstractEndNode end = mergeNode.forwardEnds().first(); + propagateFixed(end, deopt, loweringProvider, lazyLoops); + } + if (next.isAlive()) { + propagateFixed(next, deopt, loweringProvider, lazyLoops); + } + return; + } else if (current.predecessor() instanceof IfNode) { + AbstractBeginNode begin = (AbstractBeginNode) current; + IfNode ifNode = (IfNode) current.predecessor(); + if (isOsrLoopExit(begin) || isCountedLoopExit(ifNode, lazyLoops)) { + moveAsDeoptAfter(begin, deopt); + } else { + // Prioritize the source position of the IfNode + try (DebugCloseable closable = ifNode.withNodeSourcePosition()) { + StructuredGraph graph = ifNode.graph(); + LogicNode conditionNode = ifNode.condition(); + boolean negateGuardCondition = current == ifNode.trueSuccessor(); + NodeSourcePosition survivingSuccessorPosition = negateGuardCondition ? ifNode.falseSuccessor().getNodeSourcePosition() : ifNode.trueSuccessor().getNodeSourcePosition(); + FixedGuardNode guard = graph.add( + new FixedGuardNode(conditionNode, deopt.getReason(), deopt.getAction(), deopt.getSpeculation(), negateGuardCondition, survivingSuccessorPosition)); + FixedWithNextNode pred = (FixedWithNextNode) ifNode.predecessor(); + AbstractBeginNode survivingSuccessor; + if (negateGuardCondition) { + survivingSuccessor = ifNode.falseSuccessor(); + } else { + survivingSuccessor = ifNode.trueSuccessor(); + } + graph.removeSplitPropagate(ifNode, survivingSuccessor); + + Node newGuard = guard; + if (survivingSuccessor instanceof LoopExitNode) { + newGuard = ProxyNode.forGuard(guard, (LoopExitNode) survivingSuccessor, graph); + } + survivingSuccessor.replaceAtUsages(InputType.Guard, newGuard); + + graph.getDebug().log("Converting deopt on %-5s branch of %s to guard for remaining branch %s.", negateGuardCondition, ifNode, survivingSuccessor); + FixedNode next = pred.next(); + pred.setNext(guard); + guard.setNext(next); + SimplifierTool simplifierTool = GraphUtil.getDefaultSimplifier(null, null, null, false, graph.getAssumptions(), graph.getOptions(), loweringProvider); + survivingSuccessor.simplify(simplifierTool); + } + } + return; + } else if (current.predecessor() == null || current.predecessor() instanceof ControlSplitNode) { + assert current.predecessor() != null || (current instanceof StartNode && current == ((AbstractBeginNode) current).graph().start()); + moveAsDeoptAfter((AbstractBeginNode) current, deopt); + return; + } + } + current = current.predecessor(); + } + } + + @SuppressWarnings("try") + private static void moveAsDeoptAfter(FixedWithNextNode node, StaticDeoptimizingNode deopt) { + try (DebugCloseable position = deopt.asNode().withNodeSourcePosition()) { + FixedNode next = node.next(); + if (next != deopt.asNode()) { + node.setNext(node.graph().add(new DeoptimizeNode(deopt.getAction(), deopt.getReason(), deopt.getSpeculation()))); + GraphUtil.killCFG(next); + } + } + } + + private static boolean isOsrLoopExit(AbstractBeginNode node) { + if (!(node instanceof LoopExitNode)) { + return false; + } + return ((LoopExitNode) node).loopBegin().isOsrLoop(); + } + + private static boolean isCountedLoopExit(IfNode ifNode, LazyValue lazyLoops) { + LoopsData loopsData = lazyLoops.get(); + Loop loop = loopsData.getCFG().getNodeToBlock().get(ifNode).getLoop(); + if (loop != null) { + LoopEx loopEx = loopsData.loop(loop); + if (loopEx.detectCounted()) { + return ifNode == loopEx.counted().getLimitTest(); + } + } + return false; + } +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/IntegerMulHighNode.java 2019-03-12 08:10:50.756059986 +0100 @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2014, 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.nodes.calc; + +import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_2; +import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_2; + +import org.graalvm.compiler.core.common.type.ArithmeticOpTable; +import org.graalvm.compiler.core.common.type.ArithmeticOpTable.BinaryOp.MulHigh; +import org.graalvm.compiler.graph.NodeClass; +import org.graalvm.compiler.graph.spi.Canonicalizable; +import org.graalvm.compiler.graph.spi.CanonicalizerTool; +import org.graalvm.compiler.lir.gen.ArithmeticLIRGeneratorTool; +import org.graalvm.compiler.nodeinfo.NodeInfo; +import org.graalvm.compiler.nodes.ConstantNode; +import org.graalvm.compiler.nodes.NodeView; +import org.graalvm.compiler.nodes.ValueNode; +import org.graalvm.compiler.nodes.spi.NodeLIRBuilderTool; + +import jdk.vm.ci.meta.Constant; +import jdk.vm.ci.meta.PrimitiveConstant; +import jdk.vm.ci.meta.Value; + +@NodeInfo(shortName = "*H", cycles = CYCLES_2, size = SIZE_2) +public final class IntegerMulHighNode extends BinaryArithmeticNode implements Canonicalizable.BinaryCommutative { + public static final NodeClass TYPE = NodeClass.create(IntegerMulHighNode.class); + + public IntegerMulHighNode(ValueNode x, ValueNode y) { + super(TYPE, ArithmeticOpTable::getMulHigh, x, y); + } + + @Override + public void generate(NodeLIRBuilderTool nodeValueMap, ArithmeticLIRGeneratorTool gen) { + Value a = nodeValueMap.operand(getX()); + Value b = nodeValueMap.operand(getY()); + nodeValueMap.setResult(this, gen.emitMulHigh(a, b)); + } + + @Override + public ValueNode canonical(CanonicalizerTool tool, ValueNode forX, ValueNode forY) { + ValueNode ret = super.canonical(tool, forX, forY); + if (ret != this) { + return ret; + } + + if (forX.isConstant() && !forY.isConstant()) { + // we try to swap and canonicalize + ValueNode improvement = canonical(tool, forY, forX); + if (improvement != this) { + return improvement; + } + // if this fails we only swap + return new IntegerMulHighNode(forY, forX); + } + return canonical(this, forY); + } + + private static ValueNode canonical(IntegerMulHighNode self, ValueNode forY) { + if (forY.isConstant()) { + Constant c = forY.asConstant(); + if (c instanceof PrimitiveConstant && ((PrimitiveConstant) c).getJavaKind().isNumericInteger()) { + long i = ((PrimitiveConstant) c).asLong(); + if (i == 0 || i == 1) { + return ConstantNode.forIntegerStamp(self.stamp(NodeView.DEFAULT), 0); + } + } + } + return self; + } +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/memory/address/IndexAddressNode.java 2019-03-12 08:10:51.244063140 +0100 @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.nodes.memory.address; + +import org.graalvm.compiler.graph.NodeClass; +import org.graalvm.compiler.nodeinfo.InputType; +import org.graalvm.compiler.nodeinfo.NodeInfo; +import org.graalvm.compiler.nodes.ValueNode; +import org.graalvm.compiler.nodes.spi.Lowerable; +import org.graalvm.compiler.nodes.spi.LoweringTool; + +import jdk.vm.ci.meta.JavaKind; + +/** + * Represents an address that points to an element of a Java array. + */ +@NodeInfo(allowedUsageTypes = InputType.Association) +public class IndexAddressNode extends AddressNode implements Lowerable { + public static final NodeClass TYPE = NodeClass.create(IndexAddressNode.class); + + @Input ValueNode array; + @Input ValueNode index; + + private final JavaKind elementKind; + + public IndexAddressNode(ValueNode array, ValueNode index, JavaKind elementKind) { + super(TYPE); + this.array = array; + this.index = index; + this.elementKind = elementKind; + } + + @Override + public ValueNode getBase() { + return array; + } + + public ValueNode getArray() { + return array; + } + + @Override + public ValueNode getIndex() { + return index; + } + + @Override + public long getMaxConstantDisplacement() { + return Long.MAX_VALUE; + } + + public JavaKind getElementKind() { + return elementKind; + } + + @Override + public void lower(LoweringTool tool) { + tool.getLowerer().lower(this, tool); + } +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/OptimizeDivPhase.java 2019-03-12 08:10:51.720066218 +0100 @@ -0,0 +1,208 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.phases.common; + +import jdk.internal.vm.compiler.collections.Pair; +import org.graalvm.compiler.core.common.type.IntegerStamp; +import org.graalvm.compiler.nodes.ConstantNode; +import org.graalvm.compiler.nodes.FixedNode; +import org.graalvm.compiler.nodes.NodeView; +import org.graalvm.compiler.nodes.StructuredGraph; +import org.graalvm.compiler.nodes.ValueNode; +import org.graalvm.compiler.nodes.calc.BinaryArithmeticNode; +import org.graalvm.compiler.nodes.calc.IntegerDivRemNode; +import org.graalvm.compiler.nodes.calc.IntegerMulHighNode; +import org.graalvm.compiler.nodes.calc.MulNode; +import org.graalvm.compiler.nodes.calc.NarrowNode; +import org.graalvm.compiler.nodes.calc.RightShiftNode; +import org.graalvm.compiler.nodes.calc.SignExtendNode; +import org.graalvm.compiler.nodes.calc.SignedDivNode; +import org.graalvm.compiler.nodes.calc.SignedRemNode; +import org.graalvm.compiler.nodes.calc.UnsignedRightShiftNode; +import org.graalvm.compiler.phases.Phase; + +import jdk.vm.ci.code.CodeUtil; + +public class OptimizeDivPhase extends Phase { + + @Override + protected void run(StructuredGraph graph) { + for (IntegerDivRemNode rem : graph.getNodes().filter(IntegerDivRemNode.class)) { + if (rem instanceof SignedRemNode && divByNonZeroConstant(rem)) { + optimizeRem(rem); + } + } + for (IntegerDivRemNode div : graph.getNodes().filter(IntegerDivRemNode.class)) { + if (div instanceof SignedDivNode && divByNonZeroConstant(div)) { + optimizeSignedDiv((SignedDivNode) div); + } + } + } + + @Override + public float codeSizeIncrease() { + return 5.0f; + } + + protected static boolean divByNonZeroConstant(IntegerDivRemNode divRemNode) { + return divRemNode.getY().isConstant() && divRemNode.getY().asJavaConstant().asLong() != 0; + } + + protected final void optimizeRem(IntegerDivRemNode rem) { + assert rem.getOp() == IntegerDivRemNode.Op.REM; + // Java spec 15.17.3.: (a/b)*b+(a%b) == a + // so a%b == a-(a/b)*b + StructuredGraph graph = rem.graph(); + ValueNode div = findDivForRem(rem); + ValueNode mul = BinaryArithmeticNode.mul(graph, div, rem.getY(), NodeView.DEFAULT); + ValueNode result = BinaryArithmeticNode.sub(graph, rem.getX(), mul, NodeView.DEFAULT); + graph.replaceFixedWithFloating(rem, result); + } + + private ValueNode findDivForRem(IntegerDivRemNode rem) { + if (rem.next() instanceof IntegerDivRemNode) { + IntegerDivRemNode div = (IntegerDivRemNode) rem.next(); + if (div.getOp() == IntegerDivRemNode.Op.DIV && div.getType() == rem.getType() && div.getX() == rem.getX() && div.getY() == rem.getY()) { + return div; + } + } + if (rem.predecessor() instanceof IntegerDivRemNode) { + IntegerDivRemNode div = (IntegerDivRemNode) rem.predecessor(); + if (div.getOp() == IntegerDivRemNode.Op.DIV && div.getType() == rem.getType() && div.getX() == rem.getX() && div.getY() == rem.getY()) { + return div; + } + } + + // not found, create a new one (will be optimized away later) + ValueNode div = rem.graph().addOrUniqueWithInputs(createDiv(rem)); + if (div instanceof FixedNode) { + rem.graph().addAfterFixed(rem, (FixedNode) div); + } + return div; + } + + protected ValueNode createDiv(IntegerDivRemNode rem) { + assert rem instanceof SignedRemNode; + return SignedDivNode.create(rem.getX(), rem.getY(), rem.getZeroCheck(), NodeView.DEFAULT); + } + + protected static void optimizeSignedDiv(SignedDivNode div) { + ValueNode forX = div.getX(); + long c = div.getY().asJavaConstant().asLong(); + assert c != 1 && c != -1 && c != 0; + + IntegerStamp dividendStamp = (IntegerStamp) forX.stamp(NodeView.DEFAULT); + int bitSize = dividendStamp.getBits(); + Pair nums = magicDivideConstants(c, bitSize); + long magicNum = nums.getLeft().longValue(); + int shiftNum = nums.getRight().intValue(); + assert shiftNum >= 0; + ConstantNode m = ConstantNode.forLong(magicNum); + + ValueNode value; + if (bitSize == 32) { + value = new MulNode(new SignExtendNode(forX, 64), m); + if ((c > 0 && magicNum < 0) || (c < 0 && magicNum > 0)) { + // Get upper 32-bits of the result + value = NarrowNode.create(new RightShiftNode(value, ConstantNode.forInt(32)), 32, NodeView.DEFAULT); + if (c > 0) { + value = BinaryArithmeticNode.add(value, forX, NodeView.DEFAULT); + } else { + value = BinaryArithmeticNode.sub(value, forX, NodeView.DEFAULT); + } + if (shiftNum > 0) { + value = new RightShiftNode(value, ConstantNode.forInt(shiftNum)); + } + } else { + value = new RightShiftNode(value, ConstantNode.forInt(32 + shiftNum)); + value = new NarrowNode(value, Integer.SIZE); + } + } else { + assert bitSize == 64; + value = new IntegerMulHighNode(forX, m); + if (c > 0 && magicNum < 0) { + value = BinaryArithmeticNode.add(value, forX, NodeView.DEFAULT); + } else if (c < 0 && magicNum > 0) { + value = BinaryArithmeticNode.sub(value, forX, NodeView.DEFAULT); + } + if (shiftNum > 0) { + value = new RightShiftNode(value, ConstantNode.forInt(shiftNum)); + } + } + + if (c < 0) { + ConstantNode s = ConstantNode.forInt(bitSize - 1); + ValueNode sign = UnsignedRightShiftNode.create(value, s, NodeView.DEFAULT); + value = BinaryArithmeticNode.add(value, sign, NodeView.DEFAULT); + } else if (dividendStamp.canBeNegative()) { + ConstantNode s = ConstantNode.forInt(bitSize - 1); + ValueNode sign = UnsignedRightShiftNode.create(forX, s, NodeView.DEFAULT); + value = BinaryArithmeticNode.add(value, sign, NodeView.DEFAULT); + } + + StructuredGraph graph = div.graph(); + graph.replaceFixed(div, graph.addOrUniqueWithInputs(value)); + } + + /** + * Borrowed from Hacker's Delight by Henry S. Warren, Jr. Figure 10-1. + */ + private static Pair magicDivideConstants(long divisor, int size) { + final long twoW = 1L << (size - 1); // 2 ^ (size - 1). + long t = twoW + (divisor >>> 63); + long ad = Math.abs(divisor); + long anc = t - 1 - Long.remainderUnsigned(t, ad); // Absolute value of nc. + long q1 = Long.divideUnsigned(twoW, anc); // Init. q1 = 2**p/|nc|. + long r1 = Long.remainderUnsigned(twoW, anc); // Init. r1 = rem(2**p, |nc|). + long q2 = Long.divideUnsigned(twoW, ad); // Init. q2 = 2**p/|d|. + long r2 = Long.remainderUnsigned(twoW, ad); // Init. r2 = rem(2**p, |d|). + long delta; + + int p = size - 1; // Init. p. + do { + p = p + 1; + q1 = 2 * q1; // Update q1 = 2**p/|nc|. + r1 = 2 * r1; // Update r1 = rem(2**p, |nc|). + if (Long.compareUnsigned(r1, anc) >= 0) { // Must be an unsigned comparison. + q1 = q1 + 1; + r1 = r1 - anc; + } + q2 = 2 * q2; // Update q2 = 2**p/|d|. + r2 = 2 * r2; // Update r2 = rem(2**p, |d|). + if (Long.compareUnsigned(r2, ad) >= 0) { // Must be an unsigned comparison. + q2 = q2 + 1; + r2 = r2 - ad; + } + delta = ad - r2; + } while (Long.compareUnsigned(q1, delta) < 0 || (q1 == delta && r1 == 0)); + + long magic = CodeUtil.signExtend(q2 + 1, size); + if (divisor < 0) { + magic = -magic; + } + return Pair.create(magic, p - size); + } + +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/StringUTF16Substitutions.java 2019-03-12 08:10:52.208069372 +0100 @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.replacements; + +import org.graalvm.compiler.api.replacements.ClassSubstitution; +import org.graalvm.compiler.api.replacements.MethodSubstitution; +import org.graalvm.compiler.nodes.extended.JavaReadNode; +import org.graalvm.compiler.nodes.extended.JavaWriteNode; +import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugin; + +// JaCoCo Exclude + +/** + * Substitutions for {@code StringUTF16} methods for JDK9 and later. + */ +@ClassSubstitution(className = "java.lang.StringUTF16", optional = true) +public class StringUTF16Substitutions { + + @MethodSubstitution + public static char getChar(byte[] value, int i) { + ReplacementsUtil.runtimeAssert((i << 1) + 1 < value.length, "Trusted caller missed bounds check"); + return getCharDirect(value, i << 1); + } + + /** + * Will be intrinsified with an {@link InvocationPlugin} to a {@link JavaReadNode}. + */ + public static native char getCharDirect(byte[] value, int i); + + @MethodSubstitution + public static void putChar(byte[] value, int i, int c) { + ReplacementsUtil.runtimeAssert((i << 1) + 1 < value.length, "Trusted caller missed bounds check"); + putCharDirect(value, i << 1, c); + } + + /** + * Will be intrinsified with an {@link InvocationPlugin} to a {@link JavaWriteNode}. + */ + public static native void putCharDirect(byte[] value, int i, int c); +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/arithmetic/IntegerAddExactOverflowNode.java 2019-03-12 08:10:52.696072526 +0100 @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + + +package org.graalvm.compiler.replacements.nodes.arithmetic; + +import jdk.vm.ci.meta.JavaConstant; +import jdk.vm.ci.meta.JavaKind; +import org.graalvm.compiler.core.common.type.IntegerStamp; +import org.graalvm.compiler.core.common.type.Stamp; +import org.graalvm.compiler.graph.NodeClass; +import org.graalvm.compiler.graph.spi.Canonicalizable.BinaryCommutative; +import org.graalvm.compiler.graph.spi.CanonicalizerTool; +import org.graalvm.compiler.graph.spi.Simplifiable; +import org.graalvm.compiler.nodeinfo.NodeInfo; +import org.graalvm.compiler.nodes.AbstractBeginNode; +import org.graalvm.compiler.nodes.LogicConstantNode; +import org.graalvm.compiler.nodes.NodeView; +import org.graalvm.compiler.nodes.ValueNode; +import org.graalvm.compiler.nodes.calc.BinaryNode; + +import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_2; +import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_2; + +@NodeInfo(cycles = CYCLES_2, size = SIZE_2) +public final class IntegerAddExactOverflowNode extends IntegerExactOverflowNode implements Simplifiable, BinaryCommutative { + public static final NodeClass TYPE = NodeClass.create(IntegerAddExactOverflowNode.class); + + public IntegerAddExactOverflowNode(ValueNode x, ValueNode y) { + super(TYPE, x, y); + } + + @Override + public ValueNode canonical(CanonicalizerTool tool, ValueNode forX, ValueNode forY) { + if (forX.isConstant() && !forY.isConstant()) { + return new IntegerAddExactOverflowNode(forY, forX).canonical(tool); + } + if (forX.isConstant() && forY.isConstant()) { + return canonicalXYconstant(forX, forY); + } else if (forY.isConstant()) { + long c = forY.asJavaConstant().asLong(); + if (c == 0) { + return LogicConstantNode.forBoolean(false); + } + } + if (!IntegerStamp.addCanOverflow((IntegerStamp) forX.stamp(NodeView.DEFAULT), (IntegerStamp) forY.stamp(NodeView.DEFAULT))) { + return LogicConstantNode.forBoolean(false); + } + return this; + } + + private static LogicConstantNode canonicalXYconstant(ValueNode forX, ValueNode forY) { + JavaConstant xConst = forX.asJavaConstant(); + JavaConstant yConst = forY.asJavaConstant(); + assert xConst.getJavaKind() == yConst.getJavaKind(); + try { + if (xConst.getJavaKind() == JavaKind.Int) { + Math.addExact(xConst.asInt(), yConst.asInt()); + } else { + assert xConst.getJavaKind() == JavaKind.Long; + Math.addExact(xConst.asLong(), yConst.asLong()); + } + } catch (ArithmeticException ex) { + // Always overflows + return LogicConstantNode.forBoolean(true); + } + // Never overflows + return LogicConstantNode.forBoolean(false); + } + + @Override + protected IntegerExactArithmeticSplitNode createSplit(Stamp splitStamp, AbstractBeginNode next, AbstractBeginNode overflow) { + return new IntegerAddExactSplitNode(splitStamp, x, y, next, overflow); + } + + @Override + protected Class getCoupledType() { + return IntegerAddExactNode.class; + } +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/arithmetic/IntegerExactOverflowNode.java 2019-03-12 08:10:53.176075628 +0100 @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + + +package org.graalvm.compiler.replacements.nodes.arithmetic; + +import org.graalvm.compiler.core.common.type.Stamp; +import org.graalvm.compiler.graph.NodeClass; +import org.graalvm.compiler.graph.spi.Canonicalizable; +import org.graalvm.compiler.graph.spi.Simplifiable; +import org.graalvm.compiler.graph.spi.SimplifierTool; +import org.graalvm.compiler.nodeinfo.NodeInfo; +import org.graalvm.compiler.nodes.AbstractBeginNode; +import org.graalvm.compiler.nodes.IfNode; +import org.graalvm.compiler.nodes.LogicNode; +import org.graalvm.compiler.nodes.NodeView; +import org.graalvm.compiler.nodes.ValueNode; +import org.graalvm.compiler.nodes.calc.BinaryNode; + +import java.util.List; + +import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_2; +import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_2; + +@NodeInfo(cycles = CYCLES_2, size = SIZE_2) +public abstract class IntegerExactOverflowNode extends LogicNode implements Canonicalizable.Binary, Simplifiable { + public static final NodeClass TYPE = NodeClass.create(IntegerExactOverflowNode.class); + @Input protected ValueNode x; + @Input protected ValueNode y; + + public IntegerExactOverflowNode(NodeClass c, ValueNode x, ValueNode y) { + super(c); + assert x != null && y != null; + this.x = x; + this.y = y; + } + + @Override + public ValueNode getX() { + return x; + } + + @Override + public ValueNode getY() { + return y; + } + + /** + * Make sure the overflow detection nodes have the same order of inputs as the exact arithmetic + * nodes. + * + * @return the original node or another node with the same inputs, ignoring ordering. + */ + @SuppressWarnings("deprecation") + public LogicNode maybeCommuteInputs() { + assert this instanceof BinaryCommutative; + if (!y.isConstant() && (x.isConstant() || x.getId() > y.getId())) { + ValueNode tmp = x; + x = y; + y = tmp; + if (graph() != null) { + // See if this node already exists + LogicNode duplicate = graph().findDuplicate(this); + if (duplicate != null) { + return duplicate; + } + } + } + return this; + } + + protected abstract IntegerExactArithmeticSplitNode createSplit(Stamp splitStamp, AbstractBeginNode next, AbstractBeginNode overflow); + + protected abstract Class getCoupledType(); + + @Override + public void simplify(SimplifierTool tool) { + // Find all ifs that this node feeds into + for (IfNode ifNode : usages().filter(IfNode.class).snapshot()) { + // Replace the if with exact split + AbstractBeginNode next = ifNode.falseSuccessor(); + AbstractBeginNode overflow = ifNode.trueSuccessor(); + ifNode.clearSuccessors(); + + // Try to find corresponding exact nodes that could be combined with the split. They + // would be directly + // linked to the BeginNode of the false branch. + List coupledNodes = next.usages().filter(getCoupledType()).filter(n -> { + BinaryNode exact = (BinaryNode) n; + return exact.getX() == getX() && exact.getY() == getY(); + }).snapshot(); + + Stamp splitStamp = x.stamp(NodeView.DEFAULT).unrestricted(); + if (!coupledNodes.isEmpty()) { + splitStamp = coupledNodes.iterator().next().stamp(NodeView.DEFAULT); + } + IntegerExactArithmeticSplitNode split = graph().add(createSplit(splitStamp, next, overflow)); + ifNode.replaceAndDelete(split); + + coupledNodes.forEach(n -> n.replaceAndDelete(split)); + } + } +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/arithmetic/IntegerMulExactOverflowNode.java 2019-03-12 08:10:53.668078808 +0100 @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + + +package org.graalvm.compiler.replacements.nodes.arithmetic; + +import jdk.vm.ci.meta.JavaConstant; +import jdk.vm.ci.meta.JavaKind; +import org.graalvm.compiler.core.common.type.IntegerStamp; +import org.graalvm.compiler.core.common.type.Stamp; +import org.graalvm.compiler.graph.NodeClass; +import org.graalvm.compiler.graph.spi.Canonicalizable; +import org.graalvm.compiler.graph.spi.CanonicalizerTool; +import org.graalvm.compiler.graph.spi.Simplifiable; +import org.graalvm.compiler.nodeinfo.NodeInfo; +import org.graalvm.compiler.nodes.AbstractBeginNode; +import org.graalvm.compiler.nodes.LogicConstantNode; +import org.graalvm.compiler.nodes.NodeView; +import org.graalvm.compiler.nodes.ValueNode; +import org.graalvm.compiler.nodes.calc.BinaryNode; + +import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_4; +import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_2; + +@NodeInfo(cycles = CYCLES_4, cyclesRationale = "mul+cmp", size = SIZE_2) +public class IntegerMulExactOverflowNode extends IntegerExactOverflowNode implements Simplifiable, Canonicalizable.BinaryCommutative { + public static final NodeClass TYPE = NodeClass.create(IntegerMulExactOverflowNode.class); + + public IntegerMulExactOverflowNode(ValueNode x, ValueNode y) { + super(TYPE, x, y); + } + + @Override + public ValueNode canonical(CanonicalizerTool tool, ValueNode forX, ValueNode forY) { + if (forX.isConstant() && !forY.isConstant()) { + return new IntegerMulExactOverflowNode(forY, forX).canonical(tool); + } + if (forX.isConstant() && forY.isConstant()) { + return canonicalXYconstant(forX, forY); + } else if (forY.isConstant()) { + long c = forY.asJavaConstant().asLong(); + if (c == 1 || c == 0) { + return LogicConstantNode.forBoolean(false); + } + } + if (!IntegerStamp.multiplicationCanOverflow((IntegerStamp) x.stamp(NodeView.DEFAULT), (IntegerStamp) y.stamp(NodeView.DEFAULT))) { + return LogicConstantNode.forBoolean(false); + } + return this; + } + + private static LogicConstantNode canonicalXYconstant(ValueNode forX, ValueNode forY) { + JavaConstant xConst = forX.asJavaConstant(); + JavaConstant yConst = forY.asJavaConstant(); + assert xConst.getJavaKind() == yConst.getJavaKind(); + try { + if (xConst.getJavaKind() == JavaKind.Int) { + Math.multiplyExact(xConst.asInt(), yConst.asInt()); + } else { + assert xConst.getJavaKind() == JavaKind.Long; + Math.multiplyExact(xConst.asLong(), yConst.asLong()); + } + } catch (ArithmeticException ex) { + return LogicConstantNode.forBoolean(true); + } + return LogicConstantNode.forBoolean(false); + } + + @Override + protected IntegerExactArithmeticSplitNode createSplit(Stamp splitStamp, AbstractBeginNode next, AbstractBeginNode overflow) { + return new IntegerMulExactSplitNode(splitStamp, x, y, next, overflow); + } + + @Override + protected Class getCoupledType() { + return IntegerMulExactNode.class; + } + +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/arithmetic/IntegerSubExactOverflowNode.java 2019-03-12 08:10:54.164082015 +0100 @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + + +package org.graalvm.compiler.replacements.nodes.arithmetic; + +import jdk.vm.ci.meta.JavaConstant; +import jdk.vm.ci.meta.JavaKind; +import org.graalvm.compiler.core.common.type.IntegerStamp; +import org.graalvm.compiler.core.common.type.Stamp; +import org.graalvm.compiler.graph.NodeClass; +import org.graalvm.compiler.graph.spi.CanonicalizerTool; +import org.graalvm.compiler.graph.spi.Simplifiable; +import org.graalvm.compiler.nodeinfo.NodeInfo; +import org.graalvm.compiler.nodes.AbstractBeginNode; +import org.graalvm.compiler.nodes.LogicConstantNode; +import org.graalvm.compiler.nodes.NodeView; +import org.graalvm.compiler.nodes.ValueNode; +import org.graalvm.compiler.nodes.calc.BinaryNode; +import org.graalvm.compiler.nodes.util.GraphUtil; + +import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_2; +import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_2; + +@NodeInfo(cycles = CYCLES_2, size = SIZE_2) +public class IntegerSubExactOverflowNode extends IntegerExactOverflowNode implements Simplifiable { + public static final NodeClass TYPE = NodeClass.create(IntegerSubExactOverflowNode.class); + + public IntegerSubExactOverflowNode(ValueNode x, ValueNode y) { + super(TYPE, x, y); + } + + @Override + public ValueNode canonical(CanonicalizerTool tool, ValueNode forX, ValueNode forY) { + if (GraphUtil.unproxify(forX) == GraphUtil.unproxify(forY)) { + return LogicConstantNode.forBoolean(false); + } + if (forX.isConstant() && forY.isConstant()) { + return canonicalXYconstant(forX, forY); + } else if (forY.isConstant()) { + long c = forY.asJavaConstant().asLong(); + if (c == 0) { + return LogicConstantNode.forBoolean(false); + } + } + if (!IntegerStamp.subtractionCanOverflow((IntegerStamp) x.stamp(NodeView.DEFAULT), (IntegerStamp) y.stamp(NodeView.DEFAULT))) { + return LogicConstantNode.forBoolean(false); + } + return this; + } + + private static LogicConstantNode canonicalXYconstant(ValueNode forX, ValueNode forY) { + JavaConstant xConst = forX.asJavaConstant(); + JavaConstant yConst = forY.asJavaConstant(); + assert xConst.getJavaKind() == yConst.getJavaKind(); + try { + if (xConst.getJavaKind() == JavaKind.Int) { + Math.subtractExact(xConst.asInt(), yConst.asInt()); + } else { + assert xConst.getJavaKind() == JavaKind.Long; + Math.subtractExact(xConst.asLong(), yConst.asLong()); + } + } catch (ArithmeticException ex) { + // Always overflows + return LogicConstantNode.forBoolean(true); + } + // Never overflows + return LogicConstantNode.forBoolean(false); + } + + @Override + protected IntegerExactArithmeticSplitNode createSplit(Stamp splitStamp, AbstractBeginNode next, AbstractBeginNode overflow) { + return new IntegerSubExactSplitNode(splitStamp, x, y, next, overflow); + } + + @Override + protected Class getCoupledType() { + return IntegerSubExactNode.class; + } + +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.serviceprovider/src/org/graalvm/compiler/serviceprovider/JavaVersionUtil.java 2019-03-12 08:10:54.656085195 +0100 @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.serviceprovider; + +/** + * Interface to query which JDK version Graal is running on. + */ +public final class JavaVersionUtil { + + private static int getJavaSpecificationVersion() { + String value = System.getProperty("java.specification.version"); + if (value.startsWith("1.")) { + value = value.substring(2); + } + return Integer.parseInt(value); + } + + /** + * The integer value corresponding to the value of the {@code java.specification.version} system + * property after any leading {@code "1."} has been stripped. + */ + public static final int JAVA_SPECIFICATION_VERSION = getJavaSpecificationVersion(); + + /** + * Determines if the Java runtime is version 8 or earlier. + */ + public static final boolean Java8OrEarlier = JAVA_SPECIFICATION_VERSION <= 8; + + /** + * Determines if the Java runtime is version 11 or earlier. + */ + public static final boolean Java11OrEarlier = JAVA_SPECIFICATION_VERSION <= 11; + + private JavaVersionUtil() { + } +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.serviceprovider/src/org/graalvm/compiler/serviceprovider/SpeculationReasonGroup.java 2019-03-12 08:10:55.140088323 +0100 @@ -0,0 +1,169 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.serviceprovider; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; + +import jdk.vm.ci.code.BytecodePosition; +import jdk.vm.ci.meta.ResolvedJavaMethod; +import jdk.vm.ci.meta.ResolvedJavaType; +import jdk.vm.ci.meta.SpeculationLog.SpeculationReason; + +/** + * Facility for creating speculation reasons partitioned in groups. + */ +public final class SpeculationReasonGroup { + + private final int id; + private final String name; + private final Class[] signature; + + private static final AtomicInteger nextId = new AtomicInteger(1); + + /** + * Creates speculation group whose context will always match {@code signature}. + */ + public SpeculationReasonGroup(String name, Class... signature) { + this.id = nextId.get(); + this.name = name; + this.signature = signature; + for (Class c : signature) { + if (!isOfSupportedType(c)) { + throw new IllegalArgumentException("Unsupported speculation context type: " + c.getName()); + } + } + } + + @Override + public String toString() { + return String.format("%s{id:%d, sig=%s}", name, id, Arrays.toString(signature)); + } + + /** + * Creates a speculation reason described by this group. + * + * @param context the components of the reason instance being created + */ + public SpeculationReason createSpeculationReason(Object... context) { + assert checkSignature(context); + return GraalServices.createSpeculationReason(id, name, context); + } + + private static final Set> SUPPORTED_EXACT_TYPES = new HashSet<>(Arrays.asList( + String.class, + int.class, + long.class, + float.class, + double.class, + BytecodePosition.class)); + + private static boolean isOfSupportedType(Class c) { + if (SUPPORTED_EXACT_TYPES.contains(c)) { + return true; + } + if (Enum.class.isAssignableFrom(c)) { + // Trust the ordinal of an Enum to be unique + return true; + } + if (SpeculationContextObject.class.isAssignableFrom(c)) { + return true; + } + if (ResolvedJavaMethod.class.isAssignableFrom(c) || ResolvedJavaType.class.isAssignableFrom(c)) { + // Only the JVMCI implementation specific concrete subclasses + // of these types will be accepted but we cannot test for that + // here since we are in JVMCI implementation agnostic code. + return true; + } + return false; + } + + static Class toBox(Class c) { + if (c == int.class) { + return Integer.class; + } + if (c == long.class) { + return Long.class; + } + if (c == float.class) { + return Float.class; + } + if (c == double.class) { + return Double.class; + } + return c; + } + + private boolean checkSignature(Object[] context) { + assert signature.length == context.length : name + ": Incorrect number of context arguments. Expected " + signature.length + ", got " + context.length; + for (int i = 0; i < context.length; i++) { + Object o = context[i]; + Class c = signature[i]; + if (o != null) { + if (c == ResolvedJavaMethod.class || c == ResolvedJavaType.class || SpeculationContextObject.class.isAssignableFrom(c)) { + c.cast(o); + } else { + Class oClass = o.getClass(); + assert toBox(c) == oClass : name + ": Context argument " + i + " is not a " + c.getName() + " but a " + oClass.getName(); + } + } else { + if (c.isPrimitive() || Enum.class.isAssignableFrom(c)) { + throw new AssertionError(name + ": Cannot pass null for argument " + i); + } + } + } + return true; + } + + /** + * Denotes part of a {@linkplain SpeculationReasonGroup#createSpeculationReason(Object...) + * reason} that can have its attributes {@linkplain #accept(Visitor) visited}. + */ + public interface SpeculationContextObject { + void accept(Visitor v); + + public interface Visitor { + void visitBoolean(boolean v); + + void visitByte(byte v); + + void visitChar(char v); + + void visitShort(short v); + + void visitInt(int v); + + void visitLong(long v); + + void visitFloat(float v); + + void visitDouble(double v); + + void visitObject(Object v); + } + } +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.util/src/org/graalvm/util/DirectAnnotationAccess.java 2019-03-12 08:10:55.632091503 +0100 @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.util; + +//Checkstyle: allow reflection +import java.lang.annotation.Annotation; +import java.lang.reflect.AnnotatedElement; + +/** + * Wrapper class for annotation access. The purpose of this class is to encapsulate the + * AnnotatedElement.getAnnotation() to avoid the use of the "Checkstyle: allow direct annotation + * access " and "Checkstyle: disallow direct annotation access" comments for situations where the + * annotation access doesn't need to guarded, i.e., in runtime code or code that accesses annotation + * on non-user types. See {@link GuardedAnnotationAccess} for details on these checkstyle rules. + */ +public class DirectAnnotationAccess { + + public static boolean isAnnotationPresent(AnnotatedElement element, Class annotationClass) { + return element.getAnnotation(annotationClass) != null; + } + + public static T getAnnotation(AnnotatedElement element, Class annotationType) { + return element.getAnnotation(annotationType); + } +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.util/src/org/graalvm/util/GuardedAnnotationAccess.java 2019-03-12 08:10:56.124094683 +0100 @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.util; + +import java.lang.annotation.Annotation; +import java.lang.reflect.AnnotatedElement; + +/** + * Wrapper class for annotation access that defends against + * https://bugs.openjdk.java.net/browse/JDK-7183985: when an annotation declares a Class array + * parameter and one of the referenced classes is not present on the classpath parsing the + * annotations will result in an ArrayStoreException instead of caching of a + * TypeNotPresentExceptionProxy. This is a problem in JDK8 but was fixed in JDK11+. This wrapper + * class also defends against incomplete class path issues. If the element for which annotations are + * queried is a JMVCI value, i.e., a HotSpotResolvedJavaField, or HotSpotResolvedJavaMethod, the + * annotations are read via HotSpotJDKReflection using the + * getFieldAnnotation()/getMethodAnnotation() methods which first construct the field/method object + * via CompilerToVM.asReflectionField()/CompilerToVM.asReflectionExecutable() which eagerly try to + * resolve the types referenced in the element signature. If a field declared type or a method + * return type is missing then JVMCI throws a NoClassDefFoundError. + */ +public final class GuardedAnnotationAccess { + + public static boolean isAnnotationPresent(AnnotatedElement element, Class annotationClass) { + return getAnnotation(element, annotationClass) != null; + } + + public static T getAnnotation(AnnotatedElement element, Class annotationType) { + try { + return element.getAnnotation(annotationType); + } catch (ArrayStoreException | NoClassDefFoundError e) { + /* + * Returning null essentially means that the element doesn't declare the annotationType, + * but we cannot know that since the annotation parsing failed. However, this allows us + * to defend against crashing the image builder if the above JDK bug is encountered in + * user code or if the user code references types missing from the classpath. + */ + return null; + } + } + + public static Annotation[] getAnnotations(AnnotatedElement element) { + try { + return element.getAnnotations(); + } catch (ArrayStoreException | NoClassDefFoundError e) { + /* + * Returning an empty array essentially means that the element doesn't declare any + * annotations, but we know that it is not true since the reason the annotation parsing + * failed is because some annotation referenced a missing class. However, this allows us + * to defend against crashing the image builder if the above JDK bug is encountered in + * user code or if the user code references types missing from the classpath. + */ + return new Annotation[0]; + } + } + + public static T getDeclaredAnnotation(AnnotatedElement element, Class annotationType) { + try { + return element.getDeclaredAnnotation(annotationType); + } catch (ArrayStoreException | NoClassDefFoundError e) { + /* + * Returning null essentially means that the element doesn't declare the annotationType, + * but we cannot know that since the annotation parsing failed. However, this allows us + * to defend against crashing the image builder if the above JDK bug is encountered in + * user code or if the user code references types missing from the classpath. + */ + return null; + } + } + + public static Annotation[] getDeclaredAnnotations(AnnotatedElement element) { + try { + return element.getDeclaredAnnotations(); + } catch (ArrayStoreException | NoClassDefFoundError e) { + /* + * Returning an empty array essentially means that the element doesn't declare any + * annotations, but we know that it is not true since the reason the annotation parsing + * failed is because it at least one annotation referenced a missing class. However, + * this allows us to defend against crashing the image builder if the above JDK bug is + * encountered in user code or if the user code references types missing from the + * classpath. + */ + return new Annotation[0]; + } + } +} --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/test/hotspot/jtreg/compiler/graalunit/EA9Test.java 2019-03-12 08:10:56.616097861 +0100 @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/* + * @test + * @summary + * @requires vm.opt.final.EnableJVMCI == true + * + * @modules jdk.internal.vm.compiler + * + * @library /test/lib /compiler/graalunit / + * + * @build compiler.graalunit.common.GraalUnitTestLauncher + * + * @run driver jdk.test.lib.FileInstaller ../../ProblemList-graal.txt ExcludeList.txt + * + * @run main/othervm compiler.graalunit.common.GraalUnitTestLauncher -prefix org.graalvm.compiler.core.jdk9.test.ea -exclude ExcludeList.txt + */ --- /dev/null 2017-11-16 08:17:56.803999947 +0100 +++ new/test/hotspot/jtreg/compiler/graalunit/HotspotJdk9Test.java 2019-03-12 08:10:57.100100989 +0100 @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/* + * @test + * @summary + * @requires vm.opt.final.EnableJVMCI == true + * + * @modules jdk.internal.vm.compiler + * + * @library /test/lib /compiler/graalunit / + * + * @build compiler.graalunit.common.GraalUnitTestLauncher + * + * @run driver jdk.test.lib.FileInstaller ../../ProblemList-graal.txt ExcludeList.txt + * + * @run main/othervm compiler.graalunit.common.GraalUnitTestLauncher -prefix org.graalvm.compiler.hotspot.jdk9.test -exclude ExcludeList.txt + */ --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotMaths.java 2019-03-12 08:10:57.944106445 +0100 +++ /dev/null 2017-11-16 08:17:56.803999947 +0100 @@ -1,88 +0,0 @@ -/* - * Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ - - -package org.graalvm.compiler.hotspot.amd64; - -import static org.graalvm.compiler.hotspot.HotSpotBackend.Options.GraalArithmeticStubs; -import static org.graalvm.compiler.hotspot.amd64.AMD64HotSpotMathIntrinsicOp.IntrinsicOpcode.COS; -import static org.graalvm.compiler.hotspot.amd64.AMD64HotSpotMathIntrinsicOp.IntrinsicOpcode.LOG; -import static org.graalvm.compiler.hotspot.amd64.AMD64HotSpotMathIntrinsicOp.IntrinsicOpcode.LOG10; -import static org.graalvm.compiler.hotspot.amd64.AMD64HotSpotMathIntrinsicOp.IntrinsicOpcode.SIN; -import static org.graalvm.compiler.hotspot.amd64.AMD64HotSpotMathIntrinsicOp.IntrinsicOpcode.TAN; - -import org.graalvm.compiler.core.amd64.AMD64ArithmeticLIRGenerator; -import org.graalvm.compiler.core.common.LIRKind; -import org.graalvm.compiler.hotspot.HotSpotBackend.Options; -import org.graalvm.compiler.lir.Variable; -import org.graalvm.compiler.lir.gen.LIRGenerator; - -import jdk.vm.ci.meta.Value; - -/** - * Lowering of selected {@link Math} routines that depends on the value of - * {@link Options#GraalArithmeticStubs}. - */ -public class AMD64HotSpotMaths implements AMD64ArithmeticLIRGenerator.Maths { - - @Override - public Variable emitLog(LIRGenerator gen, Value input, boolean base10) { - if (GraalArithmeticStubs.getValue(gen.getResult().getLIR().getOptions())) { - return null; - } - Variable result = gen.newVariable(LIRKind.combine(input)); - gen.append(new AMD64HotSpotMathIntrinsicOp(base10 ? LOG10 : LOG, result, gen.asAllocatable(input))); - return result; - } - - @Override - public Variable emitCos(LIRGenerator gen, Value input) { - if (GraalArithmeticStubs.getValue(gen.getResult().getLIR().getOptions())) { - return null; - } - Variable result = gen.newVariable(LIRKind.combine(input)); - gen.append(new AMD64HotSpotMathIntrinsicOp(COS, result, gen.asAllocatable(input))); - return result; - } - - @Override - public Variable emitSin(LIRGenerator gen, Value input) { - if (GraalArithmeticStubs.getValue(gen.getResult().getLIR().getOptions())) { - return null; - } - Variable result = gen.newVariable(LIRKind.combine(input)); - gen.append(new AMD64HotSpotMathIntrinsicOp(SIN, result, gen.asAllocatable(input))); - return result; - } - - @Override - public Variable emitTan(LIRGenerator gen, Value input) { - if (GraalArithmeticStubs.getValue(gen.getResult().getLIR().getOptions())) { - return null; - } - Variable result = gen.newVariable(LIRKind.combine(input)); - gen.append(new AMD64HotSpotMathIntrinsicOp(TAN, result, gen.asAllocatable(input))); - return result; - } - -} --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/nodes/ArrayRangeWriteBarrier.java 2019-03-12 08:10:58.368109185 +0100 +++ /dev/null 2017-11-16 08:17:56.803999947 +0100 @@ -1,61 +0,0 @@ -/* - * Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ - - -package org.graalvm.compiler.hotspot.nodes; - -import org.graalvm.compiler.graph.NodeClass; -import org.graalvm.compiler.nodeinfo.InputType; -import org.graalvm.compiler.nodeinfo.NodeInfo; -import org.graalvm.compiler.nodes.ValueNode; -import org.graalvm.compiler.nodes.memory.address.AddressNode; -import org.graalvm.compiler.nodes.spi.Lowerable; - -@NodeInfo -public abstract class ArrayRangeWriteBarrier extends WriteBarrier implements Lowerable { - - public static final NodeClass TYPE = NodeClass.create(ArrayRangeWriteBarrier.class); - @Input(InputType.Association) AddressNode address; - @Input ValueNode length; - - private final int elementStride; - - protected ArrayRangeWriteBarrier(NodeClass c, AddressNode address, ValueNode length, int elementStride) { - super(c); - this.address = address; - this.length = length; - this.elementStride = elementStride; - } - - public AddressNode getAddress() { - return address; - } - - public ValueNode getLength() { - return length; - } - - public int getElementStride() { - return elementStride; - } -} --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/nodes/G1ArrayRangePostWriteBarrier.java 2019-03-12 08:10:58.780111847 +0100 +++ /dev/null 2017-11-16 08:17:56.803999947 +0100 @@ -1,43 +0,0 @@ -/* - * Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ - - -package org.graalvm.compiler.hotspot.nodes; - -import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_64; -import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_64; - -import org.graalvm.compiler.graph.NodeClass; -import org.graalvm.compiler.nodeinfo.NodeInfo; -import org.graalvm.compiler.nodes.ValueNode; -import org.graalvm.compiler.nodes.memory.address.AddressNode; - -@NodeInfo(cycles = CYCLES_64, size = SIZE_64) -public class G1ArrayRangePostWriteBarrier extends ArrayRangeWriteBarrier { - public static final NodeClass TYPE = NodeClass.create(G1ArrayRangePostWriteBarrier.class); - - public G1ArrayRangePostWriteBarrier(AddressNode address, ValueNode length, int elementStride) { - super(TYPE, address, length, elementStride); - } - -} --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/nodes/G1ArrayRangePreWriteBarrier.java 2019-03-12 08:10:59.200114561 +0100 +++ /dev/null 2017-11-16 08:17:56.803999947 +0100 @@ -1,43 +0,0 @@ -/* - * Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ - - -package org.graalvm.compiler.hotspot.nodes; - -import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_64; -import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_64; - -import org.graalvm.compiler.graph.NodeClass; -import org.graalvm.compiler.nodeinfo.NodeInfo; -import org.graalvm.compiler.nodes.ValueNode; -import org.graalvm.compiler.nodes.memory.address.AddressNode; - -@NodeInfo(cycles = CYCLES_64, size = SIZE_64) -public final class G1ArrayRangePreWriteBarrier extends ArrayRangeWriteBarrier { - public static final NodeClass TYPE = NodeClass.create(G1ArrayRangePreWriteBarrier.class); - - public G1ArrayRangePreWriteBarrier(AddressNode address, ValueNode length, int elementStride) { - super(TYPE, address, length, elementStride); - } - -} --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/nodes/G1PostWriteBarrier.java 2019-03-12 08:10:59.612117223 +0100 +++ /dev/null 2017-11-16 08:17:56.803999947 +0100 @@ -1,53 +0,0 @@ -/* - * Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ - - -package org.graalvm.compiler.hotspot.nodes; - -import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_64; -import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_64; - -import org.graalvm.compiler.graph.NodeClass; -import org.graalvm.compiler.nodeinfo.NodeInfo; -import org.graalvm.compiler.nodes.ValueNode; -import org.graalvm.compiler.nodes.memory.address.AddressNode; - -@NodeInfo(cycles = CYCLES_64, size = SIZE_64) -public class G1PostWriteBarrier extends ObjectWriteBarrier { - - public static final NodeClass TYPE = NodeClass.create(G1PostWriteBarrier.class); - protected final boolean alwaysNull; - - public G1PostWriteBarrier(AddressNode address, ValueNode value, boolean precise, boolean alwaysNull) { - this(TYPE, address, value, precise, alwaysNull); - } - - protected G1PostWriteBarrier(NodeClass c, AddressNode address, ValueNode value, boolean precise, boolean alwaysNull) { - super(c, address, value, precise); - this.alwaysNull = alwaysNull; - } - - public boolean alwaysNull() { - return alwaysNull; - } -} --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/nodes/G1PreWriteBarrier.java 2019-03-12 08:11:00.024119886 +0100 +++ /dev/null 2017-11-16 08:17:56.803999947 +0100 @@ -1,80 +0,0 @@ -/* - * Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ - - -package org.graalvm.compiler.hotspot.nodes; - -import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_64; -import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_64; - -import org.graalvm.compiler.graph.NodeClass; -import org.graalvm.compiler.nodeinfo.InputType; -import org.graalvm.compiler.nodeinfo.NodeInfo; -import org.graalvm.compiler.nodes.DeoptimizingNode; -import org.graalvm.compiler.nodes.FrameState; -import org.graalvm.compiler.nodes.ValueNode; -import org.graalvm.compiler.nodes.memory.address.AddressNode; - -@NodeInfo(cycles = CYCLES_64, size = SIZE_64) -public final class G1PreWriteBarrier extends ObjectWriteBarrier implements DeoptimizingNode.DeoptBefore { - - public static final NodeClass TYPE = NodeClass.create(G1PreWriteBarrier.class); - - @OptionalInput(InputType.State) FrameState stateBefore; - protected final boolean nullCheck; - protected final boolean doLoad; - - public G1PreWriteBarrier(AddressNode address, ValueNode expectedObject, boolean doLoad, boolean nullCheck) { - super(TYPE, address, expectedObject, true); - this.doLoad = doLoad; - this.nullCheck = nullCheck; - } - - public ValueNode getExpectedObject() { - return getValue(); - } - - public boolean doLoad() { - return doLoad; - } - - public boolean getNullCheck() { - return nullCheck; - } - - @Override - public boolean canDeoptimize() { - return nullCheck; - } - - @Override - public FrameState stateBefore() { - return stateBefore; - } - - @Override - public void setStateBefore(FrameState state) { - updateUsages(stateBefore, state); - stateBefore = state; - } -} --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/nodes/G1ReferentFieldReadBarrier.java 2019-03-12 08:11:00.456122677 +0100 +++ /dev/null 2017-11-16 08:17:56.803999947 +0100 @@ -1,59 +0,0 @@ -/* - * Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ - - -package org.graalvm.compiler.hotspot.nodes; - -import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_64; -import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_64; - -import org.graalvm.compiler.graph.NodeClass; -import org.graalvm.compiler.nodeinfo.NodeInfo; -import org.graalvm.compiler.nodes.ValueNode; -import org.graalvm.compiler.nodes.memory.address.AddressNode; - -/** - * The {@code G1ReferentFieldReadBarrier} is added when a read access is performed to the referent - * field of a {@link java.lang.ref.Reference} object (through a {@code LoadFieldNode} or an - * {@code UnsafeLoadNode}). The return value of the read is passed to the snippet implementing the - * read barrier and consequently is added to the SATB queue if the concurrent marker is enabled. - */ -@NodeInfo(cycles = CYCLES_64, size = SIZE_64) -public final class G1ReferentFieldReadBarrier extends ObjectWriteBarrier { - public static final NodeClass TYPE = NodeClass.create(G1ReferentFieldReadBarrier.class); - - protected final boolean doLoad; - - public G1ReferentFieldReadBarrier(AddressNode address, ValueNode expectedObject, boolean doLoad) { - super(TYPE, address, expectedObject, true); - this.doLoad = doLoad; - } - - public ValueNode getExpectedObject() { - return getValue(); - } - - public boolean doLoad() { - return doLoad; - } -} --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/nodes/ObjectWriteBarrier.java 2019-03-12 08:11:00.868125339 +0100 +++ /dev/null 2017-11-16 08:17:56.803999947 +0100 @@ -1,59 +0,0 @@ -/* - * Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ - - -package org.graalvm.compiler.hotspot.nodes; - -import org.graalvm.compiler.graph.NodeClass; -import org.graalvm.compiler.nodeinfo.InputType; -import org.graalvm.compiler.nodeinfo.NodeInfo; -import org.graalvm.compiler.nodes.ValueNode; -import org.graalvm.compiler.nodes.memory.address.AddressNode; - -@NodeInfo -public abstract class ObjectWriteBarrier extends WriteBarrier { - - public static final NodeClass TYPE = NodeClass.create(ObjectWriteBarrier.class); - @Input(InputType.Association) protected AddressNode address; - @OptionalInput protected ValueNode value; - protected final boolean precise; - - protected ObjectWriteBarrier(NodeClass c, AddressNode address, ValueNode value, boolean precise) { - super(c); - this.address = address; - this.value = value; - this.precise = precise; - } - - public ValueNode getValue() { - return value; - } - - public AddressNode getAddress() { - return address; - } - - public boolean usePrecise() { - return precise; - } -} --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/nodes/SerialArrayRangeWriteBarrier.java 2019-03-12 08:11:01.284128027 +0100 +++ /dev/null 2017-11-16 08:17:56.803999947 +0100 @@ -1,43 +0,0 @@ -/* - * Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ - - -package org.graalvm.compiler.hotspot.nodes; - -import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_8; -import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_8; - -import org.graalvm.compiler.graph.NodeClass; -import org.graalvm.compiler.nodeinfo.NodeInfo; -import org.graalvm.compiler.nodes.ValueNode; -import org.graalvm.compiler.nodes.memory.address.AddressNode; - -@NodeInfo(cycles = CYCLES_8, size = SIZE_8) -public final class SerialArrayRangeWriteBarrier extends ArrayRangeWriteBarrier { - public static final NodeClass TYPE = NodeClass.create(SerialArrayRangeWriteBarrier.class); - - public SerialArrayRangeWriteBarrier(AddressNode address, ValueNode length, int elementStride) { - super(TYPE, address, length, elementStride); - } - -} --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/nodes/SerialWriteBarrier.java 2019-03-12 08:11:01.704130740 +0100 +++ /dev/null 2017-11-16 08:17:56.803999947 +0100 @@ -1,46 +0,0 @@ -/* - * Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ - - -package org.graalvm.compiler.hotspot.nodes; - -import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_8; -import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_4; - -import org.graalvm.compiler.graph.NodeClass; -import org.graalvm.compiler.nodeinfo.NodeInfo; -import org.graalvm.compiler.nodes.memory.address.AddressNode; - -@NodeInfo(cycles = CYCLES_8, size = SIZE_4) -public class SerialWriteBarrier extends ObjectWriteBarrier { - - public static final NodeClass TYPE = NodeClass.create(SerialWriteBarrier.class); - - public SerialWriteBarrier(AddressNode address, boolean precise) { - this(TYPE, address, precise); - } - - protected SerialWriteBarrier(NodeClass c, AddressNode address, boolean precise) { - super(c, address, null, precise); - } -} --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.java/src/org/graalvm/compiler/java/IntegerExactOpSpeculation.java 2019-03-12 08:11:02.124133454 +0100 +++ /dev/null 2017-11-16 08:17:56.803999947 +0100 @@ -1,62 +0,0 @@ -/* - * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ - - -package org.graalvm.compiler.java; - -import jdk.vm.ci.meta.ResolvedJavaMethod; -import jdk.vm.ci.meta.SpeculationLog.SpeculationReason; - -public final class IntegerExactOpSpeculation implements SpeculationReason { - - public enum IntegerExactOp { - INTEGER_ADD_EXACT, - INTEGER_INCREMENT_EXACT, - INTEGER_SUBTRACT_EXACT, - INTEGER_DECREMENT_EXACT, - INTEGER_MULTIPLY_EXACT - } - - protected final String methodDescriptor; - protected final IntegerExactOp op; - - public IntegerExactOpSpeculation(ResolvedJavaMethod method, IntegerExactOp op) { - this.methodDescriptor = method.format("%H.%n(%p)%R"); - this.op = op; - } - - @Override - public int hashCode() { - return methodDescriptor.hashCode() * 31 + op.ordinal(); - } - - @Override - public boolean equals(Object obj) { - if (obj instanceof IntegerExactOpSpeculation) { - IntegerExactOpSpeculation other = (IntegerExactOpSpeculation) obj; - return op.equals(other.op) && methodDescriptor.equals(other.methodDescriptor); - } - return false; - } - -} --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/ConvertDeoptimizeToGuardPhase.java 2019-03-12 08:11:02.548136194 +0100 +++ /dev/null 2017-11-16 08:17:56.803999947 +0100 @@ -1,242 +0,0 @@ -/* - * Copyright (c) 2011, 2018, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ - - -package org.graalvm.compiler.phases.common; - -import static org.graalvm.compiler.phases.common.DeadCodeEliminationPhase.Optionality.Optional; - -import java.util.List; - -import org.graalvm.compiler.core.common.GraalOptions; -import org.graalvm.compiler.debug.DebugCloseable; -import org.graalvm.compiler.graph.Node; -import org.graalvm.compiler.graph.NodeSourcePosition; -import org.graalvm.compiler.graph.spi.SimplifierTool; -import org.graalvm.compiler.nodeinfo.InputType; -import org.graalvm.compiler.nodes.AbstractBeginNode; -import org.graalvm.compiler.nodes.AbstractEndNode; -import org.graalvm.compiler.nodes.AbstractMergeNode; -import org.graalvm.compiler.nodes.ConstantNode; -import org.graalvm.compiler.nodes.ControlSplitNode; -import org.graalvm.compiler.nodes.DeoptimizeNode; -import org.graalvm.compiler.nodes.EndNode; -import org.graalvm.compiler.nodes.FixedGuardNode; -import org.graalvm.compiler.nodes.FixedNode; -import org.graalvm.compiler.nodes.FixedWithNextNode; -import org.graalvm.compiler.nodes.GuardNode; -import org.graalvm.compiler.nodes.IfNode; -import org.graalvm.compiler.nodes.LogicNode; -import org.graalvm.compiler.nodes.LoopExitNode; -import org.graalvm.compiler.nodes.ProxyNode; -import org.graalvm.compiler.nodes.StartNode; -import org.graalvm.compiler.nodes.StaticDeoptimizingNode; -import org.graalvm.compiler.nodes.StructuredGraph; -import org.graalvm.compiler.nodes.ValueNode; -import org.graalvm.compiler.nodes.ValuePhiNode; -import org.graalvm.compiler.nodes.calc.CompareNode; -import org.graalvm.compiler.nodes.spi.LoweringProvider; -import org.graalvm.compiler.nodes.util.GraphUtil; -import org.graalvm.compiler.phases.BasePhase; -import org.graalvm.compiler.phases.tiers.PhaseContext; - -import jdk.vm.ci.meta.Constant; -import jdk.vm.ci.meta.DeoptimizationAction; - -/** - * This phase will find branches which always end with a {@link DeoptimizeNode} and replace their - * {@link ControlSplitNode ControlSplitNodes} with {@link FixedGuardNode FixedGuardNodes}. - * - * This is useful because {@link FixedGuardNode FixedGuardNodes} will be lowered to {@link GuardNode - * GuardNodes} which can later be optimized more aggressively than control-flow constructs. - * - * This is currently only done for branches that start from a {@link IfNode}. If it encounters a - * branch starting at an other kind of {@link ControlSplitNode}, it will only bring the - * {@link DeoptimizeNode} as close to the {@link ControlSplitNode} as possible. - * - */ -public class ConvertDeoptimizeToGuardPhase extends BasePhase { - @Override - @SuppressWarnings("try") - protected void run(final StructuredGraph graph, PhaseContext context) { - assert graph.hasValueProxies() : "ConvertDeoptimizeToGuardPhase always creates proxies"; - assert !graph.getGuardsStage().areFrameStatesAtDeopts() : graph.getGuardsStage(); - - for (DeoptimizeNode d : graph.getNodes(DeoptimizeNode.TYPE)) { - assert d.isAlive(); - if (d.getAction() == DeoptimizationAction.None) { - continue; - } - try (DebugCloseable closable = d.withNodeSourcePosition()) { - propagateFixed(d, d, context != null ? context.getLowerer() : null); - } - } - - if (context != null) { - for (FixedGuardNode fixedGuard : graph.getNodes(FixedGuardNode.TYPE)) { - try (DebugCloseable closable = fixedGuard.withNodeSourcePosition()) { - trySplitFixedGuard(fixedGuard, context); - } - } - } - - new DeadCodeEliminationPhase(Optional).apply(graph); - } - - private void trySplitFixedGuard(FixedGuardNode fixedGuard, PhaseContext context) { - LogicNode condition = fixedGuard.condition(); - if (condition instanceof CompareNode) { - CompareNode compare = (CompareNode) condition; - ValueNode x = compare.getX(); - ValuePhiNode xPhi = (x instanceof ValuePhiNode) ? (ValuePhiNode) x : null; - if (x instanceof ConstantNode || xPhi != null) { - ValueNode y = compare.getY(); - ValuePhiNode yPhi = (y instanceof ValuePhiNode) ? (ValuePhiNode) y : null; - if (y instanceof ConstantNode || yPhi != null) { - processFixedGuardAndPhis(fixedGuard, context, compare, x, xPhi, y, yPhi); - } - } - } - } - - private void processFixedGuardAndPhis(FixedGuardNode fixedGuard, PhaseContext context, CompareNode compare, ValueNode x, ValuePhiNode xPhi, ValueNode y, ValuePhiNode yPhi) { - AbstractBeginNode pred = AbstractBeginNode.prevBegin(fixedGuard); - if (pred instanceof AbstractMergeNode) { - AbstractMergeNode merge = (AbstractMergeNode) pred; - if (xPhi != null && xPhi.merge() != merge) { - return; - } - if (yPhi != null && yPhi.merge() != merge) { - return; - } - - processFixedGuardAndMerge(fixedGuard, context, compare, x, xPhi, y, yPhi, merge); - } - } - - @SuppressWarnings("try") - private void processFixedGuardAndMerge(FixedGuardNode fixedGuard, PhaseContext context, CompareNode compare, ValueNode x, ValuePhiNode xPhi, ValueNode y, ValuePhiNode yPhi, - AbstractMergeNode merge) { - List mergePredecessors = merge.cfgPredecessors().snapshot(); - for (int i = 0; i < mergePredecessors.size(); ++i) { - AbstractEndNode mergePredecessor = mergePredecessors.get(i); - if (!mergePredecessor.isAlive()) { - break; - } - Constant xs; - if (xPhi == null) { - xs = x.asConstant(); - } else { - xs = xPhi.valueAt(mergePredecessor).asConstant(); - } - Constant ys; - if (yPhi == null) { - ys = y.asConstant(); - } else { - ys = yPhi.valueAt(mergePredecessor).asConstant(); - } - if (xs != null && ys != null && compare.condition().foldCondition(xs, ys, context.getConstantReflection(), compare.unorderedIsTrue()) == fixedGuard.isNegated()) { - try (DebugCloseable position = fixedGuard.withNodeSourcePosition()) { - propagateFixed(mergePredecessor, fixedGuard, context.getLowerer()); - } - } - } - } - - @SuppressWarnings("try") - private void propagateFixed(FixedNode from, StaticDeoptimizingNode deopt, LoweringProvider loweringProvider) { - Node current = from; - while (current != null) { - if (GraalOptions.GuardPriorities.getValue(from.getOptions()) && current instanceof FixedGuardNode) { - FixedGuardNode otherGuard = (FixedGuardNode) current; - if (otherGuard.computePriority().isHigherPriorityThan(deopt.computePriority())) { - moveAsDeoptAfter(otherGuard, deopt); - return; - } - } else if (current instanceof AbstractBeginNode) { - if (current instanceof AbstractMergeNode) { - AbstractMergeNode mergeNode = (AbstractMergeNode) current; - FixedNode next = mergeNode.next(); - while (mergeNode.isAlive()) { - AbstractEndNode end = mergeNode.forwardEnds().first(); - propagateFixed(end, deopt, loweringProvider); - } - if (next.isAlive()) { - propagateFixed(next, deopt, loweringProvider); - } - return; - } else if (current.predecessor() instanceof IfNode) { - IfNode ifNode = (IfNode) current.predecessor(); - // Prioritize the source position of the IfNode - try (DebugCloseable closable = ifNode.withNodeSourcePosition()) { - StructuredGraph graph = ifNode.graph(); - LogicNode conditionNode = ifNode.condition(); - boolean negateGuardCondition = current == ifNode.trueSuccessor(); - NodeSourcePosition survivingSuccessorPosition = negateGuardCondition ? ifNode.falseSuccessor().getNodeSourcePosition() : ifNode.trueSuccessor().getNodeSourcePosition(); - FixedGuardNode guard = graph.add( - new FixedGuardNode(conditionNode, deopt.getReason(), deopt.getAction(), deopt.getSpeculation(), negateGuardCondition, survivingSuccessorPosition)); - - FixedWithNextNode pred = (FixedWithNextNode) ifNode.predecessor(); - AbstractBeginNode survivingSuccessor; - if (negateGuardCondition) { - survivingSuccessor = ifNode.falseSuccessor(); - } else { - survivingSuccessor = ifNode.trueSuccessor(); - } - graph.removeSplitPropagate(ifNode, survivingSuccessor); - - Node newGuard = guard; - if (survivingSuccessor instanceof LoopExitNode) { - newGuard = ProxyNode.forGuard(guard, (LoopExitNode) survivingSuccessor, graph); - } - survivingSuccessor.replaceAtUsages(InputType.Guard, newGuard); - - graph.getDebug().log("Converting deopt on %-5s branch of %s to guard for remaining branch %s.", negateGuardCondition, ifNode, survivingSuccessor); - FixedNode next = pred.next(); - pred.setNext(guard); - guard.setNext(next); - SimplifierTool simplifierTool = GraphUtil.getDefaultSimplifier(null, null, null, false, graph.getAssumptions(), graph.getOptions(), loweringProvider); - survivingSuccessor.simplify(simplifierTool); - return; - } - } else if (current.predecessor() == null || current.predecessor() instanceof ControlSplitNode) { - assert current.predecessor() != null || (current instanceof StartNode && current == ((AbstractBeginNode) current).graph().start()); - moveAsDeoptAfter((AbstractBeginNode) current, deopt); - return; - } - } - current = current.predecessor(); - } - } - - @SuppressWarnings("try") - private static void moveAsDeoptAfter(FixedWithNextNode node, StaticDeoptimizingNode deopt) { - try (DebugCloseable position = deopt.asNode().withNodeSourcePosition()) { - FixedNode next = node.next(); - if (next != deopt.asNode()) { - node.setNext(node.graph().add(new DeoptimizeNode(deopt.getAction(), deopt.getReason(), deopt.getSpeculation()))); - GraphUtil.killCFG(next); - } - } - } -} --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.amd64/src/org/graalvm/compiler/replacements/amd64/AMD64MathSubstitutions.java 2019-03-12 08:11:02.968138907 +0100 +++ /dev/null 2017-11-16 08:17:56.803999947 +0100 @@ -1,85 +0,0 @@ -/* - * Copyright (c) 2011, 2018, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ - - -package org.graalvm.compiler.replacements.amd64; - -import org.graalvm.compiler.api.replacements.ClassSubstitution; -import org.graalvm.compiler.api.replacements.MethodSubstitution; -import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor; -import org.graalvm.compiler.graph.Node.ConstantNodeParameter; -import org.graalvm.compiler.graph.Node.NodeIntrinsic; -import org.graalvm.compiler.nodes.extended.ForeignCallNode; -import org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode; -import org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.UnaryOperation; - -// JaCoCo Exclude - -/** - * Substitutions for some {@link java.lang.Math} methods that leverage AMD64 instructions for - * selected input values. - */ -@ClassSubstitution(Math.class) -public class AMD64MathSubstitutions { - - private static final double PI_4 = Math.PI / 4; - - // NOTE on snippets below: - // Math.sin(), .cos() and .tan() guarantee a value within 1 ULP of the - // exact result, but x87 trigonometric FPU instructions are only that - // accurate within [-pi/4, pi/4]. Examine the passed value and provide - // a slow path for inputs outside of that interval. - - @MethodSubstitution - public static double sin(double x) { - if (Math.abs(x) < PI_4) { - return UnaryMathIntrinsicNode.compute(x, UnaryOperation.SIN); - } else { - return callDouble1(UnaryOperation.SIN.foreignCallDescriptor, x); - } - } - - @MethodSubstitution - public static double cos(double x) { - if (Math.abs(x) < PI_4) { - return UnaryMathIntrinsicNode.compute(x, UnaryOperation.COS); - } else { - return callDouble1(UnaryOperation.COS.foreignCallDescriptor, x); - } - } - - @MethodSubstitution - public static double tan(double x) { - if (Math.abs(x) < PI_4) { - return UnaryMathIntrinsicNode.compute(x, UnaryOperation.TAN); - } else { - return callDouble1(UnaryOperation.TAN.foreignCallDescriptor, x); - } - } - - @NodeIntrinsic(value = ForeignCallNode.class) - private static native double callDouble1(@ConstantNodeParameter ForeignCallDescriptor descriptor, double value); - - @NodeIntrinsic(value = ForeignCallNode.class) - private static native double callDouble2(@ConstantNodeParameter ForeignCallDescriptor descriptor, double a, double b); -} --- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/arithmetic/IntegerMulHighNode.java 2019-03-12 08:11:03.380141568 +0100 +++ /dev/null 2017-11-16 08:17:56.803999947 +0100 @@ -1,93 +0,0 @@ -/* - * Copyright (c) 2014, 2018, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ - - -package org.graalvm.compiler.replacements.nodes.arithmetic; - -import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_2; -import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_2; - -import org.graalvm.compiler.core.common.type.ArithmeticOpTable; -import org.graalvm.compiler.core.common.type.ArithmeticOpTable.BinaryOp.MulHigh; -import org.graalvm.compiler.graph.NodeClass; -import org.graalvm.compiler.graph.spi.Canonicalizable; -import org.graalvm.compiler.graph.spi.CanonicalizerTool; -import org.graalvm.compiler.lir.gen.ArithmeticLIRGeneratorTool; -import org.graalvm.compiler.nodeinfo.NodeInfo; -import org.graalvm.compiler.nodes.ConstantNode; -import org.graalvm.compiler.nodes.NodeView; -import org.graalvm.compiler.nodes.ValueNode; -import org.graalvm.compiler.nodes.calc.BinaryArithmeticNode; -import org.graalvm.compiler.nodes.spi.NodeLIRBuilderTool; - -import jdk.vm.ci.meta.Constant; -import jdk.vm.ci.meta.PrimitiveConstant; -import jdk.vm.ci.meta.Value; - -@NodeInfo(shortName = "*H", cycles = CYCLES_2, size = SIZE_2) -public final class IntegerMulHighNode extends BinaryArithmeticNode implements Canonicalizable.BinaryCommutative { - public static final NodeClass TYPE = NodeClass.create(IntegerMulHighNode.class); - - public IntegerMulHighNode(ValueNode x, ValueNode y) { - super(TYPE, ArithmeticOpTable::getMulHigh, x, y); - } - - @Override - public void generate(NodeLIRBuilderTool nodeValueMap, ArithmeticLIRGeneratorTool gen) { - Value a = nodeValueMap.operand(getX()); - Value b = nodeValueMap.operand(getY()); - nodeValueMap.setResult(this, gen.emitMulHigh(a, b)); - } - - @Override - public ValueNode canonical(CanonicalizerTool tool, ValueNode forX, ValueNode forY) { - ValueNode ret = super.canonical(tool, forX, forY); - if (ret != this) { - return ret; - } - - if (forX.isConstant() && !forY.isConstant()) { - // we try to swap and canonicalize - ValueNode improvement = canonical(tool, forY, forX); - if (improvement != this) { - return improvement; - } - // if this fails we only swap - return new IntegerMulHighNode(forY, forX); - } - return canonical(this, forY); - } - - private static ValueNode canonical(IntegerMulHighNode self, ValueNode forY) { - if (forY.isConstant()) { - Constant c = forY.asConstant(); - if (c instanceof PrimitiveConstant && ((PrimitiveConstant) c).getJavaKind().isNumericInteger()) { - long i = ((PrimitiveConstant) c).asLong(); - if (i == 0 || i == 1) { - return ConstantNode.forIntegerStamp(self.stamp(NodeView.DEFAULT), 0); - } - } - } - return self; - } -}