--- old/src/share/vm/c1/c1_GraphBuilder.cpp 2013-10-14 20:39:24.000000000 +0400 +++ new/src/share/vm/c1/c1_GraphBuilder.cpp 2013-10-14 20:39:24.000000000 +0400 @@ -3272,7 +3272,8 @@ // negative filter: should callee NOT be inlined? returns NULL, ok to inline, or rejection msg const char* GraphBuilder::should_not_inline(ciMethod* callee) const { - if ( callee->should_exclude()) return "excluded by CompilerOracle"; + if ( callee->should_exclude() && + !callee->should_inline()) return "excluded by CompilerOracle"; if ( callee->should_not_inline()) return "disallowed by CompilerOracle"; if ( callee->dont_inline()) return "don't inline by annotation"; return NULL; @@ -3704,6 +3705,7 @@ // now perform tests that are based on flag settings if (callee->force_inline()) { + if (inline_level() > MaxForceInlineLevel) INLINE_BAILOUT("MaxForceInlineLevel"); print_inlining(callee, "force inline by annotation"); } else if (callee->should_inline()) { print_inlining(callee, "force inline by CompileOracle"); --- old/src/share/vm/c1/c1_globals.hpp 2013-10-14 20:39:25.000000000 +0400 +++ new/src/share/vm/c1/c1_globals.hpp 2013-10-14 20:39:25.000000000 +0400 @@ -341,6 +341,8 @@ diagnostic(bool, C1PatchInvokeDynamic, true, \ "Patch invokedynamic appendix not known at compile time") \ \ + product(intx, MaxForceInlineLevel, 100, \ + "maximum number of nested @ForceInline calls that are inlined") \ \ --- old/src/share/vm/opto/bytecodeInfo.cpp 2013-10-14 20:39:26.000000000 +0400 +++ new/src/share/vm/opto/bytecodeInfo.cpp 2013-10-14 20:39:25.000000000 +0400 @@ -450,11 +450,11 @@ bool pass_initial_checks(ciMethod* caller_method, int caller_bci, ciMethod* callee_method) { ciInstanceKlass *callee_holder = callee_method ? callee_method->holder() : NULL; // Check if a callee_method was suggested - if( callee_method == NULL ) return false; + if (callee_method == NULL) return false; // Check if klass of callee_method is loaded - if( !callee_holder->is_loaded() ) return false; - if( !callee_holder->is_initialized() ) return false; - if( !UseInterpreter || CompileTheWorld /* running Xcomp or CTW */ ) { + if (!callee_holder->is_loaded()) return false; + if (!callee_holder->is_initialized()) return false; + if (!UseInterpreter || CompileTheWorld /* running Xcomp or CTW */) { // Checks that constant pool's call site has been visited // stricter than callee_holder->is_initialized() ciBytecodeStream iter(caller_method); @@ -467,7 +467,7 @@ return false; } // Try to do constant pool resolution if running Xcomp - if( !caller_method->check_call(index, call_bc == Bytecodes::_invokestatic) ) { + if (!caller_method->check_call(index, call_bc == Bytecodes::_invokestatic)) { return false; } } @@ -475,7 +475,8 @@ // We will attempt to see if a class/field/etc got properly loaded. If it // did not, it may attempt to throw an exception during our probing. Catch // and ignore such exceptions and do not attempt to compile the method. - if( callee_method->should_exclude() ) return false; + if (callee_method->should_exclude() && + !callee_method->should_inline()) return false; return true; } --- /dev/null 2013-10-14 20:39:27.000000000 +0400 +++ new/test/compiler/jsr292/DeepInliningTest.java 2013-10-14 20:39:26.000000000 +0400 @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/** + * @test + * @bug 8012941 + * @summary JSR 292: too deep inlining might crash compiler because of stack overflow + * + * @run main/othervm -Xbatch + * -Djava.lang.invoke.MethodHandle.COMPILE_THRESHOLD=0 + * -XX:CompileThreshold=10 + * -XX:CompilerThreadStackSize=50 + * -XX:CompileCommand=compileonly,DeepInliningTest,m2 + * -XX:CompileCommand=inline,java/lang/invoke*,* + * DeepInliningTest + */ +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; + +public class DeepInliningTest { + public static final MethodHandle constMH; + static { + try { + MethodHandle localMH = MethodHandles.lookup().findStatic(DeepInliningTest.class, "m1", MethodType.methodType(boolean.class)); + + // Nest MH type conversions to produce long chain of lambda forms + final int DEPTH = Integer.getInteger("DEPTH", 200); + for (int i = 0; i < DEPTH; i++) { + localMH = localMH.asType(MethodType.methodType(Boolean.class)); + localMH = localMH.asType(MethodType.methodType(boolean.class)); + } + + localMH = localMH.asType(MethodType.methodType(Boolean.class)); + constMH = localMH; + } catch (Exception e) { + throw new Error(e); + } + } + + static boolean m1() { return true; } + + static boolean m2() throws Throwable { + return (Boolean) constMH.invokeExact(); + } + + public static void main(String[] args) throws Throwable { + for (int i = 0; i < 1000; i++) { + try { + boolean result = m2(); + } catch (StackOverflowError e) { + System.out.println(i+": "+e); + } + } + System.out.println("TEST PASSED"); + } +}