--- /dev/null 2013-04-15 12:30:09.841410950 +0400 +++ new/test/compiler/ciReplay/TestSA.sh 2013-04-17 20:14:24.914654602 +0400 @@ -0,0 +1,93 @@ +#!/bin/sh +# +# Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# This code is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License version 2 only, as +# published by the Free Software Foundation. +# +# This code is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# version 2 for more details (a copy is included in the LICENSE file that +# accompanied this code). +# +# You should have received a copy of the GNU General Public License version +# 2 along with this work; if not, write to the Free Software Foundation, +# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA +# or visit www.oracle.com if you need additional information or have any +# questions. +# +# + +## +## @test +## @bug 8011675 +## @summary testing of ciReplay with using generated by SA replay.txt +## @author igor.ignatyev@oracle.com +## @run shell TestSA.sh +## + +if [ "${TESTSRC}" = "" ] +then + TESTSRC=${PWD} + echo "TESTSRC not set. Using "${TESTSRC}" as default" +fi +echo "TESTSRC=${TESTSRC}" + +## Adding common setup Variables for running shell tests. +. ${TESTSRC}/../../test_env.sh + +. ${TESTSRC}/common.sh + +generate_replay + +${MV} ${replay_data} replay_vm.txt + +if [ -z "${core_file}" -o ! -r "${core_file}" ] +then + # skip test if MacOS host isn't configured for core dumping + if [ "$OS" = "Darwin" ] + then + if [ ! -d "/cores" ] + then + echo TEST SKIPPED: \'/cores\' dir doens\'t exist + exit 0 + fi + if [ ! -w "/cores" ] + then + echo TEST SKIPPED: \'/cores\' dir exists but is not writable + exit 0 + fi + fi + + test_fail 2 "CHECK :: CORE GENERATION" "core wasn't generated" +fi + +echo "dumpreplaydata -a > ${replay_data}" | \ + ${JAVA} ${TESTVMOPTS} \ + -cp ${TESTJAVA}${FS}lib${FS}sa-jdi.jar \ + sun.jvm.hotspot.CLHSDB ${JAVA} ${core_file} + +if [ ! -s ${replay_data} ] +then + test_fail 1 "CHECK :: REPLAY DATA GENERATION" \ + "replay data wasn't generated by SA" +fi + +diff --brief ${replay_data} replay_vm.txt +if [ $? -ne 0 ] +then + echo WARNING: replay.txt from SA != replay.txt from VM +fi + +common_tests 10 +${VM_TYPE}_tests 20 + +cleanup + +echo TEST PASSED + --- old/agent/src/share/classes/sun/jvm/hotspot/compiler/CompileTask.java 2013-04-17 20:14:25.242654589 +0400 +++ new/agent/src/share/classes/sun/jvm/hotspot/compiler/CompileTask.java 2013-04-17 20:14:24.938654601 +0400 @@ -46,10 +46,12 @@ Type type = db.lookupType("CompileTask"); methodField = type.getAddressField("_method"); osrBciField = new CIntField(type.getCIntegerField("_osr_bci"), 0); + compLevelField = new CIntField(type.getCIntegerField("_comp_level"), 0); } private static AddressField methodField; private static CIntField osrBciField; + private static CIntField compLevelField; public CompileTask(Address addr) { super(addr); @@ -63,4 +65,8 @@ public int osrBci() { return (int)osrBciField.getValue(getAddress()); } + + public int compLevel() { + return (int)compLevelField.getValue(getAddress()); + } } --- old/src/share/vm/ci/ciEnv.cpp 2013-04-17 20:14:25.302654587 +0400 +++ new/src/share/vm/ci/ciEnv.cpp 2013-04-17 20:14:25.014654598 +0400 @@ -1178,13 +1178,15 @@ for (int i = 0; i < objects->length(); i++) { objects->at(i)->dump_replay_data(out); } - Method* method = task()->method(); - int entry_bci = task()->osr_bci(); + CompileTask* task = this->task(); + Method* method = task->method(); + int entry_bci = task->osr_bci(); + int comp_level = task->comp_level(); // Klass holder = method->method_holder(); - out->print_cr("compile %s %s %s %d", + out->print_cr("compile %s %s %s %d %d", method->klass_name()->as_quoted_ascii(), method->name()->as_quoted_ascii(), method->signature()->as_quoted_ascii(), - entry_bci); + entry_bci, comp_level); out->flush(); } --- old/agent/doc/clhsdb.html 2013-04-17 20:14:25.270654588 +0400 +++ new/agent/doc/clhsdb.html 2013-04-17 20:14:24.958654600 +0400 @@ -15,7 +15,7 @@

There is also JavaScript based SA command line interface called jsdb. But, CLHSDB supports Unix shell-like (or dbx/gdb-like) command line interface with -support for output redirection/appending (familiar >, >>), command history and so on. +support for output redirection/appending (familiar >, >>), command history and so on. Each CLHSDB command can have zero or more arguments and optionally end with output redirection (or append) to a file. Commands may be stored in a file and run using source command. help command prints usage message for all supported commands (or a specific command) @@ -49,7 +49,7 @@ dumpheap [ file ] dump heap in hprof binary format dumpideal -a | id dump ideal graph like debug flag -XX:+PrintIdeal dumpilt -a | id dump inline tree for C2 compilation - dumpreplaydata

| -a | [>replay.txt] dump replay data into a file + dumpreplaydata <address> | -a | <thread_id> [>replay.txt] dump replay data into a file echo [ true | false ] turn on/off command echo mode examine [ address/count ] | [ address,address] show contents of memory from given address field [ type [ name fieldtype isStatic offset address ] ] print info about a field of HotSpot type @@ -96,11 +96,11 @@

JavaScript integration

-

Few CLHSDB commands are already implemented in JavaScript. It is possible to extend CLHSDB command set +

Few CLHSDB commands are already implemented in JavaScript. It is possible to extend CLHSDB command set by implementing more commands in a JavaScript file and by loading it by jsload command. jseval command may be used to evaluate arbitrary JavaScript expression from a string. Any JavaScript function may be exposed as a CLHSDB command by registering it using JavaScript registerCommand -function. This function accepts command name, usage and name of the JavaScript implementation function +function. This function accepts command name, usage and name of the JavaScript implementation function as arguments.

@@ -127,11 +127,11 @@ -

C2 Compilation Replay

+

Compilation Replay

When a java process crashes in compiled method, usually a core file is saved. -The C2 replay function can reproduce the compiling process in the core. -c2replay.html +The replay function can reproduce the compiling process in the core. +cireplay.html --- old/src/share/vm/ci/ciReplay.cpp 2013-04-17 20:14:25.258654589 +0400 +++ new/src/share/vm/ci/ciReplay.cpp 2013-04-17 20:14:24.974654600 +0400 @@ -370,11 +370,47 @@ } } - // compile + // validation of comp_level + bool is_valid_comp_level(int comp_level) { + const int msg_len = 256; + char* msg = NULL; + if (!is_compile(comp_level)) { + msg = NEW_RESOURCE_ARRAY(char, msg_len); + jio_snprintf(msg, msg_len, "%d isn't compilation level", comp_level); + } else if (!TieredCompilation && (comp_level != CompLevel_highest_tier)) { + msg = NEW_RESOURCE_ARRAY(char, msg_len); + switch (comp_level) { + case CompLevel_simple: + jio_snprintf(msg, msg_len, "compilation level %d requires Client VM or TieredCompilation", comp_level); + break; + case CompLevel_full_optimization: + jio_snprintf(msg, msg_len, "compilation level %d requires Server VM", comp_level); + break; + default: + jio_snprintf(msg, msg_len, "compilation level %d requires TieredCompilation", comp_level); + } + } + if (msg != NULL) { + report_error(msg); + return false; + } + return true; + } + + // compile void process_compile(TRAPS) { // methodHandle method; Method* method = parse_method(CHECK); int entry_bci = parse_int("entry_bci"); + const char* comp_level_label = "comp_level"; + int comp_level = parse_int(comp_level_label); + // old version w/o comp_level + if (had_error() && (error_message() == comp_level_label)) { + comp_level = CompLevel_full_optimization; + } + if (!is_valid_comp_level(comp_level)) { + return; + } Klass* k = method->method_holder(); ((InstanceKlass*)k)->initialize(THREAD); if (HAS_PENDING_EXCEPTION) { @@ -389,12 +425,12 @@ } } // Make sure the existence of a prior compile doesn't stop this one - nmethod* nm = (entry_bci != InvocationEntryBci) ? method->lookup_osr_nmethod_for(entry_bci, CompLevel_full_optimization, true) : method->code(); + nmethod* nm = (entry_bci != InvocationEntryBci) ? method->lookup_osr_nmethod_for(entry_bci, comp_level, true) : method->code(); if (nm != NULL) { nm->make_not_entrant(); } replay_state = this; - CompileBroker::compile_method(method, entry_bci, CompLevel_full_optimization, + CompileBroker::compile_method(method, entry_bci, comp_level, methodHandle(), 0, "replay", THREAD); replay_state = NULL; reset(); --- /dev/null 2013-04-15 12:30:09.841410950 +0400 +++ new/test/compiler/ciReplay/TestVM.sh 2013-04-17 20:14:24.938654601 +0400 @@ -0,0 +1,86 @@ +#!/bin/sh +# +# Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# This code is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License version 2 only, as +# published by the Free Software Foundation. +# +# This code is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# version 2 for more details (a copy is included in the LICENSE file that +# accompanied this code). +# +# You should have received a copy of the GNU General Public License version +# 2 along with this work; if not, write to the Free Software Foundation, +# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA +# or visit www.oracle.com if you need additional information or have any +# questions. +# +# + +## +## @test +## @bug 8011675 +## @summary testing of ciReplay with using generated by VM replay.txt +## @author igor.ignatyev@oracle.com +## @run shell TestVM.sh +## + +if [ "${TESTSRC}" = "" ] +then + TESTSRC=${PWD} + echo "TESTSRC not set. Using "${TESTSRC}" as default" +fi +echo "TESTSRC=${TESTSRC}" + +## Adding common setup Variables for running shell tests. +. ${TESTSRC}/../../test_env.sh + +. ${TESTSRC}/common.sh + +generate_replay + +if [ ! -s ${replay_data} ] +then + test_fail 1 "CHECK :: REPLAY DATA GENERATION" \ + "replay data wasn't generated by VM" +fi + +common_tests 10 +${VM_TYPE}_tests 20 + +cleanup + +if [ $is_tiered -eq 1 ] +then + stop_level=1 + while [ $stop_level -le $server_level ] + do + generate_replay "-XX:TieredStopAtLevel=$stop_level" + if [ ! -s ${replay_data} ] + then + test_fail `expr $stop_level + 30` \ + "TIERED LEVEL $stop_level :: REPLAY DATA GENERATION" \ + "replay data wasn't generated by VM with stop_level=$stop_level" + fi + level=`grep "^compile " $replay_data | awk '{print $6}'` + if [ $level -gt $stop_level ] + then + test_fail `expr $stop_level + 40` \ + "TIERED LEVEL $stop_level :: COMP_LEVEL VERIFICATION" \ + "comp_level in replay[$level] is greater than stop_level[$stop_level]" + fi + positive_test `expr $stop_level + 50` "TIERED LEVEL $stop_level :: REPLAY" \ + "-XX:TieredStopAtLevel=$stop_level" + stop_level=`expr $stop_level + 1` + done + cleanup +fi + +echo TEST PASSED + --- old/src/share/vm/compiler/compileBroker.cpp 2013-04-17 20:14:25.358654585 +0400 +++ new/src/share/vm/compiler/compileBroker.cpp 2013-04-17 20:14:24.974654600 +0400 @@ -1842,6 +1842,8 @@ } } } + // simulate crash during compilation + assert(task->compile_id() != CICrashAt, "just as planned"); } pop_jni_handle_block(); --- old/agent/src/share/classes/sun/jvm/hotspot/ci/ciEnv.java 2013-04-17 20:14:25.314654587 +0400 +++ new/agent/src/share/classes/sun/jvm/hotspot/ci/ciEnv.java 2013-04-17 20:14:25.006654598 +0400 @@ -93,10 +93,11 @@ CompileTask task = task(); Method method = task.method(); int entryBci = task.osrBci(); + int compLevel = task.compLevel(); Klass holder = method.getMethodHolder(); out.println("compile " + holder.getName().asString() + " " + OopUtilities.escapeString(method.getName().asString()) + " " + method.getSignature().asString() + " " + - entryBci); + entryBci + " " + compLevel); } } --- old/agent/src/share/classes/sun/jvm/hotspot/code/NMethod.java 2013-04-17 20:14:25.354654585 +0400 +++ new/agent/src/share/classes/sun/jvm/hotspot/code/NMethod.java 2013-04-17 20:14:24.970654600 +0400 @@ -78,6 +78,8 @@ current sweep traversal index. */ private static CIntegerField stackTraversalMarkField; + private static CIntegerField compLevelField; + static { VM.registerVMInitializedObserver(new Observer() { public void update(Observable o, Object data) { @@ -113,7 +115,7 @@ osrEntryPointField = type.getAddressField("_osr_entry_point"); lockCountField = type.getJIntField("_lock_count"); stackTraversalMarkField = type.getCIntegerField("_stack_traversal_mark"); - + compLevelField = type.getCIntegerField("_comp_level"); pcDescSize = db.lookupType("PcDesc").getSize(); } @@ -530,7 +532,7 @@ out.println("compile " + holder.getName().asString() + " " + OopUtilities.escapeString(method.getName().asString()) + " " + method.getSignature().asString() + " " + - getEntryBCI()); + getEntryBCI() + " " + getCompLevel()); } @@ -551,4 +553,5 @@ private int getHandlerTableOffset() { return (int) handlerTableOffsetField.getValue(addr); } private int getNulChkTableOffset() { return (int) nulChkTableOffsetField .getValue(addr); } private int getNMethodEndOffset() { return (int) nmethodEndOffsetField .getValue(addr); } + private int getCompLevel() { return (int) compLevelField .getValue(addr); } } --- old/src/share/vm/runtime/globals.hpp 2013-04-17 20:14:25.390654584 +0400 +++ new/src/share/vm/runtime/globals.hpp 2013-04-17 20:14:24.974654600 +0400 @@ -3247,7 +3247,9 @@ "Fire OutOfMemoryErrors throughout CI for testing the compiler " \ "(non-negative value throws OOM after this many CI accesses " \ "in each compile)") \ - \ + notproduct(intx, CICrashAt, -1, \ + "id of compilation to trigger assert in compiler thread for " \ + "the purpose of testing, e.g. generation of replay data") \ notproduct(bool, CIObjectFactoryVerify, false, \ "enable potentially expensive verification in ciObjectFactory") \ \ --- old/agent/doc/c2replay.html 2013-04-17 20:14:25.402654583 +0400 +++ /dev/null 2013-04-15 12:30:09.841410950 +0400 @@ -1,41 +0,0 @@ - - - -C2 Replay - - - - -

C2 compiler replay

-

-The C2 compiler replay is a function to repeat the compiling process from a crashed java process in compiled method
-This function only exists in debug version of VM -

-

Usage

-
 
-First, use SA to attach to the core file, if suceeded, do
-       clhsdb>dumpreplaydata 
| -a | [> replay.txt] - create file replay.txt, address is address of Method, or nmethod(CodeBlob) - clhsdb>buildreplayjars [all | boot | app] - create files: - all: - app.jar, boot.jar - boot: - boot.jar - app: - app.jar - exit SA now. -Second, use the obtained replay text file, replay.txt and jar files, app.jar and boot.jar, using debug version of java - java -Xbootclasspath/p:boot.jar -cp app.jar -XX:ReplayDataFile= -XX:+ReplayCompiles .... - This will replay the compiling process. - - With ReplayCompiles, the replay will recompile all the methods in app.jar, and in boot.jar to emulate the process in java app. - -notes: - 1) Most time, we don't need the boot.jar which is the classes loaded from JDK. It will be only modified when an agent(JVMDI) is running and modifies the classes. - 2) If encounter error as "" not found, that means the SA is using a VMStructs which is different from the one with corefile. In this case, SA has a utility tool vmstructsdump which is located at agent/src/os//proc/ - - Use this tool to dump VM type library: - vmstructsdump libjvm.so > .db - - set env SA_TYPEDB=.db (refer different shell for set envs) --- /dev/null 2013-04-15 12:30:09.841410950 +0400 +++ new/agent/doc/cireplay.html 2013-04-17 20:14:24.930654601 +0400 @@ -0,0 +1,41 @@ + + + +Replay + + + + +

Compiler replay

+

+The compiler replay is a function to repeat the compiling process from a crashed java process in compiled method
+This function only exists in debug version of VM +

+

Usage

+
+First, use SA to attach to the core file, if succeeded, do
+       hsdb> dumpreplaydata <address> | -a | <thread_id> [> replay.txt]
+       create file replay.txt, address is address of Method, or nmethod(CodeBlob)
+       hsdb> buildreplayjars [all | boot | app]
+       create files:
+         all:
+           app.jar, boot.jar
+         boot:
+           boot.jar
+         app:
+           app.jar
+       exit SA now.
+Second, use the obtained replay text file, replay.txt and jar files, app.jar and boot.jar, using debug version of java
+       java -Xbootclasspath/p:boot.jar -cp app.jar -XX:ReplayDataFile=<datafile> -XX:+ReplayCompiles ....
+       This will replay the compiling process.
+
+       With ReplayCompiles, the replay will recompile all the methods in app.jar, and in boot.jar to emulate the process in java app.
+
+notes:
+       1) Most time, we don't need the boot.jar which is the classes loaded from JDK. It will be only modified when an agent(JVMDI) is running and modifies the classes.
+       2) If encounter error as "<flag>" not found, that means the SA is using a VMStructs which is different from the one with corefile. In this case, SA has a utility tool vmstructsdump which is located at agent/src/os/<os>/proc/<os_platform>
+
+       Use this tool to dump VM type library:
+       vmstructsdump libjvm.so > <type_name>.db
+
+       set env SA_TYPEDB=<type_name>.db (refer different shell for set envs)
--- old/src/share/vm/runtime/vmStructs.cpp	2013-04-17 20:14:25.414654583 +0400
+++ new/src/share/vm/runtime/vmStructs.cpp	2013-04-17 20:14:24.982654599 +0400
@@ -826,6 +826,7 @@
   nonstatic_field(nmethod,             _lock_count,                                   jint)                                  \
   nonstatic_field(nmethod,             _stack_traversal_mark,                         long)                                  \
   nonstatic_field(nmethod,             _compile_id,                                   int)                                   \
+  nonstatic_field(nmethod,             _comp_level,                                   int)                                   \
   nonstatic_field(nmethod,             _exception_cache,                              ExceptionCache*)                       \
   nonstatic_field(nmethod,             _marked_for_deoptimization,                    bool)                                  \
                                                                                                                              \
--- /dev/null	2013-04-15 12:30:09.841410950 +0400
+++ new/test/compiler/ciReplay/TestVM_no_comp_level.sh	2013-04-17 20:14:25.594654576 +0400
@@ -0,0 +1,74 @@
+#!/bin/sh
+# 
+# Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+# 
+# This code is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 2 only, as
+# published by the Free Software Foundation.
+# 
+# This code is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# version 2 for more details (a copy is included in the LICENSE file that
+# accompanied this code).
+# 
+# You should have received a copy of the GNU General Public License version
+# 2 along with this work; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+# 
+# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+# or visit www.oracle.com if you need additional information or have any
+# questions.
+# 
+# 
+
+##
+## @test
+## @bug 8011675
+## @summary testing of ciReplay with using generated by VM replay.txt w/o comp_level
+## @author igor.ignatyev@oracle.com
+## @run shell TestVM_no_comp_level.sh
+##
+
+if [ "${TESTSRC}" = "" ]
+then
+  TESTSRC=${PWD}
+  echo "TESTSRC not set.  Using "${TESTSRC}" as default"
+fi
+echo "TESTSRC=${TESTSRC}"
+
+## Adding common setup Variables for running shell tests.
+. ${TESTSRC}/../../test_env.sh
+
+. ${TESTSRC}/common.sh
+
+generate_replay
+
+if [ ! -s ${replay_data} ]
+then
+    test_fail 1 "CHECK :: REPLAY DATA GENERATION" \
+            "replay data wasn't generated by VM"
+fi
+
+${CP} ${replay_data} replay_vm.txt
+
+sed 's/^\(compile *[^ ][^ ]* *[^ ][^ ]* [^ ][^ ]* [^ ][^ ]*\).*$/\1/' \
+        replay_vm.txt > ${replay_data}
+
+if [ $client_available -eq 1 ]
+then
+    # tiered is unavailable in client vm, so results w/ flags will be the same as w/o flags
+    negative_test 10 "CLIENT" -client
+fi
+
+if [ $server_available -eq 1 ]
+then
+    positive_test 21 "SERVER :: NON-TIERED" -XX:-TieredCompilation -server
+    positive_test 22 "SERVER :: TIERED" -XX:+TieredCompilation -server
+fi
+
+cleanup
+
+echo TEST PASSED
+
--- /dev/null	2013-04-15 12:30:09.841410950 +0400
+++ new/test/compiler/ciReplay/common.sh	2013-04-17 20:14:26.790654531 +0400
@@ -0,0 +1,232 @@
+#!/bin/sh
+# 
+# Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+# 
+# This code is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 2 only, as
+# published by the Free Software Foundation.
+# 
+# This code is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# version 2 for more details (a copy is included in the LICENSE file that
+# accompanied this code).
+# 
+# You should have received a copy of the GNU General Public License version
+# 2 along with this work; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+# 
+# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+# or visit www.oracle.com if you need additional information or have any
+# questions.
+# 
+# 
+
+# $1 - error code
+# $2 - test name
+# $3,.. - decription
+test_fail() {
+    error=$1
+    shift
+    name=$1
+    shift
+    echo "TEST [$name] FAILED:"
+    echo "$@"
+    exit $error
+}
+
+# $@ - additional vm opts
+start_test() {
+    # disable core dump on *nix
+    ulimit -S -c 0
+    # disable core dump on windows
+    VMOPTS="$@ -XX:-CreateMinidumpOnCrash"
+    cmd="${JAVA} ${VMOPTS} -XX:+ReplayCompiles -XX:ReplayDataFile=${replay_data}"
+    echo $cmd
+    $cmd
+    return $?
+}
+
+# $1 - error_code
+# $2 - test name
+# $3,.. - additional vm opts
+positive_test() {
+    error=$1
+    shift
+    name=$1
+    shift
+    VMOPTS="${TESTVMOPTS} $@"
+    echo "POSITIVE TEST [$name]"
+    start_test ${VMOPTS}
+    exit_code=$?
+    if [ ${exit_code} -ne 0 ]
+    then
+        test_fail $error $name "exit_code[${exit_code}] != 0 during replay "\
+                "w/ vmopts: ${VMOPTS}"
+    fi
+}
+
+# $1 - error_code
+# $2 - test name
+# $2,.. - additional vm opts
+negative_test() {
+    error=$1
+    shift
+    name=$1
+    shift
+    VMOPTS="${TESTVMOPTS} $@"
+    echo "NEGATIVE TEST [$name]"
+    start_test ${VMOPTS}
+    exit_code=$?
+    if [ ${exit_code} -eq 0 ]
+    then
+        test_fail $error $name "exit_code[${exit_code}] == 0 during replay "\
+                "w/ vmopts: ${VMOPTS}"
+    fi
+}
+
+# $1 - initial error_code
+common_tests() {
+    positive_test $1 "COMMON :: THE SAME FLAGS"
+    positive_test `expr $1 + 1` "COMMON :: TIERED" -XX:+TieredCompilation
+}
+
+# $1 - initial error_code
+# $2 - non-tiered comp_level 
+nontiered_tests() {
+    level=`grep "^compile " $replay_data | awk '{print $6}'`
+    # is level available in non-tiere
+    if [ "$level" -eq $2 ]
+    then
+        positive_test $1 "NON-TIERED :: AVAILABLE COMP_LEVEL" \
+                -XX:-TieredCompilation
+    else
+        negative_test `expr $1 + 1` "NON-TIERED :: UNAVAILABLE COMP_LEVEL" \
+        negative_test `expr $1 + 1` "NON-TIERED :: UNAVAILABLE COMP_LEVEL" \
+                -XX:-TieredCompilation
+    fi
+}
+
+# $1 - initial error_code
+client_tests() {
+    # testing in opposite VM
+    if [ $server_available -eq 1 ]
+    then
+        negative_test $1 "SERVER :: NON-TIERED" -XX:-TieredCompilation \
+                -server
+        positive_test `expr $1 + 1` "SERVER :: TIERED" -XX:+TieredCompilation \
+                -server
+    fi
+    nontiered_tests `expr $1 + 2` $client_level 
+}
+
+# $1 - initial error_code
+server_tests() {
+    # testing in opposite VM
+    if [ $client_available -eq 1 ]
+    then
+        # tiered is unavailable in client vm, so results w/ flags will be the same as w/o flags
+        negative_test $1 "CLIENT" -client
+    fi
+    nontiered_tests `expr $1 + 2` $server_level
+}
+
+cleanup() {
+    ${RM} -f core*
+    ${RM} -f replay*.txt
+    ${RM} -f hs_err_pid*.log
+    ${RM} -f test_core
+    ${RM} -f test_replay.txt
+}
+
+JAVA=${TESTJAVA}${FS}bin${FS}java
+
+replay_data=test_replay.txt
+
+${JAVA} ${TESTVMOPTS} -Xinternalversion 2>&1 | grep debug
+
+# Only test fastdebug 
+if [ $? -ne 0 ]
+then
+    echo TEST SKIPPED: product build
+    exit 0
+fi
+
+is_int=`${JAVA} ${TESTVMOPTS} -version 2>&1 | grep -c "interpreted mode"`
+# Not applicable for Xint
+if [ $is_int -ne 0 ]
+then
+    echo TEST SKIPPED: interpreted mode
+    exit 0
+fi
+
+cleanup
+
+client_available=`${JAVA} ${TESTVMOPTS} -client -Xinternalversion 2>&1 | \
+        grep -c Client`
+server_available=`${JAVA} ${TESTVMOPTS} -server -Xinternalversion 2>&1 | \
+        grep -c Server`
+is_tiered=`${JAVA} ${TESTVMOPTS} -XX:+PrintFlagsFinal -version | \
+        grep TieredCompilation | \
+        grep -c true`
+# CompLevel_simple -- C1
+client_level=1
+# CompLevel_full_optimization -- C2 or Shark 
+server_level=4
+
+echo "client_available=$client_available"
+echo "server_available=$server_available"
+echo "is_tiered=$is_tiered"
+
+# crash vm in compiler thread with generation replay data and 'small' dump-file
+# $@ - additional vm opts
+generate_replay() {
+    # enable core dump
+    ulimit -c unlimited
+
+    cmd="${JAVA} ${TESTVMOPTS} $@ \
+            -Xms8m \
+            -Xmx32m \
+            -XX:MetaspaceSize=4m \
+            -XX:MaxMetaspaceSize=16m \
+            -XX:InitialCodeCacheSize=512k \
+            -XX:ReservedCodeCacheSize=4m \
+            -XX:ThreadStackSize=512 \
+            -XX:VMThreadStackSize=512 \
+            -XX:CompilerThreadStackSize=512 \
+            -XX:ParallelGCThreads=1 \
+            -XX:CICompilerCount=1 \
+            -Xcomp \
+            -XX:CICrashAt=1 \
+            -XX:+CreateMinidumpOnCrash \
+            -XX:+DumpReplayDataOnError \
+            -XX:ReplayDataFile=${replay_data} \
+            -version"
+    echo GENERATION OF REPLAY.TXT:
+    echo $cmd
+
+    core_locations=`${cmd} 2>&1 | \
+            grep -i core | \
+            grep "location:" | \
+            sed -e 's/location: \([^ ]*\) or \([^ ]*\)/\1 \2/'`
+
+    for core in $core_locations
+    do
+        if [ -r "$core" ]
+        then
+            core_file=$core
+        fi
+    done
+
+    # core-file was found
+    if [ -n "$core_file" ]
+    then
+        ${MV} "${core_file}" test_core
+        core_file=test_core
+    fi
+
+    ${RM} -f hs_err_pid*.log
+}
+
+