--- old/src/share/vm/ci/ciMethod.hpp 2015-01-16 19:37:15.000000000 +0300 +++ new/src/share/vm/ci/ciMethod.hpp 2015-01-16 19:37:15.000000000 +0300 @@ -180,6 +180,7 @@ bool caller_sensitive() { return get_Method()->caller_sensitive(); } bool force_inline() { return get_Method()->force_inline(); } bool dont_inline() { return get_Method()->dont_inline(); } + bool ignore_profile() { return get_Method()->ignore_profile(); } int comp_level(); int highest_osr_comp_level(); --- old/src/share/vm/classfile/classFileParser.cpp 2015-01-16 19:37:16.000000000 +0300 +++ new/src/share/vm/classfile/classFileParser.cpp 2015-01-16 19:37:16.000000000 +0300 @@ -1786,6 +1786,10 @@ if (_location != _in_method) break; // only allow for methods if (!privileged) break; // only allow in privileged code return _method_LambdaForm_Compiled; + case vmSymbols::VM_SYMBOL_ENUM_NAME(java_lang_invoke_LambdaForm_IgnoreProfile_signature): + if (_location != _in_method) break; // only allow for methods + if (!privileged) break; // only allow in privileged code + return _method_LambdaForm_IgnoreProfile; case vmSymbols::VM_SYMBOL_ENUM_NAME(java_lang_invoke_LambdaForm_Hidden_signature): if (_location != _in_method) break; // only allow for methods if (!privileged) break; // only allow in privileged code @@ -1827,6 +1831,8 @@ m->set_intrinsic_id(vmIntrinsics::_compiledLambdaForm); if (has_annotation(_method_LambdaForm_Hidden)) m->set_hidden(true); + if (has_annotation(_method_LambdaForm_IgnoreProfile)) + m->set_ignore_profile(true); } void ClassFileParser::ClassAnnotationCollector::apply_to(instanceKlassHandle k) { --- old/src/share/vm/classfile/classFileParser.hpp 2015-01-16 19:37:18.000000000 +0300 +++ new/src/share/vm/classfile/classFileParser.hpp 2015-01-16 19:37:18.000000000 +0300 @@ -124,6 +124,7 @@ _method_DontInline, _method_LambdaForm_Compiled, _method_LambdaForm_Hidden, + _method_LambdaForm_IgnoreProfile, _sun_misc_Contended, _field_Stable, _annotation_LIMIT --- old/src/share/vm/classfile/vmSymbols.hpp 2015-01-16 19:37:19.000000000 +0300 +++ new/src/share/vm/classfile/vmSymbols.hpp 2015-01-16 19:37:19.000000000 +0300 @@ -279,6 +279,7 @@ template(java_lang_invoke_DontInline_signature, "Ljava/lang/invoke/DontInline;") \ template(java_lang_invoke_Stable_signature, "Ljava/lang/invoke/Stable;") \ template(java_lang_invoke_LambdaForm_Compiled_signature, "Ljava/lang/invoke/LambdaForm$Compiled;") \ + template(java_lang_invoke_LambdaForm_IgnoreProfile_signature, "Ljava/lang/invoke/LambdaForm$IgnoreProfile;") \ template(java_lang_invoke_LambdaForm_Hidden_signature, "Ljava/lang/invoke/LambdaForm$Hidden;") \ /* internal up-calls made only by the JVM, via class sun.invoke.MethodHandleNatives: */ \ template(findMethodHandleType_name, "findMethodHandleType") \ @@ -867,6 +868,12 @@ do_name( fullFence_name, "fullFence") \ do_alias( fullFence_signature, void_method_signature) \ \ + /* Custom branch frequencies profiling support for JSR292 */ \ + do_class(java_lang_invoke_MethodHandleImpl, "java/lang/invoke/MethodHandleImpl") \ + do_intrinsic(_profileBranch, java_lang_invoke_MethodHandleImpl, profileBranch_name, profileBranch_signature, F_S) \ + do_name( profileBranch_name, "profileBranch") \ + do_signature(profileBranch_signature, "(Z[I)Z") \ + \ /* unsafe memory references (there are a lot of them...) */ \ do_signature(getObject_signature, "(Ljava/lang/Object;J)Ljava/lang/Object;") \ do_signature(putObject_signature, "(Ljava/lang/Object;JLjava/lang/Object;)V") \ --- old/src/share/vm/oops/method.hpp 2015-01-16 19:37:21.000000000 +0300 +++ new/src/share/vm/oops/method.hpp 2015-01-16 19:37:21.000000000 +0300 @@ -81,7 +81,8 @@ _force_inline = 1 << 2, _dont_inline = 1 << 3, _hidden = 1 << 4, - _running_emcp = 1 << 5 + _running_emcp = 1 << 5, + _ignore_profile = 1 << 6 }; u1 _flags; @@ -811,6 +812,13 @@ _flags = x ? (_flags | _hidden) : (_flags & ~_hidden); } + bool ignore_profile() { + return (_flags & _ignore_profile) != 0; + } + void set_ignore_profile(bool x) { + _flags = x ? (_flags | _ignore_profile) : (_flags & ~_ignore_profile); + } + ConstMethod::MethodType method_type() const { return _constMethod->method_type(); } --- old/src/share/vm/opto/classes.hpp 2015-01-16 19:37:23.000000000 +0300 +++ new/src/share/vm/opto/classes.hpp 2015-01-16 19:37:23.000000000 +0300 @@ -200,6 +200,7 @@ macro(Opaque1) macro(Opaque2) macro(Opaque3) +macro(Opaque4) macro(OrI) macro(OrL) macro(OverflowAddI) --- old/src/share/vm/opto/compile.cpp 2015-01-16 19:37:25.000000000 +0300 +++ new/src/share/vm/opto/compile.cpp 2015-01-16 19:37:24.000000000 +0300 @@ -2651,6 +2651,7 @@ case Op_Opaque1: // Remove Opaque Nodes before matching case Op_Opaque2: // Remove Opaque Nodes before matching case Op_Opaque3: + case Op_Opaque4: n->subsume_by(n->in(1), this); break; case Op_CallStaticJava: @@ -3317,6 +3318,9 @@ bool Compile::too_many_traps(ciMethod* method, int bci, Deoptimization::DeoptReason reason) { + if (method->ignore_profile()) { + return false; + } ciMethodData* md = method->method_data(); if (md->is_empty()) { // Assume the trap has not occurred, or that it occurred only @@ -3366,6 +3370,9 @@ bool Compile::too_many_recompiles(ciMethod* method, int bci, Deoptimization::DeoptReason reason) { + if (method->ignore_profile()) { + return false; + } ciMethodData* md = method->method_data(); if (md->is_empty()) { // Assume the trap has not occurred, or that it occurred only --- old/src/share/vm/opto/graphKit.hpp 2015-01-16 19:37:26.000000000 +0300 +++ new/src/share/vm/opto/graphKit.hpp 2015-01-16 19:37:26.000000000 +0300 @@ -713,6 +713,15 @@ klass, reason_string, must_throw, keep_exact_action); } + // Bail out to the interpreter and keep exact action (avoid switching to Action_none). + void uncommon_trap_exact(Deoptimization::DeoptReason reason, + Deoptimization::DeoptAction action, + ciKlass* klass = NULL, const char* reason_string = NULL, + bool must_throw = false) { + uncommon_trap(Deoptimization::make_trap_request(reason, action), + klass, reason_string, must_throw, /*keep_exact_action=*/true); + } + // SP when bytecode needs to be reexecuted. virtual int reexecute_sp() { return sp(); } --- old/src/share/vm/opto/library_call.cpp 2015-01-16 19:37:27.000000000 +0300 +++ new/src/share/vm/opto/library_call.cpp 2015-01-16 19:37:27.000000000 +0300 @@ -41,6 +41,7 @@ #include "opto/movenode.hpp" #include "opto/mulnode.hpp" #include "opto/narrowptrnode.hpp" +#include "opto/opaquenode.hpp" #include "opto/parse.hpp" #include "opto/runtime.hpp" #include "opto/subnode.hpp" @@ -287,6 +288,8 @@ bool inline_updateBytesCRC32(); bool inline_updateByteBufferCRC32(); bool inline_multiplyToLen(); + + bool inline_profileBranch(); }; @@ -900,6 +903,9 @@ case vmIntrinsics::_updateByteBufferCRC32: return inline_updateByteBufferCRC32(); + case vmIntrinsics::_profileBranch: + return inline_profileBranch(); + default: // If you get here, it may be that someone has added a new intrinsic // to the list in vmSymbols.hpp without implementing it here. @@ -5807,3 +5813,36 @@ return instof_false; // even if it is NULL } + +bool LibraryCallKit::inline_profileBranch() { + Node* counts = argument(1); + const TypeAryPtr* ary = NULL; + ciArray* aobj = NULL; + if (counts->is_Con() + && (ary = counts->bottom_type()->isa_aryptr()) != NULL + && (aobj = ary->const_oop()->as_array()) != NULL) { + assert(aobj->length() == 2, ""); + jint taken = aobj->element_value(1).as_int(); + jint not_taken = aobj->element_value(0).as_int(); + + if (C->log() != NULL) { + C->log()->elem("profile_branch taken='%d' not_taken='%d'", + taken, not_taken); + } + + if (taken + not_taken == 0) { + // According to profile, never executed. + uncommon_trap_exact(Deoptimization::Reason_intrinsic, + Deoptimization::Action_reinterpret); + return true; + } + // Stop profiling + Node* opq = _gvn.transform(new Opaque4Node(argument(0), taken, not_taken)); + C->record_for_igvn(opq); + set_result(opq); + return true; + } else { + // Continue profiling + return false; + } +} --- old/src/share/vm/opto/macro.cpp 2015-01-16 19:37:29.000000000 +0300 +++ new/src/share/vm/opto/macro.cpp 2015-01-16 19:37:29.000000000 +0300 @@ -2475,7 +2475,8 @@ assert(n->Opcode() == Op_LoopLimit || n->Opcode() == Op_Opaque1 || n->Opcode() == Op_Opaque2 || - n->Opcode() == Op_Opaque3, "unknown node type in macro list"); + n->Opcode() == Op_Opaque3 || + n->Opcode() == Op_Opaque4, "unknown node type in macro list"); } assert(success == (C->macro_count() < old_macro_count), "elimination reduces macro count"); progress = progress || success; @@ -2513,7 +2514,7 @@ C->remove_macro_node(n); _igvn._worklist.push(n); success = true; - } else if (n->Opcode() == Op_Opaque1 || n->Opcode() == Op_Opaque2) { + } else if (n->Opcode() == Op_Opaque1 || n->Opcode() == Op_Opaque2 || n->Opcode() == Op_Opaque4) { _igvn.replace_node(n, n->in(1)); success = true; #if INCLUDE_RTM_OPT --- old/src/share/vm/opto/opaquenode.cpp 2015-01-16 19:37:30.000000000 +0300 +++ new/src/share/vm/opto/opaquenode.cpp 2015-01-16 19:37:30.000000000 +0300 @@ -60,4 +60,26 @@ return (&n == this); // Always fail except on self } +//============================================================================= +uint Opaque4Node::hash() const { return NO_HASH; } +uint Opaque4Node::cmp( const Node &n ) const { + return (&n == this); +} + +Node *Opaque4Node::Ideal(PhaseGVN *phase, bool can_reshape) { + if (can_reshape & _delay_removal) { + _delay_removal = false; + return this; + } else { + return NULL; + } +} + +Node *Opaque4Node::Identity( PhaseTransform *phase ) { + if (_delay_removal) { + return this; + } else { + return in(1); + } +} --- old/src/share/vm/opto/opaquenode.hpp 2015-01-16 19:37:31.000000000 +0300 +++ new/src/share/vm/opto/opaquenode.hpp 2015-01-16 19:37:31.000000000 +0300 @@ -87,5 +87,28 @@ bool rtm_opt() const { return (_opt == RTM_OPT); } }; +//------------------------------Opaque4Node------------------------------------ +// The node can store branch frequencies during parsing. +// Once parsing is over, the node goes away (during IGVN). +class Opaque4Node : public Node { + uint _taken; + uint _not_taken; + bool _consumed; + bool _delay_removal; + virtual uint hash() const ; // { return NO_HASH; } + virtual uint cmp( const Node &n ) const; + public: + Opaque4Node(Node *n, uint taken, uint not_taken) : Node(0, n), + _taken(taken), _not_taken(not_taken), _delay_removal(true), _consumed(false) {} + virtual int Opcode() const; + uint taken() const { return _taken; } + uint not_taken() const { return _not_taken; } + void consume() { _consumed = true; } + + virtual Node *Ideal(PhaseGVN *phase, bool can_reshape); + virtual Node *Identity(PhaseTransform *phase); + virtual const Type *bottom_type() const { return TypeInt::INT; } +}; + #endif // SHARE_VM_OPTO_OPAQUENODE_HPP --- old/src/share/vm/opto/parse.hpp 2015-01-16 19:37:32.000000000 +0300 +++ new/src/share/vm/opto/parse.hpp 2015-01-16 19:37:32.000000000 +0300 @@ -555,8 +555,8 @@ void do_jsr(); void do_ret(); - float dynamic_branch_prediction(float &cnt); - float branch_prediction(float &cnt, BoolTest::mask btest, int target_bci); + float dynamic_branch_prediction(float &cnt, BoolTest::mask btest, Node* test); + float branch_prediction(float &cnt, BoolTest::mask btest, int target_bci, Node* test); bool seems_never_taken(float prob) const; bool path_is_suitable_for_uncommon_trap(float prob) const; bool seems_stable_comparison() const; --- old/src/share/vm/opto/parse2.cpp 2015-01-16 19:37:33.000000000 +0300 +++ new/src/share/vm/opto/parse2.cpp 2015-01-16 19:37:33.000000000 +0300 @@ -37,6 +37,7 @@ #include "opto/matcher.hpp" #include "opto/memnode.hpp" #include "opto/mulnode.hpp" +#include "opto/opaquenode.hpp" #include "opto/parse.hpp" #include "opto/runtime.hpp" #include "runtime/deoptimization.hpp" @@ -767,28 +768,45 @@ // Try to gather dynamic branch prediction behavior. Return a probability // of the branch being taken and set the "cnt" field. Returns a -1.0 // if we need to use static prediction for some reason. -float Parse::dynamic_branch_prediction(float &cnt) { +float Parse::dynamic_branch_prediction(float &cnt, BoolTest::mask btest, Node* test) { ResourceMark rm; cnt = COUNT_UNKNOWN; - // Use MethodData information if it is available - // FIXME: free the ProfileData structure - ciMethodData* methodData = method()->method_data(); - if (!methodData->is_mature()) return PROB_UNKNOWN; - ciProfileData* data = methodData->bci_to_data(bci()); - if (!data->is_JumpData()) return PROB_UNKNOWN; - - // get taken and not taken values - int taken = data->as_JumpData()->taken(); + int taken = 0; int not_taken = 0; - if (data->is_BranchData()) { - not_taken = data->as_BranchData()->not_taken(); - } - // scale the counts to be commensurate with invocation counts: - taken = method()->scale_count(taken); - not_taken = method()->scale_count(not_taken); + if (method()->ignore_profile()) { + if (btest == BoolTest::eq && test->is_Cmp() && test->in(1)->Opcode() == Op_Opaque4) { + Opaque4Node* opq = (Opaque4Node*)test->in(1); + taken = opq->taken(); + not_taken = opq->not_taken(); + opq->consume(); + } else { + // No profile info. Be conservative. + int cnt = method()->interpreter_invocation_count(); + taken = cnt / 2; + not_taken = cnt - taken; + } + } else { + // Use MethodData information if it is available + // FIXME: free the ProfileData structure + ciMethodData* methodData = method()->method_data(); + if (!methodData->is_mature()) return PROB_UNKNOWN; + ciProfileData* data = methodData->bci_to_data(bci()); + if (!data->is_JumpData()) return PROB_UNKNOWN; + + // get taken and not taken values + taken = data->as_JumpData()->taken(); + not_taken = 0; + if (data->is_BranchData()) { + not_taken = data->as_BranchData()->not_taken(); + } + + // scale the counts to be commensurate with invocation counts: + taken = method()->scale_count(taken); + not_taken = method()->scale_count(not_taken); + } // Give up if too few (or too many, in which case the sum will overflow) counts to be meaningful. // We also check that individual counters are positive first, overwise the sum can become positive. @@ -841,8 +859,9 @@ //-----------------------------branch_prediction------------------------------- float Parse::branch_prediction(float& cnt, BoolTest::mask btest, - int target_bci) { - float prob = dynamic_branch_prediction(cnt); + int target_bci, + Node* test) { + float prob = dynamic_branch_prediction(cnt, btest, test); // If prob is unknown, switch to static prediction if (prob != PROB_UNKNOWN) return prob; @@ -932,7 +951,7 @@ Block* next_block = successor_for_bci(iter().next_bci()); float cnt; - float prob = branch_prediction(cnt, btest, target_bci); + float prob = branch_prediction(cnt, btest, target_bci, c); if (prob == PROB_UNKNOWN) { // (An earlier version of do_ifnull omitted this trap for OSR methods.) #ifndef PRODUCT @@ -1013,7 +1032,7 @@ Block* next_block = successor_for_bci(iter().next_bci()); float cnt; - float prob = branch_prediction(cnt, btest, target_bci); + float prob = branch_prediction(cnt, btest, target_bci, c); float untaken_prob = 1.0 - prob; if (prob == PROB_UNKNOWN) {