--- old/src/share/vm/ci/ciMethod.cpp 2015-01-26 19:16:03.000000000 +0300 +++ new/src/share/vm/ci/ciMethod.cpp 2015-01-26 19:16:03.000000000 +0300 @@ -70,7 +70,8 @@ // Loaded method. ciMethod::ciMethod(methodHandle h_m, ciInstanceKlass* holder) : ciMetadata(h_m()), - _holder(holder) + _holder(holder), + _has_injected_profile(false) { assert(h_m() != NULL, "no null method"); @@ -168,7 +169,8 @@ _liveness( NULL), _can_be_statically_bound(false), _method_blocks( NULL), - _method_data( NULL) + _method_data( NULL), + _has_injected_profile( false) #if defined(COMPILER2) || defined(SHARK) , _flow( NULL), --- old/src/share/vm/ci/ciMethod.hpp 2015-01-26 19:16:04.000000000 +0300 +++ new/src/share/vm/ci/ciMethod.hpp 2015-01-26 19:16:04.000000000 +0300 @@ -79,6 +79,7 @@ bool _is_c1_compilable; bool _is_c2_compilable; bool _can_be_statically_bound; + bool _has_injected_profile; // Lazy fields, filled in on demand address _code; @@ -286,6 +287,9 @@ int instructions_size(); int scale_count(int count, float prof_factor = 1.); // make MDO count commensurate with IIC + bool has_injected_profile() const { return _has_injected_profile; } + void set_injected_profile(bool x) { _has_injected_profile = x; } + // Stack walking support bool is_ignored_by_security_stack_walk() const; --- old/src/share/vm/classfile/vmSymbols.hpp 2015-01-26 19:16:09.000000000 +0300 +++ new/src/share/vm/classfile/vmSymbols.hpp 2015-01-26 19:16:09.000000000 +0300 @@ -242,7 +242,6 @@ template(returnType_name, "returnType") \ template(signature_name, "signature") \ template(slot_name, "slot") \ - template(selectAlternative_name, "selectAlternative") \ \ /* Support for annotations (JDK 1.5 and above) */ \ \ @@ -294,8 +293,7 @@ template(setTarget_signature, "(Ljava/lang/invoke/MethodHandle;)V") \ NOT_LP64( do_alias(intptr_signature, int_signature) ) \ LP64_ONLY( do_alias(intptr_signature, long_signature) ) \ - template(selectAlternative_signature, "(ZLjava/lang/invoke/MethodHandle;Ljava/lang/invoke/MethodHandle;)Ljava/lang/invoke/MethodHandle;") \ - \ + \ /* common method and field names */ \ template(object_initializer_name, "") \ template(class_initializer_name, "") \ @@ -867,6 +865,12 @@ do_name( fullFence_name, "fullFence") \ do_alias( fullFence_signature, void_method_signature) \ \ + /* Custom branch frequencies profiling support for JSR292 */ \ + do_class(java_lang_invoke_MethodHandleImpl, "java/lang/invoke/MethodHandleImpl") \ + do_intrinsic(_profileBranch, java_lang_invoke_MethodHandleImpl, profileBranch_name, profileBranch_signature, F_S) \ + do_name( profileBranch_name, "profileBranch") \ + do_signature(profileBranch_signature, "(Z[I)Z") \ + \ /* unsafe memory references (there are a lot of them...) */ \ do_signature(getObject_signature, "(Ljava/lang/Object;J)Ljava/lang/Object;") \ do_signature(putObject_signature, "(Ljava/lang/Object;JLjava/lang/Object;)V") \ --- old/src/share/vm/opto/classes.hpp 2015-01-26 19:16:12.000000000 +0300 +++ new/src/share/vm/opto/classes.hpp 2015-01-26 19:16:12.000000000 +0300 @@ -200,6 +200,7 @@ macro(Opaque1) macro(Opaque2) macro(Opaque3) +macro(ProfileBranch) macro(OrI) macro(OrL) macro(OverflowAddI) --- old/src/share/vm/opto/compile.cpp 2015-01-26 19:16:14.000000000 +0300 +++ new/src/share/vm/opto/compile.cpp 2015-01-26 19:16:14.000000000 +0300 @@ -3105,6 +3105,7 @@ default: assert( !n->is_Call(), "" ); assert( !n->is_Mem(), "" ); + assert( nop != Op_ProfileBranch, "should be eliminated during IGVN"); break; } @@ -3321,6 +3322,9 @@ bool Compile::too_many_traps(ciMethod* method, int bci, Deoptimization::DeoptReason reason) { + if (method->has_injected_profile()) { + return false; + } ciMethodData* md = method->method_data(); if (md->is_empty()) { // Assume the trap has not occurred, or that it occurred only @@ -3370,6 +3374,9 @@ bool Compile::too_many_recompiles(ciMethod* method, int bci, Deoptimization::DeoptReason reason) { + if (method->has_injected_profile()) { + return false; + } ciMethodData* md = method->method_data(); if (md->is_empty()) { // Assume the trap has not occurred, or that it occurred only --- old/src/share/vm/opto/graphKit.hpp 2015-01-26 19:16:15.000000000 +0300 +++ new/src/share/vm/opto/graphKit.hpp 2015-01-26 19:16:15.000000000 +0300 @@ -714,6 +714,15 @@ klass, reason_string, must_throw, keep_exact_action); } + // Bail out to the interpreter and keep exact action (avoid switching to Action_none). + void uncommon_trap_exact(Deoptimization::DeoptReason reason, + Deoptimization::DeoptAction action, + ciKlass* klass = NULL, const char* reason_string = NULL, + bool must_throw = false) { + uncommon_trap(Deoptimization::make_trap_request(reason, action), + klass, reason_string, must_throw, /*keep_exact_action=*/true); + } + // SP when bytecode needs to be reexecuted. virtual int reexecute_sp() { return sp(); } --- old/src/share/vm/opto/library_call.cpp 2015-01-26 19:16:16.000000000 +0300 +++ new/src/share/vm/opto/library_call.cpp 2015-01-26 19:16:16.000000000 +0300 @@ -41,6 +41,7 @@ #include "opto/movenode.hpp" #include "opto/mulnode.hpp" #include "opto/narrowptrnode.hpp" +#include "opto/opaquenode.hpp" #include "opto/parse.hpp" #include "opto/runtime.hpp" #include "opto/subnode.hpp" @@ -287,6 +288,8 @@ bool inline_updateBytesCRC32(); bool inline_updateByteBufferCRC32(); bool inline_multiplyToLen(); + + bool inline_profileBranch(); }; @@ -900,6 +903,9 @@ case vmIntrinsics::_updateByteBufferCRC32: return inline_updateByteBufferCRC32(); + case vmIntrinsics::_profileBranch: + return inline_profileBranch(); + default: // If you get here, it may be that someone has added a new intrinsic // to the list in vmSymbols.hpp without implementing it here. @@ -5873,3 +5879,46 @@ return instof_false; // even if it is NULL } + +bool LibraryCallKit::inline_profileBranch() { + Node* counts = argument(1); + const TypeAryPtr* ary = NULL; + ciArray* aobj = NULL; + if (counts->is_Con() + && (ary = counts->bottom_type()->isa_aryptr()) != NULL + && (aobj = ary->const_oop()->as_array()) != NULL + && (aobj->length() == 2)) { + jint taken = aobj->element_value(1).as_int(); + jint not_taken = aobj->element_value(0).as_int(); + + method()->set_injected_profile(true); + + if (C->log() != NULL) { + C->log()->elem("observe source='profileBranch' taken='%d' not_taken='%d'", + taken, not_taken); + } + + if (taken + not_taken == 0) { + // According to profile, never executed. + uncommon_trap_exact(Deoptimization::Reason_intrinsic, + Deoptimization::Action_reinterpret); + return true; + } + // Stop profiling. + // MethodHandleImpl::profileBranch() has profiling logic in it's bytecode. + // By replacing method's body with profile data (represented as ProfileBranchNode + // on IR level) we effectively disable profiling. + // It enables full speed execution once optimized code is generated. + Node* profile = _gvn.transform(new ProfileBranchNode(argument(0), taken, not_taken)); + C->record_for_igvn(profile); + set_result(profile); + return true; + } else { + // Continue profiling. + // Profile data isn't available at the moment. So, execute method's bytecode version. + // Usually, when GWT LambdaForms are profiled it means that a stand-alone nmethod + // is compiled and counters aren't available since corresponding MethodHandle + // isn't a compile-time constant. + return false; + } +} --- old/src/share/vm/opto/opaquenode.cpp 2015-01-26 19:16:19.000000000 +0300 +++ new/src/share/vm/opto/opaquenode.cpp 2015-01-26 19:16:19.000000000 +0300 @@ -60,4 +60,26 @@ return (&n == this); // Always fail except on self } +//============================================================================= +uint ProfileBranchNode::hash() const { return NO_HASH; } +uint ProfileBranchNode::cmp( const Node &n ) const { + return (&n == this); +} + +Node *ProfileBranchNode::Ideal(PhaseGVN *phase, bool can_reshape) { + if (can_reshape && _delay_removal) { + _delay_removal = false; + return this; + } else { + return NULL; + } +} + +Node *ProfileBranchNode::Identity( PhaseTransform *phase ) { + if (_delay_removal) { + return this; + } else { + return in(1); + } +} --- old/src/share/vm/opto/opaquenode.hpp 2015-01-26 19:16:20.000000000 +0300 +++ new/src/share/vm/opto/opaquenode.hpp 2015-01-26 19:16:20.000000000 +0300 @@ -87,5 +87,28 @@ bool rtm_opt() const { return (_opt == RTM_OPT); } }; +//------------------------------ProfileBranchNode------------------------------- +// The node can store branch frequencies during parsing. +// Once parsing is over, the node goes away (during IGVN). +class ProfileBranchNode : public Node { + uint _taken; + uint _not_taken; + bool _consumed; + bool _delay_removal; + virtual uint hash() const ; // { return NO_HASH; } + virtual uint cmp( const Node &n ) const; + public: + ProfileBranchNode(Node *n, uint taken, uint not_taken) : Node(0, n), + _taken(taken), _not_taken(not_taken), _delay_removal(true), _consumed(false) {} + virtual int Opcode() const; + uint taken() const { return _taken; } + uint not_taken() const { return _not_taken; } + void consume() { _consumed = true; } + + virtual Node *Ideal(PhaseGVN *phase, bool can_reshape); + virtual Node *Identity(PhaseTransform *phase); + virtual const Type *bottom_type() const { return TypeInt::INT; } +}; + #endif // SHARE_VM_OPTO_OPAQUENODE_HPP --- old/src/share/vm/opto/parse.hpp 2015-01-26 19:16:21.000000000 +0300 +++ new/src/share/vm/opto/parse.hpp 2015-01-26 19:16:20.000000000 +0300 @@ -555,8 +555,8 @@ void do_jsr(); void do_ret(); - float dynamic_branch_prediction(float &cnt); - float branch_prediction(float &cnt, BoolTest::mask btest, int target_bci); + float dynamic_branch_prediction(float &cnt, BoolTest::mask btest, Node* test); + float branch_prediction(float &cnt, BoolTest::mask btest, int target_bci, Node* test); bool seems_never_taken(float prob) const; bool path_is_suitable_for_uncommon_trap(float prob) const; bool seems_stable_comparison() const; --- old/src/share/vm/opto/parse2.cpp 2015-01-26 19:16:22.000000000 +0300 +++ new/src/share/vm/opto/parse2.cpp 2015-01-26 19:16:21.000000000 +0300 @@ -37,6 +37,7 @@ #include "opto/matcher.hpp" #include "opto/memnode.hpp" #include "opto/mulnode.hpp" +#include "opto/opaquenode.hpp" #include "opto/parse.hpp" #include "opto/runtime.hpp" #include "runtime/deoptimization.hpp" @@ -767,28 +768,52 @@ // Try to gather dynamic branch prediction behavior. Return a probability // of the branch being taken and set the "cnt" field. Returns a -1.0 // if we need to use static prediction for some reason. -float Parse::dynamic_branch_prediction(float &cnt) { +float Parse::dynamic_branch_prediction(float &cnt, BoolTest::mask btest, Node* test) { ResourceMark rm; cnt = COUNT_UNKNOWN; - // Use MethodData information if it is available - // FIXME: free the ProfileData structure - ciMethodData* methodData = method()->method_data(); - if (!methodData->is_mature()) return PROB_UNKNOWN; - ciProfileData* data = methodData->bci_to_data(bci()); - if (!data->is_JumpData()) return PROB_UNKNOWN; - - // get taken and not taken values - int taken = data->as_JumpData()->taken(); + int taken = 0; int not_taken = 0; - if (data->is_BranchData()) { - not_taken = data->as_BranchData()->not_taken(); + + ProfileBranchNode* profile = NULL; + if (btest == BoolTest::eq && test->is_Cmp()) { + Node* n = test->in(1); + switch (n->Opcode()) { + case Op_ProfileBranch: + profile = (ProfileBranchNode*) n; + break; + case Op_AndI: { + if (n->in(1)->Opcode() == Op_ProfileBranch) { + profile = (ProfileBranchNode*) n->in(1); + } + break; + } + } } + if (profile != NULL) { + taken = profile->taken(); + not_taken = profile->not_taken(); + profile->consume(); + } else { + // Use MethodData information if it is available + // FIXME: free the ProfileData structure + ciMethodData* methodData = method()->method_data(); + if (!methodData->is_mature()) return PROB_UNKNOWN; + ciProfileData* data = methodData->bci_to_data(bci()); + if (!data->is_JumpData()) return PROB_UNKNOWN; - // scale the counts to be commensurate with invocation counts: - taken = method()->scale_count(taken); - not_taken = method()->scale_count(not_taken); + // get taken and not taken values + taken = data->as_JumpData()->taken(); + not_taken = 0; + if (data->is_BranchData()) { + not_taken = data->as_BranchData()->not_taken(); + } + + // scale the counts to be commensurate with invocation counts: + taken = method()->scale_count(taken); + not_taken = method()->scale_count(not_taken); + } // Give up if too few (or too many, in which case the sum will overflow) counts to be meaningful. // We also check that individual counters are positive first, overwise the sum can become positive. @@ -841,8 +866,9 @@ //-----------------------------branch_prediction------------------------------- float Parse::branch_prediction(float& cnt, BoolTest::mask btest, - int target_bci) { - float prob = dynamic_branch_prediction(cnt); + int target_bci, + Node* test) { + float prob = dynamic_branch_prediction(cnt, btest, test); // If prob is unknown, switch to static prediction if (prob != PROB_UNKNOWN) return prob; @@ -932,7 +958,7 @@ Block* next_block = successor_for_bci(iter().next_bci()); float cnt; - float prob = branch_prediction(cnt, btest, target_bci); + float prob = branch_prediction(cnt, btest, target_bci, c); if (prob == PROB_UNKNOWN) { // (An earlier version of do_ifnull omitted this trap for OSR methods.) #ifndef PRODUCT @@ -1013,7 +1039,7 @@ Block* next_block = successor_for_bci(iter().next_bci()); float cnt; - float prob = branch_prediction(cnt, btest, target_bci); + float prob = branch_prediction(cnt, btest, target_bci, c); float untaken_prob = 1.0 - prob; if (prob == PROB_UNKNOWN) {