--- old/src/hotspot/share/runtime/deoptimization.cpp 2019-08-27 14:06:34.228639294 +0200 +++ new/src/hotspot/share/runtime/deoptimization.cpp 2019-08-27 14:06:33.928628902 +0200 @@ -157,6 +157,92 @@ return fetch_unroll_info_helper(thread, exec_mode); JRT_END +#if COMPILER2_OR_JVMCI +static bool eliminate_allocations(JavaThread* thread, int exec_mode, CompiledMethod* compiled_method, + frame& deoptee, RegisterMap& map, GrowableArray* chunk) { + bool realloc_failures = false; + assert (chunk->at(0)->scope() != NULL,"expect only compiled java frames"); + + GrowableArray* objects = chunk->at(0)->scope()->objects(); + + // The flag return_oop() indicates call sites which return oop + // in compiled code. Such sites include java method calls, + // runtime calls (for example, used to allocate new objects/arrays + // on slow code path) and any other calls generated in compiled code. + // It is not guaranteed that we can get such information here only + // by analyzing bytecode in deoptimized frames. This is why this flag + // is set during method compilation (see Compile::Process_OopMap_Node()). + // If the previous frame was popped or if we are dispatching an exception, + // we don't have an oop result. + bool save_oop_result = chunk->at(0)->scope()->return_oop() && !thread->popframe_forcing_deopt_reexecution() && (exec_mode == Deoptimization::Unpack_deopt); + Handle return_value; + if (save_oop_result) { + // Reallocation may trigger GC. If deoptimization happened on return from + // call which returns oop we need to save it since it is not in oopmap. + oop result = deoptee.saved_oop_result(&map); + assert(oopDesc::is_oop_or_null(result), "must be oop"); + return_value = Handle(thread, result); + assert(Universe::heap()->is_in_or_null(result), "must be heap pointer"); + if (TraceDeoptimization) { + ttyLocker ttyl; + tty->print_cr("SAVED OOP RESULT " INTPTR_FORMAT " in thread " INTPTR_FORMAT, p2i(result), p2i(thread)); + } + } + if (objects != NULL) { + JRT_BLOCK + realloc_failures = Deoptimization::realloc_objects(thread, &deoptee, &map, objects, THREAD); + JRT_END + bool skip_internal = (compiled_method != NULL) && !compiled_method->is_compiled_by_jvmci(); + Deoptimization::reassign_fields(&deoptee, &map, objects, realloc_failures, skip_internal); +#ifndef PRODUCT + if (TraceDeoptimization) { + ttyLocker ttyl; + tty->print_cr("REALLOC OBJECTS in thread " INTPTR_FORMAT, p2i(thread)); + Deoptimization::print_objects(objects, realloc_failures); + } +#endif + } + if (save_oop_result) { + // Restore result. + deoptee.set_saved_oop_result(&map, return_value()); + } + return realloc_failures; +} + +static void eliminate_locks(JavaThread* thread, GrowableArray* chunk, bool realloc_failures) { +#ifndef PRODUCT + bool first = true; +#endif + for (int i = 0; i < chunk->length(); i++) { + compiledVFrame* cvf = chunk->at(i); + assert (cvf->scope() != NULL,"expect only compiled java frames"); + GrowableArray* monitors = cvf->monitors(); + if (monitors->is_nonempty()) { + Deoptimization::relock_objects(monitors, thread, realloc_failures); +#ifndef PRODUCT + if (PrintDeoptimizationDetails) { + ttyLocker ttyl; + for (int j = 0; j < monitors->length(); j++) { + MonitorInfo* mi = monitors->at(j); + if (mi->eliminated()) { + if (first) { + first = false; + tty->print_cr("RELOCK OBJECTS in thread " INTPTR_FORMAT, p2i(thread)); + } + if (mi->owner_is_scalar_replaced()) { + Klass* k = java_lang_Class::as_Klass(mi->owner_klass()); + tty->print_cr(" failed reallocation for klass %s", k->external_name()); + } else { + tty->print_cr(" object <" INTPTR_FORMAT "> locked", p2i(mi->owner())); + } + } + } + } +#endif // !PRODUCT + } + } +} +#endif // COMPILER2_OR_JVMCI // This is factored, since it is both called from a JRT_LEAF (deoptimization) and a JRT_ENTRY (uncommon_trap) Deoptimization::UnrollBlock* Deoptimization::fetch_unroll_info_helper(JavaThread* thread, int exec_mode) { @@ -201,95 +287,31 @@ bool realloc_failures = false; #if COMPILER2_OR_JVMCI +#if INCLUDE_JVMCI + bool jvmci_enabled = true; +#else + bool jvmci_enabled = false; +#endif + // Reallocate the non-escaping objects and restore their fields. Then // relock objects if synchronization on them was eliminated. -#if !INCLUDE_JVMCI - if (DoEscapeAnalysis || EliminateNestedLocks) { - if (EliminateAllocations) { -#endif // INCLUDE_JVMCI - assert (chunk->at(0)->scope() != NULL,"expect only compiled java frames"); - GrowableArray* objects = chunk->at(0)->scope()->objects(); - - // The flag return_oop() indicates call sites which return oop - // in compiled code. Such sites include java method calls, - // runtime calls (for example, used to allocate new objects/arrays - // on slow code path) and any other calls generated in compiled code. - // It is not guaranteed that we can get such information here only - // by analyzing bytecode in deoptimized frames. This is why this flag - // is set during method compilation (see Compile::Process_OopMap_Node()). - // If the previous frame was popped or if we are dispatching an exception, - // we don't have an oop result. - bool save_oop_result = chunk->at(0)->scope()->return_oop() && !thread->popframe_forcing_deopt_reexecution() && (exec_mode == Unpack_deopt); - Handle return_value; - if (save_oop_result) { - // Reallocation may trigger GC. If deoptimization happened on return from - // call which returns oop we need to save it since it is not in oopmap. - oop result = deoptee.saved_oop_result(&map); - assert(oopDesc::is_oop_or_null(result), "must be oop"); - return_value = Handle(thread, result); - assert(Universe::heap()->is_in_or_null(result), "must be heap pointer"); - if (TraceDeoptimization) { - ttyLocker ttyl; - tty->print_cr("SAVED OOP RESULT " INTPTR_FORMAT " in thread " INTPTR_FORMAT, p2i(result), p2i(thread)); - } - } - if (objects != NULL) { - JRT_BLOCK - realloc_failures = realloc_objects(thread, &deoptee, &map, objects, THREAD); - JRT_END - bool skip_internal = (cm != NULL) && !cm->is_compiled_by_jvmci(); - reassign_fields(&deoptee, &map, objects, realloc_failures, skip_internal); -#ifndef PRODUCT - if (TraceDeoptimization) { - ttyLocker ttyl; - tty->print_cr("REALLOC OBJECTS in thread " INTPTR_FORMAT, p2i(thread)); - print_objects(objects, realloc_failures); - } -#endif - } - if (save_oop_result) { - // Restore result. - deoptee.set_saved_oop_result(&map, return_value()); - } -#if !INCLUDE_JVMCI - } - if (EliminateLocks) { -#endif // INCLUDE_JVMCI -#ifndef PRODUCT - bool first = true; -#endif - for (int i = 0; i < chunk->length(); i++) { - compiledVFrame* cvf = chunk->at(i); - assert (cvf->scope() != NULL,"expect only compiled java frames"); - GrowableArray* monitors = cvf->monitors(); - if (monitors->is_nonempty()) { - relock_objects(monitors, thread, realloc_failures); -#ifndef PRODUCT - if (PrintDeoptimizationDetails) { - ttyLocker ttyl; - for (int j = 0; j < monitors->length(); j++) { - MonitorInfo* mi = monitors->at(j); - if (mi->eliminated()) { - if (first) { - first = false; - tty->print_cr("RELOCK OBJECTS in thread " INTPTR_FORMAT, p2i(thread)); - } - if (mi->owner_is_scalar_replaced()) { - Klass* k = java_lang_Class::as_Klass(mi->owner_klass()); - tty->print_cr(" failed reallocation for klass %s", k->external_name()); - } else { - tty->print_cr(" object <" INTPTR_FORMAT "> locked", p2i(mi->owner())); - } - } - } - } -#endif // !PRODUCT - } - } -#if !INCLUDE_JVMCI - } + if (jvmci_enabled || ((DoEscapeAnalysis || EliminateNestedLocks) && EliminateAllocations)) { + realloc_failures = eliminate_allocations(thread, exec_mode, cm, deoptee, map, chunk); + } + + // Revoke biases, done with in java state. + // No safepoints allowed after this + revoke_from_deopt_handler(thread, deoptee, &map); + + // Ensure that no safepoint is taken after pointers have been stored + // in fields of rematerialized objects. If a safepoint occurs from here on + // out the java state residing in the vframeArray will be missed. + // Locks may be rebaised in a safepoint. + NoSafepointVerifier no_safepoint; + + if (jvmci_enabled || ((DoEscapeAnalysis || EliminateNestedLocks) && EliminateLocks)) { + eliminate_locks(thread, chunk, realloc_failures); } -#endif // INCLUDE_JVMCI #endif // COMPILER2_OR_JVMCI ScopeDesc* trap_scope = chunk->at(0)->scope(); @@ -305,11 +327,6 @@ guarantee(exceptionObject() != NULL, "exception oop can not be null"); } - // Ensure that no safepoint is taken after pointers have been stored - // in fields of rematerialized objects. If a safepoint occurs from here on - // out the java state residing in the vframeArray will be missed. - NoSafepointVerifier no_safepoint; - vframeArray* array = create_vframeArray(thread, deoptee, &map, chunk, realloc_failures); #if COMPILER2_OR_JVMCI if (realloc_failures) { @@ -779,10 +796,33 @@ return bt; JRT_END +class DeoptimizeMarkedTC : public ThreadClosure { + public: + virtual void do_thread(Thread* thread) { + assert(thread->is_Java_thread(), "must be"); + JavaThread* jt = (JavaThread*)thread; + jt->deoptimize_marked_methods(); + } +}; -int Deoptimization::deoptimize_dependents() { - Threads::deoptimized_wrt_marked_nmethods(); - return 0; +void Deoptimization::deoptimize_all_marked() { + ResourceMark rm; + DeoptimizationMarker dm; + + if (SafepointSynchronize::is_at_safepoint()) { + DeoptimizeMarkedTC deopt; + // Make the dependent methods not entrant + CodeCache::make_marked_nmethods_not_entrant(); + Threads::java_threads_do(&deopt); + } else { + // Make the dependent methods not entrant + { + MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag); + CodeCache::make_marked_nmethods_not_entrant(); + } + DeoptimizeMarkedTC deopt; + Handshake::execute(&deopt); + } } Deoptimization::DeoptAction Deoptimization::_unloaded_action @@ -1397,14 +1437,7 @@ } } - -void Deoptimization::revoke_biases_of_monitors(JavaThread* thread, frame fr, RegisterMap* map) { - if (!UseBiasedLocking) { - return; - } - - GrowableArray* objects_to_revoke = new GrowableArray(); - +static void get_monitors_from_stack(GrowableArray* objects_to_revoke, JavaThread* thread, frame fr, RegisterMap* map) { // Unfortunately we don't have a RegisterMap available in most of // the places we want to call this routine so we need to walk the // stack again to update the register map. @@ -1428,11 +1461,20 @@ cvf = compiledVFrame::cast(cvf->sender()); } collect_monitors(cvf, objects_to_revoke); +} - if (SafepointSynchronize::is_at_safepoint()) { - BiasedLocking::revoke_at_safepoint(objects_to_revoke); - } else { - BiasedLocking::revoke(objects_to_revoke, thread); +void Deoptimization::revoke_from_deopt_handler(JavaThread* thread, frame fr, RegisterMap* map) { + if (!UseBiasedLocking) { + return; + } + GrowableArray* objects_to_revoke = new GrowableArray(); + get_monitors_from_stack(objects_to_revoke, thread, fr, map); + + int len = objects_to_revoke->length(); + for (int i = 0; i < len; i++) { + oop obj = (objects_to_revoke->at(i))(); + BiasedLocking::revoke_own_locks(objects_to_revoke->at(i), thread); + assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now"); } } @@ -1464,10 +1506,6 @@ fr.deoptimize(thread); } -void Deoptimization::deoptimize(JavaThread* thread, frame fr, RegisterMap *map) { - deoptimize(thread, fr, map, Reason_constraint); -} - void Deoptimization::deoptimize(JavaThread* thread, frame fr, RegisterMap *map, DeoptReason reason) { // Deoptimize only if the frame comes from compile code. // Do not deoptimize the frame which is already patched @@ -1477,11 +1515,7 @@ } ResourceMark rm; DeoptimizationMarker dm; - if (UseBiasedLocking) { - revoke_biases_of_monitors(thread, fr, map); - } deoptimize_single_frame(thread, fr, reason); - } #if INCLUDE_JVMCI @@ -1642,9 +1676,6 @@ { ResourceMark rm; - // Revoke biases of any monitors in the frame to ensure we can migrate them - revoke_biases_of_monitors(thread, fr, ®_map); - DeoptReason reason = trap_request_reason(trap_request); DeoptAction action = trap_request_action(trap_request); #if INCLUDE_JVMCI