--- old/src/hotspot/share/gc/z/zNMethodTable.cpp 2019-02-21 09:29:20.582906969 +0100 +++ new/src/hotspot/share/gc/z/zNMethodTable.cpp 2019-02-21 09:29:19.986896874 +0100 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -27,205 +27,38 @@ #include "code/icBuffer.hpp" #include "gc/shared/barrierSet.hpp" #include "gc/shared/barrierSetNMethod.hpp" -#include "gc/z/zArray.inline.hpp" #include "gc/z/zGlobals.hpp" #include "gc/z/zHash.inline.hpp" #include "gc/z/zLock.inline.hpp" +#include "gc/z/zNMethodAllocator.hpp" +#include "gc/z/zNMethodClosure.hpp" +#include "gc/z/zNMethodData.hpp" #include "gc/z/zNMethodTable.hpp" +#include "gc/z/zNMethodTableEntry.hpp" +#include "gc/z/zNMethodTableIteration.hpp" #include "gc/z/zOopClosures.inline.hpp" #include "gc/z/zTask.hpp" #include "gc/z/zWorkers.hpp" #include "logging/log.hpp" -#include "memory/allocation.inline.hpp" +#include "memory/allocation.hpp" #include "memory/resourceArea.hpp" #include "runtime/atomic.hpp" #include "runtime/orderAccess.hpp" -#include "runtime/os.hpp" #include "utilities/debug.hpp" -class ZNMethodDataImmediateOops { -private: - const size_t _nimmediate_oops; - - static size_t header_size(); - - ZNMethodDataImmediateOops(const GrowableArray& immediate_oops); - -public: - static ZNMethodDataImmediateOops* create(const GrowableArray& immediate_oops); - static void destroy(ZNMethodDataImmediateOops* data_immediate_oops); - - size_t immediate_oops_count() const; - oop** immediate_oops_begin() const; - oop** immediate_oops_end() const; -}; - -size_t ZNMethodDataImmediateOops::header_size() { - const size_t size = sizeof(ZNMethodDataImmediateOops); - assert(is_aligned(size, sizeof(oop*)), "Header misaligned"); - return size; -} - -ZNMethodDataImmediateOops* ZNMethodDataImmediateOops::create(const GrowableArray& immediate_oops) { - // Allocate memory for the ZNMethodDataImmediateOops object - // plus the immediate oop* array that follows right after. - const size_t size = ZNMethodDataImmediateOops::header_size() + (sizeof(oop*) * immediate_oops.length()); - void* const data_immediate_oops = NEW_C_HEAP_ARRAY(uint8_t, size, mtGC); - return ::new (data_immediate_oops) ZNMethodDataImmediateOops(immediate_oops); -} - -void ZNMethodDataImmediateOops::destroy(ZNMethodDataImmediateOops* data_immediate_oops) { - ZNMethodTable::safe_delete(data_immediate_oops); -} - -ZNMethodDataImmediateOops::ZNMethodDataImmediateOops(const GrowableArray& immediate_oops) : - _nimmediate_oops(immediate_oops.length()) { - // Save all immediate oops - for (size_t i = 0; i < _nimmediate_oops; i++) { - immediate_oops_begin()[i] = immediate_oops.at(i); - } -} - -size_t ZNMethodDataImmediateOops::immediate_oops_count() const { - return _nimmediate_oops; -} - -oop** ZNMethodDataImmediateOops::immediate_oops_begin() const { - // The immediate oop* array starts immediately after this object - return (oop**)((uintptr_t)this + header_size()); -} - -oop** ZNMethodDataImmediateOops::immediate_oops_end() const { - return immediate_oops_begin() + immediate_oops_count(); -} - -class ZNMethodData { -private: - ZReentrantLock _lock; - ZNMethodDataImmediateOops* volatile _immediate_oops; - - ZNMethodData(nmethod* nm); - -public: - static ZNMethodData* create(nmethod* nm); - static void destroy(ZNMethodData* data); - - ZReentrantLock* lock(); - - ZNMethodDataImmediateOops* immediate_oops() const; - ZNMethodDataImmediateOops* swap_immediate_oops(const GrowableArray& immediate_oops); -}; - -ZNMethodData* ZNMethodData::create(nmethod* nm) { - void* const method = NEW_C_HEAP_ARRAY(uint8_t, sizeof(ZNMethodData), mtGC); - return ::new (method) ZNMethodData(nm); -} - -void ZNMethodData::destroy(ZNMethodData* data) { - ZNMethodDataImmediateOops::destroy(data->immediate_oops()); - ZNMethodTable::safe_delete(data); -} - -ZNMethodData::ZNMethodData(nmethod* nm) : - _lock(), - _immediate_oops(NULL) {} - -ZReentrantLock* ZNMethodData::lock() { - return &_lock; -} - -ZNMethodDataImmediateOops* ZNMethodData::immediate_oops() const { - return OrderAccess::load_acquire(&_immediate_oops); -} - -ZNMethodDataImmediateOops* ZNMethodData::swap_immediate_oops(const GrowableArray& immediate_oops) { - ZNMethodDataImmediateOops* const data_immediate_oops = - immediate_oops.is_empty() ? NULL : ZNMethodDataImmediateOops::create(immediate_oops); - return Atomic::xchg(data_immediate_oops, &_immediate_oops); -} - -static ZNMethodData* gc_data(const nmethod* nm) { - return nm->gc_data(); -} - -static void set_gc_data(nmethod* nm, ZNMethodData* data) { - return nm->set_gc_data(data); -} - ZNMethodTableEntry* ZNMethodTable::_table = NULL; size_t ZNMethodTable::_size = 0; -ZLock ZNMethodTable::_iter_lock; -ZNMethodTableEntry* ZNMethodTable::_iter_table = NULL; -size_t ZNMethodTable::_iter_table_size = 0; -ZArray ZNMethodTable::_iter_deferred_deletes; size_t ZNMethodTable::_nregistered = 0; size_t ZNMethodTable::_nunregistered = 0; -volatile size_t ZNMethodTable::_claimed = 0; - -void ZNMethodTable::safe_delete(void* data) { - if (data == NULL) { - return; - } +ZNMethodTableIteration ZNMethodTable::_iteration; - ZLocker locker(&_iter_lock); - if (_iter_table != NULL) { - // Iteration in progress, defer delete - _iter_deferred_deletes.add(data); - } else { - // Iteration not in progress, delete now - FREE_C_HEAP_ARRAY(uint8_t, data); - } +ZNMethodTableEntry* ZNMethodTable::create(size_t size) { + void* const mem = ZNMethodAllocator::allocate(size * sizeof(ZNMethodTableEntry)); + return ::new (mem) ZNMethodTableEntry[size]; } -ZNMethodTableEntry ZNMethodTable::create_entry(nmethod* nm) { - GrowableArray immediate_oops; - bool non_immediate_oops = false; - - // Find all oops relocations - RelocIterator iter(nm); - while (iter.next()) { - if (iter.type() != relocInfo::oop_type) { - // Not an oop - continue; - } - - oop_Relocation* r = iter.oop_reloc(); - - if (!r->oop_is_immediate()) { - // Non-immediate oop found - non_immediate_oops = true; - continue; - } - - if (r->oop_value() != NULL) { - // Non-NULL immediate oop found. NULL oops can safely be - // ignored since the method will be re-registered if they - // are later patched to be non-NULL. - immediate_oops.push(r->oop_addr()); - } - } - - // Attach GC data to nmethod - ZNMethodData* data = gc_data(nm); - if (data == NULL) { - data = ZNMethodData::create(nm); - set_gc_data(nm, data); - } - - // Attach immediate oops in GC data - ZNMethodDataImmediateOops* const old_data_immediate_oops = data->swap_immediate_oops(immediate_oops); - ZNMethodDataImmediateOops::destroy(old_data_immediate_oops); - - // Create entry - return ZNMethodTableEntry(nm, non_immediate_oops, !immediate_oops.is_empty()); -} - -ZReentrantLock* ZNMethodTable::lock_for_nmethod(nmethod* nm) { - ZNMethodData* const data = gc_data(nm); - if (data == NULL) { - return NULL; - } - return data->lock(); +void ZNMethodTable::destroy(ZNMethodTableEntry* table) { + ZNMethodAllocator::free(table); } size_t ZNMethodTable::first_index(const nmethod* nm, size_t size) { @@ -241,8 +74,8 @@ return (prev_index + 1) & mask; } -bool ZNMethodTable::register_entry(ZNMethodTableEntry* table, size_t size, ZNMethodTableEntry entry) { - const nmethod* const nm = entry.method(); +bool ZNMethodTable::register_entry(ZNMethodTableEntry* table, size_t size, nmethod* nm) { + const ZNMethodTableEntry entry(nm); size_t index = first_index(nm, size); for (;;) { @@ -265,11 +98,6 @@ } void ZNMethodTable::unregister_entry(ZNMethodTableEntry* table, size_t size, nmethod* nm) { - if (size == 0) { - // Table is empty - return; - } - size_t index = first_index(nm, size); for (;;) { @@ -279,10 +107,6 @@ if (table_entry.registered() && table_entry.method() == nm) { // Remove entry table[index] = ZNMethodTableEntry(true /* unregistered */); - - // Destroy GC data - ZNMethodData::destroy(gc_data(nm)); - set_gc_data(nm, NULL); return; } @@ -291,7 +115,8 @@ } void ZNMethodTable::rebuild(size_t new_size) { - ZLocker locker(&_iter_lock); + assert(CodeCache_lock->owned_by_self(), "Lock must be held"); + assert(is_power_of_2(new_size), "Invalid size"); log_debug(gc, nmethod)("Rebuilding NMethod Table: " @@ -303,20 +128,18 @@ _nunregistered, percent_of(_nunregistered, _size), 0.0); // Allocate new table - ZNMethodTableEntry* const new_table = new ZNMethodTableEntry[new_size]; + ZNMethodTableEntry* const new_table = ZNMethodTable::create(new_size); // Transfer all registered entries for (size_t i = 0; i < _size; i++) { const ZNMethodTableEntry entry = _table[i]; if (entry.registered()) { - register_entry(new_table, new_size, entry); + register_entry(new_table, new_size, entry.method()); } } - if (_iter_table != _table) { - // Delete old table - delete [] _table; - } + // Free old table + ZNMethodTable::destroy(_table); // Install new table _table = new_table; @@ -353,61 +176,6 @@ } } -void ZNMethodTable::log_register(const nmethod* nm, ZNMethodTableEntry entry) { - LogTarget(Trace, gc, nmethod) log; - if (!log.is_enabled()) { - return; - } - - log.print("Register NMethod: %s.%s (" PTR_FORMAT "), " - "Compiler: %s, Oops: %d, ImmediateOops: " SIZE_FORMAT ", NonImmediateOops: %s", - nm->method()->method_holder()->external_name(), - nm->method()->name()->as_C_string(), - p2i(nm), - nm->compiler_name(), - nm->oops_count() - 1, - entry.immediate_oops() ? gc_data(nm)->immediate_oops()->immediate_oops_count() : 0, - entry.non_immediate_oops() ? "Yes" : "No"); - - LogTarget(Trace, gc, nmethod, oops) log_oops; - if (!log_oops.is_enabled()) { - return; - } - - // Print nmethod oops table - oop* const begin = nm->oops_begin(); - oop* const end = nm->oops_end(); - for (oop* p = begin; p < end; p++) { - log_oops.print(" Oop[" SIZE_FORMAT "] " PTR_FORMAT " (%s)", - (p - begin), p2i(*p), (*p)->klass()->external_name()); - } - - if (entry.immediate_oops()) { - // Print nmethod immediate oops - const ZNMethodDataImmediateOops* const nmi = gc_data(nm)->immediate_oops(); - if (nmi != NULL) { - oop** const begin = nmi->immediate_oops_begin(); - oop** const end = nmi->immediate_oops_end(); - for (oop** p = begin; p < end; p++) { - log_oops.print(" ImmediateOop[" SIZE_FORMAT "] " PTR_FORMAT " @ " PTR_FORMAT " (%s)", - (p - begin), p2i(**p), p2i(*p), (**p)->klass()->external_name()); - } - } - } -} - -void ZNMethodTable::log_unregister(const nmethod* nm) { - LogTarget(Debug, gc, nmethod) log; - if (!log.is_enabled()) { - return; - } - - log.print("Unregister NMethod: %s.%s (" PTR_FORMAT ")", - nm->method()->method_holder()->external_name(), - nm->method()->name()->as_C_string(), - p2i(nm)); -} - size_t ZNMethodTable::registered_nmethods() { return _nregistered; } @@ -418,48 +186,29 @@ void ZNMethodTable::register_nmethod(nmethod* nm) { assert(CodeCache_lock->owned_by_self(), "Lock must be held"); - ResourceMark rm; // Grow/Shrink/Prune table if needed rebuild_if_needed(); - // Create entry - const ZNMethodTableEntry entry = create_entry(nm); - - log_register(nm, entry); - // Insert new entry - if (register_entry(_table, _size, entry)) { + if (register_entry(_table, _size, nm)) { // New entry registered. When register_entry() instead returns // false the nmethod was already in the table so we do not want // to increase number of registered entries in that case. _nregistered++; } - - // Disarm nmethod entry barrier - disarm_nmethod(nm); } -void ZNMethodTable::sweeper_wait_for_iteration() { - // The sweeper must wait for any ongoing iteration to complete - // before it can unregister an nmethod. - if (!Thread::current()->is_Code_cache_sweeper_thread()) { - return; - } +void ZNMethodTable::wait_until_iteration_done() { + assert(CodeCache_lock->owned_by_self(), "Lock must be held"); - while (_iter_table != NULL) { - MutexUnlockerEx mu(CodeCache_lock, Mutex::_no_safepoint_check_flag); - os::naked_short_sleep(1); + while (_iteration.in_progress()) { + CodeCache_lock->wait(Monitor::_no_safepoint_check_flag); } } void ZNMethodTable::unregister_nmethod(nmethod* nm) { assert(CodeCache_lock->owned_by_self(), "Lock must be held"); - ResourceMark rm; - - sweeper_wait_for_iteration(); - - log_unregister(nm); // Remove entry unregister_entry(_table, _size, nm); @@ -467,248 +216,29 @@ _nregistered--; } -void ZNMethodTable::disarm_nmethod(nmethod* nm) { - BarrierSetNMethod* const bs = BarrierSet::barrier_set()->barrier_set_nmethod(); - if (bs != NULL) { - bs->disarm(nm); - } -} - -void ZNMethodTable::nmethod_entries_do_begin() { +void ZNMethodTable::nmethods_do_begin() { MutexLockerEx mu(CodeCache_lock, Mutex::_no_safepoint_check_flag); - ZLocker locker(&_iter_lock); + + // Make sure we don't free data while iterating + ZNMethodAllocator::activate_deferred_frees(); // Prepare iteration - _iter_table = _table; - _iter_table_size = _size; - _claimed = 0; - assert(_iter_deferred_deletes.is_empty(), "Should be emtpy"); + _iteration.nmethods_do_begin(_table, _size); } -void ZNMethodTable::nmethod_entries_do_end() { +void ZNMethodTable::nmethods_do_end() { MutexLockerEx mu(CodeCache_lock, Mutex::_no_safepoint_check_flag); - ZLocker locker(&_iter_lock); // Finish iteration - if (_iter_table != _table) { - delete [] _iter_table; - } - _iter_table = NULL; - assert(_claimed >= _iter_table_size, "Failed to claim all table entries"); - - // Process deferred deletes - ZArrayIterator iter(&_iter_deferred_deletes); - for (void* data; iter.next(&data);) { - FREE_C_HEAP_ARRAY(uint8_t, data); - } - _iter_deferred_deletes.clear(); -} - -void ZNMethodTable::entry_oops_do(ZNMethodTableEntry entry, OopClosure* cl) { - nmethod* const nm = entry.method(); - - // Process oops table - oop* const begin = nm->oops_begin(); - oop* const end = nm->oops_end(); - for (oop* p = begin; p < end; p++) { - if (*p != Universe::non_oop_word()) { - cl->do_oop(p); - } - } - - // Process immediate oops - if (entry.immediate_oops()) { - const ZNMethodDataImmediateOops* const nmi = gc_data(nm)->immediate_oops(); - if (nmi != NULL) { - oop** const begin = nmi->immediate_oops_begin(); - oop** const end = nmi->immediate_oops_end(); - for (oop** p = begin; p < end; p++) { - if (**p != Universe::non_oop_word()) { - cl->do_oop(*p); - } - } - } - } - - // Process non-immediate oops - if (entry.non_immediate_oops()) { - nmethod* const nm = entry.method(); - nm->fix_oop_relocations(); - } -} - -class ZNMethodTableEntryToOopsDo : public ZNMethodTableEntryClosure { -private: - OopClosure* _cl; - -public: - ZNMethodTableEntryToOopsDo(OopClosure* cl) : - _cl(cl) {} - - void do_nmethod_entry(ZNMethodTableEntry entry) { - ZNMethodTable::entry_oops_do(entry, _cl); - } -}; - -void ZNMethodTable::oops_do(OopClosure* cl) { - ZNMethodTableEntryToOopsDo entry_cl(cl); - nmethod_entries_do(&entry_cl); -} - -void ZNMethodTable::nmethod_entries_do(ZNMethodTableEntryClosure* cl) { - for (;;) { - // Claim table partition. Each partition is currently sized to span - // two cache lines. This number is just a guess, but seems to work well. - const size_t partition_size = (ZCacheLineSize * 2) / sizeof(ZNMethodTableEntry); - const size_t partition_start = MIN2(Atomic::add(partition_size, &_claimed) - partition_size, _iter_table_size); - const size_t partition_end = MIN2(partition_start + partition_size, _iter_table_size); - if (partition_start == partition_end) { - // End of table - break; - } - - // Process table partition - for (size_t i = partition_start; i < partition_end; i++) { - const ZNMethodTableEntry entry = _iter_table[i]; - if (entry.registered()) { - cl->do_nmethod_entry(entry); - } - } - } -} - -class ZNMethodTableUnlinkClosure : public ZNMethodTableEntryClosure { -private: - bool _unloading_occurred; - volatile bool _failed; - - void set_failed() { - Atomic::store(true, &_failed); - } - -public: - ZNMethodTableUnlinkClosure(bool unloading_occurred) : - _unloading_occurred(unloading_occurred), - _failed(false) {} - - virtual void do_nmethod_entry(ZNMethodTableEntry entry) { - if (failed()) { - return; - } - - nmethod* const nm = entry.method(); - if (!nm->is_alive()) { - return; - } - - ZLocker locker(ZNMethodTable::lock_for_nmethod(nm)); - - if (nm->is_unloading()) { - // Unlinking of the dependencies must happen before the - // handshake separating unlink and purge. - nm->flush_dependencies(false /* delete_immediately */); - - // We don't need to take the lock when unlinking nmethods from - // the Method, because it is only concurrently unlinked by - // the entry barrier, which acquires the per nmethod lock. - nm->unlink_from_method(false /* acquire_lock */); - return; - } - - // Heal oops and disarm - ZNMethodOopClosure cl; - ZNMethodTable::entry_oops_do(entry, &cl); - ZNMethodTable::disarm_nmethod(nm); - - // Clear compiled ICs and exception caches - if (!nm->unload_nmethod_caches(_unloading_occurred)) { - set_failed(); - } - } - - bool failed() const { - return Atomic::load(&_failed); - } -}; - -class ZNMethodTableUnlinkTask : public ZTask { -private: - ZNMethodTableUnlinkClosure _cl; - ICRefillVerifier* _verifier; - -public: - ZNMethodTableUnlinkTask(bool unloading_occurred, ICRefillVerifier* verifier) : - ZTask("ZNMethodTableUnlinkTask"), - _cl(unloading_occurred), - _verifier(verifier) { - ZNMethodTable::nmethod_entries_do_begin(); - } + _iteration.nmethods_do_end(); - ~ZNMethodTableUnlinkTask() { - ZNMethodTable::nmethod_entries_do_end(); - } - - virtual void work() { - ICRefillVerifierMark mark(_verifier); - ZNMethodTable::nmethod_entries_do(&_cl); - } - - bool success() const { - return !_cl.failed(); - } -}; + // Process deferred frees + ZNMethodAllocator::deactivate_and_process_deferred_frees(); -void ZNMethodTable::unlink(ZWorkers* workers, bool unloading_occurred) { - for (;;) { - ICRefillVerifier verifier; - - { - ZNMethodTableUnlinkTask task(unloading_occurred, &verifier); - workers->run_concurrent(&task); - if (task.success()) { - return; - } - } - - // Cleaning failed because we ran out of transitional IC stubs, - // so we have to refill and try again. Refilling requires taking - // a safepoint, so we temporarily leave the suspendible thread set. - SuspendibleThreadSetLeaver sts; - InlineCacheBuffer::refill_ic_stubs(); - } + // Notify iteration done + CodeCache_lock->notify_all(); } -class ZNMethodTablePurgeClosure : public ZNMethodTableEntryClosure { -public: - virtual void do_nmethod_entry(ZNMethodTableEntry entry) { - nmethod* const nm = entry.method(); - if (nm->is_alive() && nm->is_unloading()) { - nm->make_unloaded(); - } - } -}; - -class ZNMethodTablePurgeTask : public ZTask { -private: - ZNMethodTablePurgeClosure _cl; - -public: - ZNMethodTablePurgeTask() : - ZTask("ZNMethodTablePurgeTask"), - _cl() { - ZNMethodTable::nmethod_entries_do_begin(); - } - - ~ZNMethodTablePurgeTask() { - ZNMethodTable::nmethod_entries_do_end(); - } - - virtual void work() { - ZNMethodTable::nmethod_entries_do(&_cl); - } -}; - -void ZNMethodTable::purge(ZWorkers* workers) { - ZNMethodTablePurgeTask task; - workers->run_concurrent(&task); +void ZNMethodTable::nmethods_do(ZNMethodClosure* cl) { + _iteration.nmethods_do(cl); }