--- old/src/hotspot/share/gc/z/zBarrierSetNMethod.cpp 2019-02-21 09:29:19.118882172 +0100 +++ new/src/hotspot/share/gc/z/zBarrierSetNMethod.cpp 2019-02-21 09:29:18.546872484 +0100 @@ -27,12 +27,12 @@ #include "gc/z/zGlobals.hpp" #include "gc/z/zLock.inline.hpp" #include "gc/z/zOopClosures.hpp" -#include "gc/z/zNMethodTable.hpp" +#include "gc/z/zNMethod.hpp" #include "gc/z/zThreadLocalData.hpp" #include "logging/log.hpp" bool ZBarrierSetNMethod::nmethod_entry_barrier(nmethod* nm) { - ZLocker locker(ZNMethodTable::lock_for_nmethod(nm)); + ZLocker locker(ZNMethod::lock_for_nmethod(nm)); log_trace(nmethod, barrier)("Entered critical zone for %p", nm); if (!is_armed(nm)) { --- old/src/hotspot/share/gc/z/zCollectedHeap.cpp 2019-02-21 09:29:19.838894367 +0100 +++ new/src/hotspot/share/gc/z/zCollectedHeap.cpp 2019-02-21 09:29:19.250884408 +0100 @@ -27,7 +27,7 @@ #include "gc/z/zCollectedHeap.hpp" #include "gc/z/zGlobals.hpp" #include "gc/z/zHeap.inline.hpp" -#include "gc/z/zNMethodTable.hpp" +#include "gc/z/zNMethod.hpp" #include "gc/z/zServiceability.hpp" #include "gc/z/zStat.hpp" #include "gc/z/zUtils.inline.hpp" @@ -255,11 +255,11 @@ } void ZCollectedHeap::register_nmethod(nmethod* nm) { - ZNMethodTable::register_nmethod(nm); + ZNMethod::register_nmethod(nm); } void ZCollectedHeap::unregister_nmethod(nmethod* nm) { - ZNMethodTable::unregister_nmethod(nm); + ZNMethod::unregister_nmethod(nm); } void ZCollectedHeap::verify_nmethod(nmethod* nm) { --- old/src/hotspot/share/gc/z/zNMethodTable.cpp 2019-02-21 09:29:20.582906969 +0100 +++ new/src/hotspot/share/gc/z/zNMethodTable.cpp 2019-02-21 09:29:19.986896874 +0100 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -27,205 +27,38 @@ #include "code/icBuffer.hpp" #include "gc/shared/barrierSet.hpp" #include "gc/shared/barrierSetNMethod.hpp" -#include "gc/z/zArray.inline.hpp" #include "gc/z/zGlobals.hpp" #include "gc/z/zHash.inline.hpp" #include "gc/z/zLock.inline.hpp" +#include "gc/z/zNMethodAllocator.hpp" +#include "gc/z/zNMethodClosure.hpp" +#include "gc/z/zNMethodData.hpp" #include "gc/z/zNMethodTable.hpp" +#include "gc/z/zNMethodTableEntry.hpp" +#include "gc/z/zNMethodTableIteration.hpp" #include "gc/z/zOopClosures.inline.hpp" #include "gc/z/zTask.hpp" #include "gc/z/zWorkers.hpp" #include "logging/log.hpp" -#include "memory/allocation.inline.hpp" +#include "memory/allocation.hpp" #include "memory/resourceArea.hpp" #include "runtime/atomic.hpp" #include "runtime/orderAccess.hpp" -#include "runtime/os.hpp" #include "utilities/debug.hpp" -class ZNMethodDataImmediateOops { -private: - const size_t _nimmediate_oops; - - static size_t header_size(); - - ZNMethodDataImmediateOops(const GrowableArray& immediate_oops); - -public: - static ZNMethodDataImmediateOops* create(const GrowableArray& immediate_oops); - static void destroy(ZNMethodDataImmediateOops* data_immediate_oops); - - size_t immediate_oops_count() const; - oop** immediate_oops_begin() const; - oop** immediate_oops_end() const; -}; - -size_t ZNMethodDataImmediateOops::header_size() { - const size_t size = sizeof(ZNMethodDataImmediateOops); - assert(is_aligned(size, sizeof(oop*)), "Header misaligned"); - return size; -} - -ZNMethodDataImmediateOops* ZNMethodDataImmediateOops::create(const GrowableArray& immediate_oops) { - // Allocate memory for the ZNMethodDataImmediateOops object - // plus the immediate oop* array that follows right after. - const size_t size = ZNMethodDataImmediateOops::header_size() + (sizeof(oop*) * immediate_oops.length()); - void* const data_immediate_oops = NEW_C_HEAP_ARRAY(uint8_t, size, mtGC); - return ::new (data_immediate_oops) ZNMethodDataImmediateOops(immediate_oops); -} - -void ZNMethodDataImmediateOops::destroy(ZNMethodDataImmediateOops* data_immediate_oops) { - ZNMethodTable::safe_delete(data_immediate_oops); -} - -ZNMethodDataImmediateOops::ZNMethodDataImmediateOops(const GrowableArray& immediate_oops) : - _nimmediate_oops(immediate_oops.length()) { - // Save all immediate oops - for (size_t i = 0; i < _nimmediate_oops; i++) { - immediate_oops_begin()[i] = immediate_oops.at(i); - } -} - -size_t ZNMethodDataImmediateOops::immediate_oops_count() const { - return _nimmediate_oops; -} - -oop** ZNMethodDataImmediateOops::immediate_oops_begin() const { - // The immediate oop* array starts immediately after this object - return (oop**)((uintptr_t)this + header_size()); -} - -oop** ZNMethodDataImmediateOops::immediate_oops_end() const { - return immediate_oops_begin() + immediate_oops_count(); -} - -class ZNMethodData { -private: - ZReentrantLock _lock; - ZNMethodDataImmediateOops* volatile _immediate_oops; - - ZNMethodData(nmethod* nm); - -public: - static ZNMethodData* create(nmethod* nm); - static void destroy(ZNMethodData* data); - - ZReentrantLock* lock(); - - ZNMethodDataImmediateOops* immediate_oops() const; - ZNMethodDataImmediateOops* swap_immediate_oops(const GrowableArray& immediate_oops); -}; - -ZNMethodData* ZNMethodData::create(nmethod* nm) { - void* const method = NEW_C_HEAP_ARRAY(uint8_t, sizeof(ZNMethodData), mtGC); - return ::new (method) ZNMethodData(nm); -} - -void ZNMethodData::destroy(ZNMethodData* data) { - ZNMethodDataImmediateOops::destroy(data->immediate_oops()); - ZNMethodTable::safe_delete(data); -} - -ZNMethodData::ZNMethodData(nmethod* nm) : - _lock(), - _immediate_oops(NULL) {} - -ZReentrantLock* ZNMethodData::lock() { - return &_lock; -} - -ZNMethodDataImmediateOops* ZNMethodData::immediate_oops() const { - return OrderAccess::load_acquire(&_immediate_oops); -} - -ZNMethodDataImmediateOops* ZNMethodData::swap_immediate_oops(const GrowableArray& immediate_oops) { - ZNMethodDataImmediateOops* const data_immediate_oops = - immediate_oops.is_empty() ? NULL : ZNMethodDataImmediateOops::create(immediate_oops); - return Atomic::xchg(data_immediate_oops, &_immediate_oops); -} - -static ZNMethodData* gc_data(const nmethod* nm) { - return nm->gc_data(); -} - -static void set_gc_data(nmethod* nm, ZNMethodData* data) { - return nm->set_gc_data(data); -} - ZNMethodTableEntry* ZNMethodTable::_table = NULL; size_t ZNMethodTable::_size = 0; -ZLock ZNMethodTable::_iter_lock; -ZNMethodTableEntry* ZNMethodTable::_iter_table = NULL; -size_t ZNMethodTable::_iter_table_size = 0; -ZArray ZNMethodTable::_iter_deferred_deletes; size_t ZNMethodTable::_nregistered = 0; size_t ZNMethodTable::_nunregistered = 0; -volatile size_t ZNMethodTable::_claimed = 0; - -void ZNMethodTable::safe_delete(void* data) { - if (data == NULL) { - return; - } +ZNMethodTableIteration ZNMethodTable::_iteration; - ZLocker locker(&_iter_lock); - if (_iter_table != NULL) { - // Iteration in progress, defer delete - _iter_deferred_deletes.add(data); - } else { - // Iteration not in progress, delete now - FREE_C_HEAP_ARRAY(uint8_t, data); - } +ZNMethodTableEntry* ZNMethodTable::create(size_t size) { + void* const mem = ZNMethodAllocator::allocate(size * sizeof(ZNMethodTableEntry)); + return ::new (mem) ZNMethodTableEntry[size]; } -ZNMethodTableEntry ZNMethodTable::create_entry(nmethod* nm) { - GrowableArray immediate_oops; - bool non_immediate_oops = false; - - // Find all oops relocations - RelocIterator iter(nm); - while (iter.next()) { - if (iter.type() != relocInfo::oop_type) { - // Not an oop - continue; - } - - oop_Relocation* r = iter.oop_reloc(); - - if (!r->oop_is_immediate()) { - // Non-immediate oop found - non_immediate_oops = true; - continue; - } - - if (r->oop_value() != NULL) { - // Non-NULL immediate oop found. NULL oops can safely be - // ignored since the method will be re-registered if they - // are later patched to be non-NULL. - immediate_oops.push(r->oop_addr()); - } - } - - // Attach GC data to nmethod - ZNMethodData* data = gc_data(nm); - if (data == NULL) { - data = ZNMethodData::create(nm); - set_gc_data(nm, data); - } - - // Attach immediate oops in GC data - ZNMethodDataImmediateOops* const old_data_immediate_oops = data->swap_immediate_oops(immediate_oops); - ZNMethodDataImmediateOops::destroy(old_data_immediate_oops); - - // Create entry - return ZNMethodTableEntry(nm, non_immediate_oops, !immediate_oops.is_empty()); -} - -ZReentrantLock* ZNMethodTable::lock_for_nmethod(nmethod* nm) { - ZNMethodData* const data = gc_data(nm); - if (data == NULL) { - return NULL; - } - return data->lock(); +void ZNMethodTable::destroy(ZNMethodTableEntry* table) { + ZNMethodAllocator::free(table); } size_t ZNMethodTable::first_index(const nmethod* nm, size_t size) { @@ -241,8 +74,8 @@ return (prev_index + 1) & mask; } -bool ZNMethodTable::register_entry(ZNMethodTableEntry* table, size_t size, ZNMethodTableEntry entry) { - const nmethod* const nm = entry.method(); +bool ZNMethodTable::register_entry(ZNMethodTableEntry* table, size_t size, nmethod* nm) { + const ZNMethodTableEntry entry(nm); size_t index = first_index(nm, size); for (;;) { @@ -265,11 +98,6 @@ } void ZNMethodTable::unregister_entry(ZNMethodTableEntry* table, size_t size, nmethod* nm) { - if (size == 0) { - // Table is empty - return; - } - size_t index = first_index(nm, size); for (;;) { @@ -279,10 +107,6 @@ if (table_entry.registered() && table_entry.method() == nm) { // Remove entry table[index] = ZNMethodTableEntry(true /* unregistered */); - - // Destroy GC data - ZNMethodData::destroy(gc_data(nm)); - set_gc_data(nm, NULL); return; } @@ -291,7 +115,8 @@ } void ZNMethodTable::rebuild(size_t new_size) { - ZLocker locker(&_iter_lock); + assert(CodeCache_lock->owned_by_self(), "Lock must be held"); + assert(is_power_of_2(new_size), "Invalid size"); log_debug(gc, nmethod)("Rebuilding NMethod Table: " @@ -303,20 +128,18 @@ _nunregistered, percent_of(_nunregistered, _size), 0.0); // Allocate new table - ZNMethodTableEntry* const new_table = new ZNMethodTableEntry[new_size]; + ZNMethodTableEntry* const new_table = ZNMethodTable::create(new_size); // Transfer all registered entries for (size_t i = 0; i < _size; i++) { const ZNMethodTableEntry entry = _table[i]; if (entry.registered()) { - register_entry(new_table, new_size, entry); + register_entry(new_table, new_size, entry.method()); } } - if (_iter_table != _table) { - // Delete old table - delete [] _table; - } + // Free old table + ZNMethodTable::destroy(_table); // Install new table _table = new_table; @@ -353,61 +176,6 @@ } } -void ZNMethodTable::log_register(const nmethod* nm, ZNMethodTableEntry entry) { - LogTarget(Trace, gc, nmethod) log; - if (!log.is_enabled()) { - return; - } - - log.print("Register NMethod: %s.%s (" PTR_FORMAT "), " - "Compiler: %s, Oops: %d, ImmediateOops: " SIZE_FORMAT ", NonImmediateOops: %s", - nm->method()->method_holder()->external_name(), - nm->method()->name()->as_C_string(), - p2i(nm), - nm->compiler_name(), - nm->oops_count() - 1, - entry.immediate_oops() ? gc_data(nm)->immediate_oops()->immediate_oops_count() : 0, - entry.non_immediate_oops() ? "Yes" : "No"); - - LogTarget(Trace, gc, nmethod, oops) log_oops; - if (!log_oops.is_enabled()) { - return; - } - - // Print nmethod oops table - oop* const begin = nm->oops_begin(); - oop* const end = nm->oops_end(); - for (oop* p = begin; p < end; p++) { - log_oops.print(" Oop[" SIZE_FORMAT "] " PTR_FORMAT " (%s)", - (p - begin), p2i(*p), (*p)->klass()->external_name()); - } - - if (entry.immediate_oops()) { - // Print nmethod immediate oops - const ZNMethodDataImmediateOops* const nmi = gc_data(nm)->immediate_oops(); - if (nmi != NULL) { - oop** const begin = nmi->immediate_oops_begin(); - oop** const end = nmi->immediate_oops_end(); - for (oop** p = begin; p < end; p++) { - log_oops.print(" ImmediateOop[" SIZE_FORMAT "] " PTR_FORMAT " @ " PTR_FORMAT " (%s)", - (p - begin), p2i(**p), p2i(*p), (**p)->klass()->external_name()); - } - } - } -} - -void ZNMethodTable::log_unregister(const nmethod* nm) { - LogTarget(Debug, gc, nmethod) log; - if (!log.is_enabled()) { - return; - } - - log.print("Unregister NMethod: %s.%s (" PTR_FORMAT ")", - nm->method()->method_holder()->external_name(), - nm->method()->name()->as_C_string(), - p2i(nm)); -} - size_t ZNMethodTable::registered_nmethods() { return _nregistered; } @@ -418,48 +186,29 @@ void ZNMethodTable::register_nmethod(nmethod* nm) { assert(CodeCache_lock->owned_by_self(), "Lock must be held"); - ResourceMark rm; // Grow/Shrink/Prune table if needed rebuild_if_needed(); - // Create entry - const ZNMethodTableEntry entry = create_entry(nm); - - log_register(nm, entry); - // Insert new entry - if (register_entry(_table, _size, entry)) { + if (register_entry(_table, _size, nm)) { // New entry registered. When register_entry() instead returns // false the nmethod was already in the table so we do not want // to increase number of registered entries in that case. _nregistered++; } - - // Disarm nmethod entry barrier - disarm_nmethod(nm); } -void ZNMethodTable::sweeper_wait_for_iteration() { - // The sweeper must wait for any ongoing iteration to complete - // before it can unregister an nmethod. - if (!Thread::current()->is_Code_cache_sweeper_thread()) { - return; - } +void ZNMethodTable::wait_until_iteration_done() { + assert(CodeCache_lock->owned_by_self(), "Lock must be held"); - while (_iter_table != NULL) { - MutexUnlockerEx mu(CodeCache_lock, Mutex::_no_safepoint_check_flag); - os::naked_short_sleep(1); + while (_iteration.in_progress()) { + CodeCache_lock->wait(Monitor::_no_safepoint_check_flag); } } void ZNMethodTable::unregister_nmethod(nmethod* nm) { assert(CodeCache_lock->owned_by_self(), "Lock must be held"); - ResourceMark rm; - - sweeper_wait_for_iteration(); - - log_unregister(nm); // Remove entry unregister_entry(_table, _size, nm); @@ -467,248 +216,29 @@ _nregistered--; } -void ZNMethodTable::disarm_nmethod(nmethod* nm) { - BarrierSetNMethod* const bs = BarrierSet::barrier_set()->barrier_set_nmethod(); - if (bs != NULL) { - bs->disarm(nm); - } -} - -void ZNMethodTable::nmethod_entries_do_begin() { +void ZNMethodTable::nmethods_do_begin() { MutexLockerEx mu(CodeCache_lock, Mutex::_no_safepoint_check_flag); - ZLocker locker(&_iter_lock); + + // Make sure we don't free data while iterating + ZNMethodAllocator::activate_deferred_frees(); // Prepare iteration - _iter_table = _table; - _iter_table_size = _size; - _claimed = 0; - assert(_iter_deferred_deletes.is_empty(), "Should be emtpy"); + _iteration.nmethods_do_begin(_table, _size); } -void ZNMethodTable::nmethod_entries_do_end() { +void ZNMethodTable::nmethods_do_end() { MutexLockerEx mu(CodeCache_lock, Mutex::_no_safepoint_check_flag); - ZLocker locker(&_iter_lock); // Finish iteration - if (_iter_table != _table) { - delete [] _iter_table; - } - _iter_table = NULL; - assert(_claimed >= _iter_table_size, "Failed to claim all table entries"); - - // Process deferred deletes - ZArrayIterator iter(&_iter_deferred_deletes); - for (void* data; iter.next(&data);) { - FREE_C_HEAP_ARRAY(uint8_t, data); - } - _iter_deferred_deletes.clear(); -} - -void ZNMethodTable::entry_oops_do(ZNMethodTableEntry entry, OopClosure* cl) { - nmethod* const nm = entry.method(); - - // Process oops table - oop* const begin = nm->oops_begin(); - oop* const end = nm->oops_end(); - for (oop* p = begin; p < end; p++) { - if (*p != Universe::non_oop_word()) { - cl->do_oop(p); - } - } - - // Process immediate oops - if (entry.immediate_oops()) { - const ZNMethodDataImmediateOops* const nmi = gc_data(nm)->immediate_oops(); - if (nmi != NULL) { - oop** const begin = nmi->immediate_oops_begin(); - oop** const end = nmi->immediate_oops_end(); - for (oop** p = begin; p < end; p++) { - if (**p != Universe::non_oop_word()) { - cl->do_oop(*p); - } - } - } - } - - // Process non-immediate oops - if (entry.non_immediate_oops()) { - nmethod* const nm = entry.method(); - nm->fix_oop_relocations(); - } -} - -class ZNMethodTableEntryToOopsDo : public ZNMethodTableEntryClosure { -private: - OopClosure* _cl; - -public: - ZNMethodTableEntryToOopsDo(OopClosure* cl) : - _cl(cl) {} - - void do_nmethod_entry(ZNMethodTableEntry entry) { - ZNMethodTable::entry_oops_do(entry, _cl); - } -}; - -void ZNMethodTable::oops_do(OopClosure* cl) { - ZNMethodTableEntryToOopsDo entry_cl(cl); - nmethod_entries_do(&entry_cl); -} - -void ZNMethodTable::nmethod_entries_do(ZNMethodTableEntryClosure* cl) { - for (;;) { - // Claim table partition. Each partition is currently sized to span - // two cache lines. This number is just a guess, but seems to work well. - const size_t partition_size = (ZCacheLineSize * 2) / sizeof(ZNMethodTableEntry); - const size_t partition_start = MIN2(Atomic::add(partition_size, &_claimed) - partition_size, _iter_table_size); - const size_t partition_end = MIN2(partition_start + partition_size, _iter_table_size); - if (partition_start == partition_end) { - // End of table - break; - } - - // Process table partition - for (size_t i = partition_start; i < partition_end; i++) { - const ZNMethodTableEntry entry = _iter_table[i]; - if (entry.registered()) { - cl->do_nmethod_entry(entry); - } - } - } -} - -class ZNMethodTableUnlinkClosure : public ZNMethodTableEntryClosure { -private: - bool _unloading_occurred; - volatile bool _failed; - - void set_failed() { - Atomic::store(true, &_failed); - } - -public: - ZNMethodTableUnlinkClosure(bool unloading_occurred) : - _unloading_occurred(unloading_occurred), - _failed(false) {} - - virtual void do_nmethod_entry(ZNMethodTableEntry entry) { - if (failed()) { - return; - } - - nmethod* const nm = entry.method(); - if (!nm->is_alive()) { - return; - } - - ZLocker locker(ZNMethodTable::lock_for_nmethod(nm)); - - if (nm->is_unloading()) { - // Unlinking of the dependencies must happen before the - // handshake separating unlink and purge. - nm->flush_dependencies(false /* delete_immediately */); - - // We don't need to take the lock when unlinking nmethods from - // the Method, because it is only concurrently unlinked by - // the entry barrier, which acquires the per nmethod lock. - nm->unlink_from_method(false /* acquire_lock */); - return; - } - - // Heal oops and disarm - ZNMethodOopClosure cl; - ZNMethodTable::entry_oops_do(entry, &cl); - ZNMethodTable::disarm_nmethod(nm); - - // Clear compiled ICs and exception caches - if (!nm->unload_nmethod_caches(_unloading_occurred)) { - set_failed(); - } - } - - bool failed() const { - return Atomic::load(&_failed); - } -}; - -class ZNMethodTableUnlinkTask : public ZTask { -private: - ZNMethodTableUnlinkClosure _cl; - ICRefillVerifier* _verifier; - -public: - ZNMethodTableUnlinkTask(bool unloading_occurred, ICRefillVerifier* verifier) : - ZTask("ZNMethodTableUnlinkTask"), - _cl(unloading_occurred), - _verifier(verifier) { - ZNMethodTable::nmethod_entries_do_begin(); - } + _iteration.nmethods_do_end(); - ~ZNMethodTableUnlinkTask() { - ZNMethodTable::nmethod_entries_do_end(); - } - - virtual void work() { - ICRefillVerifierMark mark(_verifier); - ZNMethodTable::nmethod_entries_do(&_cl); - } - - bool success() const { - return !_cl.failed(); - } -}; + // Process deferred frees + ZNMethodAllocator::deactivate_and_process_deferred_frees(); -void ZNMethodTable::unlink(ZWorkers* workers, bool unloading_occurred) { - for (;;) { - ICRefillVerifier verifier; - - { - ZNMethodTableUnlinkTask task(unloading_occurred, &verifier); - workers->run_concurrent(&task); - if (task.success()) { - return; - } - } - - // Cleaning failed because we ran out of transitional IC stubs, - // so we have to refill and try again. Refilling requires taking - // a safepoint, so we temporarily leave the suspendible thread set. - SuspendibleThreadSetLeaver sts; - InlineCacheBuffer::refill_ic_stubs(); - } + // Notify iteration done + CodeCache_lock->notify_all(); } -class ZNMethodTablePurgeClosure : public ZNMethodTableEntryClosure { -public: - virtual void do_nmethod_entry(ZNMethodTableEntry entry) { - nmethod* const nm = entry.method(); - if (nm->is_alive() && nm->is_unloading()) { - nm->make_unloaded(); - } - } -}; - -class ZNMethodTablePurgeTask : public ZTask { -private: - ZNMethodTablePurgeClosure _cl; - -public: - ZNMethodTablePurgeTask() : - ZTask("ZNMethodTablePurgeTask"), - _cl() { - ZNMethodTable::nmethod_entries_do_begin(); - } - - ~ZNMethodTablePurgeTask() { - ZNMethodTable::nmethod_entries_do_end(); - } - - virtual void work() { - ZNMethodTable::nmethod_entries_do(&_cl); - } -}; - -void ZNMethodTable::purge(ZWorkers* workers) { - ZNMethodTablePurgeTask task; - workers->run_concurrent(&task); +void ZNMethodTable::nmethods_do(ZNMethodClosure* cl) { + _iteration.nmethods_do(cl); } --- old/src/hotspot/share/gc/z/zNMethodTable.hpp 2019-02-21 09:29:21.338919774 +0100 +++ new/src/hotspot/share/gc/z/zNMethodTable.hpp 2019-02-21 09:29:20.734909544 +0100 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -24,66 +24,46 @@ #ifndef SHARE_GC_Z_ZNMETHODTABLE_HPP #define SHARE_GC_Z_ZNMETHODTABLE_HPP -#include "gc/z/zArray.hpp" -#include "gc/z/zGlobals.hpp" -#include "gc/z/zLock.hpp" -#include "gc/z/zNMethodTableEntry.hpp" +#include "gc/z/zNMethodTableIteration.hpp" #include "memory/allocation.hpp" +class nmethod; +class ZNMethodClosure; +class ZNMethodTableEntry; class ZWorkers; -class ZNMethodTableEntryClosure { -public: - virtual void do_nmethod_entry(ZNMethodTableEntry entry) = 0; -}; - class ZNMethodTable : public AllStatic { private: - static ZNMethodTableEntry* _table; - static size_t _size; - static ZLock _iter_lock; - static ZNMethodTableEntry* _iter_table; - static size_t _iter_table_size; - static ZArray _iter_deferred_deletes; - static size_t _nregistered; - static size_t _nunregistered; - static volatile size_t _claimed ATTRIBUTE_ALIGNED(ZCacheLineSize); + static ZNMethodTableEntry* _table; + static size_t _size; + static size_t _nregistered; + static size_t _nunregistered; + static ZNMethodTableIteration _iteration; - static ZNMethodTableEntry create_entry(nmethod* nm); + static ZNMethodTableEntry* create(size_t size); + static void destroy(ZNMethodTableEntry* table); static size_t first_index(const nmethod* nm, size_t size); static size_t next_index(size_t prev_index, size_t size); - static void sweeper_wait_for_iteration(); - - static bool register_entry(ZNMethodTableEntry* table, size_t size, ZNMethodTableEntry entry); + static bool register_entry(ZNMethodTableEntry* table, size_t size, nmethod* nm); static void unregister_entry(ZNMethodTableEntry* table, size_t size, nmethod* nm); static void rebuild(size_t new_size); static void rebuild_if_needed(); - static void log_register(const nmethod* nm, ZNMethodTableEntry entry); - static void log_unregister(const nmethod* nm); - public: - static void safe_delete(void* data); - static size_t registered_nmethods(); static size_t unregistered_nmethods(); static void register_nmethod(nmethod* nm); static void unregister_nmethod(nmethod* nm); - static void disarm_nmethod(nmethod* nm); - - static ZReentrantLock* lock_for_nmethod(nmethod* nm); - - static void oops_do(OopClosure* cl); - static void entry_oops_do(ZNMethodTableEntry entry, OopClosure* cl); + static void wait_until_iteration_done(); - static void nmethod_entries_do_begin(); - static void nmethod_entries_do_end(); - static void nmethod_entries_do(ZNMethodTableEntryClosure* cl); + static void nmethods_do_begin(); + static void nmethods_do_end(); + static void nmethods_do(ZNMethodClosure* cl); static void unlink(ZWorkers* workers, bool unloading_occurred); static void purge(ZWorkers* workers); --- old/src/hotspot/share/gc/z/zNMethodTableEntry.hpp 2019-02-21 09:29:22.050931834 +0100 +++ new/src/hotspot/share/gc/z/zNMethodTableEntry.hpp 2019-02-21 09:29:21.462921874 +0100 @@ -32,16 +32,16 @@ // -------------------------- // // 6 -// 3 3 2 1 0 -// +--------------------------------------------------------------------+-+-+-+ -// |11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111|1|1|1| -// +--------------------------------------------------------------------+-+-+-+ -// | | | | -// | 2-2 Non-immediate Oops Flag (1-bits) * | | -// | | | -// | 1-1 Immediate Oops/Unregistered Flag (1-bits) * | -// | | -// | 0-0 Registered Flag (1-bits) * +// 3 2 1 0 +// +---------------------------------------------------------------------+-+-+ +// |11111111 11111111 11111111 11111111 11111111 11111111 11111111 111111|1|1| +// +---------------------------------------------------------------------+-+-+ +// | | | +// | | | +// | | | +// | 1-1 Unregistered Flag (1-bits) * | +// | | +// | 0-0 Registered Flag (1-bits) * // | // * 63-3 NMethod Address (61-bits) // @@ -52,22 +52,20 @@ private: typedef ZBitField field_registered; typedef ZBitField field_unregistered; - typedef ZBitField field_immediate_oops; - typedef ZBitField field_non_immediate_oops; - typedef ZBitField field_method; + typedef ZBitField field_method; uint64_t _entry; public: explicit ZNMethodTableEntry(bool unregistered = false) : - _entry(field_unregistered::encode(unregistered) | - field_registered::encode(false)) {} - - ZNMethodTableEntry(nmethod* method, bool non_immediate_oops, bool immediate_oops) : - _entry(field_method::encode(method) | - field_non_immediate_oops::encode(non_immediate_oops) | - field_immediate_oops::encode(immediate_oops) | - field_registered::encode(true)) {} + _entry(field_registered::encode(false) | + field_unregistered::encode(unregistered) | + field_method::encode(NULL)) {} + + explicit ZNMethodTableEntry(nmethod* method) : + _entry(field_registered::encode(true) | + field_unregistered::encode(false) | + field_method::encode(method)) {} bool registered() const { return field_registered::decode(_entry); @@ -77,14 +75,6 @@ return field_unregistered::decode(_entry); } - bool immediate_oops() const { - return field_immediate_oops::decode(_entry); - } - - bool non_immediate_oops() const { - return field_non_immediate_oops::decode(_entry); - } - nmethod* method() const { return field_method::decode(_entry); } --- old/src/hotspot/share/gc/z/zRootsIterator.cpp 2019-02-21 09:29:22.718943148 +0100 +++ new/src/hotspot/share/gc/z/zRootsIterator.cpp 2019-02-21 09:29:22.178934002 +0100 @@ -33,7 +33,7 @@ #include "gc/shared/suspendibleThreadSet.hpp" #include "gc/z/zBarrierSetNMethod.hpp" #include "gc/z/zGlobals.hpp" -#include "gc/z/zNMethodTable.hpp" +#include "gc/z/zNMethod.hpp" #include "gc/z/zOopClosures.inline.hpp" #include "gc/z/zRootsIterator.hpp" #include "gc/z/zStat.hpp" @@ -175,7 +175,7 @@ if (ClassUnloading) { nmethod::oops_do_marking_prologue(); } else { - ZNMethodTable::nmethod_entries_do_begin(); + ZNMethod::oops_do_begin(); } } @@ -185,7 +185,7 @@ if (ClassUnloading) { nmethod::oops_do_marking_epilogue(); } else { - ZNMethodTable::nmethod_entries_do_end(); + ZNMethod::oops_do_end(); } JvmtiExport::gc_epilogue(); @@ -232,7 +232,7 @@ void ZRootsIterator::do_code_cache(ZRootsIteratorClosure* cl) { ZStatTimer timer(ZSubPhasePauseRootsCodeCache); - ZNMethodTable::oops_do(cl); + ZNMethod::oops_do(cl); } void ZRootsIterator::oops_do(ZRootsIteratorClosure* cl, bool visit_jvmti_weak_export) { --- old/src/hotspot/share/gc/z/zUnload.cpp 2019-02-21 09:29:23.378954327 +0100 +++ new/src/hotspot/share/gc/z/zUnload.cpp 2019-02-21 09:29:22.842945248 +0100 @@ -30,7 +30,7 @@ #include "gc/shared/gcBehaviours.hpp" #include "gc/shared/suspendibleThreadSet.hpp" #include "gc/z/zLock.inline.hpp" -#include "gc/z/zNMethodTable.hpp" +#include "gc/z/zNMethod.hpp" #include "gc/z/zOopClosures.hpp" #include "gc/z/zStat.hpp" #include "gc/z/zUnload.hpp" @@ -75,7 +75,7 @@ public: virtual bool is_unloading(CompiledMethod* method) const { nmethod* const nm = method->as_nmethod(); - ZReentrantLock* const lock = ZNMethodTable::lock_for_nmethod(nm); + ZReentrantLock* const lock = ZNMethod::lock_for_nmethod(nm); if (lock == NULL) { return is_unloading(nm); } else { @@ -89,7 +89,7 @@ public: virtual bool lock(CompiledMethod* method) { nmethod* const nm = method->as_nmethod(); - ZReentrantLock* const lock = ZNMethodTable::lock_for_nmethod(nm); + ZReentrantLock* const lock = ZNMethod::lock_for_nmethod(nm); if (lock != NULL) { lock->lock(); } @@ -98,7 +98,7 @@ virtual void unlock(CompiledMethod* method) { nmethod* const nm = method->as_nmethod(); - ZReentrantLock* const lock = ZNMethodTable::lock_for_nmethod(nm); + ZReentrantLock* const lock = ZNMethod::lock_for_nmethod(nm); if (lock != NULL) { lock->unlock(); } @@ -110,7 +110,7 @@ } nmethod* const nm = method->as_nmethod(); - ZReentrantLock* const lock = ZNMethodTable::lock_for_nmethod(nm); + ZReentrantLock* const lock = ZNMethod::lock_for_nmethod(nm); return lock == NULL || lock->is_owned(); } }; @@ -149,7 +149,7 @@ Klass::clean_weak_klass_links(unloading_occurred); - ZNMethodTable::unlink(_workers, unloading_occurred); + ZNMethod::unlink(_workers, unloading_occurred); DependencyContext::cleaning_end(); } @@ -157,7 +157,7 @@ void ZUnload::purge() { { SuspendibleThreadSetJoiner sts; - ZNMethodTable::purge(_workers); + ZNMethod::purge(_workers); } ClassLoaderDataGraph::purge(); --- /dev/null 2019-02-14 11:31:37.500399000 +0100 +++ new/src/hotspot/share/gc/z/zNMethod.cpp 2019-02-21 09:29:23.506956495 +0100 @@ -0,0 +1,390 @@ +/* + * Copyright (c) 2017, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "code/relocInfo.hpp" +#include "code/nmethod.hpp" +#include "code/icBuffer.hpp" +#include "gc/shared/barrierSet.hpp" +#include "gc/shared/barrierSetNMethod.hpp" +#include "gc/z/zGlobals.hpp" +#include "gc/z/zLock.inline.hpp" +#include "gc/z/zNMethod.hpp" +#include "gc/z/zNMethodClosure.hpp" +#include "gc/z/zNMethodData.hpp" +#include "gc/z/zNMethodTable.hpp" +#include "gc/z/zOopClosures.inline.hpp" +#include "gc/z/zTask.hpp" +#include "gc/z/zWorkers.hpp" +#include "logging/log.hpp" +#include "memory/allocation.inline.hpp" +#include "memory/resourceArea.hpp" +#include "runtime/atomic.hpp" +#include "runtime/orderAccess.hpp" +#include "utilities/debug.hpp" + +static ZNMethodData* gc_data(const nmethod* nm) { + return nm->gc_data(); +} + +static void set_gc_data(nmethod* nm, ZNMethodData* data) { + return nm->set_gc_data(data); +} + +void ZNMethod::attach_gc_data(nmethod* nm) { + GrowableArray immediate_oops; + bool non_immediate_oops = false; + + // Find all oops relocations + RelocIterator iter(nm); + while (iter.next()) { + if (iter.type() != relocInfo::oop_type) { + // Not an oop + continue; + } + + oop_Relocation* r = iter.oop_reloc(); + + if (!r->oop_is_immediate()) { + // Non-immediate oop found + non_immediate_oops = true; + continue; + } + + if (r->oop_value() != NULL) { + // Non-NULL immediate oop found. NULL oops can safely be + // ignored since the method will be re-registered if they + // are later patched to be non-NULL. + immediate_oops.push(r->oop_addr()); + } + } + + // Attach GC data to nmethod + ZNMethodData* data = gc_data(nm); + if (data == NULL) { + data = ZNMethodData::create(nm); + set_gc_data(nm, data); + } + + // Attach oops in GC data + ZNMethodDataOops* const new_oops = ZNMethodDataOops::create(immediate_oops, non_immediate_oops); + ZNMethodDataOops* const old_oops = data->swap_oops(new_oops); + ZNMethodDataOops::destroy(old_oops); +} + +void ZNMethod::detach_gc_data(nmethod* nm) { + // Destroy GC data + ZNMethodData::destroy(gc_data(nm)); + set_gc_data(nm, NULL); +} + +ZReentrantLock* ZNMethod::lock_for_nmethod(nmethod* nm) { + ZNMethodData* const data = gc_data(nm); + if (data == NULL) { + return NULL; + } + return data->lock(); +} + +void ZNMethod::log_register(const nmethod* nm) { + LogTarget(Trace, gc, nmethod) log; + if (!log.is_enabled()) { + return; + } + + const ZNMethodDataOops* const oops = gc_data(nm)->oops(); + + log.print("Register NMethod: %s.%s (" PTR_FORMAT "), " + "Compiler: %s, Oops: %d, ImmediateOops: " SIZE_FORMAT ", NonImmediateOops: %s", + nm->method()->method_holder()->external_name(), + nm->method()->name()->as_C_string(), + p2i(nm), + nm->compiler_name(), + nm->oops_count() - 1, + oops->immediates_count(), + oops->has_non_immediates() ? "Yes" : "No"); + + LogTarget(Trace, gc, nmethod, oops) log_oops; + if (!log_oops.is_enabled()) { + return; + } + + // Print nmethod oops table + oop* const begin = nm->oops_begin(); + oop* const end = nm->oops_end(); + for (oop* p = begin; p < end; p++) { + log_oops.print(" Oop[" SIZE_FORMAT "] " PTR_FORMAT " (%s)", + (p - begin), p2i(*p), (*p)->klass()->external_name()); + } + + // Print nmethod immediate oops + if (oops->immediates_count() > 0) { + oop** const begin = oops->immediates_begin(); + oop** const end = oops->immediates_end(); + for (oop** p = begin; p < end; p++) { + log_oops.print(" ImmediateOop[" SIZE_FORMAT "] " PTR_FORMAT " @ " PTR_FORMAT " (%s)", + (p - begin), p2i(**p), p2i(*p), (**p)->klass()->external_name()); + } + } +} + +void ZNMethod::log_unregister(const nmethod* nm) { + LogTarget(Debug, gc, nmethod) log; + if (!log.is_enabled()) { + return; + } + + log.print("Unregister NMethod: %s.%s (" PTR_FORMAT ")", + nm->method()->method_holder()->external_name(), + nm->method()->name()->as_C_string(), + p2i(nm)); +} + +void ZNMethod::register_nmethod(nmethod* nm) { + ResourceMark rm; + + // Create and attach gc data + attach_gc_data(nm); + + log_register(nm); + + ZNMethodTable::register_nmethod(nm); + + // Disarm nmethod entry barrier + disarm_nmethod(nm); +} + +void ZNMethod::unregister_nmethod(nmethod* nm) { + assert(CodeCache_lock->owned_by_self(), "Lock must be held"); + + if (Thread::current()->is_Code_cache_sweeper_thread()) { + // The sweeper must wait for any ongoing iteration to complete + // before it can unregister an nmethod. + ZNMethodTable::wait_until_iteration_done(); + } + + ResourceMark rm; + + log_unregister(nm); + + ZNMethodTable::unregister_nmethod(nm); + + // Destroy and detach gc data + detach_gc_data(nm); +} + +void ZNMethod::disarm_nmethod(nmethod* nm) { + BarrierSetNMethod* const bs = BarrierSet::barrier_set()->barrier_set_nmethod(); + if (bs != NULL) { + bs->disarm(nm); + } +} + +void ZNMethod::nmethod_oops_do(nmethod* nm, OopClosure* cl) { + // Process oops table + oop* const begin = nm->oops_begin(); + oop* const end = nm->oops_end(); + for (oop* p = begin; p < end; p++) { + if (*p != Universe::non_oop_word()) { + cl->do_oop(p); + } + } + + ZNMethodDataOops* const oops = gc_data(nm)->oops(); + + // Process immediate oops + if (oops->immediates_count() > 0) { + oop** const begin = oops->immediates_begin(); + oop** const end = oops->immediates_end(); + for (oop** p = begin; p < end; p++) { + if (**p != Universe::non_oop_word()) { + cl->do_oop(*p); + } + } + } + + // Process non-immediate oops + if (oops->has_non_immediates()) { + nm->fix_oop_relocations(); + } +} + +class ZNMethodToOopsDo : public ZNMethodClosure { +private: + OopClosure* _cl; + +public: + ZNMethodToOopsDo(OopClosure* cl) : + _cl(cl) {} + + void do_nmethod(nmethod* nm) { + ZNMethod::nmethod_oops_do(nm, _cl); + } +}; + +void ZNMethod::oops_do_begin() { + ZNMethodTable::nmethods_do_begin(); +} + +void ZNMethod::oops_do_end() { + ZNMethodTable::nmethods_do_end(); +} + +void ZNMethod::oops_do(OopClosure* cl) { + ZNMethodToOopsDo nmethod_cl(cl); + ZNMethodTable::nmethods_do(&nmethod_cl); +} + +class ZNMethodUnlinkClosure : public ZNMethodClosure { +private: + bool _unloading_occurred; + volatile bool _failed; + + void set_failed() { + Atomic::store(true, &_failed); + } + +public: + ZNMethodUnlinkClosure(bool unloading_occurred) : + _unloading_occurred(unloading_occurred), + _failed(false) {} + + virtual void do_nmethod(nmethod* nm) { + if (failed()) { + return; + } + + if (!nm->is_alive()) { + return; + } + + ZLocker locker(ZNMethod::lock_for_nmethod(nm)); + + if (nm->is_unloading()) { + // Unlinking of the dependencies must happen before the + // handshake separating unlink and purge. + nm->flush_dependencies(false /* delete_immediately */); + + // We don't need to take the lock when unlinking nmethods from + // the Method, because it is only concurrently unlinked by + // the entry barrier, which acquires the per nmethod lock. + nm->unlink_from_method(false /* acquire_lock */); + return; + } + + // Heal oops and disarm + ZNMethodOopClosure cl; + ZNMethod::nmethod_oops_do(nm, &cl); + ZNMethod::disarm_nmethod(nm); + + // Clear compiled ICs and exception caches + if (!nm->unload_nmethod_caches(_unloading_occurred)) { + set_failed(); + } + } + + bool failed() const { + return Atomic::load(&_failed); + } +}; + +class ZNMethodUnlinkTask : public ZTask { +private: + ZNMethodUnlinkClosure _cl; + ICRefillVerifier* _verifier; + +public: + ZNMethodUnlinkTask(bool unloading_occurred, ICRefillVerifier* verifier) : + ZTask("ZNMethodUnlinkTask"), + _cl(unloading_occurred), + _verifier(verifier) { + ZNMethodTable::nmethods_do_begin(); + } + + ~ZNMethodUnlinkTask() { + ZNMethodTable::nmethods_do_end(); + } + + virtual void work() { + ICRefillVerifierMark mark(_verifier); + ZNMethodTable::nmethods_do(&_cl); + } + + bool success() const { + return !_cl.failed(); + } +}; + +void ZNMethod::unlink(ZWorkers* workers, bool unloading_occurred) { + for (;;) { + ICRefillVerifier verifier; + + { + ZNMethodUnlinkTask task(unloading_occurred, &verifier); + workers->run_concurrent(&task); + if (task.success()) { + return; + } + } + + // Cleaning failed because we ran out of transitional IC stubs, + // so we have to refill and try again. Refilling requires taking + // a safepoint, so we temporarily leave the suspendible thread set. + SuspendibleThreadSetLeaver sts; + InlineCacheBuffer::refill_ic_stubs(); + } +} + +class ZNMethodPurgeClosure : public ZNMethodClosure { +public: + virtual void do_nmethod(nmethod* nm) { + if (nm->is_alive() && nm->is_unloading()) { + nm->make_unloaded(); + } + } +}; + +class ZNMethodPurgeTask : public ZTask { +private: + ZNMethodPurgeClosure _cl; + +public: + ZNMethodPurgeTask() : + ZTask("ZNMethodPurgeTask"), + _cl() { + ZNMethodTable::nmethods_do_begin(); + } + + ~ZNMethodPurgeTask() { + ZNMethodTable::nmethods_do_end(); + } + + virtual void work() { + ZNMethodTable::nmethods_do(&_cl); + } +}; + +void ZNMethod::purge(ZWorkers* workers) { + ZNMethodPurgeTask task; + workers->run_concurrent(&task); +} --- /dev/null 2019-02-14 11:31:37.500399000 +0100 +++ new/src/hotspot/share/gc/z/zNMethod.hpp 2019-02-21 09:29:24.270969435 +0100 @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2017, 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZNMETHOD_HPP +#define SHARE_GC_Z_ZNMETHOD_HPP + +#include "memory/allocation.hpp" + +class nmethod; +class OopClosure; +class ZReentrantLock; +class ZWorkers; + +class ZNMethod : public AllStatic { +private: + static void attach_gc_data(nmethod* nm); + static void detach_gc_data(nmethod* nm); + + static void log_register(const nmethod* nm); + static void log_unregister(const nmethod* nm); + +public: + static void register_nmethod(nmethod* nm); + static void unregister_nmethod(nmethod* nm); + + static void disarm_nmethod(nmethod* nm); + + static void nmethod_oops_do(nmethod* nm, OopClosure* cl); + + static void oops_do_begin(); + static void oops_do_end(); + static void oops_do(OopClosure* cl); + + static ZReentrantLock* lock_for_nmethod(nmethod* nm); + + static void unlink(ZWorkers* workers, bool unloading_occurred); + static void purge(ZWorkers* workers); +}; + +#endif // SHARE_GC_Z_ZNMETHOD_HPP --- /dev/null 2019-02-14 11:31:37.500399000 +0100 +++ new/src/hotspot/share/gc/z/zNMethodAllocator.cpp 2019-02-21 09:29:24.986981563 +0100 @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "gc/z/zArray.inline.hpp" +#include "gc/z/zNMethodAllocator.hpp" +#include "memory/allocation.hpp" + +ZArray ZNMethodAllocator::_deferred_frees; +bool ZNMethodAllocator::_defer_frees(false); + +void ZNMethodAllocator::immediate_free(void* data) { + FREE_C_HEAP_ARRAY(uint8_t, data); +} + +void ZNMethodAllocator::deferred_free(void* data) { + _deferred_frees.add(data); +} + +void* ZNMethodAllocator::allocate(size_t size) { + return NEW_C_HEAP_ARRAY(uint8_t, size, mtGC); +} + +void ZNMethodAllocator::free(void* data) { + if (data == NULL) { + return; + } + + if (_defer_frees) { + deferred_free(data); + } else { + immediate_free(data); + } +} + +void ZNMethodAllocator::activate_deferred_frees() { + assert(_deferred_frees.is_empty(), "precondition"); + _defer_frees = true; +} + +void ZNMethodAllocator::deactivate_and_process_deferred_frees() { + _defer_frees = false; + + ZArrayIterator iter(&_deferred_frees); + for (void* data; iter.next(&data);) { + immediate_free(data); + } + _deferred_frees.clear(); +} --- /dev/null 2019-02-14 11:31:37.500399000 +0100 +++ new/src/hotspot/share/gc/z/zNMethodAllocator.hpp 2019-02-21 09:29:25.702993690 +0100 @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZNMETHODALLOCATOR_HPP +#define SHARE_GC_Z_ZNMETHODALLOCATOR_HPP + +#include "memory/allocation.hpp" +#include "gc/z/zArray.hpp" + +class ZNMethodAllocator : AllStatic { +private: + static ZArray _deferred_frees; + static bool _defer_frees; + + static void immediate_free(void* data); + static void deferred_free(void* data); + +public: + static void* allocate(size_t size); + static void free(void* data); + + static void activate_deferred_frees(); + static void deactivate_and_process_deferred_frees(); +}; + +#endif // SHARE_GC_Z_ZNMETHODALLOCATOR_HPP --- /dev/null 2019-02-14 11:31:37.500399000 +0100 +++ new/src/hotspot/share/gc/z/zNMethodClosure.hpp 2019-02-21 09:29:26.479006834 +0100 @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZNMETHODCLOSURE_HPP +#define SHARE_GC_Z_ZNMETHODCLOSURE_HPP + +class nmethod; + +class ZNMethodClosure { +public: + virtual void do_nmethod(nmethod* nm) = 0; +}; + +#endif // SHARE_GC_Z_ZNMETHODCLOSURE_HPP --- /dev/null 2019-02-14 11:31:37.500399000 +0100 +++ new/src/hotspot/share/gc/z/zNMethodData.cpp 2019-02-21 09:29:27.251019910 +0100 @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2017, 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "gc/z/zLock.inline.hpp" +#include "gc/z/zNMethodAllocator.hpp" +#include "gc/z/zNMethodData.hpp" +#include "memory/allocation.hpp" +#include "runtime/atomic.hpp" +#include "runtime/orderAccess.hpp" +#include "utilities/align.hpp" +#include "utilities/debug.hpp" +#include "utilities/growableArray.hpp" + +size_t ZNMethodDataOops::header_size() { + const size_t size = sizeof(ZNMethodDataOops); + assert(is_aligned(size, sizeof(oop*)), "Header misaligned"); + return size; +} + +ZNMethodDataOops* ZNMethodDataOops::create(const GrowableArray& immediates, bool has_non_immediates) { + // Allocate memory for the ZNMethodDataOops object + // plus the immediate oop* array that follows right after. + const size_t size = ZNMethodDataOops::header_size() + (sizeof(oop*) * immediates.length()); + void* const mem = ZNMethodAllocator::allocate(size); + return ::new (mem) ZNMethodDataOops(immediates, has_non_immediates); +} + +void ZNMethodDataOops::destroy(ZNMethodDataOops* oops) { + ZNMethodAllocator::free(oops); +} + +ZNMethodDataOops::ZNMethodDataOops(const GrowableArray& immediates, bool has_non_immediates) : + _nimmediates(immediates.length()), + _has_non_immediates(has_non_immediates) { + // Save all immediate oops + for (size_t i = 0; i < _nimmediates; i++) { + immediates_begin()[i] = immediates.at(i); + } +} + +size_t ZNMethodDataOops::immediates_count() const { + return _nimmediates; +} + +oop** ZNMethodDataOops::immediates_begin() const { + // The immediate oop* array starts immediately after this object + return (oop**)((uintptr_t)this + header_size()); +} + +oop** ZNMethodDataOops::immediates_end() const { + return immediates_begin() + immediates_count(); +} + +bool ZNMethodDataOops::has_non_immediates() const { + return _has_non_immediates; +} + +ZNMethodData* ZNMethodData::create(nmethod* nm) { + void* const mem = ZNMethodAllocator::allocate(sizeof(ZNMethodData)); + return ::new (mem) ZNMethodData(nm); +} + +void ZNMethodData::destroy(ZNMethodData* data) { + ZNMethodAllocator::free(data->oops()); + ZNMethodAllocator::free(data); +} + +ZNMethodData::ZNMethodData(nmethod* nm) : + _lock(), + _oops(NULL) {} + +ZReentrantLock* ZNMethodData::lock() { + return &_lock; +} + +ZNMethodDataOops* ZNMethodData::oops() const { + return OrderAccess::load_acquire(&_oops); +} + +ZNMethodDataOops* ZNMethodData::swap_oops(ZNMethodDataOops* new_oops) { + return Atomic::xchg(new_oops, &_oops); +} --- /dev/null 2019-02-14 11:31:37.500399000 +0100 +++ new/src/hotspot/share/gc/z/zNMethodData.hpp 2019-02-21 09:29:27.971032105 +0100 @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2017, 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "gc/z/zLock.hpp" +#include "oops/oopsHierarchy.hpp" +#include "utilities/globalDefinitions.hpp" + +class nmethod; +template class GrowableArray; + +class ZNMethodDataOops { +private: + const size_t _nimmediates; + bool _has_non_immediates; + + static size_t header_size(); + + ZNMethodDataOops(const GrowableArray& immediates, bool has_non_immediates); + +public: + static ZNMethodDataOops* create(const GrowableArray& immediates, bool has_non_immediates); + static void destroy(ZNMethodDataOops* oops); + + size_t immediates_count() const; + oop** immediates_begin() const; + oop** immediates_end() const; + + bool has_non_immediates() const; +}; + +class ZNMethodData { +private: + ZReentrantLock _lock; + ZNMethodDataOops* volatile _oops; + + ZNMethodData(nmethod* nm); + +public: + static ZNMethodData* create(nmethod* nm); + static void destroy(ZNMethodData* data); + + ZReentrantLock* lock(); + + ZNMethodDataOops* oops() const; + ZNMethodDataOops* swap_oops(ZNMethodDataOops* oops); +}; --- /dev/null 2019-02-14 11:31:37.500399000 +0100 +++ new/src/hotspot/share/gc/z/zNMethodTableIteration.cpp 2019-02-21 09:29:28.715044706 +0100 @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2017, 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "gc/z/zNMethodClosure.hpp" +#include "gc/z/zNMethodTableEntry.hpp" +#include "gc/z/zNMethodTableIteration.hpp" +#include "runtime/atomic.hpp" +#include "utilities/debug.hpp" +#include "utilities/globalDefinitions.hpp" + +ZNMethodTableIteration::ZNMethodTableIteration() : + _table(NULL), + _size(0), + _claimed(0) {} + +bool ZNMethodTableIteration::in_progress() const { + return _table != NULL; +} + +void ZNMethodTableIteration::nmethods_do_begin(ZNMethodTableEntry* table, size_t size) { + assert(!in_progress(), "precondition"); + + _table = table; + _size = size; + _claimed = 0; +} + +void ZNMethodTableIteration::nmethods_do_end() { + assert(_claimed >= _size, "Failed to claim all table entries"); + + // Finish iteration + _table = NULL; +} + +void ZNMethodTableIteration::nmethods_do(ZNMethodClosure* cl) { + for (;;) { + // Claim table partition. Each partition is currently sized to span + // two cache lines. This number is just a guess, but seems to work well. + const size_t partition_size = (ZCacheLineSize * 2) / sizeof(ZNMethodTableEntry); + const size_t partition_start = MIN2(Atomic::add(partition_size, &_claimed) - partition_size, _size); + const size_t partition_end = MIN2(partition_start + partition_size, _size); + if (partition_start == partition_end) { + // End of table + break; + } + + // Process table partition + for (size_t i = partition_start; i < partition_end; i++) { + const ZNMethodTableEntry entry = _table[i]; + if (entry.registered()) { + cl->do_nmethod(entry.method()); + } + } + } +} --- /dev/null 2019-02-14 11:31:37.500399000 +0100 +++ new/src/hotspot/share/gc/z/zNMethodTableIteration.hpp 2019-02-21 09:29:29.483057715 +0100 @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZNMETHODTABLEITERATION_HPP +#define SHARE_GC_Z_ZNMETHODTABLEITERATION_HPP + +#include "gc/z/zGlobals.hpp" + +class ZNMethodClosure; +class ZNMethodTableEntry; + +class ZNMethodTableIteration { +private: + ZNMethodTableEntry* _table; + size_t _size; + volatile size_t _claimed ATTRIBUTE_ALIGNED(ZCacheLineSize); + +public: + ZNMethodTableIteration(); + + bool in_progress() const; + + void nmethods_do_begin(ZNMethodTableEntry* table, size_t size); + void nmethods_do_end(); + void nmethods_do(ZNMethodClosure* cl); +}; + +#endif // SHARE_GC_Z_ZNMETHODTABLEITERATION_HPP