--- /dev/null 2018-04-03 12:55:20.301839954 +0200 +++ new/src/hotspot/share/gc/z/zMarkStack.inline.hpp 2018-06-08 19:46:18.688655246 +0200 @@ -0,0 +1,249 @@ +/* + * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZMARKSTACK_INLINE_HPP +#define SHARE_GC_Z_ZMARKSTACK_INLINE_HPP + +#include "gc/z/zMarkStack.hpp" +#include "utilities/debug.hpp" +#include "runtime/atomic.hpp" + +template +inline ZStack::ZStack() : + _top(0), + _next(NULL) {} + +template +inline bool ZStack::is_empty() const { + return _top == 0; +} + +template +inline bool ZStack::is_full() const { + return _top == S; +} + +template +inline bool ZStack::push(T value) { + if (is_full()) { + return false; + } + + _slots[_top++] = value; + return true; +} + +template +inline bool ZStack::pop(T& value) { + if (is_empty()) { + return false; + } + + value = _slots[--_top]; + return true; +} + +template +inline ZStack* ZStack::next() const { + return _next; +} + +template +inline ZStack** ZStack::next_addr() { + return &_next; +} + +template +inline ZStackList::ZStackList() : + _head(encode_versioned_pointer(NULL, 0)) {} + +template +inline T* ZStackList::encode_versioned_pointer(const T* stack, uint32_t version) const { + uint64_t addr; + + if (stack == NULL) { + addr = (uint32_t)-1; + } else { + addr = ((uint64_t)stack - ZMarkStackSpaceStart) >> ZMarkStackSizeShift; + } + + return (T*)((addr << 32) | (uint64_t)version); +} + +template +inline void ZStackList::decode_versioned_pointer(const T* vstack, T** stack, uint32_t* version) const { + const uint64_t addr = (uint64_t)vstack >> 32; + + if (addr == (uint32_t)-1) { + *stack = NULL; + } else { + *stack = (T*)((addr << ZMarkStackSizeShift) + ZMarkStackSpaceStart); + } + + *version = (uint32_t)(uint64_t)vstack; +} + +template +inline bool ZStackList::is_empty() const { + const T* vstack = _head; + T* stack = NULL; + uint32_t version = 0; + + decode_versioned_pointer(vstack, &stack, &version); + return stack == NULL; +} + +template +inline void ZStackList::push_atomic(T* stack) { + T* vstack = _head; + uint32_t version = 0; + + for (;;) { + decode_versioned_pointer(vstack, stack->next_addr(), &version); + T* const new_vstack = encode_versioned_pointer(stack, version + 1); + T* const prev_vstack = Atomic::cmpxchg(new_vstack, &_head, vstack); + if (prev_vstack == vstack) { + // Success + break; + } + + // Retry + vstack = prev_vstack; + } +} + +template +inline T* ZStackList::pop_atomic() { + T* vstack = _head; + T* stack = NULL; + uint32_t version = 0; + + for (;;) { + decode_versioned_pointer(vstack, &stack, &version); + if (stack == NULL) { + return NULL; + } + + T* const new_vstack = encode_versioned_pointer(stack->next(), version + 1); + T* const prev_vstack = Atomic::cmpxchg(new_vstack, &_head, vstack); + if (prev_vstack == vstack) { + // Success + return stack; + } + + // Retry + vstack = prev_vstack; + } +} + +inline bool ZMarkStripe::is_empty() const { + return _published.is_empty() && _overflowed.is_empty(); +} + +inline void ZMarkStripe::publish_stack(ZMarkStack* stack, bool publish) { + // A stack is published either on the published list or the overflowed + // list. The published list is used by mutators publishing stacks for GC + // workers to work on, while the overflowed list is used by GC workers + // to publish stacks that overflowed. The intention here is to avoid + // contention between mutators and GC workers as much as possible, while + // still allowing GC workers to help out and steal work from each other. + if (publish) { + _published.push_atomic(stack); + } else { + _overflowed.push_atomic(stack); + } +} + +inline ZMarkStack* ZMarkStripe::steal_stack() { + // Steal overflowed stacks first, then published stacks + ZMarkStack* const stack = _overflowed.pop_atomic(); + if (stack != NULL) { + return stack; + } + + return _published.pop_atomic(); +} + +inline size_t ZMarkStripeSet::nstripes() const { + return _nstripes; +} + +inline size_t ZMarkStripeSet::stripe_id(const ZMarkStripe* stripe) const { + const size_t index = ((uintptr_t)stripe - (uintptr_t)_stripes) / sizeof(ZMarkStripe); + assert(index < _nstripes, "Invalid index"); + return index; +} + +inline ZMarkStripe* ZMarkStripeSet::stripe_at(size_t index) { + assert(index < _nstripes, "Invalid index"); + return &_stripes[index]; +} + +inline ZMarkStripe* ZMarkStripeSet::stripe_next(ZMarkStripe* stripe) { + const size_t index = (stripe_id(stripe) + 1) & _nstripes_mask; + assert(index < _nstripes, "Invalid index"); + return &_stripes[index]; +} + +inline ZMarkStripe* ZMarkStripeSet::stripe_for_addr(uintptr_t addr) { + const size_t index = (addr >> ZMarkStripeShift) & _nstripes_mask; + assert(index < _nstripes, "Invalid index"); + return &_stripes[index]; +} + +inline void ZMarkThreadLocalStacks::install(ZMarkStripeSet* stripes, + ZMarkStripe* stripe, + ZMarkStack* stack) { + ZMarkStack** const stackp = &_stacks[stripes->stripe_id(stripe)]; + assert(*stackp == NULL, "Should be empty"); + *stackp = stack; +} + +inline bool ZMarkThreadLocalStacks::push(ZMarkStackAllocator* allocator, + ZMarkStripeSet* stripes, + ZMarkStripe* stripe, + ZMarkStackEntry entry, + bool publish) { + ZMarkStack** const stackp = &_stacks[stripes->stripe_id(stripe)]; + ZMarkStack* const stack = *stackp; + if (stack != NULL && stack->push(entry)) { + return true; + } + + return push_slow(allocator, stripe, stackp, entry, publish); +} + +inline bool ZMarkThreadLocalStacks::pop(ZMarkStackAllocator* allocator, + ZMarkStripeSet* stripes, + ZMarkStripe* stripe, + ZMarkStackEntry& entry) { + ZMarkStack** const stackp = &_stacks[stripes->stripe_id(stripe)]; + ZMarkStack* const stack = *stackp; + if (stack != NULL && stack->pop(entry)) { + return true; + } + + return pop_slow(allocator, stripe, stackp, entry); +} + +#endif // SHARE_GC_Z_ZMARKSTACK_INLINE_HPP