1 /*
2 * Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 #include "precompiled.hpp"
25 #include "gc/shared/gcLogPrecious.hpp"
26 #include "gc/z/zLock.inline.hpp"
27 #include "gc/z/zMarkStack.inline.hpp"
28 #include "gc/z/zMarkStackAllocator.hpp"
29 #include "logging/log.hpp"
30 #include "runtime/atomic.hpp"
31 #include "runtime/os.hpp"
32 #include "utilities/debug.hpp"
33
34 uintptr_t ZMarkStackSpaceStart;
35
36 ZMarkStackSpace::ZMarkStackSpace() :
37 _expand_lock(),
38 _start(0),
39 _top(0),
40 _end(0) {
41 assert(ZMarkStackSpaceLimit >= ZMarkStackSpaceExpandSize, "ZMarkStackSpaceLimit too small");
42
43 // Reserve address space
44 const size_t size = ZMarkStackSpaceLimit;
45 const size_t alignment = (size_t)os::vm_allocation_granularity();
46 const uintptr_t addr = (uintptr_t)os::reserve_memory(size, NULL, alignment, mtGC);
47 if (addr == 0) {
48 log_error_p(gc, marking)("Failed to reserve address space for mark stacks");
49 return;
50 }
51
52 // Successfully initialized
53 _start = _top = _end = addr;
54
55 // Register mark stack space start
56 ZMarkStackSpaceStart = _start;
57 }
58
59 bool ZMarkStackSpace::is_initialized() const {
60 return _start != 0;
61 }
62
63 uintptr_t ZMarkStackSpace::alloc_space(size_t size) {
64 uintptr_t top = Atomic::load(&_top);
65
66 for (;;) {
67 const uintptr_t end = Atomic::load(&_end);
68 const uintptr_t new_top = top + size;
69 if (new_top > end) {
70 // Not enough space left
71 return 0;
72 }
73
74 const uintptr_t prev_top = Atomic::cmpxchg(&_top, top, new_top);
75 if (prev_top == top) {
76 // Success
77 return top;
78 }
79
80 // Retry
81 top = prev_top;
82 }
83 }
84
85 uintptr_t ZMarkStackSpace::expand_and_alloc_space(size_t size) {
86 ZLocker<ZLock> locker(&_expand_lock);
87
88 // Retry allocation before expanding
89 uintptr_t addr = alloc_space(size);
90 if (addr != 0) {
91 return addr;
92 }
93
94 // Check expansion limit
95 const size_t expand_size = ZMarkStackSpaceExpandSize;
96 const size_t old_size = _end - _start;
97 const size_t new_size = old_size + expand_size;
98 if (new_size > ZMarkStackSpaceLimit) {
99 // Expansion limit reached. This is a fatal error since we
100 // currently can't recover from running out of mark stack space.
101 fatal("Mark stack space exhausted. Use -XX:ZMarkStackSpaceLimit=<size> to increase the "
102 "maximum number of bytes allocated for mark stacks. Current limit is " SIZE_FORMAT "M.",
103 ZMarkStackSpaceLimit / M);
104 }
105
106 log_debug(gc, marking)("Expanding mark stack space: " SIZE_FORMAT "M->" SIZE_FORMAT "M",
107 old_size / M, new_size / M);
108
109 // Expand
110 os::commit_memory_or_exit((char*)_end, expand_size, false /* executable */, "Mark stack space");
111
112 // Increment top before end to make sure another
113 // thread can't steal out newly expanded space.
114 addr = Atomic::fetch_and_add(&_top, size);
115 Atomic::add(&_end, expand_size);
116
117 return addr;
118 }
119
120 uintptr_t ZMarkStackSpace::alloc(size_t size) {
121 const uintptr_t addr = alloc_space(size);
122 if (addr != 0) {
123 return addr;
124 }
125
126 return expand_and_alloc_space(size);
127 }
128
129 ZMarkStackAllocator::ZMarkStackAllocator() :
130 _freelist(),
131 _space() {
132 guarantee(sizeof(ZMarkStack) == ZMarkStackSize, "Size mismatch");
133 guarantee(sizeof(ZMarkStackMagazine) <= ZMarkStackSize, "Size mismatch");
134
135 // Prime free list to avoid an immediate space
136 // expansion when marking starts.
137 if (_space.is_initialized()) {
138 prime_freelist();
139 }
140 }
141
142 bool ZMarkStackAllocator::is_initialized() const {
143 return _space.is_initialized();
144 }
145
146 void ZMarkStackAllocator::prime_freelist() {
147 for (size_t size = 0; size < ZMarkStackSpaceExpandSize; size += ZMarkStackMagazineSize) {
148 const uintptr_t addr = _space.alloc(ZMarkStackMagazineSize);
149 ZMarkStackMagazine* const magazine = create_magazine_from_space(addr, ZMarkStackMagazineSize);
150 free_magazine(magazine);
151 }
152 }
153
154 ZMarkStackMagazine* ZMarkStackAllocator::create_magazine_from_space(uintptr_t addr, size_t size) {
155 assert(is_aligned(size, ZMarkStackSize), "Invalid size");
156
157 // Use first stack as magazine
158 ZMarkStackMagazine* const magazine = new ((void*)addr) ZMarkStackMagazine();
159 for (size_t i = ZMarkStackSize; i < size; i += ZMarkStackSize) {
160 ZMarkStack* const stack = new ((void*)(addr + i)) ZMarkStack();
161 const bool success = magazine->push(stack);
162 assert(success, "Magazine should never get full");
163 }
164
165 return magazine;
166 }
167
168 ZMarkStackMagazine* ZMarkStackAllocator::alloc_magazine() {
169 // Try allocating from the free list first
170 ZMarkStackMagazine* const magazine = _freelist.pop();
171 if (magazine != NULL) {
172 return magazine;
173 }
174
175 // Allocate new magazine
176 const uintptr_t addr = _space.alloc(ZMarkStackMagazineSize);
177 if (addr == 0) {
178 return NULL;
179 }
180
181 return create_magazine_from_space(addr, ZMarkStackMagazineSize);
182 }
183
184 void ZMarkStackAllocator::free_magazine(ZMarkStackMagazine* magazine) {
185 _freelist.push(magazine);
186 }