1 /*
2 * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 #include "precompiled.hpp"
25 #include "gc/z/zCollectedHeap.hpp"
26 #include "gc/z/zGlobals.hpp"
27 #include "gc/z/zHeap.inline.hpp"
28 #include "gc/z/zObjectAllocator.hpp"
29 #include "gc/z/zPage.inline.hpp"
30 #include "gc/z/zStat.hpp"
31 #include "gc/z/zThread.hpp"
32 #include "gc/z/zUtils.inline.hpp"
33 #include "logging/log.hpp"
34 #include "runtime/atomic.hpp"
35 #include "runtime/safepoint.hpp"
36 #include "runtime/thread.hpp"
37 #include "runtime/threadSMR.hpp"
38 #include "utilities/align.hpp"
39 #include "utilities/debug.hpp"
40
41 static const ZStatCounter ZCounterUndoObjectAllocationSucceeded("Memory", "Undo Object Allocation Succeeded", ZStatUnitOpsPerSecond);
42 static const ZStatCounter ZCounterUndoObjectAllocationFailed("Memory", "Undo Object Allocation Failed", ZStatUnitOpsPerSecond);
43
44 ZObjectAllocator::ZObjectAllocator(uint nworkers) :
45 _nworkers(nworkers),
46 _used(0),
47 _shared_medium_page(NULL),
48 _shared_small_page(NULL),
49 _worker_small_page(NULL) {}
50
51 ZPage* ZObjectAllocator::alloc_page(uint8_t type, size_t size, ZAllocationFlags flags) {
52 ZPage* const page = ZHeap::heap()->alloc_page(type, size, flags);
53 if (page != NULL) {
54 // Increment used bytes
55 Atomic::add(size, _used.addr());
56 }
57
58 return page;
59 }
60
61 uintptr_t ZObjectAllocator::alloc_object_in_shared_page(ZPage** shared_page,
62 uint8_t page_type,
63 size_t page_size,
64 size_t size,
65 ZAllocationFlags flags) {
66 uintptr_t addr = 0;
67 ZPage* page = *shared_page;
68
69 if (page != NULL) {
70 addr = page->alloc_object_atomic(size);
71 }
72
73 if (addr == 0) {
74 // Allocate new page
75 ZPage* const new_page = alloc_page(page_type, page_size, flags);
76 if (new_page != NULL) {
77 // Allocate object before installing the new page
78 addr = new_page->alloc_object(size);
79
80 retry:
81 // Install new page
82 ZPage* const prev_page = Atomic::cmpxchg(new_page, shared_page, page);
83 if (prev_page != page) {
84 if (prev_page == NULL) {
85 // Previous page was retired, retry installing the new page
86 page = prev_page;
87 goto retry;
88 }
89
90 // Another page already installed, try allocation there first
91 const uintptr_t prev_addr = prev_page->alloc_object_atomic(size);
92 if (prev_addr == 0) {
93 // Allocation failed, retry installing the new page
94 page = prev_page;
95 goto retry;
96 }
97
98 // Allocation succeeded in already installed page
99 addr = prev_addr;
100
101 // Undo new page allocation
102 ZHeap::heap()->undo_alloc_page(new_page);
103 }
104 }
105 }
106
107 return addr;
108 }
109
110 uintptr_t ZObjectAllocator::alloc_large_object(size_t size, ZAllocationFlags flags) {
111 assert(ZThread::is_java(), "Should be a Java thread");
112
113 uintptr_t addr = 0;
114
115 // Allocate new large page
116 const size_t page_size = align_up(size, ZPageSizeMin);
117 ZPage* const page = alloc_page(ZPageTypeLarge, page_size, flags);
118 if (page != NULL) {
119 // Allocate the object
120 addr = page->alloc_object(size);
121 }
122
123 return addr;
124 }
125
126 uintptr_t ZObjectAllocator::alloc_medium_object(size_t size, ZAllocationFlags flags) {
127 return alloc_object_in_shared_page(_shared_medium_page.addr(), ZPageTypeMedium, ZPageSizeMedium, size, flags);
128 }
129
130 uintptr_t ZObjectAllocator::alloc_small_object_from_nonworker(size_t size, ZAllocationFlags flags) {
131 assert(ZThread::is_java() || ZThread::is_vm(), "Should be a Java or VM thread");
132
133 // Non-worker small page allocation can never use the reserve
134 flags.set_no_reserve();
135
136 return alloc_object_in_shared_page(_shared_small_page.addr(), ZPageTypeSmall, ZPageSizeSmall, size, flags);
137 }
138
139 uintptr_t ZObjectAllocator::alloc_small_object_from_worker(size_t size, ZAllocationFlags flags) {
140 assert(ZThread::is_worker(), "Should be a worker thread");
141
142 ZPage* page = _worker_small_page.get();
143 uintptr_t addr = 0;
144
145 if (page != NULL) {
146 addr = page->alloc_object(size);
147 }
148
149 if (addr == 0) {
150 // Allocate new page
151 page = alloc_page(ZPageTypeSmall, ZPageSizeSmall, flags);
152 if (page != NULL) {
153 addr = page->alloc_object(size);
154 }
155 _worker_small_page.set(page);
156 }
157
158 return addr;
159 }
160
161 uintptr_t ZObjectAllocator::alloc_small_object(size_t size, ZAllocationFlags flags) {
162 if (flags.worker_thread()) {
163 return alloc_small_object_from_worker(size, flags);
164 } else {
165 return alloc_small_object_from_nonworker(size, flags);
166 }
167 }
168
169 uintptr_t ZObjectAllocator::alloc_object(size_t size, ZAllocationFlags flags) {
170 if (size <= ZObjectSizeLimitSmall) {
171 // Small
172 return alloc_small_object(size, flags);
173 } else if (size <= ZObjectSizeLimitMedium) {
174 // Medium
175 return alloc_medium_object(size, flags);
176 } else {
177 // Large
178 return alloc_large_object(size, flags);
179 }
180 }
181
182 uintptr_t ZObjectAllocator::alloc_object(size_t size) {
183 assert(ZThread::is_java(), "Must be a Java thread");
184
185 ZAllocationFlags flags;
186 flags.set_no_reserve();
187
188 if (!ZStallOnOutOfMemory) {
189 flags.set_non_blocking();
190 }
191
192 return alloc_object(size, flags);
193 }
194
195 uintptr_t ZObjectAllocator::alloc_object_for_relocation(size_t size) {
196 assert(ZThread::is_java() || ZThread::is_worker() || ZThread::is_vm(), "Unknown thread");
197
198 ZAllocationFlags flags;
199 flags.set_relocation();
200 flags.set_non_blocking();
201
202 if (ZThread::is_worker()) {
203 flags.set_worker_thread();
204 }
205
206 return alloc_object(size, flags);
207 }
208
209 bool ZObjectAllocator::undo_alloc_large_object(ZPage* page) {
210 assert(page->type() == ZPageTypeLarge, "Invalid page type");
211
212 // Undo page allocation
213 ZHeap::heap()->undo_alloc_page(page);
214 return true;
215 }
216
217 bool ZObjectAllocator::undo_alloc_medium_object(ZPage* page, uintptr_t addr, size_t size) {
218 assert(page->type() == ZPageTypeMedium, "Invalid page type");
219
220 // Try atomic undo on shared page
221 return page->undo_alloc_object_atomic(addr, size);
222 }
223
224 bool ZObjectAllocator::undo_alloc_small_object_from_nonworker(ZPage* page, uintptr_t addr, size_t size) {
225 assert(page->type() == ZPageTypeSmall, "Invalid page type");
226
227 // Try atomic undo on shared page
228 return page->undo_alloc_object_atomic(addr, size);
229 }
230
231 bool ZObjectAllocator::undo_alloc_small_object_from_worker(ZPage* page, uintptr_t addr, size_t size) {
232 assert(page->type() == ZPageTypeSmall, "Invalid page type");
233 assert(page == _worker_small_page.get(), "Invalid page");
234
235 // Non-atomic undo on worker-local page
236 const bool success = page->undo_alloc_object(addr, size);
237 assert(success, "Should always succeed");
238 return success;
239 }
240
241 bool ZObjectAllocator::undo_alloc_small_object(ZPage* page, uintptr_t addr, size_t size) {
242 if (ZThread::is_worker()) {
243 return undo_alloc_small_object_from_worker(page, addr, size);
244 } else {
245 return undo_alloc_small_object_from_nonworker(page, addr, size);
246 }
247 }
248
249 bool ZObjectAllocator::undo_alloc_object(ZPage* page, uintptr_t addr, size_t size) {
250 const uint8_t type = page->type();
251
252 if (type == ZPageTypeSmall) {
253 return undo_alloc_small_object(page, addr, size);
254 } else if (type == ZPageTypeMedium) {
255 return undo_alloc_medium_object(page, addr, size);
256 } else {
257 return undo_alloc_large_object(page);
258 }
259 }
260
261 void ZObjectAllocator::undo_alloc_object_for_relocation(ZPage* page, uintptr_t addr, size_t size) {
262 if (undo_alloc_object(page, addr, size)) {
263 ZStatInc(ZCounterUndoObjectAllocationSucceeded);
264 } else {
265 ZStatInc(ZCounterUndoObjectAllocationFailed);
266 log_trace(gc)("Failed to undo object allocation: " PTR_FORMAT ", Size: " SIZE_FORMAT ", Thread: " PTR_FORMAT " (%s)",
267 addr, size, ZThread::id(), ZThread::name());
268 }
269 }
270
271 size_t ZObjectAllocator::used() const {
272 size_t total_used = 0;
273
274 ZPerCPUConstIterator<size_t> iter(&_used);
275 for (const size_t* cpu_used; iter.next(&cpu_used);) {
276 total_used += *cpu_used;
277 }
278
279 return total_used;
280 }
281
282 size_t ZObjectAllocator::remaining() const {
283 assert(ZThread::is_java(), "Should be a Java thread");
284
285 ZPage* page = _shared_small_page.get();
286 if (page != NULL) {
287 return page->remaining();
288 }
289
290 return 0;
291 }
292
293 void ZObjectAllocator::retire_pages() {
294 assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint");
295
296 // Reset used
297 _used.set_all(0);
298
299 // Reset allocation pages
300 _shared_medium_page.set(NULL);
301 _shared_small_page.set_all(NULL);
302 _worker_small_page.set_all(NULL);
303 }