1 /*
2 * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 #ifndef SHARE_GC_Z_ZPAGE_INLINE_HPP
25 #define SHARE_GC_Z_ZPAGE_INLINE_HPP
26
27 #include "gc/z/zAddress.inline.hpp"
28 #include "gc/z/zGlobals.hpp"
29 #include "gc/z/zLiveMap.inline.hpp"
30 #include "gc/z/zMark.hpp"
31 #include "gc/z/zNUMA.hpp"
32 #include "gc/z/zPage.hpp"
33 #include "gc/z/zPhysicalMemory.inline.hpp"
34 #include "gc/z/zVirtualMemory.inline.hpp"
35 #include "oops/oop.inline.hpp"
36 #include "runtime/atomic.hpp"
37 #include "runtime/os.hpp"
38 #include "utilities/align.hpp"
39 #include "utilities/debug.hpp"
40
41 inline uint8_t ZPage::type_from_size(size_t size) const {
42 if (size == ZPageSizeSmall) {
43 return ZPageTypeSmall;
44 } else if (size == ZPageSizeMedium) {
45 return ZPageTypeMedium;
46 } else {
47 return ZPageTypeLarge;
48 }
49 }
50
51 inline const char* ZPage::type_to_string() const {
52 switch (type()) {
53 case ZPageTypeSmall:
54 return "Small";
55
56 case ZPageTypeMedium:
57 return "Medium";
58
59 default:
60 assert(type() == ZPageTypeLarge, "Invalid page type");
61 return "Large";
62 }
63 }
64
65 inline uint32_t ZPage::object_max_count() const {
66 switch (type()) {
67 case ZPageTypeLarge:
68 // A large page can only contain a single
69 // object aligned to the start of the page.
70 return 1;
71
72 default:
73 return (uint32_t)(size() >> object_alignment_shift());
74 }
75 }
76
77 inline size_t ZPage::object_alignment_shift() const {
78 switch (type()) {
79 case ZPageTypeSmall:
80 return ZObjectAlignmentSmallShift;
81
82 case ZPageTypeMedium:
83 return ZObjectAlignmentMediumShift;
84
85 default:
86 assert(type() == ZPageTypeLarge, "Invalid page type");
87 return ZObjectAlignmentLargeShift;
88 }
89 }
90
91 inline size_t ZPage::object_alignment() const {
92 switch (type()) {
93 case ZPageTypeSmall:
94 return ZObjectAlignmentSmall;
95
96 case ZPageTypeMedium:
97 return ZObjectAlignmentMedium;
98
99 default:
100 assert(type() == ZPageTypeLarge, "Invalid page type");
101 return ZObjectAlignmentLarge;
102 }
103 }
104
105 inline uint8_t ZPage::type() const {
106 return _type;
107 }
108
109 inline uintptr_t ZPage::start() const {
110 return _virtual.start();
111 }
112
113 inline uintptr_t ZPage::end() const {
114 return _virtual.end();
115 }
116
117 inline size_t ZPage::size() const {
118 return _virtual.size();
119 }
120
121 inline uintptr_t ZPage::top() const {
122 return _top;
123 }
124
125 inline size_t ZPage::remaining() const {
126 return end() - top();
127 }
128
129 inline const ZVirtualMemory& ZPage::virtual_memory() const {
130 return _virtual;
131 }
132
133 inline const ZPhysicalMemory& ZPage::physical_memory() const {
134 return _physical;
135 }
136
137 inline ZPhysicalMemory& ZPage::physical_memory() {
138 return _physical;
139 }
140
141 inline uint8_t ZPage::numa_id() {
142 if (_numa_id == (uint8_t)-1) {
143 _numa_id = ZNUMA::memory_id(ZAddress::good(start()));
144 }
145
146 return _numa_id;
147 }
148
149 inline bool ZPage::is_allocating() const {
150 return _seqnum == ZGlobalSeqNum;
151 }
152
153 inline bool ZPage::is_relocatable() const {
154 return _seqnum < ZGlobalSeqNum;
155 }
156
157 inline uint64_t ZPage::last_used() const {
158 return _last_used;
159 }
160
161 inline void ZPage::set_last_used() {
162 _last_used = os::elapsedTime();
163 }
164
165 inline bool ZPage::is_in(uintptr_t addr) const {
166 const uintptr_t offset = ZAddress::offset(addr);
167 return offset >= start() && offset < top();
168 }
169
170 inline bool ZPage::is_marked() const {
171 assert(is_relocatable(), "Invalid page state");
172 return _livemap.is_marked();
173 }
174
175 inline bool ZPage::is_object_marked(uintptr_t addr) const {
176 const size_t index = ((ZAddress::offset(addr) - start()) >> object_alignment_shift()) * 2;
177 return _livemap.get(index);
178 }
179
180 inline bool ZPage::is_object_strongly_marked(uintptr_t addr) const {
181 const size_t index = ((ZAddress::offset(addr) - start()) >> object_alignment_shift()) * 2;
182 return _livemap.get(index + 1);
183 }
184
185 inline bool ZPage::is_object_live(uintptr_t addr) const {
186 return is_allocating() || is_object_marked(addr);
187 }
188
189 inline bool ZPage::is_object_strongly_live(uintptr_t addr) const {
190 return is_allocating() || is_object_strongly_marked(addr);
191 }
192
193 inline bool ZPage::mark_object(uintptr_t addr, bool finalizable, bool& inc_live) {
194 assert(ZAddress::is_marked(addr), "Invalid address");
195 assert(is_relocatable(), "Invalid page state");
196 assert(is_in(addr), "Invalid address");
197
198 // Set mark bit
199 const size_t index = ((ZAddress::offset(addr) - start()) >> object_alignment_shift()) * 2;
200 return _livemap.set(index, finalizable, inc_live);
201 }
202
203 inline void ZPage::inc_live(uint32_t objects, size_t bytes) {
204 _livemap.inc_live(objects, bytes);
205 }
206
207 inline uint32_t ZPage::live_objects() const {
208 assert(is_marked(), "Should be marked");
209 return _livemap.live_objects();
210 }
211
212 inline size_t ZPage::live_bytes() const {
213 assert(is_marked(), "Should be marked");
214 return _livemap.live_bytes();
215 }
216
217 inline void ZPage::object_iterate(ObjectClosure* cl) {
218 _livemap.iterate(cl, ZAddress::good(start()), object_alignment_shift());
219 }
220
221 inline uintptr_t ZPage::alloc_object(size_t size) {
222 assert(is_allocating(), "Invalid state");
223
224 const size_t aligned_size = align_up(size, object_alignment());
225 const uintptr_t addr = top();
226 const uintptr_t new_top = addr + aligned_size;
227
228 if (new_top > end()) {
229 // Not enough space left
230 return 0;
231 }
232
233 _top = new_top;
234
235 return ZAddress::good(addr);
236 }
237
238 inline uintptr_t ZPage::alloc_object_atomic(size_t size) {
239 assert(is_allocating(), "Invalid state");
240
241 const size_t aligned_size = align_up(size, object_alignment());
242 uintptr_t addr = top();
243
244 for (;;) {
245 const uintptr_t new_top = addr + aligned_size;
246 if (new_top > end()) {
247 // Not enough space left
248 return 0;
249 }
250
251 const uintptr_t prev_top = Atomic::cmpxchg(&_top, addr, new_top);
252 if (prev_top == addr) {
253 // Success
254 return ZAddress::good(addr);
255 }
256
257 // Retry
258 addr = prev_top;
259 }
260 }
261
262 inline bool ZPage::undo_alloc_object(uintptr_t addr, size_t size) {
263 assert(is_allocating(), "Invalid state");
264
265 const uintptr_t offset = ZAddress::offset(addr);
266 const size_t aligned_size = align_up(size, object_alignment());
267 const uintptr_t old_top = top();
268 const uintptr_t new_top = old_top - aligned_size;
269
270 if (new_top != offset) {
271 // Failed to undo allocation, not the last allocated object
272 return false;
273 }
274
275 _top = new_top;
276
277 // Success
278 return true;
279 }
280
281 inline bool ZPage::undo_alloc_object_atomic(uintptr_t addr, size_t size) {
282 assert(is_allocating(), "Invalid state");
283
284 const uintptr_t offset = ZAddress::offset(addr);
285 const size_t aligned_size = align_up(size, object_alignment());
286 uintptr_t old_top = top();
287
288 for (;;) {
289 const uintptr_t new_top = old_top - aligned_size;
290 if (new_top != offset) {
291 // Failed to undo allocation, not the last allocated object
292 return false;
293 }
294
295 const uintptr_t prev_top = Atomic::cmpxchg(&_top, old_top, new_top);
296 if (prev_top == old_top) {
297 // Success
298 return true;
299 }
300
301 // Retry
302 old_top = prev_top;
303 }
304 }
305
306 #endif // SHARE_GC_Z_ZPAGE_INLINE_HPP