1 /*
2 * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 #ifndef SHARE_GC_Z_ZPAGE_INLINE_HPP
25 #define SHARE_GC_Z_ZPAGE_INLINE_HPP
26
27 #include "gc/z/zAddress.inline.hpp"
28 #include "gc/z/zGlobals.hpp"
29 #include "gc/z/zLiveMap.inline.hpp"
30 #include "gc/z/zMark.hpp"
31 #include "gc/z/zNUMA.hpp"
32 #include "gc/z/zPage.hpp"
33 #include "gc/z/zPhysicalMemory.inline.hpp"
34 #include "gc/z/zVirtualMemory.inline.hpp"
35 #include "oops/oop.inline.hpp"
36 #include "runtime/atomic.hpp"
37 #include "runtime/os.hpp"
38 #include "utilities/align.hpp"
39 #include "utilities/debug.hpp"
40
41 inline uint8_t ZPage::type_from_size(size_t size) const {
42 if (size == ZPageSizeSmall) {
43 return ZPageTypeSmall;
44 } else if (size == ZPageSizeMedium) {
45 return ZPageTypeMedium;
46 } else {
47 return ZPageTypeLarge;
48 }
49 }
50
51 inline const char* ZPage::type_to_string() const {
52 switch (type()) {
53 case ZPageTypeSmall:
54 return "Small";
55
56 case ZPageTypeMedium:
57 return "Medium";
58
59 default:
60 assert(type() == ZPageTypeLarge, "Invalid page type");
61 return "Large";
62 }
63 }
64
65 inline uint32_t ZPage::object_max_count() const {
66 switch (type()) {
67 case ZPageTypeLarge:
68 // A large page can only contain a single
69 // object aligned to the start of the page.
70 return 1;
71
72 default:
73 return (uint32_t)(size() >> object_alignment_shift());
74 }
75 }
76
77 inline size_t ZPage::object_alignment_shift() const {
78 switch (type()) {
79 case ZPageTypeSmall:
80 return ZObjectAlignmentSmallShift;
81
82 case ZPageTypeMedium:
83 return ZObjectAlignmentMediumShift;
84
85 default:
86 assert(type() == ZPageTypeLarge, "Invalid page type");
87 return ZObjectAlignmentLargeShift;
88 }
89 }
90
91 inline size_t ZPage::object_alignment() const {
92 switch (type()) {
93 case ZPageTypeSmall:
94 return ZObjectAlignmentSmall;
95
96 case ZPageTypeMedium:
97 return ZObjectAlignmentMedium;
98
99 default:
100 assert(type() == ZPageTypeLarge, "Invalid page type");
101 return ZObjectAlignmentLarge;
102 }
103 }
104
105 inline uint8_t ZPage::type() const {
106 return _type;
107 }
108
109 inline uintptr_t ZPage::start() const {
110 return _virtual.start();
111 }
112
113 inline uintptr_t ZPage::end() const {
114 return _virtual.end();
115 }
116
117 inline size_t ZPage::size() const {
118 return _virtual.size();
119 }
120
121 inline uintptr_t ZPage::top() const {
122 return _top;
123 }
124
125 inline size_t ZPage::remaining() const {
126 return end() - top();
127 }
128
129 inline const ZPhysicalMemory& ZPage::physical_memory() const {
130 return _physical;
131 }
132
133 inline const ZVirtualMemory& ZPage::virtual_memory() const {
134 return _virtual;
135 }
136
137 inline uint8_t ZPage::numa_id() {
138 if (_numa_id == (uint8_t)-1) {
139 _numa_id = (uint8_t)ZNUMA::memory_id(ZAddress::good(start()));
140 }
141
142 return _numa_id;
143 }
144
145 inline bool ZPage::is_allocating() const {
146 return _seqnum == ZGlobalSeqNum;
147 }
148
149 inline bool ZPage::is_relocatable() const {
150 return _seqnum < ZGlobalSeqNum;
151 }
152
153 inline bool ZPage::is_mapped() const {
154 return _seqnum > 0;
155 }
156
157 inline void ZPage::set_pre_mapped() {
158 // The _seqnum variable is also used to signal that the virtual and physical
159 // memory has been mapped. So, we need to set it to non-zero when the memory
160 // has been pre-mapped.
161 _seqnum = 1;
162 }
163
164 inline uint64_t ZPage::last_used() const {
165 return _last_used;
166 }
167
168 inline void ZPage::set_last_used() {
169 _last_used = os::elapsedTime();
170 }
171
172 inline bool ZPage::is_in(uintptr_t addr) const {
173 const uintptr_t offset = ZAddress::offset(addr);
174 return offset >= start() && offset < top();
175 }
176
177 inline bool ZPage::is_marked() const {
178 assert(is_relocatable(), "Invalid page state");
179 return _livemap.is_marked();
180 }
181
182 inline bool ZPage::is_object_marked(uintptr_t addr) const {
183 const size_t index = ((ZAddress::offset(addr) - start()) >> object_alignment_shift()) * 2;
184 return _livemap.get(index);
185 }
186
187 inline bool ZPage::is_object_strongly_marked(uintptr_t addr) const {
188 const size_t index = ((ZAddress::offset(addr) - start()) >> object_alignment_shift()) * 2;
189 return _livemap.get(index + 1);
190 }
191
192 inline bool ZPage::is_object_live(uintptr_t addr) const {
193 return is_allocating() || is_object_marked(addr);
194 }
195
196 inline bool ZPage::is_object_strongly_live(uintptr_t addr) const {
197 return is_allocating() || is_object_strongly_marked(addr);
198 }
199
200 inline bool ZPage::mark_object(uintptr_t addr, bool finalizable, bool& inc_live) {
201 assert(ZAddress::is_marked(addr), "Invalid address");
202 assert(is_relocatable(), "Invalid page state");
203 assert(is_in(addr), "Invalid address");
204
205 // Set mark bit
206 const size_t index = ((ZAddress::offset(addr) - start()) >> object_alignment_shift()) * 2;
207 return _livemap.set(index, finalizable, inc_live);
208 }
209
210 inline void ZPage::inc_live(uint32_t objects, size_t bytes) {
211 _livemap.inc_live(objects, bytes);
212 }
213
214 inline uint32_t ZPage::live_objects() const {
215 assert(is_marked(), "Should be marked");
216 return _livemap.live_objects();
217 }
218
219 inline size_t ZPage::live_bytes() const {
220 assert(is_marked(), "Should be marked");
221 return _livemap.live_bytes();
222 }
223
224 inline void ZPage::object_iterate(ObjectClosure* cl) {
225 _livemap.iterate(cl, ZAddress::good(start()), object_alignment_shift());
226 }
227
228 inline uintptr_t ZPage::alloc_object(size_t size) {
229 assert(is_allocating(), "Invalid state");
230
231 const size_t aligned_size = align_up(size, object_alignment());
232 const uintptr_t addr = top();
233 const uintptr_t new_top = addr + aligned_size;
234
235 if (new_top > end()) {
236 // Not enough space left
237 return 0;
238 }
239
240 _top = new_top;
241
242 return ZAddress::good(addr);
243 }
244
245 inline uintptr_t ZPage::alloc_object_atomic(size_t size) {
246 assert(is_allocating(), "Invalid state");
247
248 const size_t aligned_size = align_up(size, object_alignment());
249 uintptr_t addr = top();
250
251 for (;;) {
252 const uintptr_t new_top = addr + aligned_size;
253 if (new_top > end()) {
254 // Not enough space left
255 return 0;
256 }
257
258 const uintptr_t prev_top = Atomic::cmpxchg(&_top, addr, new_top);
259 if (prev_top == addr) {
260 // Success
261 return ZAddress::good(addr);
262 }
263
264 // Retry
265 addr = prev_top;
266 }
267 }
268
269 inline bool ZPage::undo_alloc_object(uintptr_t addr, size_t size) {
270 assert(is_allocating(), "Invalid state");
271
272 const uintptr_t offset = ZAddress::offset(addr);
273 const size_t aligned_size = align_up(size, object_alignment());
274 const uintptr_t old_top = top();
275 const uintptr_t new_top = old_top - aligned_size;
276
277 if (new_top != offset) {
278 // Failed to undo allocation, not the last allocated object
279 return false;
280 }
281
282 _top = new_top;
283
284 // Success
285 return true;
286 }
287
288 inline bool ZPage::undo_alloc_object_atomic(uintptr_t addr, size_t size) {
289 assert(is_allocating(), "Invalid state");
290
291 const uintptr_t offset = ZAddress::offset(addr);
292 const size_t aligned_size = align_up(size, object_alignment());
293 uintptr_t old_top = top();
294
295 for (;;) {
296 const uintptr_t new_top = old_top - aligned_size;
297 if (new_top != offset) {
298 // Failed to undo allocation, not the last allocated object
299 return false;
300 }
301
302 const uintptr_t prev_top = Atomic::cmpxchg(&_top, old_top, new_top);
303 if (prev_top == old_top) {
304 // Success
305 return true;
306 }
307
308 // Retry
309 old_top = prev_top;
310 }
311 }
312
313 #endif // SHARE_GC_Z_ZPAGE_INLINE_HPP