1 /*
2 * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 #ifndef SHARE_GC_Z_ZPAGE_INLINE_HPP
25 #define SHARE_GC_Z_ZPAGE_INLINE_HPP
26
27 #include "gc/z/zAddress.inline.hpp"
28 #include "gc/z/zGlobals.hpp"
29 #include "gc/z/zLiveMap.inline.hpp"
30 #include "gc/z/zMark.hpp"
31 #include "gc/z/zNUMA.hpp"
32 #include "gc/z/zPage.hpp"
33 #include "gc/z/zPhysicalMemory.inline.hpp"
34 #include "gc/z/zVirtualMemory.inline.hpp"
35 #include "oops/oop.inline.hpp"
36 #include "runtime/atomic.hpp"
37 #include "runtime/orderAccess.hpp"
38 #include "utilities/align.hpp"
39 #include "utilities/debug.hpp"
40
41 inline const char* ZPage::type_to_string() const {
42 switch (type()) {
43 case ZPageTypeSmall:
44 return "Small";
45
46 case ZPageTypeMedium:
47 return "Medium";
48
49 default:
50 assert(type() == ZPageTypeLarge, "Invalid page type");
51 return "Large";
52 }
53 }
54
55 inline uint32_t ZPage::object_max_count() const {
56 switch (type()) {
57 case ZPageTypeLarge:
58 // A large page can only contain a single
59 // object aligned to the start of the page.
60 return 1;
61
62 default:
63 return (uint32_t)(size() >> object_alignment_shift());
64 }
65 }
66
67 inline size_t ZPage::object_alignment_shift() const {
68 switch (type()) {
69 case ZPageTypeSmall:
70 return ZObjectAlignmentSmallShift;
71
72 case ZPageTypeMedium:
73 return ZObjectAlignmentMediumShift;
74
75 default:
76 assert(type() == ZPageTypeLarge, "Invalid page type");
77 return ZObjectAlignmentLargeShift;
78 }
79 }
80
81 inline size_t ZPage::object_alignment() const {
82 switch (type()) {
83 case ZPageTypeSmall:
84 return ZObjectAlignmentSmall;
85
86 case ZPageTypeMedium:
87 return ZObjectAlignmentMedium;
88
89 default:
90 assert(type() == ZPageTypeLarge, "Invalid page type");
91 return ZObjectAlignmentLarge;
92 }
93 }
94
95 inline uint8_t ZPage::type() const {
96 return _type;
97 }
98
99 inline uintptr_t ZPage::start() const {
100 return _virtual.start();
101 }
102
103 inline uintptr_t ZPage::end() const {
104 return _virtual.end();
105 }
106
107 inline size_t ZPage::size() const {
108 return _virtual.size();
109 }
110
111 inline uintptr_t ZPage::top() const {
112 return _top;
113 }
114
115 inline size_t ZPage::remaining() const {
116 return end() - top();
117 }
118
119 inline ZPhysicalMemory& ZPage::physical_memory() {
120 return _physical;
121 }
122
123 inline const ZVirtualMemory& ZPage::virtual_memory() const {
124 return _virtual;
125 }
126
127 inline uint8_t ZPage::numa_id() {
128 if (_numa_id == (uint8_t)-1) {
129 _numa_id = (uint8_t)ZNUMA::memory_id(ZAddress::good(start()));
130 }
131
132 return _numa_id;
133 }
134
135 inline bool ZPage::is_in(uintptr_t addr) const {
136 const uintptr_t offset = ZAddress::offset(addr);
137 return offset >= start() && offset < top();
138 }
139
140 inline uintptr_t ZPage::block_start(uintptr_t addr) const {
141 if (block_is_obj(addr)) {
142 return addr;
143 } else {
144 return ZAddress::good(top());
145 }
146 }
147
148 inline bool ZPage::block_is_obj(uintptr_t addr) const {
149 return ZAddress::offset(addr) < top();
150 }
151
152 inline bool ZPage::is_allocating() const {
153 return _seqnum == ZGlobalSeqNum;
154 }
155
156 inline bool ZPage::is_relocatable() const {
157 return _seqnum < ZGlobalSeqNum;
158 }
159
160 inline bool ZPage::is_mapped() const {
161 return _seqnum > 0;
162 }
163
164 inline void ZPage::set_pre_mapped() {
165 // The _seqnum variable is also used to signal that the virtual and physical
166 // memory has been mapped. So, we need to set it to non-zero when the memory
167 // has been pre-mapped.
168 _seqnum = 1;
169 }
170
171 inline bool ZPage::is_marked() const {
172 assert(is_relocatable(), "Invalid page state");
173 return _livemap.is_marked();
174 }
175
176 inline bool ZPage::is_object_marked(uintptr_t addr) const {
177 const size_t index = ((ZAddress::offset(addr) - start()) >> object_alignment_shift()) * 2;
178 return _livemap.get(index);
179 }
180
181 inline bool ZPage::is_object_strongly_marked(uintptr_t addr) const {
182 const size_t index = ((ZAddress::offset(addr) - start()) >> object_alignment_shift()) * 2;
183 return _livemap.get(index + 1);
184 }
185
186 inline bool ZPage::is_object_live(uintptr_t addr) const {
187 return is_allocating() || is_object_marked(addr);
188 }
189
190 inline bool ZPage::is_object_strongly_live(uintptr_t addr) const {
191 return is_allocating() || is_object_strongly_marked(addr);
192 }
193
194 inline bool ZPage::mark_object(uintptr_t addr, bool finalizable, bool& inc_live) {
195 assert(ZAddress::is_marked(addr), "Invalid address");
196 assert(is_relocatable(), "Invalid page state");
197 assert(is_in(addr), "Invalid address");
198
199 // Set mark bit
200 const size_t index = ((ZAddress::offset(addr) - start()) >> object_alignment_shift()) * 2;
201 return _livemap.set_atomic(index, finalizable, inc_live);
202 }
203
204 inline void ZPage::inc_live_atomic(uint32_t objects, size_t bytes) {
205 _livemap.inc_live_atomic(objects, bytes);
206 }
207
208 inline uint32_t ZPage::live_objects() const {
209 assert(is_marked(), "Should be marked");
210 return _livemap.live_objects();
211 }
212
213 inline size_t ZPage::live_bytes() const {
214 assert(is_marked(), "Should be marked");
215 return _livemap.live_bytes();
216 }
217
218 inline void ZPage::object_iterate(ObjectClosure* cl) {
219 _livemap.iterate(cl, ZAddress::good(start()), object_alignment_shift());
220 }
221
222 inline uintptr_t ZPage::alloc_object(size_t size) {
223 assert(is_allocating(), "Invalid state");
224
225 const size_t aligned_size = align_up(size, object_alignment());
226 const uintptr_t addr = top();
227 const uintptr_t new_top = addr + aligned_size;
228
229 if (new_top > end()) {
230 // Not enough space left
231 return 0;
232 }
233
234 _top = new_top;
235
236 return ZAddress::good(addr);
237 }
238
239 inline uintptr_t ZPage::alloc_object_atomic(size_t size) {
240 assert(is_allocating(), "Invalid state");
241
242 const size_t aligned_size = align_up(size, object_alignment());
243 uintptr_t addr = top();
244
245 for (;;) {
246 const uintptr_t new_top = addr + aligned_size;
247 if (new_top > end()) {
248 // Not enough space left
249 return 0;
250 }
251
252 const uintptr_t prev_top = Atomic::cmpxchg(new_top, &_top, addr);
253 if (prev_top == addr) {
254 // Success
255 return ZAddress::good(addr);
256 }
257
258 // Retry
259 addr = prev_top;
260 }
261 }
262
263 inline bool ZPage::undo_alloc_object(uintptr_t addr, size_t size) {
264 assert(is_allocating(), "Invalid state");
265
266 const uintptr_t offset = ZAddress::offset(addr);
267 const size_t aligned_size = align_up(size, object_alignment());
268 const uintptr_t old_top = top();
269 const uintptr_t new_top = old_top - aligned_size;
270
271 if (new_top != offset) {
272 // Failed to undo allocation, not the last allocated object
273 return false;
274 }
275
276 _top = new_top;
277
278 // Success
279 return true;
280 }
281
282 inline bool ZPage::undo_alloc_object_atomic(uintptr_t addr, size_t size) {
283 assert(is_allocating(), "Invalid state");
284
285 const uintptr_t offset = ZAddress::offset(addr);
286 const size_t aligned_size = align_up(size, object_alignment());
287 uintptr_t old_top = top();
288
289 for (;;) {
290 const uintptr_t new_top = old_top - aligned_size;
291 if (new_top != offset) {
292 // Failed to undo allocation, not the last allocated object
293 return false;
294 }
295
296 const uintptr_t prev_top = Atomic::cmpxchg(new_top, &_top, old_top);
297 if (prev_top == old_top) {
298 // Success
299 return true;
300 }
301
302 // Retry
303 old_top = prev_top;
304 }
305 }
306
307 #endif // SHARE_GC_Z_ZPAGE_INLINE_HPP