1 /* 2 * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 */ 23 24 #ifndef SHARE_GC_Z_ZPAGE_INLINE_HPP 25 #define SHARE_GC_Z_ZPAGE_INLINE_HPP 26 27 #include "gc/z/zAddress.inline.hpp" 28 #include "gc/z/zGlobals.hpp" 29 #include "gc/z/zLiveMap.inline.hpp" 30 #include "gc/z/zMark.hpp" 31 #include "gc/z/zNUMA.hpp" 32 #include "gc/z/zPage.hpp" 33 #include "gc/z/zPhysicalMemory.inline.hpp" 34 #include "gc/z/zVirtualMemory.inline.hpp" 35 #include "oops/oop.inline.hpp" 36 #include "runtime/atomic.hpp" 37 #include "runtime/os.hpp" 38 #include "utilities/align.hpp" 39 #include "utilities/debug.hpp" 40 41 inline uint8_t ZPage::type_from_size(size_t size) const { 42 switch (size) { 43 case ZPageSizeSmall: 44 return ZPageTypeSmall; 45 46 case ZPageSizeMedium: 47 return ZPageTypeMedium; 48 49 default: 50 return ZPageTypeLarge; 51 } 52 } 53 54 inline const char* ZPage::type_to_string() const { 55 switch (type()) { 56 case ZPageTypeSmall: 57 return "Small"; 58 59 case ZPageTypeMedium: 60 return "Medium"; 61 62 default: 63 assert(type() == ZPageTypeLarge, "Invalid page type"); 64 return "Large"; 65 } 66 } 67 68 inline uint32_t ZPage::object_max_count() const { 69 switch (type()) { 70 case ZPageTypeLarge: 71 // A large page can only contain a single 72 // object aligned to the start of the page. 73 return 1; 74 75 default: 76 return (uint32_t)(size() >> object_alignment_shift()); 77 } 78 } 79 80 inline size_t ZPage::object_alignment_shift() const { 81 switch (type()) { 82 case ZPageTypeSmall: 83 return ZObjectAlignmentSmallShift; 84 85 case ZPageTypeMedium: 86 return ZObjectAlignmentMediumShift; 87 88 default: 89 assert(type() == ZPageTypeLarge, "Invalid page type"); 90 return ZObjectAlignmentLargeShift; 91 } 92 } 93 94 inline size_t ZPage::object_alignment() const { 95 switch (type()) { 96 case ZPageTypeSmall: 97 return ZObjectAlignmentSmall; 98 99 case ZPageTypeMedium: 100 return ZObjectAlignmentMedium; 101 102 default: 103 assert(type() == ZPageTypeLarge, "Invalid page type"); 104 return ZObjectAlignmentLarge; 105 } 106 } 107 108 inline uint8_t ZPage::type() const { 109 return _type; 110 } 111 112 inline uintptr_t ZPage::start() const { 113 return _virtual.start(); 114 } 115 116 inline uintptr_t ZPage::end() const { 117 return _virtual.end(); 118 } 119 120 inline size_t ZPage::size() const { 121 return _virtual.size(); 122 } 123 124 inline uintptr_t ZPage::top() const { 125 return _top; 126 } 127 128 inline size_t ZPage::remaining() const { 129 return end() - top(); 130 } 131 132 inline const ZPhysicalMemory& ZPage::physical_memory() const { 133 return _physical; 134 } 135 136 inline const ZVirtualMemory& ZPage::virtual_memory() const { 137 return _virtual; 138 } 139 140 inline uint8_t ZPage::numa_id() { 141 if (_numa_id == (uint8_t)-1) { 142 _numa_id = (uint8_t)ZNUMA::memory_id(ZAddress::good(start())); 143 } 144 145 return _numa_id; 146 } 147 148 inline bool ZPage::is_allocating() const { 149 return _seqnum == ZGlobalSeqNum; 150 } 151 152 inline bool ZPage::is_relocatable() const { 153 return _seqnum < ZGlobalSeqNum; 154 } 155 156 inline bool ZPage::is_mapped() const { 157 return _seqnum > 0; 158 } 159 160 inline void ZPage::set_pre_mapped() { 161 // The _seqnum variable is also used to signal that the virtual and physical 162 // memory has been mapped. So, we need to set it to non-zero when the memory 163 // has been pre-mapped. 164 _seqnum = 1; 165 } 166 167 inline uint64_t ZPage::last_used() const { 168 return _last_used; 169 } 170 171 inline void ZPage::set_last_used() { 172 _last_used = os::elapsedTime(); 173 } 174 175 inline bool ZPage::is_in(uintptr_t addr) const { 176 const uintptr_t offset = ZAddress::offset(addr); 177 return offset >= start() && offset < top(); 178 } 179 180 inline bool ZPage::is_marked() const { 181 assert(is_relocatable(), "Invalid page state"); 182 return _livemap.is_marked(); 183 } 184 185 inline bool ZPage::is_object_marked(uintptr_t addr) const { 186 const size_t index = ((ZAddress::offset(addr) - start()) >> object_alignment_shift()) * 2; 187 return _livemap.get(index); 188 } 189 190 inline bool ZPage::is_object_strongly_marked(uintptr_t addr) const { 191 const size_t index = ((ZAddress::offset(addr) - start()) >> object_alignment_shift()) * 2; 192 return _livemap.get(index + 1); 193 } 194 195 inline bool ZPage::is_object_live(uintptr_t addr) const { 196 return is_allocating() || is_object_marked(addr); 197 } 198 199 inline bool ZPage::is_object_strongly_live(uintptr_t addr) const { 200 return is_allocating() || is_object_strongly_marked(addr); 201 } 202 203 inline bool ZPage::mark_object(uintptr_t addr, bool finalizable, bool& inc_live) { 204 assert(ZAddress::is_marked(addr), "Invalid address"); 205 assert(is_relocatable(), "Invalid page state"); 206 assert(is_in(addr), "Invalid address"); 207 208 // Set mark bit 209 const size_t index = ((ZAddress::offset(addr) - start()) >> object_alignment_shift()) * 2; 210 return _livemap.set_atomic(index, finalizable, inc_live); 211 } 212 213 inline void ZPage::inc_live_atomic(uint32_t objects, size_t bytes) { 214 _livemap.inc_live_atomic(objects, bytes); 215 } 216 217 inline uint32_t ZPage::live_objects() const { 218 assert(is_marked(), "Should be marked"); 219 return _livemap.live_objects(); 220 } 221 222 inline size_t ZPage::live_bytes() const { 223 assert(is_marked(), "Should be marked"); 224 return _livemap.live_bytes(); 225 } 226 227 inline void ZPage::object_iterate(ObjectClosure* cl) { 228 _livemap.iterate(cl, ZAddress::good(start()), object_alignment_shift()); 229 } 230 231 inline uintptr_t ZPage::alloc_object(size_t size) { 232 assert(is_allocating(), "Invalid state"); 233 234 const size_t aligned_size = align_up(size, object_alignment()); 235 const uintptr_t addr = top(); 236 const uintptr_t new_top = addr + aligned_size; 237 238 if (new_top > end()) { 239 // Not enough space left 240 return 0; 241 } 242 243 _top = new_top; 244 245 return ZAddress::good(addr); 246 } 247 248 inline uintptr_t ZPage::alloc_object_atomic(size_t size) { 249 assert(is_allocating(), "Invalid state"); 250 251 const size_t aligned_size = align_up(size, object_alignment()); 252 uintptr_t addr = top(); 253 254 for (;;) { 255 const uintptr_t new_top = addr + aligned_size; 256 if (new_top > end()) { 257 // Not enough space left 258 return 0; 259 } 260 261 const uintptr_t prev_top = Atomic::cmpxchg(new_top, &_top, addr); 262 if (prev_top == addr) { 263 // Success 264 return ZAddress::good(addr); 265 } 266 267 // Retry 268 addr = prev_top; 269 } 270 } 271 272 inline bool ZPage::undo_alloc_object(uintptr_t addr, size_t size) { 273 assert(is_allocating(), "Invalid state"); 274 275 const uintptr_t offset = ZAddress::offset(addr); 276 const size_t aligned_size = align_up(size, object_alignment()); 277 const uintptr_t old_top = top(); 278 const uintptr_t new_top = old_top - aligned_size; 279 280 if (new_top != offset) { 281 // Failed to undo allocation, not the last allocated object 282 return false; 283 } 284 285 _top = new_top; 286 287 // Success 288 return true; 289 } 290 291 inline bool ZPage::undo_alloc_object_atomic(uintptr_t addr, size_t size) { 292 assert(is_allocating(), "Invalid state"); 293 294 const uintptr_t offset = ZAddress::offset(addr); 295 const size_t aligned_size = align_up(size, object_alignment()); 296 uintptr_t old_top = top(); 297 298 for (;;) { 299 const uintptr_t new_top = old_top - aligned_size; 300 if (new_top != offset) { 301 // Failed to undo allocation, not the last allocated object 302 return false; 303 } 304 305 const uintptr_t prev_top = Atomic::cmpxchg(new_top, &_top, old_top); 306 if (prev_top == old_top) { 307 // Success 308 return true; 309 } 310 311 // Retry 312 old_top = prev_top; 313 } 314 } 315 316 #endif // SHARE_GC_Z_ZPAGE_INLINE_HPP