1 /*
   2  * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  */
  23 
  24 #ifndef SHARE_GC_Z_ZPAGE_INLINE_HPP
  25 #define SHARE_GC_Z_ZPAGE_INLINE_HPP
  26 
  27 #include "gc/z/zAddress.inline.hpp"
  28 #include "gc/z/zGlobals.hpp"
  29 #include "gc/z/zLiveMap.inline.hpp"
  30 #include "gc/z/zMark.hpp"
  31 #include "gc/z/zNUMA.hpp"
  32 #include "gc/z/zPage.hpp"
  33 #include "gc/z/zPhysicalMemory.inline.hpp"
  34 #include "gc/z/zVirtualMemory.inline.hpp"
  35 #include "oops/oop.inline.hpp"
  36 #include "runtime/atomic.hpp"
  37 #include "runtime/os.hpp"
  38 #include "utilities/align.hpp"
  39 #include "utilities/debug.hpp"
  40 
  41 inline uint8_t ZPage::type_from_size(size_t size) const {
  42   if (size == ZPageSizeSmall) {
  43     return ZPageTypeSmall;
  44   } else if (size == ZPageSizeMedium) {
  45     return ZPageTypeMedium;
  46   } else {
  47     return ZPageTypeLarge;
  48   }
  49 }
  50 
  51 inline const char* ZPage::type_to_string() const {
  52   switch (type()) {
  53   case ZPageTypeSmall:
  54     return "Small";
  55 
  56   case ZPageTypeMedium:
  57     return "Medium";
  58 
  59   default:
  60     assert(type() == ZPageTypeLarge, "Invalid page type");
  61     return "Large";
  62   }
  63 }
  64 
  65 inline uint32_t ZPage::object_max_count() const {
  66   switch (type()) {
  67   case ZPageTypeLarge:
  68     // A large page can only contain a single
  69     // object aligned to the start of the page.
  70     return 1;
  71 
  72   default:
  73     return (uint32_t)(size() >> object_alignment_shift());
  74   }
  75 }
  76 
  77 inline size_t ZPage::object_alignment_shift() const {
  78   switch (type()) {
  79   case ZPageTypeSmall:
  80     return ZObjectAlignmentSmallShift;
  81 
  82   case ZPageTypeMedium:
  83     return ZObjectAlignmentMediumShift;
  84 
  85   default:
  86     assert(type() == ZPageTypeLarge, "Invalid page type");
  87     return ZObjectAlignmentLargeShift;
  88   }
  89 }
  90 
  91 inline size_t ZPage::object_alignment() const {
  92   switch (type()) {
  93   case ZPageTypeSmall:
  94     return ZObjectAlignmentSmall;
  95 
  96   case ZPageTypeMedium:
  97     return ZObjectAlignmentMedium;
  98 
  99   default:
 100     assert(type() == ZPageTypeLarge, "Invalid page type");
 101     return ZObjectAlignmentLarge;
 102   }
 103 }
 104 
 105 inline uint8_t ZPage::type() const {
 106   return _type;
 107 }
 108 
 109 inline uintptr_t ZPage::start() const {
 110   return _virtual.start();
 111 }
 112 
 113 inline uintptr_t ZPage::end() const {
 114   return _virtual.end();
 115 }
 116 
 117 inline size_t ZPage::size() const {
 118   return _virtual.size();
 119 }
 120 
 121 inline uintptr_t ZPage::top() const {
 122   return _top;
 123 }
 124 
 125 inline size_t ZPage::remaining() const {
 126   return end() - top();
 127 }
 128 
 129 inline const ZVirtualMemory& ZPage::virtual_memory() const {
 130   return _virtual;
 131 }
 132 
 133 inline const ZPhysicalMemory& ZPage::physical_memory() const {
 134   return _physical;
 135 }
 136 
 137 inline ZPhysicalMemory& ZPage::physical_memory() {
 138   return _physical;
 139 }
 140 
 141 inline uint8_t ZPage::numa_id() {
 142   if (_numa_id == (uint8_t)-1) {
 143     _numa_id = ZNUMA::memory_id(ZAddress::good(start()));
 144   }
 145 
 146   return _numa_id;
 147 }
 148 
 149 inline bool ZPage::is_allocating() const {
 150   return _seqnum == ZGlobalSeqNum;
 151 }
 152 
 153 inline bool ZPage::is_relocatable() const {
 154   return _seqnum < ZGlobalSeqNum;
 155 }
 156 
 157 inline uint64_t ZPage::last_used() const {
 158   return _last_used;
 159 }
 160 
 161 inline void ZPage::set_last_used() {
 162   _last_used = ceil(os::elapsedTime());
 163 }
 164 
 165 inline bool ZPage::is_in(uintptr_t addr) const {
 166   const uintptr_t offset = ZAddress::offset(addr);
 167   return offset >= start() && offset < top();
 168 }
 169 
 170 inline bool ZPage::is_marked() const {
 171   assert(is_relocatable(), "Invalid page state");
 172   return _livemap.is_marked();
 173 }
 174 
 175 inline bool ZPage::is_object_marked(uintptr_t addr) const {
 176   const size_t index = ((ZAddress::offset(addr) - start()) >> object_alignment_shift()) * 2;
 177   return _livemap.get(index);
 178 }
 179 
 180 inline bool ZPage::is_object_strongly_marked(uintptr_t addr) const {
 181   const size_t index = ((ZAddress::offset(addr) - start()) >> object_alignment_shift()) * 2;
 182   return _livemap.get(index + 1);
 183 }
 184 
 185 inline bool ZPage::is_object_live(uintptr_t addr) const {
 186   return is_allocating() || is_object_marked(addr);
 187 }
 188 
 189 inline bool ZPage::is_object_strongly_live(uintptr_t addr) const {
 190   return is_allocating() || is_object_strongly_marked(addr);
 191 }
 192 
 193 inline bool ZPage::mark_object(uintptr_t addr, bool finalizable, bool& inc_live) {
 194   assert(ZAddress::is_marked(addr), "Invalid address");
 195   assert(is_relocatable(), "Invalid page state");
 196   assert(is_in(addr), "Invalid address");
 197 
 198   // Set mark bit
 199   const size_t index = ((ZAddress::offset(addr) - start()) >> object_alignment_shift()) * 2;
 200   return _livemap.set(index, finalizable, inc_live);
 201 }
 202 
 203 inline void ZPage::inc_live(uint32_t objects, size_t bytes) {
 204   _livemap.inc_live(objects, bytes);
 205 }
 206 
 207 inline uint32_t ZPage::live_objects() const {
 208   assert(is_marked(), "Should be marked");
 209   return _livemap.live_objects();
 210 }
 211 
 212 inline size_t ZPage::live_bytes() const {
 213   assert(is_marked(), "Should be marked");
 214   return _livemap.live_bytes();
 215 }
 216 
 217 inline void ZPage::object_iterate(ObjectClosure* cl) {
 218   _livemap.iterate(cl, ZAddress::good(start()), object_alignment_shift());
 219 }
 220 
 221 inline uintptr_t ZPage::alloc_object(size_t size) {
 222   assert(is_allocating(), "Invalid state");
 223 
 224   const size_t aligned_size = align_up(size, object_alignment());
 225   const uintptr_t addr = top();
 226   const uintptr_t new_top = addr + aligned_size;
 227 
 228   if (new_top > end()) {
 229     // Not enough space left
 230     return 0;
 231   }
 232 
 233   _top = new_top;
 234 
 235   return ZAddress::good(addr);
 236 }
 237 
 238 inline uintptr_t ZPage::alloc_object_atomic(size_t size) {
 239   assert(is_allocating(), "Invalid state");
 240 
 241   const size_t aligned_size = align_up(size, object_alignment());
 242   uintptr_t addr = top();
 243 
 244   for (;;) {
 245     const uintptr_t new_top = addr + aligned_size;
 246     if (new_top > end()) {
 247       // Not enough space left
 248       return 0;
 249     }
 250 
 251     const uintptr_t prev_top = Atomic::cmpxchg(&_top, addr, new_top);
 252     if (prev_top == addr) {
 253       // Success
 254       return ZAddress::good(addr);
 255     }
 256 
 257     // Retry
 258     addr = prev_top;
 259   }
 260 }
 261 
 262 inline bool ZPage::undo_alloc_object(uintptr_t addr, size_t size) {
 263   assert(is_allocating(), "Invalid state");
 264 
 265   const uintptr_t offset = ZAddress::offset(addr);
 266   const size_t aligned_size = align_up(size, object_alignment());
 267   const uintptr_t old_top = top();
 268   const uintptr_t new_top = old_top - aligned_size;
 269 
 270   if (new_top != offset) {
 271     // Failed to undo allocation, not the last allocated object
 272     return false;
 273   }
 274 
 275   _top = new_top;
 276 
 277   // Success
 278   return true;
 279 }
 280 
 281 inline bool ZPage::undo_alloc_object_atomic(uintptr_t addr, size_t size) {
 282   assert(is_allocating(), "Invalid state");
 283 
 284   const uintptr_t offset = ZAddress::offset(addr);
 285   const size_t aligned_size = align_up(size, object_alignment());
 286   uintptr_t old_top = top();
 287 
 288   for (;;) {
 289     const uintptr_t new_top = old_top - aligned_size;
 290     if (new_top != offset) {
 291       // Failed to undo allocation, not the last allocated object
 292       return false;
 293     }
 294 
 295     const uintptr_t prev_top = Atomic::cmpxchg(&_top, old_top, new_top);
 296     if (prev_top == old_top) {
 297       // Success
 298       return true;
 299     }
 300 
 301     // Retry
 302     old_top = prev_top;
 303   }
 304 }
 305 
 306 #endif // SHARE_GC_Z_ZPAGE_INLINE_HPP