1 /*
   2  * Copyright (c) 2017, 2018, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "memory/allocation.hpp"
  27 #include "memory/allocation.inline.hpp"
  28 #include "memory/metaspaceShared.hpp"
  29 #include "memory/resourceArea.hpp"
  30 #include "memory/universe.hpp"
  31 #include "runtime/atomic.hpp"
  32 #include "runtime/os.hpp"
  33 #include "runtime/task.hpp"
  34 #include "runtime/threadCritical.hpp"
  35 #include "services/memTracker.hpp"
  36 #include "utilities/ostream.hpp"
  37 
  38 //--------------------------------------------------------------------------------------
  39 // ChunkPool implementation
  40 
  41 // MT-safe pool of chunks to reduce malloc/free thrashing
  42 // NB: not using Mutex because pools are used before Threads are initialized
  43 class ChunkPool: public CHeapObj<mtInternal> {
  44   Chunk*       _first;        // first cached Chunk; its first word points to next chunk
  45   size_t       _num_chunks;   // number of unused chunks in pool
  46   size_t       _num_used;     // number of chunks currently checked out
  47   const size_t _size;         // size of each chunk (must be uniform)
  48 
  49   // Our four static pools
  50   static ChunkPool* _large_pool;
  51   static ChunkPool* _medium_pool;
  52   static ChunkPool* _small_pool;
  53   static ChunkPool* _tiny_pool;
  54 
  55   // return first element or null
  56   void* get_first() {
  57     Chunk* c = _first;
  58     if (_first) {
  59       _first = _first->next();
  60       _num_chunks--;
  61     }
  62     return c;
  63   }
  64 
  65  public:
  66   // All chunks in a ChunkPool has the same size
  67    ChunkPool(size_t size) : _size(size) { _first = NULL; _num_chunks = _num_used = 0; }
  68 
  69   // Allocate a new chunk from the pool (might expand the pool)
  70   NOINLINE void* allocate(size_t bytes, AllocFailType alloc_failmode) {
  71     assert(bytes == _size, "bad size");
  72     void* p = NULL;
  73     // No VM lock can be taken inside ThreadCritical lock, so os::malloc
  74     // should be done outside ThreadCritical lock due to NMT
  75     { ThreadCritical tc;
  76       _num_used++;
  77       p = get_first();
  78     }
  79     if (p == NULL) p = os::malloc(bytes, mtChunk, CURRENT_PC);
  80     if (p == NULL && alloc_failmode == AllocFailStrategy::EXIT_OOM) {
  81       vm_exit_out_of_memory(bytes, OOM_MALLOC_ERROR, "ChunkPool::allocate");
  82     }
  83     return p;
  84   }
  85 
  86   // Return a chunk to the pool
  87   void free(Chunk* chunk) {
  88     assert(chunk->length() + Chunk::aligned_overhead_size() == _size, "bad size");
  89     ThreadCritical tc;
  90     _num_used--;
  91 
  92     // Add chunk to list
  93     chunk->set_next(_first);
  94     _first = chunk;
  95     _num_chunks++;
  96   }
  97 
  98   // Prune the pool
  99   void free_all_but(size_t n) {
 100     Chunk* cur = NULL;
 101     Chunk* next;
 102     {
 103       // if we have more than n chunks, free all of them
 104       ThreadCritical tc;
 105       if (_num_chunks > n) {
 106         // free chunks at end of queue, for better locality
 107         cur = _first;
 108         for (size_t i = 0; i < (n - 1) && cur != NULL; i++) cur = cur->next();
 109 
 110         if (cur != NULL) {
 111           next = cur->next();
 112           cur->set_next(NULL);
 113           cur = next;
 114 
 115           // Free all remaining chunks while in ThreadCritical lock
 116           // so NMT adjustment is stable.
 117           while(cur != NULL) {
 118             next = cur->next();
 119             os::free(cur);
 120             _num_chunks--;
 121             cur = next;
 122           }
 123         }
 124       }
 125     }
 126   }
 127 
 128   // Accessors to preallocated pool's
 129   static ChunkPool* large_pool()  { assert(_large_pool  != NULL, "must be initialized"); return _large_pool;  }
 130   static ChunkPool* medium_pool() { assert(_medium_pool != NULL, "must be initialized"); return _medium_pool; }
 131   static ChunkPool* small_pool()  { assert(_small_pool  != NULL, "must be initialized"); return _small_pool;  }
 132   static ChunkPool* tiny_pool()   { assert(_tiny_pool   != NULL, "must be initialized"); return _tiny_pool;   }
 133 
 134   static void initialize() {
 135     _large_pool  = new ChunkPool(Chunk::size        + Chunk::aligned_overhead_size());
 136     _medium_pool = new ChunkPool(Chunk::medium_size + Chunk::aligned_overhead_size());
 137     _small_pool  = new ChunkPool(Chunk::init_size   + Chunk::aligned_overhead_size());
 138     _tiny_pool   = new ChunkPool(Chunk::tiny_size   + Chunk::aligned_overhead_size());
 139   }
 140 
 141   static void clean() {
 142     enum { BlocksToKeep = 5 };
 143      _tiny_pool->free_all_but(BlocksToKeep);
 144      _small_pool->free_all_but(BlocksToKeep);
 145      _medium_pool->free_all_but(BlocksToKeep);
 146      _large_pool->free_all_but(BlocksToKeep);
 147   }
 148 };
 149 
 150 ChunkPool* ChunkPool::_large_pool  = NULL;
 151 ChunkPool* ChunkPool::_medium_pool = NULL;
 152 ChunkPool* ChunkPool::_small_pool  = NULL;
 153 ChunkPool* ChunkPool::_tiny_pool   = NULL;
 154 
 155 void chunkpool_init() {
 156   ChunkPool::initialize();
 157 }
 158 
 159 void
 160 Chunk::clean_chunk_pool() {
 161   ChunkPool::clean();
 162 }
 163 
 164 
 165 //--------------------------------------------------------------------------------------
 166 // ChunkPoolCleaner implementation
 167 //
 168 
 169 class ChunkPoolCleaner : public PeriodicTask {
 170   enum { CleaningInterval = 5000 };      // cleaning interval in ms
 171 
 172  public:
 173    ChunkPoolCleaner() : PeriodicTask(CleaningInterval) {}
 174    void task() {
 175      ChunkPool::clean();
 176    }
 177 };
 178 
 179 //--------------------------------------------------------------------------------------
 180 // Chunk implementation
 181 
 182 void* Chunk::operator new (size_t requested_size, AllocFailType alloc_failmode, size_t length) throw() {
 183   // requested_size is equal to sizeof(Chunk) but in order for the arena
 184   // allocations to come out aligned as expected the size must be aligned
 185   // to expected arena alignment.
 186   // expect requested_size but if sizeof(Chunk) doesn't match isn't proper size we must align it.
 187   assert(ARENA_ALIGN(requested_size) == aligned_overhead_size(), "Bad alignment");
 188   size_t bytes = ARENA_ALIGN(requested_size) + length;
 189   switch (length) {
 190    case Chunk::size:        return ChunkPool::large_pool()->allocate(bytes, alloc_failmode);
 191    case Chunk::medium_size: return ChunkPool::medium_pool()->allocate(bytes, alloc_failmode);
 192    case Chunk::init_size:   return ChunkPool::small_pool()->allocate(bytes, alloc_failmode);
 193    case Chunk::tiny_size:   return ChunkPool::tiny_pool()->allocate(bytes, alloc_failmode);
 194    default: {
 195      void* p = os::malloc(bytes, mtChunk, CALLER_PC);
 196      if (p == NULL && alloc_failmode == AllocFailStrategy::EXIT_OOM) {
 197        vm_exit_out_of_memory(bytes, OOM_MALLOC_ERROR, "Chunk::new");
 198      }
 199      return p;
 200    }
 201   }
 202 }
 203 
 204 void Chunk::operator delete(void* p) {
 205   Chunk* c = (Chunk*)p;
 206   switch (c->length()) {
 207    case Chunk::size:        ChunkPool::large_pool()->free(c); break;
 208    case Chunk::medium_size: ChunkPool::medium_pool()->free(c); break;
 209    case Chunk::init_size:   ChunkPool::small_pool()->free(c); break;
 210    case Chunk::tiny_size:   ChunkPool::tiny_pool()->free(c); break;
 211    default:
 212      ThreadCritical tc;  // Free chunks under TC lock so that NMT adjustment is stable.
 213      os::free(c);
 214   }
 215 }
 216 
 217 Chunk::Chunk(size_t length) : _len(length) {
 218   _next = NULL;         // Chain on the linked list
 219 }
 220 
 221 void Chunk::chop() {
 222   Chunk *k = this;
 223   while( k ) {
 224     Chunk *tmp = k->next();
 225     // clear out this chunk (to detect allocation bugs)
 226     if (ZapResourceArea) memset(k->bottom(), badResourceValue, k->length());
 227     delete k;                   // Free chunk (was malloc'd)
 228     k = tmp;
 229   }
 230 }
 231 
 232 void Chunk::next_chop() {
 233   _next->chop();
 234   _next = NULL;
 235 }
 236 
 237 void Chunk::start_chunk_pool_cleaner_task() {
 238 #ifdef ASSERT
 239   static bool task_created = false;
 240   assert(!task_created, "should not start chuck pool cleaner twice");
 241   task_created = true;
 242 #endif
 243   ChunkPoolCleaner* cleaner = new ChunkPoolCleaner();
 244   cleaner->enroll();
 245 }
 246 
 247 //------------------------------Arena------------------------------------------
 248 
 249 Arena::Arena(MEMFLAGS flag, size_t init_size) : _flags(flag), _size_in_bytes(0)  {
 250   size_t round_size = (sizeof (char *)) - 1;
 251   init_size = (init_size+round_size) & ~round_size;
 252   _first = _chunk = new (AllocFailStrategy::EXIT_OOM, init_size) Chunk(init_size);
 253   _hwm = _chunk->bottom();      // Save the cached hwm, max
 254   _max = _chunk->top();
 255   MemTracker::record_new_arena(flag);
 256   set_size_in_bytes(init_size);
 257 }
 258 
 259 Arena::Arena(MEMFLAGS flag) : _flags(flag), _size_in_bytes(0) {
 260   _first = _chunk = new (AllocFailStrategy::EXIT_OOM, Chunk::init_size) Chunk(Chunk::init_size);
 261   _hwm = _chunk->bottom();      // Save the cached hwm, max
 262   _max = _chunk->top();
 263   MemTracker::record_new_arena(flag);
 264   set_size_in_bytes(Chunk::init_size);
 265 }
 266 
 267 Arena *Arena::move_contents(Arena *copy) {
 268   copy->destruct_contents();
 269   copy->_chunk = _chunk;
 270   copy->_hwm   = _hwm;
 271   copy->_max   = _max;
 272   copy->_first = _first;
 273 
 274   // workaround rare racing condition, which could double count
 275   // the arena size by native memory tracking
 276   size_t size = size_in_bytes();
 277   set_size_in_bytes(0);
 278   copy->set_size_in_bytes(size);
 279   // Destroy original arena
 280   reset();
 281   return copy;            // Return Arena with contents
 282 }
 283 
 284 Arena::~Arena() {
 285   destruct_contents();
 286   MemTracker::record_arena_free(_flags);
 287 }
 288 
 289 void* Arena::operator new(size_t size) throw() {
 290   assert(false, "Use dynamic memory type binding");
 291   return NULL;
 292 }
 293 
 294 void* Arena::operator new (size_t size, const std::nothrow_t&  nothrow_constant) throw() {
 295   assert(false, "Use dynamic memory type binding");
 296   return NULL;
 297 }
 298 
 299   // dynamic memory type binding
 300 void* Arena::operator new(size_t size, MEMFLAGS flags) throw() {
 301   return (void *) AllocateHeap(size, flags, CALLER_PC);
 302 }
 303 
 304 void* Arena::operator new(size_t size, const std::nothrow_t& nothrow_constant, MEMFLAGS flags) throw() {
 305   return (void*)AllocateHeap(size, flags, CALLER_PC, AllocFailStrategy::RETURN_NULL);
 306 }
 307 
 308 void Arena::operator delete(void* p) {
 309   FreeHeap(p);
 310 }
 311 
 312 // Destroy this arenas contents and reset to empty
 313 void Arena::destruct_contents() {
 314   if (UseMallocOnly && _first != NULL) {
 315     char* end = _first->next() ? _first->top() : _hwm;
 316     free_malloced_objects(_first, _first->bottom(), end, _hwm);
 317   }
 318   // reset size before chop to avoid a rare racing condition
 319   // that can have total arena memory exceed total chunk memory
 320   set_size_in_bytes(0);
 321   _first->chop();
 322   reset();
 323 }
 324 
 325 // This is high traffic method, but many calls actually don't
 326 // change the size
 327 void Arena::set_size_in_bytes(size_t size) {
 328   if (_size_in_bytes != size) {
 329     long delta = (long)(size - size_in_bytes());
 330     _size_in_bytes = size;
 331     MemTracker::record_arena_size_change(delta, _flags);
 332   }
 333 }
 334 
 335 // Total of all Chunks in arena
 336 size_t Arena::used() const {
 337   size_t sum = _chunk->length() - (_max-_hwm); // Size leftover in this Chunk
 338   register Chunk *k = _first;
 339   while( k != _chunk) {         // Whilst have Chunks in a row
 340     sum += k->length();         // Total size of this Chunk
 341     k = k->next();              // Bump along to next Chunk
 342   }
 343   return sum;                   // Return total consumed space.
 344 }
 345 
 346 void Arena::signal_out_of_memory(size_t sz, const char* whence) const {
 347   vm_exit_out_of_memory(sz, OOM_MALLOC_ERROR, "%s", whence);
 348 }
 349 
 350 // Grow a new Chunk
 351 void* Arena::grow(size_t x, AllocFailType alloc_failmode) {
 352   // Get minimal required size.  Either real big, or even bigger for giant objs
 353   size_t len = MAX2(x, (size_t) Chunk::size);
 354 
 355   Chunk *k = _chunk;            // Get filled-up chunk address
 356   _chunk = new (alloc_failmode, len) Chunk(len);
 357 
 358   if (_chunk == NULL) {
 359     _chunk = k;                 // restore the previous value of _chunk
 360     return NULL;
 361   }
 362   if (k) k->set_next(_chunk);   // Append new chunk to end of linked list
 363   else _first = _chunk;
 364   _hwm  = _chunk->bottom();     // Save the cached hwm, max
 365   _max =  _chunk->top();
 366   set_size_in_bytes(size_in_bytes() + len);
 367   void* result = _hwm;
 368   _hwm += x;
 369   return result;
 370 }
 371 
 372 
 373 
 374 // Reallocate storage in Arena.
 375 void *Arena::Arealloc(void* old_ptr, size_t old_size, size_t new_size, AllocFailType alloc_failmode) {
 376   if (new_size == 0) return NULL;
 377 #ifdef ASSERT
 378   if (UseMallocOnly) {
 379     // always allocate a new object  (otherwise we'll free this one twice)
 380     char* copy = (char*)Amalloc(new_size, alloc_failmode);
 381     if (copy == NULL) {
 382       return NULL;
 383     }
 384     size_t n = MIN2(old_size, new_size);
 385     if (n > 0) memcpy(copy, old_ptr, n);
 386     Afree(old_ptr,old_size);    // Mostly done to keep stats accurate
 387     return copy;
 388   }
 389 #endif
 390   char *c_old = (char*)old_ptr; // Handy name
 391   // Stupid fast special case
 392   if( new_size <= old_size ) {  // Shrink in-place
 393     if( c_old+old_size == _hwm) // Attempt to free the excess bytes
 394       _hwm = c_old+new_size;    // Adjust hwm
 395     return c_old;
 396   }
 397 
 398   // make sure that new_size is legal
 399   size_t corrected_new_size = ARENA_ALIGN(new_size);
 400 
 401   // See if we can resize in-place
 402   if( (c_old+old_size == _hwm) &&       // Adjusting recent thing
 403       (c_old+corrected_new_size <= _max) ) {      // Still fits where it sits
 404     _hwm = c_old+corrected_new_size;      // Adjust hwm
 405     return c_old;               // Return old pointer
 406   }
 407 
 408   // Oops, got to relocate guts
 409   void *new_ptr = Amalloc(new_size, alloc_failmode);
 410   if (new_ptr == NULL) {
 411     return NULL;
 412   }
 413   memcpy( new_ptr, c_old, old_size );
 414   Afree(c_old,old_size);        // Mostly done to keep stats accurate
 415   return new_ptr;
 416 }
 417 
 418 
 419 // Determine if pointer belongs to this Arena or not.
 420 bool Arena::contains( const void *ptr ) const {
 421 #ifdef ASSERT
 422   if (UseMallocOnly) {
 423     // really slow, but not easy to make fast
 424     if (_chunk == NULL) return false;
 425     char** bottom = (char**)_chunk->bottom();
 426     for (char** p = (char**)_hwm - 1; p >= bottom; p--) {
 427       if (*p == ptr) return true;
 428     }
 429     for (Chunk *c = _first; c != NULL; c = c->next()) {
 430       if (c == _chunk) continue;  // current chunk has been processed
 431       char** bottom = (char**)c->bottom();
 432       for (char** p = (char**)c->top() - 1; p >= bottom; p--) {
 433         if (*p == ptr) return true;
 434       }
 435     }
 436     return false;
 437   }
 438 #endif
 439   if( (void*)_chunk->bottom() <= ptr && ptr < (void*)_hwm )
 440     return true;                // Check for in this chunk
 441   for (Chunk *c = _first; c; c = c->next()) {
 442     if (c == _chunk) continue;  // current chunk has been processed
 443     if ((void*)c->bottom() <= ptr && ptr < (void*)c->top()) {
 444       return true;              // Check for every chunk in Arena
 445     }
 446   }
 447   return false;                 // Not in any Chunk, so not in Arena
 448 }
 449 
 450 
 451 #ifdef ASSERT
 452 void* Arena::malloc(size_t size) {
 453   assert(UseMallocOnly, "shouldn't call");
 454   // use malloc, but save pointer in res. area for later freeing
 455   char** save = (char**)internal_malloc_4(sizeof(char*));
 456   return (*save = (char*)os::malloc(size, mtChunk));
 457 }
 458 
 459 // for debugging with UseMallocOnly
 460 void* Arena::internal_malloc_4(size_t x) {
 461   assert( (x&(sizeof(char*)-1)) == 0, "misaligned size" );
 462   check_for_overflow(x, "Arena::internal_malloc_4");
 463   if (_hwm + x > _max) {
 464     return grow(x);
 465   } else {
 466     char *old = _hwm;
 467     _hwm += x;
 468     return old;
 469   }
 470 }
 471 #endif
 472 
 473 
 474 //--------------------------------------------------------------------------------------
 475 // Non-product code
 476 
 477 #ifndef PRODUCT
 478 
 479 julong Arena::_bytes_allocated = 0;
 480 
 481 void Arena::inc_bytes_allocated(size_t x) { inc_stat_counter(&_bytes_allocated, x); }
 482 
 483 // debugging code
 484 inline void Arena::free_all(char** start, char** end) {
 485   for (char** p = start; p < end; p++) if (*p) os::free(*p);
 486 }
 487 
 488 void Arena::free_malloced_objects(Chunk* chunk, char* hwm, char* max, char* hwm2) {
 489   assert(UseMallocOnly, "should not call");
 490   // free all objects malloced since resource mark was created; resource area
 491   // contains their addresses
 492   if (chunk->next()) {
 493     // this chunk is full, and some others too
 494     for (Chunk* c = chunk->next(); c != NULL; c = c->next()) {
 495       char* top = c->top();
 496       if (c->next() == NULL) {
 497         top = hwm2;     // last junk is only used up to hwm2
 498         assert(c->contains(hwm2), "bad hwm2");
 499       }
 500       free_all((char**)c->bottom(), (char**)top);
 501     }
 502     assert(chunk->contains(hwm), "bad hwm");
 503     assert(chunk->contains(max), "bad max");
 504     free_all((char**)hwm, (char**)max);
 505   } else {
 506     // this chunk was partially used
 507     assert(chunk->contains(hwm), "bad hwm");
 508     assert(chunk->contains(hwm2), "bad hwm2");
 509     free_all((char**)hwm, (char**)hwm2);
 510   }
 511 }
 512 
 513 #endif // Non-product