1 /*
   2  * Copyright (c) 2012, 2018, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "jfr/jfrEvents.hpp"
  27 #include "jfr/jni/jfrJavaSupport.hpp"
  28 #include "jfr/recorder/jfrRecorder.hpp"
  29 #include "jfr/recorder/repository/jfrChunkWriter.hpp"
  30 #include "jfr/recorder/service/jfrOptionSet.hpp"
  31 #include "jfr/recorder/service/jfrPostBox.hpp"
  32 #include "jfr/recorder/storage/jfrMemorySpace.inline.hpp"
  33 #include "jfr/recorder/storage/jfrStorage.hpp"
  34 #include "jfr/recorder/storage/jfrStorageControl.hpp"
  35 #include "jfr/recorder/storage/jfrStorageUtils.inline.hpp"
  36 #include "jfr/utilities/jfrIterator.hpp"
  37 #include "jfr/utilities/jfrTime.hpp"
  38 #include "jfr/writers/jfrNativeEventWriter.hpp"
  39 #include "logging/log.hpp"
  40 #include "runtime/mutexLocker.hpp"
  41 #include "runtime/orderAccess.hpp"
  42 #include "runtime/os.inline.hpp"
  43 #include "runtime/safepoint.hpp"
  44 #include "runtime/thread.hpp"
  45 
  46 typedef JfrStorage::Buffer* BufferPtr;
  47 
  48 static JfrStorage* _instance = NULL;
  49 static JfrStorageControl* _control;
  50 
  51 JfrStorage& JfrStorage::instance() {
  52   return *_instance;
  53 }
  54 
  55 JfrStorage* JfrStorage::create(JfrChunkWriter& chunkwriter, JfrPostBox& post_box) {
  56   assert(_instance == NULL, "invariant");
  57   _instance = new JfrStorage(chunkwriter, post_box);
  58   return _instance;
  59 }
  60 
  61 void JfrStorage::destroy() {
  62   if (_instance != NULL) {
  63     delete _instance;
  64     _instance = NULL;
  65   }
  66 }
  67 
  68 JfrStorage::JfrStorage(JfrChunkWriter& chunkwriter, JfrPostBox& post_box) :
  69   _control(NULL),
  70   _global_mspace(NULL),
  71   _thread_local_mspace(NULL),
  72   _transient_mspace(NULL),
  73   _age_mspace(NULL),
  74   _chunkwriter(chunkwriter),
  75   _post_box(post_box) {}
  76 
  77 JfrStorage::~JfrStorage() {
  78   if (_control != NULL) {
  79     delete _control;
  80   }
  81   if (_global_mspace != NULL) {
  82     delete _global_mspace;
  83   }
  84   if (_thread_local_mspace != NULL) {
  85     delete _thread_local_mspace;
  86   }
  87   if (_transient_mspace != NULL) {
  88     delete _transient_mspace;
  89   }
  90   if (_age_mspace != NULL) {
  91     delete _age_mspace;
  92   }
  93   _instance = NULL;
  94 }
  95 
  96 static const size_t in_memory_discard_threshold_delta = 2; // start to discard data when the only this number of free buffers are left
  97 static const size_t unlimited_mspace_size = 0;
  98 static const size_t thread_local_cache_count = 8;
  99 static const size_t thread_local_scavenge_threshold = thread_local_cache_count / 2;
 100 static const size_t transient_buffer_size_multiplier = 8; // against thread local buffer size
 101 
 102 template <typename Mspace>
 103 static Mspace* create_mspace(size_t buffer_size, size_t limit, size_t cache_count, JfrStorage* storage_instance) {
 104   Mspace* mspace = new Mspace(buffer_size, limit, cache_count, storage_instance);
 105   if (mspace != NULL) {
 106     mspace->initialize();
 107   }
 108   return mspace;
 109 }
 110 
 111 bool JfrStorage::initialize() {
 112   assert(_control == NULL, "invariant");
 113   assert(_global_mspace == NULL, "invariant");
 114   assert(_thread_local_mspace == NULL, "invariant");
 115   assert(_transient_mspace == NULL, "invariant");
 116   assert(_age_mspace == NULL, "invariant");
 117 
 118   const size_t num_global_buffers = (size_t)JfrOptionSet::num_global_buffers();
 119   assert(num_global_buffers >= in_memory_discard_threshold_delta, "invariant");
 120   const size_t memory_size = (size_t)JfrOptionSet::memory_size();
 121   const size_t global_buffer_size = (size_t)JfrOptionSet::global_buffer_size();
 122   const size_t thread_buffer_size = (size_t)JfrOptionSet::thread_buffer_size();
 123 
 124   _control = new JfrStorageControl(num_global_buffers, num_global_buffers - in_memory_discard_threshold_delta);
 125   if (_control == NULL) {
 126     return false;
 127   }
 128   _global_mspace = create_mspace<JfrStorageMspace>(global_buffer_size, memory_size, num_global_buffers, this);
 129   if (_global_mspace == NULL) {
 130     return false;
 131   }
 132   _thread_local_mspace = create_mspace<JfrThreadLocalMspace>(thread_buffer_size, unlimited_mspace_size, thread_local_cache_count, this);
 133   if (_thread_local_mspace == NULL) {
 134     return false;
 135   }
 136   _transient_mspace = create_mspace<JfrStorageMspace>(thread_buffer_size * transient_buffer_size_multiplier, unlimited_mspace_size, 0, this);
 137   if (_transient_mspace == NULL) {
 138     return false;
 139   }
 140   _age_mspace = create_mspace<JfrStorageAgeMspace>(0 /* no extra size except header */, unlimited_mspace_size, num_global_buffers, this);
 141   if (_age_mspace == NULL) {
 142     return false;
 143   }
 144   control().set_scavenge_threshold(thread_local_scavenge_threshold);
 145   return true;
 146 }
 147 
 148 JfrStorageControl& JfrStorage::control() {
 149   return *instance()._control;
 150 }
 151 
 152 static void log_allocation_failure(const char* msg, size_t size) {
 153   log_warning(jfr)("Unable to allocate " SIZE_FORMAT " bytes of %s.", size, msg);
 154 }
 155 
 156 BufferPtr JfrStorage::acquire_thread_local(Thread* thread, size_t size /* 0 */) {
 157   BufferPtr buffer = mspace_get_to_full(size, instance()._thread_local_mspace, thread);
 158   if (buffer == NULL) {
 159     log_allocation_failure("thread local_memory", size);
 160     return NULL;
 161   }
 162   assert(buffer->acquired_by_self(), "invariant");
 163   return buffer;
 164 }
 165 
 166 BufferPtr JfrStorage::acquire_transient(size_t size, Thread* thread) {
 167   BufferPtr buffer = mspace_allocate_transient_lease_to_full(size, instance()._transient_mspace, thread);
 168   if (buffer == NULL) {
 169     log_allocation_failure("transient memory", size);
 170     return NULL;
 171   }
 172   assert(buffer->acquired_by_self(), "invariant");
 173   assert(buffer->transient(), "invariant");
 174   assert(buffer->lease(), "invariant");
 175   return buffer;
 176 }
 177 
 178 static BufferPtr get_lease(size_t size, JfrStorageMspace* mspace, JfrStorage& storage_instance, size_t retry_count, Thread* thread) {
 179   assert(size <= mspace->min_elem_size(), "invariant");
 180   while (true) {
 181     BufferPtr t = mspace_get_free_lease_with_retry(size, mspace, retry_count, thread);
 182     if (t == NULL && storage_instance.control().should_discard()) {
 183       storage_instance.discard_oldest(thread);
 184       continue;
 185     }
 186     return t;
 187   }
 188 }
 189 
 190 static BufferPtr get_promotion_buffer(size_t size, JfrStorageMspace* mspace, JfrStorage& storage_instance, size_t retry_count, Thread* thread) {
 191   assert(size <= mspace->min_elem_size(), "invariant");
 192   while (true) {
 193     BufferPtr t = mspace_get_free_with_retry(size, mspace, retry_count, thread);
 194     if (t == NULL && storage_instance.control().should_discard()) {
 195       storage_instance.discard_oldest(thread);
 196       continue;
 197     }
 198     return t;
 199   }
 200 }
 201 
 202 static const size_t lease_retry = 10;
 203 
 204 BufferPtr JfrStorage::acquire_large(size_t size, Thread* thread) {
 205   JfrStorage& storage_instance = instance();
 206   const size_t max_elem_size = storage_instance._global_mspace->min_elem_size(); // min is also max
 207   // if not too large and capacity is still available, ask for a lease from the global system
 208   if (size < max_elem_size && storage_instance.control().is_global_lease_allowed()) {
 209     BufferPtr const buffer = get_lease(size, storage_instance._global_mspace, storage_instance, lease_retry, thread);
 210     if (buffer != NULL) {
 211       assert(buffer->acquired_by_self(), "invariant");
 212       assert(!buffer->transient(), "invariant");
 213       assert(buffer->lease(), "invariant");
 214       storage_instance.control().increment_leased();
 215       return buffer;
 216     }
 217   }
 218   return acquire_transient(size, thread);
 219 }
 220 
 221 static void write_data_loss_event(JfrBuffer* buffer, u8 unflushed_size, Thread* thread) {
 222   assert(buffer != NULL, "invariant");
 223   assert(buffer->empty(), "invariant");
 224   const u8 total_data_loss = thread->jfr_thread_local()->add_data_lost(unflushed_size);
 225   if (EventDataLoss::is_enabled()) {
 226     JfrNativeEventWriter writer(buffer, thread);
 227     writer.write<u8>(EventDataLoss::eventId);
 228     writer.write(JfrTicks::now());
 229     writer.write(unflushed_size);
 230     writer.write(total_data_loss);
 231   }
 232 }
 233 
 234 static void write_data_loss(BufferPtr buffer, Thread* thread) {
 235   assert(buffer != NULL, "invariant");
 236   const size_t unflushed_size = buffer->unflushed_size();
 237   buffer->concurrent_reinitialization();
 238   if (unflushed_size == 0) {
 239     return;
 240   }
 241   write_data_loss_event(buffer, unflushed_size, thread);
 242 }
 243 
 244 static const size_t promotion_retry = 100;
 245 
 246 bool JfrStorage::flush_regular_buffer(BufferPtr buffer, Thread* thread) {
 247   assert(buffer != NULL, "invariant");
 248   assert(!buffer->lease(), "invariant");
 249   assert(!buffer->transient(), "invariant");
 250   const size_t unflushed_size = buffer->unflushed_size();
 251   if (unflushed_size == 0) {
 252     buffer->concurrent_reinitialization();
 253     assert(buffer->empty(), "invariant");
 254     return true;
 255   }
 256   BufferPtr const promotion_buffer = get_promotion_buffer(unflushed_size, _global_mspace, *this, promotion_retry, thread);
 257   if (promotion_buffer == NULL) {
 258     write_data_loss(buffer, thread);
 259     return false;
 260   }
 261   assert(promotion_buffer->acquired_by_self(), "invariant");
 262   assert(promotion_buffer->free_size() >= unflushed_size, "invariant");
 263   buffer->concurrent_move_and_reinitialize(promotion_buffer, unflushed_size);
 264   assert(buffer->empty(), "invariant");
 265   return true;
 266 }
 267 
 268 /*
 269 * 1. If the buffer was a "lease" from the global system, release back.
 270 * 2. If the buffer is transient (temporal dynamically allocated), retire and register full.
 271 *
 272 * The buffer is effectively invalidated for the thread post-return,
 273 * and the caller should take means to ensure that it is not referenced any longer.
 274 */
 275 void JfrStorage::release_large(BufferPtr buffer, Thread* thread) {
 276   assert(buffer != NULL, "invariant");
 277   assert(buffer->lease(), "invariant");
 278   assert(buffer->acquired_by_self(), "invariant");
 279   buffer->clear_lease();
 280   if (buffer->transient()) {
 281     buffer->set_retired();
 282     register_full(buffer, thread);
 283   } else {
 284     buffer->release();
 285     control().decrement_leased();
 286   }
 287 }
 288 
 289 static JfrAgeNode* new_age_node(BufferPtr buffer, JfrStorageAgeMspace* age_mspace, Thread* thread) {
 290   assert(buffer != NULL, "invariant");
 291   assert(age_mspace != NULL, "invariant");
 292   return mspace_allocate_transient(0, age_mspace, thread);
 293 }
 294 
 295 static void log_registration_failure(size_t unflushed_size) {
 296   log_warning(jfr)("Unable to register a full buffer of " SIZE_FORMAT " bytes.", unflushed_size);
 297   log_debug(jfr, system)("Cleared 1 full buffer of " SIZE_FORMAT " bytes.", unflushed_size);
 298 }
 299 
 300 static void handle_registration_failure(BufferPtr buffer) {
 301   assert(buffer != NULL, "invariant");
 302   assert(buffer->retired(), "invariant");
 303   const size_t unflushed_size = buffer->unflushed_size();
 304   buffer->reinitialize();
 305   log_registration_failure(unflushed_size);
 306 }
 307 
 308 static JfrAgeNode* get_free_age_node(JfrStorageAgeMspace* age_mspace, Thread* thread) {
 309   assert(JfrBuffer_lock->owned_by_self(), "invariant");
 310   return mspace_get_free_with_detach(0, age_mspace, thread);
 311 }
 312 
 313 static bool insert_full_age_node(JfrAgeNode* age_node, JfrStorageAgeMspace* age_mspace, Thread* thread) {
 314   assert(JfrBuffer_lock->owned_by_self(), "invariant");
 315   assert(age_node->retired_buffer()->retired(), "invariant");
 316   age_mspace->insert_full_head(age_node);
 317   return true;
 318 }
 319 
 320 static bool full_buffer_registration(BufferPtr buffer, JfrStorageAgeMspace* age_mspace, JfrStorageControl& control, Thread* thread) {
 321   assert(buffer != NULL, "invariant");
 322   assert(buffer->retired(), "invariant");
 323   assert(age_mspace != NULL, "invariant");
 324   MutexLockerEx lock(JfrBuffer_lock, Mutex::_no_safepoint_check_flag);
 325   JfrAgeNode* age_node = get_free_age_node(age_mspace, thread);
 326   if (age_node == NULL) {
 327     age_node = new_age_node(buffer, age_mspace, thread);
 328     if (age_node == NULL) {
 329       return false;
 330     }
 331   }
 332   assert(age_node->acquired_by_self(), "invariant");
 333   assert(age_node != NULL, "invariant");
 334   age_node->set_retired_buffer(buffer);
 335   control.increment_full();
 336   return insert_full_age_node(age_node, age_mspace, thread);
 337 }
 338 
 339 void JfrStorage::register_full(BufferPtr buffer, Thread* thread) {
 340   assert(buffer != NULL, "invariant");
 341   assert(buffer->retired(), "invariant");
 342   if (!full_buffer_registration(buffer, _age_mspace, control(), thread)) {
 343     handle_registration_failure(buffer);
 344     buffer->release();
 345   }
 346   if (control().should_post_buffer_full_message()) {
 347     _post_box.post(MSG_FULLBUFFER);
 348   }
 349 }
 350 
 351 void JfrStorage::lock() {
 352   assert(!JfrBuffer_lock->owned_by_self(), "invariant");
 353   JfrBuffer_lock->lock_without_safepoint_check();
 354 }
 355 
 356 void JfrStorage::unlock() {
 357   assert(JfrBuffer_lock->owned_by_self(), "invariant");
 358   JfrBuffer_lock->unlock();
 359 }
 360 
 361 #ifdef ASSERT
 362 bool JfrStorage::is_locked() const {
 363   return JfrBuffer_lock->owned_by_self();
 364 }
 365 #endif
 366 
 367 // don't use buffer on return, it is gone
 368 void JfrStorage::release(BufferPtr buffer, Thread* thread) {
 369   assert(buffer != NULL, "invariant");
 370   assert(!buffer->lease(), "invariant");
 371   assert(!buffer->transient(), "invariant");
 372   assert(!buffer->retired(), "invariant");
 373   if (!buffer->empty()) {
 374     if (!flush_regular_buffer(buffer, thread)) {
 375       buffer->concurrent_reinitialization();
 376     }
 377   }
 378   assert(buffer->empty(), "invariant");
 379   control().increment_dead();
 380   buffer->release();
 381   buffer->set_retired();
 382 }
 383 
 384 void JfrStorage::release_thread_local(BufferPtr buffer, Thread* thread) {
 385   assert(buffer != NULL, "invariant");
 386   JfrStorage& storage_instance = instance();
 387   storage_instance.release(buffer, thread);
 388   if (storage_instance.control().should_scavenge()) {
 389     storage_instance._post_box.post(MSG_DEADBUFFER);
 390   }
 391 }
 392 
 393 static void log_discard(size_t count, size_t amount, size_t current) {
 394   if (log_is_enabled(Debug, jfr, system)) {
 395     assert(count > 0, "invariant");
 396     log_debug(jfr, system)("Cleared " SIZE_FORMAT " full buffer(s) of " SIZE_FORMAT" bytes.", count, amount);
 397     log_debug(jfr, system)("Current number of full buffers " SIZE_FORMAT "", current);
 398   }
 399 }
 400 
 401 void JfrStorage::discard_oldest(Thread* thread) {
 402   if (JfrBuffer_lock->try_lock()) {
 403     if (!control().should_discard()) {
 404       // another thread handled it
 405       return;
 406     }
 407     const size_t num_full_pre_discard = control().full_count();
 408     size_t num_full_post_discard = 0;
 409     size_t discarded_size = 0;
 410     while (true) {
 411       JfrAgeNode* const oldest_age_node = _age_mspace->full_tail();
 412       if (oldest_age_node == NULL) {
 413         break;
 414       }
 415       BufferPtr const buffer = oldest_age_node->retired_buffer();
 416       assert(buffer->retired(), "invariant");
 417       discarded_size += buffer->unflushed_size();
 418       num_full_post_discard = control().decrement_full();
 419       if (buffer->transient()) {
 420         mspace_release_full(buffer, _transient_mspace);
 421         mspace_release_full(oldest_age_node, _age_mspace);
 422         continue;
 423       } else {
 424         mspace_release_full(oldest_age_node, _age_mspace);
 425         buffer->reinitialize();
 426         buffer->release(); // pusb
 427         break;
 428       }
 429     }
 430     JfrBuffer_lock->unlock();
 431     const size_t number_of_discards = num_full_pre_discard - num_full_post_discard;
 432     if (number_of_discards > 0) {
 433       log_discard(number_of_discards, discarded_size, num_full_post_discard);
 434     }
 435   }
 436 }
 437 
 438 #ifdef ASSERT
 439 typedef const BufferPtr ConstBufferPtr;
 440 
 441 static void assert_flush_precondition(ConstBufferPtr cur, size_t used, bool native, const Thread* t) {
 442   assert(t != NULL, "invariant");
 443   assert(cur != NULL, "invariant");
 444   assert(cur->pos() + used <= cur->end(), "invariant");
 445   assert(native ? t->jfr_thread_local()->native_buffer() == cur : t->jfr_thread_local()->java_buffer() == cur, "invariant");
 446 }
 447 
 448 static void assert_flush_regular_precondition(ConstBufferPtr cur, const u1* const cur_pos, size_t used, size_t req, const Thread* t) {
 449   assert(t != NULL, "invariant");
 450   assert(t->jfr_thread_local()->shelved_buffer() == NULL, "invariant");
 451   assert(cur != NULL, "invariant");
 452   assert(!cur->lease(), "invariant");
 453   assert(cur_pos != NULL, "invariant");
 454   assert(req >= used, "invariant");
 455 }
 456 
 457 static void assert_provision_large_precondition(ConstBufferPtr cur, size_t used, size_t req, const Thread* t) {
 458   assert(cur != NULL, "invariant");
 459   assert(t != NULL, "invariant");
 460   assert(t->jfr_thread_local()->shelved_buffer() != NULL, "invariant");
 461   assert(req >= used, "invariant");
 462 }
 463 
 464 static void assert_flush_large_precondition(ConstBufferPtr cur, const u1* const cur_pos, size_t used, size_t req, bool native, Thread* t) {
 465   assert(t != NULL, "invariant");
 466   assert(cur != NULL, "invariant");
 467   assert(cur->lease(), "invariant");
 468   assert(cur_pos != NULL, "invariant");
 469   assert(native ? t->jfr_thread_local()->native_buffer() == cur : t->jfr_thread_local()->java_buffer() == cur, "invariant");
 470   assert(t->jfr_thread_local()->shelved_buffer() != NULL, "invariant");
 471   assert(req >= used, "invariant");
 472   assert(cur != t->jfr_thread_local()->shelved_buffer(), "invariant");
 473 }
 474 #endif // ASSERT
 475 
 476 BufferPtr JfrStorage::flush(BufferPtr cur, size_t used, size_t req, bool native, Thread* t) {
 477   debug_only(assert_flush_precondition(cur, used, native, t);)
 478   const u1* const cur_pos = cur->pos();
 479   req += used;
 480   // requested size now encompass the outstanding used size
 481   return cur->lease() ? instance().flush_large(cur, cur_pos, used, req, native, t) :
 482                           instance().flush_regular(cur, cur_pos, used, req, native, t);
 483 }
 484 
 485 BufferPtr JfrStorage::flush_regular(BufferPtr cur, const u1* const cur_pos, size_t used, size_t req, bool native, Thread* t) {
 486   debug_only(assert_flush_regular_precondition(cur, cur_pos, used, req, t);)
 487   // A flush is needed before memcpy since a non-large buffer is thread stable
 488   // (thread local). The flush will not modify memory in addresses above pos()
 489   // which is where the "used / uncommitted" data resides. It is therefore both
 490   // possible and valid to migrate data after the flush. This is however only
 491   // the case for stable thread local buffers; it is not the case for large buffers.
 492   if (!cur->empty()) {
 493     flush_regular_buffer(cur, t);
 494   }
 495   assert(t->jfr_thread_local()->shelved_buffer() == NULL, "invariant");
 496   if (cur->free_size() >= req) {
 497     // simplest case, no switching of buffers
 498     if (used > 0) {
 499       memcpy(cur->pos(), (void*)cur_pos, used);
 500     }
 501     assert(native ? t->jfr_thread_local()->native_buffer() == cur : t->jfr_thread_local()->java_buffer() == cur, "invariant");
 502     return cur;
 503   }
 504   // Going for a "larger-than-regular" buffer.
 505   // Shelve the current buffer to make room for a temporary lease.
 506   t->jfr_thread_local()->shelve_buffer(cur);
 507   return provision_large(cur, cur_pos, used, req, native, t);
 508 }
 509 
 510 static BufferPtr store_buffer_to_thread_local(BufferPtr buffer, JfrThreadLocal* jfr_thread_local, bool native) {
 511   assert(buffer != NULL, "invariant");
 512   if (native) {
 513     jfr_thread_local->set_native_buffer(buffer);
 514   } else {
 515     jfr_thread_local->set_java_buffer(buffer);
 516   }
 517   return buffer;
 518 }
 519 
 520 static BufferPtr restore_shelved_buffer(bool native, Thread* t) {
 521   JfrThreadLocal* const tl = t->jfr_thread_local();
 522   BufferPtr shelved = tl->shelved_buffer();
 523   assert(shelved != NULL, "invariant");
 524   tl->shelve_buffer(NULL);
 525   // restore shelved buffer back as primary
 526   return store_buffer_to_thread_local(shelved, tl, native);
 527 }
 528 
 529 BufferPtr JfrStorage::flush_large(BufferPtr cur, const u1* const cur_pos, size_t used, size_t req, bool native, Thread* t) {
 530   debug_only(assert_flush_large_precondition(cur, cur_pos, used, req, native, t);)
 531   // Can the "regular" buffer (now shelved) accommodate the requested size?
 532   BufferPtr shelved = t->jfr_thread_local()->shelved_buffer();
 533   assert(shelved != NULL, "invariant");
 534   if (shelved->free_size() >= req) {
 535     if (req > 0) {
 536       memcpy(shelved->pos(), (void*)cur_pos, (size_t)used);
 537     }
 538     // release and invalidate
 539     release_large(cur, t);
 540     return restore_shelved_buffer(native, t);
 541   }
 542   // regular too small
 543   return provision_large(cur, cur_pos,  used, req, native, t);
 544 }
 545 
 546 static BufferPtr large_fail(BufferPtr cur, bool native, JfrStorage& storage_instance, Thread* t) {
 547   assert(cur != NULL, "invariant");
 548   assert(t != NULL, "invariant");
 549   if (cur->lease()) {
 550     storage_instance.release_large(cur, t);
 551   }
 552   return restore_shelved_buffer(native, t);
 553 }
 554 
 555 // Always returns a non-null buffer.
 556 // If accommodating the large request fails, the shelved buffer is returned
 557 // even though it might be smaller than the requested size.
 558 // Caller needs to ensure if the size was successfully accommodated.
 559 BufferPtr JfrStorage::provision_large(BufferPtr cur, const u1* const cur_pos, size_t used, size_t req, bool native, Thread* t) {
 560   debug_only(assert_provision_large_precondition(cur, used, req, t);)
 561   assert(t->jfr_thread_local()->shelved_buffer() != NULL, "invariant");
 562   BufferPtr const buffer = acquire_large(req, t);
 563   if (buffer == NULL) {
 564     // unable to allocate and serve the request
 565     return large_fail(cur, native, *this, t);
 566   }
 567   // ok managed to acquire a "large" buffer for the requested size
 568   assert(buffer->free_size() >= req, "invariant");
 569   assert(buffer->lease(), "invariant");
 570   // transfer outstanding data
 571   memcpy(buffer->pos(), (void*)cur_pos, used);
 572   if (cur->lease()) {
 573     release_large(cur, t);
 574     // don't use current anymore, it is gone
 575   }
 576   return store_buffer_to_thread_local(buffer, t->jfr_thread_local(), native);
 577 }
 578 
 579 typedef UnBufferedWriteToChunk<JfrBuffer> WriteOperation;
 580 typedef MutexedWriteOp<WriteOperation> MutexedWriteOperation;
 581 typedef ConcurrentWriteOp<WriteOperation> ConcurrentWriteOperation;
 582 typedef ConcurrentWriteOpExcludeRetired<WriteOperation> ThreadLocalConcurrentWriteOperation;
 583 
 584 size_t JfrStorage::write() {
 585   const size_t full_size_processed = write_full();
 586   WriteOperation wo(_chunkwriter);
 587   ThreadLocalConcurrentWriteOperation tlwo(wo);
 588   process_full_list(tlwo, _thread_local_mspace);
 589   ConcurrentWriteOperation cwo(wo);
 590   process_free_list(cwo, _global_mspace);
 591   return full_size_processed + wo.processed();
 592 }
 593 
 594 size_t JfrStorage::write_at_safepoint() {
 595   assert(SafepointSynchronize::is_at_safepoint(), "invariant");
 596   WriteOperation wo(_chunkwriter);
 597   MutexedWriteOperation writer(wo); // mutexed write mode
 598   process_full_list(writer, _thread_local_mspace);
 599   assert(_transient_mspace->is_free_empty(), "invariant");
 600   process_full_list(writer, _transient_mspace);
 601   assert(_global_mspace->is_full_empty(), "invariant");
 602   process_free_list(writer, _global_mspace);
 603   return wo.processed();
 604 }
 605 
 606 typedef DiscardOp<DefaultDiscarder<JfrStorage::Buffer> > DiscardOperation;
 607 typedef ReleaseOp<JfrStorageMspace> ReleaseOperation;
 608 typedef CompositeOperation<MutexedWriteOperation, ReleaseOperation> FullOperation;
 609 
 610 size_t JfrStorage::clear() {
 611   const size_t full_size_processed = clear_full();
 612   DiscardOperation discarder(concurrent); // concurrent discard mode
 613   process_full_list(discarder, _thread_local_mspace);
 614   assert(_transient_mspace->is_free_empty(), "invariant");
 615   process_full_list(discarder, _transient_mspace);
 616   assert(_global_mspace->is_full_empty(), "invariant");
 617   process_free_list(discarder, _global_mspace);
 618   return full_size_processed + discarder.processed();
 619 }
 620 
 621 static void insert_free_age_nodes(JfrStorageAgeMspace* age_mspace, JfrAgeNode* head, JfrAgeNode* tail, size_t count) {
 622   if (tail != NULL) {
 623     assert(tail->next() == NULL, "invariant");
 624     assert(head != NULL, "invariant");
 625     assert(head->prev() == NULL, "invariant");
 626     MutexLockerEx buffer_lock(JfrBuffer_lock, Mutex::_no_safepoint_check_flag);
 627     age_mspace->insert_free_tail(head, tail, count);
 628   }
 629 }
 630 
 631 template <typename Processor>
 632 static void process_age_list(Processor& processor, JfrStorageAgeMspace* age_mspace, JfrAgeNode* head, size_t count) {
 633   assert(age_mspace != NULL, "invariant");
 634   assert(head != NULL, "invariant");
 635   assert(count > 0, "invariant");
 636   JfrAgeNode* node = head;
 637   JfrAgeNode* last = NULL;
 638   while (node != NULL) {
 639     last = node;
 640     BufferPtr const buffer = node->retired_buffer();
 641     assert(buffer != NULL, "invariant");
 642     assert(buffer->retired(), "invariant");
 643     processor.process(buffer);
 644     // at this point, buffer is already live or destroyed
 645     node->clear_identity();
 646     JfrAgeNode* const next = (JfrAgeNode*)node->next();
 647     if (node->transient()) {
 648       // detach
 649       last = (JfrAgeNode*)last->prev();
 650       if (last != NULL) {
 651         last->set_next(next);
 652       } else {
 653         head = next;
 654       }
 655       if (next != NULL) {
 656         next->set_prev(last);
 657       }
 658       --count;
 659       age_mspace->deallocate(node);
 660     }
 661     node = next;
 662   }
 663   insert_free_age_nodes(age_mspace, head, last, count);
 664 }
 665 
 666 template <typename Processor>
 667 static size_t process_full(Processor& processor, JfrStorageControl& control, JfrStorageAgeMspace* age_mspace) {
 668   assert(age_mspace != NULL, "invariant");
 669   if (age_mspace->is_full_empty()) {
 670     // nothing to do
 671     return 0;
 672   }
 673   size_t count;
 674   JfrAgeNode* head;
 675   {
 676     // fetch age list
 677     MutexLockerEx buffer_lock(JfrBuffer_lock, Mutex::_no_safepoint_check_flag);
 678     count = age_mspace->full_count();
 679     head = age_mspace->clear_full();
 680     control.reset_full();
 681   }
 682   assert(head != NULL, "invariant");
 683   assert(count > 0, "invariant");
 684   process_age_list(processor, age_mspace, head, count);
 685   return count;
 686 }
 687 
 688 static void log(size_t count, size_t amount, bool clear = false) {
 689   if (log_is_enabled(Debug, jfr, system)) {
 690     if (count > 0) {
 691       log_debug(jfr, system)("%s " SIZE_FORMAT " full buffer(s) of " SIZE_FORMAT" B of data%s",
 692         clear ? "Discarded" : "Wrote", count, amount, clear ? "." : " to chunk.");
 693     }
 694   }
 695 }
 696 
 697 // full writer
 698 // Assumption is retired only; exclusive access
 699 // MutexedWriter -> ReleaseOp
 700 //
 701 size_t JfrStorage::write_full() {
 702   assert(_chunkwriter.is_valid(), "invariant");
 703   Thread* const thread = Thread::current();
 704   WriteOperation wo(_chunkwriter);
 705   MutexedWriteOperation writer(wo); // a retired buffer implies mutexed access
 706   ReleaseOperation ro(_transient_mspace, thread);
 707   FullOperation cmd(&writer, &ro);
 708   const size_t count = process_full(cmd, control(), _age_mspace);
 709   log(count, writer.processed());
 710   return writer.processed();
 711 }
 712 
 713 size_t JfrStorage::clear_full() {
 714   DiscardOperation discarder(mutexed); // a retired buffer implies mutexed access
 715   const size_t count = process_full(discarder, control(), _age_mspace);
 716   log(count, discarder.processed(), true);
 717   return discarder.processed();
 718 }
 719 
 720 static void scavenge_log(size_t count, size_t amount, size_t current) {
 721   if (count > 0) {
 722     if (log_is_enabled(Debug, jfr, system)) {
 723       log_debug(jfr, system)("Released " SIZE_FORMAT " dead buffer(s) of " SIZE_FORMAT" B of data.", count, amount);
 724       log_debug(jfr, system)("Current number of dead buffers " SIZE_FORMAT "", current);
 725     }
 726   }
 727 }
 728 
 729 template <typename Mspace>
 730 class Scavenger {
 731 private:
 732   JfrStorageControl& _control;
 733   Mspace* _mspace;
 734   size_t _count;
 735   size_t _amount;
 736 public:
 737   typedef typename Mspace::Type Type;
 738   Scavenger(JfrStorageControl& control, Mspace* mspace) : _control(control), _mspace(mspace), _count(0), _amount(0) {}
 739   bool process(Type* t) {
 740     if (t->retired()) {
 741       assert(!t->transient(), "invariant");
 742       assert(!t->lease(), "invariant");
 743       assert(t->empty(), "invariant");
 744       assert(t->identity() == NULL, "invariant");
 745       ++_count;
 746       _amount += t->total_size();
 747       t->clear_retired();
 748       _control.decrement_dead();
 749       mspace_release_full_critical(t, _mspace);
 750     }
 751     return true;
 752   }
 753   size_t processed() const { return _count; }
 754   size_t amount() const { return _amount; }
 755 };
 756 
 757 size_t JfrStorage::scavenge() {
 758   JfrStorageControl& ctrl = control();
 759   if (ctrl.dead_count() == 0) {
 760     return 0;
 761   }
 762   Scavenger<JfrThreadLocalMspace> scavenger(ctrl, _thread_local_mspace);
 763   process_full_list(scavenger, _thread_local_mspace);
 764   scavenge_log(scavenger.processed(), scavenger.amount(), ctrl.dead_count());
 765   return scavenger.processed();
 766 }