1 /*
2 * Copyright (c) 2012, 2019, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "jfr/jfrEvents.hpp"
27 #include "jfr/jni/jfrJavaSupport.hpp"
28 #include "jfr/recorder/jfrRecorder.hpp"
29 #include "jfr/recorder/checkpoint/jfrCheckpointManager.hpp"
30 #include "jfr/recorder/repository/jfrChunkWriter.hpp"
31 #include "jfr/recorder/service/jfrOptionSet.hpp"
32 #include "jfr/recorder/service/jfrPostBox.hpp"
33 #include "jfr/recorder/storage/jfrMemorySpace.inline.hpp"
34 #include "jfr/recorder/storage/jfrStorage.hpp"
35 #include "jfr/recorder/storage/jfrStorageControl.hpp"
36 #include "jfr/recorder/storage/jfrStorageUtils.inline.hpp"
37 #include "jfr/utilities/jfrIterator.hpp"
38 #include "jfr/utilities/jfrTime.hpp"
39 #include "jfr/writers/jfrNativeEventWriter.hpp"
40 #include "logging/log.hpp"
41 #include "runtime/mutexLocker.hpp"
42 #include "runtime/os.inline.hpp"
43 #include "runtime/safepoint.hpp"
44 #include "runtime/thread.hpp"
45
46 typedef JfrStorage::Buffer* BufferPtr;
47
48 static JfrStorage* _instance = NULL;
49 static JfrStorageControl* _control;
50
51 JfrStorage& JfrStorage::instance() {
52 return *_instance;
53 }
54
55 JfrStorage* JfrStorage::create(JfrChunkWriter& chunkwriter, JfrPostBox& post_box) {
56 assert(_instance == NULL, "invariant");
57 _instance = new JfrStorage(chunkwriter, post_box);
58 return _instance;
59 }
60
61 void JfrStorage::destroy() {
62 if (_instance != NULL) {
63 delete _instance;
64 _instance = NULL;
65 }
66 }
67
68 JfrStorage::JfrStorage(JfrChunkWriter& chunkwriter, JfrPostBox& post_box) :
69 _control(NULL),
70 _global_mspace(NULL),
71 _thread_local_mspace(NULL),
72 _transient_mspace(NULL),
73 _age_mspace(NULL),
74 _chunkwriter(chunkwriter),
75 _post_box(post_box) {}
76
77 JfrStorage::~JfrStorage() {
78 if (_control != NULL) {
79 delete _control;
80 }
81 if (_global_mspace != NULL) {
82 delete _global_mspace;
83 }
84 if (_thread_local_mspace != NULL) {
85 delete _thread_local_mspace;
86 }
87 if (_transient_mspace != NULL) {
88 delete _transient_mspace;
89 }
90 if (_age_mspace != NULL) {
91 delete _age_mspace;
92 }
93 _instance = NULL;
94 }
95
96 static const size_t in_memory_discard_threshold_delta = 2; // start to discard data when the only this number of free buffers are left
97 static const size_t unlimited_mspace_size = 0;
98 static const size_t thread_local_cache_count = 8;
99 static const size_t thread_local_scavenge_threshold = thread_local_cache_count / 2;
100 static const size_t transient_buffer_size_multiplier = 8; // against thread local buffer size
101
102 template <typename Mspace>
103 static Mspace* create_mspace(size_t buffer_size, size_t limit, size_t cache_count, JfrStorage* storage_instance) {
104 Mspace* mspace = new Mspace(buffer_size, limit, cache_count, storage_instance);
105 if (mspace != NULL) {
106 mspace->initialize();
107 }
108 return mspace;
109 }
110
111 bool JfrStorage::initialize() {
112 assert(_control == NULL, "invariant");
113 assert(_global_mspace == NULL, "invariant");
114 assert(_thread_local_mspace == NULL, "invariant");
115 assert(_transient_mspace == NULL, "invariant");
116 assert(_age_mspace == NULL, "invariant");
117
118 const size_t num_global_buffers = (size_t)JfrOptionSet::num_global_buffers();
119 assert(num_global_buffers >= in_memory_discard_threshold_delta, "invariant");
120 const size_t memory_size = (size_t)JfrOptionSet::memory_size();
121 const size_t global_buffer_size = (size_t)JfrOptionSet::global_buffer_size();
122 const size_t thread_buffer_size = (size_t)JfrOptionSet::thread_buffer_size();
123
124 _control = new JfrStorageControl(num_global_buffers, num_global_buffers - in_memory_discard_threshold_delta);
125 if (_control == NULL) {
126 return false;
127 }
128 _global_mspace = create_mspace<JfrStorageMspace>(global_buffer_size, memory_size, num_global_buffers, this);
129 if (_global_mspace == NULL) {
130 return false;
131 }
132 _thread_local_mspace = create_mspace<JfrThreadLocalMspace>(thread_buffer_size, unlimited_mspace_size, thread_local_cache_count, this);
133 if (_thread_local_mspace == NULL) {
134 return false;
135 }
136 _transient_mspace = create_mspace<JfrStorageMspace>(thread_buffer_size * transient_buffer_size_multiplier, unlimited_mspace_size, 0, this);
137 if (_transient_mspace == NULL) {
138 return false;
139 }
140 _age_mspace = create_mspace<JfrStorageAgeMspace>(0 /* no extra size except header */, unlimited_mspace_size, num_global_buffers, this);
141 if (_age_mspace == NULL) {
142 return false;
143 }
144 control().set_scavenge_threshold(thread_local_scavenge_threshold);
145 return true;
146 }
147
148 JfrStorageControl& JfrStorage::control() {
149 return *instance()._control;
150 }
151
152 static void log_allocation_failure(const char* msg, size_t size) {
153 log_warning(jfr)("Unable to allocate " SIZE_FORMAT " bytes of %s.", size, msg);
154 }
155
156 BufferPtr JfrStorage::acquire_thread_local(Thread* thread, size_t size /* 0 */) {
157 BufferPtr buffer = mspace_get_to_full(size, instance()._thread_local_mspace, thread);
158 if (buffer == NULL) {
159 log_allocation_failure("thread local_memory", size);
160 return NULL;
161 }
162 assert(buffer->acquired_by_self(), "invariant");
163 return buffer;
164 }
165
166 BufferPtr JfrStorage::acquire_transient(size_t size, Thread* thread) {
167 BufferPtr buffer = mspace_allocate_transient_lease_to_full(size, instance()._transient_mspace, thread);
168 if (buffer == NULL) {
169 log_allocation_failure("transient memory", size);
170 return NULL;
171 }
172 assert(buffer->acquired_by_self(), "invariant");
173 assert(buffer->transient(), "invariant");
174 assert(buffer->lease(), "invariant");
175 return buffer;
176 }
177
178 static BufferPtr get_lease(size_t size, JfrStorageMspace* mspace, JfrStorage& storage_instance, size_t retry_count, Thread* thread) {
179 assert(size <= mspace->min_elem_size(), "invariant");
180 while (true) {
181 BufferPtr t = mspace_get_free_lease_with_retry(size, mspace, retry_count, thread);
182 if (t == NULL && storage_instance.control().should_discard()) {
183 storage_instance.discard_oldest(thread);
184 continue;
185 }
186 return t;
187 }
188 }
189
190 static BufferPtr get_promotion_buffer(size_t size, JfrStorageMspace* mspace, JfrStorage& storage_instance, size_t retry_count, Thread* thread) {
191 assert(size <= mspace->min_elem_size(), "invariant");
192 while (true) {
193 BufferPtr t = mspace_get_free_with_retry(size, mspace, retry_count, thread);
194 if (t == NULL && storage_instance.control().should_discard()) {
195 storage_instance.discard_oldest(thread);
196 continue;
197 }
198 return t;
199 }
200 }
201
202 static const size_t lease_retry = 10;
203
204 BufferPtr JfrStorage::acquire_large(size_t size, Thread* thread) {
205 JfrStorage& storage_instance = instance();
206 const size_t max_elem_size = storage_instance._global_mspace->min_elem_size(); // min is also max
207 // if not too large and capacity is still available, ask for a lease from the global system
208 if (size < max_elem_size && storage_instance.control().is_global_lease_allowed()) {
209 BufferPtr const buffer = get_lease(size, storage_instance._global_mspace, storage_instance, lease_retry, thread);
210 if (buffer != NULL) {
211 assert(buffer->acquired_by_self(), "invariant");
212 assert(!buffer->transient(), "invariant");
213 assert(buffer->lease(), "invariant");
214 storage_instance.control().increment_leased();
215 return buffer;
216 }
217 }
218 return acquire_transient(size, thread);
219 }
220
221 static void write_data_loss_event(JfrBuffer* buffer, u8 unflushed_size, Thread* thread) {
222 assert(buffer != NULL, "invariant");
223 assert(buffer->empty(), "invariant");
224 const u8 total_data_loss = thread->jfr_thread_local()->add_data_lost(unflushed_size);
225 if (EventDataLoss::is_enabled()) {
226 JfrNativeEventWriter writer(buffer, thread);
227 writer.write<u8>(EventDataLoss::eventId);
228 writer.write(JfrTicks::now());
229 writer.write(unflushed_size);
230 writer.write(total_data_loss);
231 }
232 }
233
234 static void write_data_loss(BufferPtr buffer, Thread* thread) {
235 assert(buffer != NULL, "invariant");
236 const size_t unflushed_size = buffer->unflushed_size();
237 buffer->concurrent_reinitialization();
238 if (unflushed_size == 0) {
239 return;
240 }
241 write_data_loss_event(buffer, unflushed_size, thread);
242 }
243
244 static const size_t promotion_retry = 100;
245
246 bool JfrStorage::flush_regular_buffer(BufferPtr buffer, Thread* thread) {
247 assert(buffer != NULL, "invariant");
248 assert(!buffer->lease(), "invariant");
249 assert(!buffer->transient(), "invariant");
250 const size_t unflushed_size = buffer->unflushed_size();
251 if (unflushed_size == 0) {
252 buffer->concurrent_reinitialization();
253 assert(buffer->empty(), "invariant");
254 return true;
255 }
256
257 if (buffer->excluded()) {
258 const bool thread_is_excluded = thread->jfr_thread_local()->is_excluded();
259 buffer->reinitialize(thread_is_excluded);
260 assert(buffer->empty(), "invariant");
261 if (!thread_is_excluded) {
262 // state change from exclusion to inclusion requires a thread checkpoint
263 JfrCheckpointManager::write_thread_checkpoint(thread);
264 }
265 return true;
266 }
267
268 BufferPtr const promotion_buffer = get_promotion_buffer(unflushed_size, _global_mspace, *this, promotion_retry, thread);
269 if (promotion_buffer == NULL) {
270 write_data_loss(buffer, thread);
271 return false;
272 }
273 assert(promotion_buffer->acquired_by_self(), "invariant");
274 assert(promotion_buffer->free_size() >= unflushed_size, "invariant");
275 buffer->concurrent_move_and_reinitialize(promotion_buffer, unflushed_size);
276 assert(buffer->empty(), "invariant");
277 return true;
278 }
279
280 /*
281 * 1. If the buffer was a "lease" from the global system, release back.
282 * 2. If the buffer is transient (temporal dynamically allocated), retire and register full.
283 *
284 * The buffer is effectively invalidated for the thread post-return,
285 * and the caller should take means to ensure that it is not referenced any longer.
286 */
287 void JfrStorage::release_large(BufferPtr buffer, Thread* thread) {
288 assert(buffer != NULL, "invariant");
289 assert(buffer->lease(), "invariant");
290 assert(buffer->acquired_by_self(), "invariant");
291 buffer->clear_lease();
292 if (buffer->transient()) {
293 buffer->set_retired();
294 register_full(buffer, thread);
295 } else {
296 buffer->release();
297 control().decrement_leased();
298 }
299 }
300
301 static JfrAgeNode* new_age_node(BufferPtr buffer, JfrStorageAgeMspace* age_mspace, Thread* thread) {
302 assert(buffer != NULL, "invariant");
303 assert(age_mspace != NULL, "invariant");
304 return mspace_allocate_transient(0, age_mspace, thread);
305 }
306
307 static void log_registration_failure(size_t unflushed_size) {
308 log_warning(jfr)("Unable to register a full buffer of " SIZE_FORMAT " bytes.", unflushed_size);
309 log_debug(jfr, system)("Cleared 1 full buffer of " SIZE_FORMAT " bytes.", unflushed_size);
310 }
311
312 static void handle_registration_failure(BufferPtr buffer) {
313 assert(buffer != NULL, "invariant");
314 assert(buffer->retired(), "invariant");
315 const size_t unflushed_size = buffer->unflushed_size();
316 buffer->concurrent_reinitialization();
317 log_registration_failure(unflushed_size);
318 }
319
320 static JfrAgeNode* get_free_age_node(JfrStorageAgeMspace* age_mspace, Thread* thread) {
321 assert(JfrBuffer_lock->owned_by_self(), "invariant");
322 return mspace_get_free_with_detach(0, age_mspace, thread);
323 }
324
325 static bool insert_full_age_node(JfrAgeNode* age_node, JfrStorageAgeMspace* age_mspace, Thread* thread) {
326 assert(JfrBuffer_lock->owned_by_self(), "invariant");
327 assert(age_node != NULL, "invariant");
328 assert(age_node->acquired_by_self(), "invariant");
329 assert(age_node->retired_buffer()->retired(), "invariant");
330 age_node->release(); // drop identity claim on age node when inserting to full list
331 assert(age_node->identity() == NULL, "invariant");
332 age_mspace->insert_full_head(age_node);
333 return true;
334 }
335
336 static bool full_buffer_registration(BufferPtr buffer, JfrStorageAgeMspace* age_mspace, JfrStorageControl& control, Thread* thread) {
337 assert(buffer != NULL, "invariant");
338 assert(buffer->retired(), "invariant");
339 assert(age_mspace != NULL, "invariant");
340 MutexLocker lock(JfrBuffer_lock, Mutex::_no_safepoint_check_flag);
341 JfrAgeNode* age_node = get_free_age_node(age_mspace, thread);
342 if (age_node == NULL) {
343 age_node = new_age_node(buffer, age_mspace, thread);
344 if (age_node == NULL) {
345 return false;
346 }
347 }
348 assert(age_node != NULL, "invariant");
349 assert(age_node->acquired_by_self(), "invariant");
350 age_node->set_retired_buffer(buffer);
351 control.increment_full();
352 return insert_full_age_node(age_node, age_mspace, thread);
353 }
354
355 void JfrStorage::register_full(BufferPtr buffer, Thread* thread) {
356 assert(buffer != NULL, "invariant");
357 assert(buffer->retired(), "invariant");
358 assert(buffer->acquired_by(thread), "invariant");
359 if (!full_buffer_registration(buffer, _age_mspace, control(), thread)) {
360 handle_registration_failure(buffer);
361 }
362 if (control().should_post_buffer_full_message()) {
363 _post_box.post(MSG_FULLBUFFER);
364 }
365 }
366
367 void JfrStorage::lock() {
368 assert(!JfrBuffer_lock->owned_by_self(), "invariant");
369 JfrBuffer_lock->lock_without_safepoint_check();
370 }
371
372 void JfrStorage::unlock() {
373 assert(JfrBuffer_lock->owned_by_self(), "invariant");
374 JfrBuffer_lock->unlock();
375 }
376
377 #ifdef ASSERT
378 bool JfrStorage::is_locked() const {
379 return JfrBuffer_lock->owned_by_self();
380 }
381 #endif
382
383 // don't use buffer on return, it is gone
384 void JfrStorage::release(BufferPtr buffer, Thread* thread) {
385 assert(buffer != NULL, "invariant");
386 assert(!buffer->lease(), "invariant");
387 assert(!buffer->transient(), "invariant");
388 assert(!buffer->retired(), "invariant");
389 if (!buffer->empty()) {
390 if (!flush_regular_buffer(buffer, thread)) {
391 buffer->concurrent_reinitialization();
392 }
393 }
394 assert(buffer->empty(), "invariant");
395 assert(buffer->identity() != NULL, "invariant");
396 control().increment_dead();
397 buffer->set_retired();
398 }
399
400 void JfrStorage::release_thread_local(BufferPtr buffer, Thread* thread) {
401 assert(buffer != NULL, "invariant");
402 JfrStorage& storage_instance = instance();
403 storage_instance.release(buffer, thread);
404 if (storage_instance.control().should_scavenge()) {
405 storage_instance._post_box.post(MSG_DEADBUFFER);
406 }
407 }
408
409 static void log_discard(size_t count, size_t amount, size_t current) {
410 if (log_is_enabled(Debug, jfr, system)) {
411 assert(count > 0, "invariant");
412 log_debug(jfr, system)("Cleared " SIZE_FORMAT " full buffer(s) of " SIZE_FORMAT" bytes.", count, amount);
413 log_debug(jfr, system)("Current number of full buffers " SIZE_FORMAT "", current);
414 }
415 }
416
417 void JfrStorage::discard_oldest(Thread* thread) {
418 if (JfrBuffer_lock->try_lock()) {
419 if (!control().should_discard()) {
420 // another thread handled it
421 return;
422 }
423 const size_t num_full_pre_discard = control().full_count();
424 size_t num_full_post_discard = 0;
425 size_t discarded_size = 0;
426 while (true) {
427 JfrAgeNode* const oldest_age_node = _age_mspace->full_tail();
428 if (oldest_age_node == NULL) {
429 break;
430 }
431 assert(oldest_age_node->identity() == NULL, "invariant");
432 BufferPtr const buffer = oldest_age_node->retired_buffer();
433 assert(buffer->retired(), "invariant");
434 discarded_size += buffer->discard();
435 assert(buffer->unflushed_size() == 0, "invariant");
436 num_full_post_discard = control().decrement_full();
437 mspace_release_full(oldest_age_node, _age_mspace);
438 if (buffer->transient()) {
439 mspace_release_full(buffer, _transient_mspace);
440 continue;
441 }
442 buffer->reinitialize();
443 buffer->release(); // publish
444 break;
445 }
446 JfrBuffer_lock->unlock();
447 const size_t number_of_discards = num_full_pre_discard - num_full_post_discard;
448 if (number_of_discards > 0) {
449 log_discard(number_of_discards, discarded_size, num_full_post_discard);
450 }
451 }
452 }
453
454 #ifdef ASSERT
455 typedef const BufferPtr ConstBufferPtr;
456
457 static void assert_flush_precondition(ConstBufferPtr cur, size_t used, bool native, const Thread* t) {
458 assert(t != NULL, "invariant");
459 assert(cur != NULL, "invariant");
460 assert(cur->pos() + used <= cur->end(), "invariant");
461 assert(native ? t->jfr_thread_local()->native_buffer() == cur : t->jfr_thread_local()->java_buffer() == cur, "invariant");
462 }
463
464 static void assert_flush_regular_precondition(ConstBufferPtr cur, const u1* const cur_pos, size_t used, size_t req, const Thread* t) {
465 assert(t != NULL, "invariant");
466 assert(t->jfr_thread_local()->shelved_buffer() == NULL, "invariant");
467 assert(cur != NULL, "invariant");
468 assert(!cur->lease(), "invariant");
469 assert(cur_pos != NULL, "invariant");
470 assert(req >= used, "invariant");
471 }
472
473 static void assert_provision_large_precondition(ConstBufferPtr cur, size_t used, size_t req, const Thread* t) {
474 assert(cur != NULL, "invariant");
475 assert(t != NULL, "invariant");
476 assert(t->jfr_thread_local()->shelved_buffer() != NULL, "invariant");
477 assert(req >= used, "invariant");
478 }
479
480 static void assert_flush_large_precondition(ConstBufferPtr cur, const u1* const cur_pos, size_t used, size_t req, bool native, Thread* t) {
481 assert(t != NULL, "invariant");
482 assert(cur != NULL, "invariant");
483 assert(cur->lease(), "invariant");
484 assert(!cur->excluded(), "invariant");
485 assert(cur_pos != NULL, "invariant");
486 assert(native ? t->jfr_thread_local()->native_buffer() == cur : t->jfr_thread_local()->java_buffer() == cur, "invariant");
487 assert(t->jfr_thread_local()->shelved_buffer() != NULL, "invariant");
488 assert(req >= used, "invariant");
489 assert(cur != t->jfr_thread_local()->shelved_buffer(), "invariant");
490 }
491 #endif // ASSERT
492
493 BufferPtr JfrStorage::flush(BufferPtr cur, size_t used, size_t req, bool native, Thread* t) {
494 debug_only(assert_flush_precondition(cur, used, native, t);)
495 const u1* const cur_pos = cur->pos();
496 req += used;
497 // requested size now encompass the outstanding used size
498 return cur->lease() ? instance().flush_large(cur, cur_pos, used, req, native, t) :
499 instance().flush_regular(cur, cur_pos, used, req, native, t);
500 }
501
502 BufferPtr JfrStorage::flush_regular(BufferPtr cur, const u1* const cur_pos, size_t used, size_t req, bool native, Thread* t) {
503 debug_only(assert_flush_regular_precondition(cur, cur_pos, used, req, t);)
504 // A flush is needed before memcpy since a non-large buffer is thread stable
505 // (thread local). The flush will not modify memory in addresses above pos()
506 // which is where the "used / uncommitted" data resides. It is therefore both
507 // possible and valid to migrate data after the flush. This is however only
508 // the case for stable thread local buffers; it is not the case for large buffers.
509 if (!cur->empty()) {
510 flush_regular_buffer(cur, t);
511 if (cur->excluded()) {
512 return cur;
513 }
514 }
515 assert(t->jfr_thread_local()->shelved_buffer() == NULL, "invariant");
516 if (cur->free_size() >= req) {
517 // simplest case, no switching of buffers
518 if (used > 0) {
519 memcpy(cur->pos(), (void*)cur_pos, used);
520 }
521 assert(native ? t->jfr_thread_local()->native_buffer() == cur : t->jfr_thread_local()->java_buffer() == cur, "invariant");
522 return cur;
523 }
524 // Going for a "larger-than-regular" buffer.
525 // Shelve the current buffer to make room for a temporary lease.
526 t->jfr_thread_local()->shelve_buffer(cur);
527 return provision_large(cur, cur_pos, used, req, native, t);
528 }
529
530 static BufferPtr store_buffer_to_thread_local(BufferPtr buffer, JfrThreadLocal* jfr_thread_local, bool native) {
531 assert(buffer != NULL, "invariant");
532 if (native) {
533 jfr_thread_local->set_native_buffer(buffer);
534 } else {
535 jfr_thread_local->set_java_buffer(buffer);
536 }
537 return buffer;
538 }
539
540 static BufferPtr restore_shelved_buffer(bool native, Thread* t) {
541 JfrThreadLocal* const tl = t->jfr_thread_local();
542 BufferPtr shelved = tl->shelved_buffer();
543 assert(shelved != NULL, "invariant");
544 tl->shelve_buffer(NULL);
545 // restore shelved buffer back as primary
546 return store_buffer_to_thread_local(shelved, tl, native);
547 }
548
549 BufferPtr JfrStorage::flush_large(BufferPtr cur, const u1* const cur_pos, size_t used, size_t req, bool native, Thread* t) {
550 debug_only(assert_flush_large_precondition(cur, cur_pos, used, req, native, t);)
551 // Can the "regular" buffer (now shelved) accommodate the requested size?
552 BufferPtr shelved = t->jfr_thread_local()->shelved_buffer();
553 assert(shelved != NULL, "invariant");
554 if (shelved->free_size() >= req) {
555 if (req > 0) {
556 memcpy(shelved->pos(), (void*)cur_pos, (size_t)used);
557 }
558 // release and invalidate
559 release_large(cur, t);
560 return restore_shelved_buffer(native, t);
561 }
562 // regular too small
563 return provision_large(cur, cur_pos, used, req, native, t);
564 }
565
566 static BufferPtr large_fail(BufferPtr cur, bool native, JfrStorage& storage_instance, Thread* t) {
567 assert(cur != NULL, "invariant");
568 assert(t != NULL, "invariant");
569 if (cur->lease()) {
570 storage_instance.release_large(cur, t);
571 }
572 return restore_shelved_buffer(native, t);
573 }
574
575 // Always returns a non-null buffer.
576 // If accommodating the large request fails, the shelved buffer is returned
577 // even though it might be smaller than the requested size.
578 // Caller needs to ensure if the size was successfully accommodated.
579 BufferPtr JfrStorage::provision_large(BufferPtr cur, const u1* const cur_pos, size_t used, size_t req, bool native, Thread* t) {
580 debug_only(assert_provision_large_precondition(cur, used, req, t);)
581 assert(t->jfr_thread_local()->shelved_buffer() != NULL, "invariant");
582 BufferPtr const buffer = acquire_large(req, t);
583 if (buffer == NULL) {
584 // unable to allocate and serve the request
585 return large_fail(cur, native, *this, t);
586 }
587 // ok managed to acquire a "large" buffer for the requested size
588 assert(buffer->free_size() >= req, "invariant");
589 assert(buffer->lease(), "invariant");
590 // transfer outstanding data
591 memcpy(buffer->pos(), (void*)cur_pos, used);
592 if (cur->lease()) {
593 release_large(cur, t);
594 // don't use current anymore, it is gone
595 }
596 return store_buffer_to_thread_local(buffer, t->jfr_thread_local(), native);
597 }
598
599 typedef UnBufferedWriteToChunk<JfrBuffer> WriteOperation;
600 typedef MutexedWriteOp<WriteOperation> MutexedWriteOperation;
601 typedef ConcurrentWriteOp<WriteOperation> ConcurrentWriteOperation;
602
603 typedef Retired<JfrBuffer, true> NonRetired;
604 typedef Excluded<JfrBuffer, true> NonExcluded;
605 typedef CompositeOperation<NonRetired, NonExcluded> BufferPredicate;
606 typedef PredicatedMutexedWriteOp<WriteOperation, BufferPredicate> ThreadLocalMutexedWriteOperation;
607 typedef PredicatedConcurrentWriteOp<WriteOperation, BufferPredicate> ThreadLocalConcurrentWriteOperation;
608
609 size_t JfrStorage::write() {
610 const size_t full_elements = write_full();
611 WriteOperation wo(_chunkwriter);
612 NonRetired nr;
613 NonExcluded ne;
614 BufferPredicate bp(&nr, &ne);
615 ThreadLocalConcurrentWriteOperation tlwo(wo, bp);
616 process_full_list(tlwo, _thread_local_mspace);
617 ConcurrentWriteOperation cwo(wo);
618 process_free_list(cwo, _global_mspace);
619 return full_elements + wo.elements();
620 }
621
622 size_t JfrStorage::write_at_safepoint() {
623 assert(SafepointSynchronize::is_at_safepoint(), "invariant");
624 WriteOperation wo(_chunkwriter);
625 MutexedWriteOperation writer(wo); // mutexed write mode
626 NonRetired nr;
627 NonExcluded ne;
628 BufferPredicate bp(&nr, &ne);
629 ThreadLocalMutexedWriteOperation tlmwo(wo, bp);
630 process_full_list(tlmwo, _thread_local_mspace);
631 assert(_transient_mspace->is_free_empty(), "invariant");
632 process_full_list(writer, _transient_mspace);
633 assert(_global_mspace->is_full_empty(), "invariant");
634 process_free_list(writer, _global_mspace);
635 return wo.elements();
636 }
637
638 typedef DiscardOp<DefaultDiscarder<JfrStorage::Buffer> > DiscardOperation;
639 typedef ReleaseOp<JfrStorageMspace> ReleaseOperation;
640 typedef CompositeOperation<MutexedWriteOperation, ReleaseOperation> FullOperation;
641
642 size_t JfrStorage::clear() {
643 const size_t full_elements = clear_full();
644 DiscardOperation discarder(concurrent); // concurrent discard mode
645 process_full_list(discarder, _thread_local_mspace);
646 assert(_transient_mspace->is_free_empty(), "invariant");
647 process_full_list(discarder, _transient_mspace);
648 assert(_global_mspace->is_full_empty(), "invariant");
649 process_free_list(discarder, _global_mspace);
650 return full_elements + discarder.elements();
651 }
652
653 static void insert_free_age_nodes(JfrStorageAgeMspace* age_mspace, JfrAgeNode* head, JfrAgeNode* tail, size_t count) {
654 if (tail != NULL) {
655 assert(tail->next() == NULL, "invariant");
656 assert(head != NULL, "invariant");
657 assert(head->prev() == NULL, "invariant");
658 MutexLocker buffer_lock(JfrBuffer_lock, Mutex::_no_safepoint_check_flag);
659 age_mspace->insert_free_tail(head, tail, count);
660 }
661 }
662
663 template <typename Processor>
664 static void process_age_list(Processor& processor, JfrStorageAgeMspace* age_mspace, JfrAgeNode* head, size_t count) {
665 assert(age_mspace != NULL, "invariant");
666 assert(head != NULL, "invariant");
667 assert(count > 0, "invariant");
668 JfrAgeNode* node = head;
669 JfrAgeNode* last = NULL;
670 while (node != NULL) {
671 last = node;
672 assert(node->identity() == NULL, "invariant");
673 BufferPtr const buffer = node->retired_buffer();
674 assert(buffer != NULL, "invariant");
675 assert(buffer->retired(), "invariant");
676 processor.process(buffer);
677 // at this point, buffer is already live or destroyed
678 JfrAgeNode* const next = (JfrAgeNode*)node->next();
679 if (node->transient()) {
680 // detach
681 last = (JfrAgeNode*)last->prev();
682 if (last != NULL) {
683 last->set_next(next);
684 } else {
685 head = next;
686 }
687 if (next != NULL) {
688 next->set_prev(last);
689 }
690 --count;
691 age_mspace->deallocate(node);
692 }
693 node = next;
694 }
695 insert_free_age_nodes(age_mspace, head, last, count);
696 }
697
698 template <typename Processor>
699 static size_t process_full(Processor& processor, JfrStorageControl& control, JfrStorageAgeMspace* age_mspace) {
700 assert(age_mspace != NULL, "invariant");
701 if (age_mspace->is_full_empty()) {
702 // nothing to do
703 return 0;
704 }
705 size_t count;
706 JfrAgeNode* head;
707 {
708 // fetch age list
709 MutexLocker buffer_lock(JfrBuffer_lock, Mutex::_no_safepoint_check_flag);
710 count = age_mspace->full_count();
711 head = age_mspace->clear_full();
712 control.reset_full();
713 }
714 assert(head != NULL, "invariant");
715 assert(count > 0, "invariant");
716 process_age_list(processor, age_mspace, head, count);
717 return count;
718 }
719
720 static void log(size_t count, size_t amount, bool clear = false) {
721 if (log_is_enabled(Debug, jfr, system)) {
722 if (count > 0) {
723 log_debug(jfr, system)("%s " SIZE_FORMAT " full buffer(s) of " SIZE_FORMAT" B of data%s",
724 clear ? "Discarded" : "Wrote", count, amount, clear ? "." : " to chunk.");
725 }
726 }
727 }
728
729 // full writer
730 // Assumption is retired only; exclusive access
731 // MutexedWriter -> ReleaseOp
732 //
733 size_t JfrStorage::write_full() {
734 assert(_chunkwriter.is_valid(), "invariant");
735 Thread* const thread = Thread::current();
736 WriteOperation wo(_chunkwriter);
737 MutexedWriteOperation writer(wo); // a retired buffer implies mutexed access
738 ReleaseOperation ro(_transient_mspace, thread);
739 FullOperation cmd(&writer, &ro);
740 const size_t count = process_full(cmd, control(), _age_mspace);
741 if (0 == count) {
742 assert(0 == writer.elements(), "invariant");
743 return 0;
744 }
745 const size_t size = writer.size();
746 log(count, size);
747 return count;
748 }
749
750 size_t JfrStorage::clear_full() {
751 DiscardOperation discarder(mutexed); // a retired buffer implies mutexed access
752 const size_t count = process_full(discarder, control(), _age_mspace);
753 if (0 == count) {
754 assert(0 == discarder.elements(), "invariant");
755 return 0;
756 }
757 const size_t size = discarder.size();
758 log(count, size, true);
759 return count;
760 }
761
762 static void scavenge_log(size_t count, size_t amount, size_t current) {
763 if (count > 0) {
764 if (log_is_enabled(Debug, jfr, system)) {
765 log_debug(jfr, system)("Released " SIZE_FORMAT " dead buffer(s) of " SIZE_FORMAT" B of data.", count, amount);
766 log_debug(jfr, system)("Current number of dead buffers " SIZE_FORMAT "", current);
767 }
768 }
769 }
770
771 template <typename Mspace>
772 class Scavenger {
773 private:
774 JfrStorageControl& _control;
775 Mspace* _mspace;
776 size_t _count;
777 size_t _amount;
778 public:
779 typedef typename Mspace::Type Type;
780 Scavenger(JfrStorageControl& control, Mspace* mspace) : _control(control), _mspace(mspace), _count(0), _amount(0) {}
781 bool process(Type* t) {
782 if (t->retired()) {
783 assert(t->identity() != NULL, "invariant");
784 assert(t->empty(), "invariant");
785 assert(!t->transient(), "invariant");
786 assert(!t->lease(), "invariant");
787 ++_count;
788 _amount += t->total_size();
789 if (t->excluded()) {
790 t->clear_excluded();
791 }
792 assert(!t->excluded(), "invariant");
793 t->clear_retired();
794 t->release();
795 _control.decrement_dead();
796 mspace_release_full_critical(t, _mspace);
797 }
798 return true;
799 }
800 size_t processed() const { return _count; }
801 size_t amount() const { return _amount; }
802 };
803
804 size_t JfrStorage::scavenge() {
805 JfrStorageControl& ctrl = control();
806 if (ctrl.dead_count() == 0) {
807 return 0;
808 }
809 Scavenger<JfrThreadLocalMspace> scavenger(ctrl, _thread_local_mspace);
810 process_full_list(scavenger, _thread_local_mspace);
811 const size_t count = scavenger.processed();
812 if (0 == count) {
813 assert(0 == scavenger.amount(), "invariant");
814 return 0;
815 }
816 scavenge_log(count, scavenger.amount(), ctrl.dead_count());
817 return count;
818 }