1 /*
2 * Copyright (c) 2016, 2019, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "classfile/javaClasses.inline.hpp"
27 #include "jfr/leakprofiler/checkpoint/objectSampleCheckpoint.hpp"
28 #include "jfr/leakprofiler/leakProfiler.hpp"
29 #include "jfr/recorder/checkpoint/jfrCheckpointManager.hpp"
30 #include "jfr/recorder/checkpoint/jfrCheckpointWriter.hpp"
31 #include "jfr/recorder/checkpoint/types/jfrTypeManager.hpp"
32 #include "jfr/recorder/checkpoint/types/jfrTypeSet.hpp"
33 #include "jfr/recorder/checkpoint/types/traceid/jfrTraceIdEpoch.hpp"
34 #include "jfr/recorder/jfrRecorder.hpp"
35 #include "jfr/recorder/repository/jfrChunkWriter.hpp"
36 #include "jfr/recorder/service/jfrOptionSet.hpp"
37 #include "jfr/recorder/storage/jfrMemorySpace.inline.hpp"
38 #include "jfr/recorder/storage/jfrStorageUtils.inline.hpp"
39 #include "jfr/utilities/jfrBigEndian.hpp"
40 #include "jfr/utilities/jfrIterator.hpp"
41 #include "jfr/utilities/jfrThreadIterator.hpp"
42 #include "jfr/utilities/jfrTypes.hpp"
43 #include "jfr/writers/jfrJavaEventWriter.hpp"
44 #include "logging/log.hpp"
45 #include "memory/iterator.hpp"
46 #include "memory/resourceArea.hpp"
47 #include "runtime/handles.inline.hpp"
48 #include "runtime/mutex.hpp"
49 #include "runtime/orderAccess.hpp"
50 #include "runtime/os.inline.hpp"
51 #include "runtime/safepoint.hpp"
52
53 typedef JfrCheckpointManager::Buffer* BufferPtr;
54
55 static JfrCheckpointManager* _instance = NULL;
56
57 JfrCheckpointManager& JfrCheckpointManager::instance() {
58 return *_instance;
59 }
60
61 JfrCheckpointManager* JfrCheckpointManager::create(JfrChunkWriter& cw) {
62 assert(_instance == NULL, "invariant");
63 _instance = new JfrCheckpointManager(cw);
64 return _instance;
65 }
66
67 void JfrCheckpointManager::destroy() {
68 assert(_instance != NULL, "invariant");
69 delete _instance;
70 _instance = NULL;
71 }
72
73 JfrCheckpointManager::JfrCheckpointManager(JfrChunkWriter& cw) :
74 _free_list_mspace(NULL),
75 _epoch_transition_mspace(NULL),
76 _lock(NULL),
77 _service_thread(NULL),
78 _chunkwriter(cw),
79 _checkpoint_epoch_state(JfrTraceIdEpoch::epoch()) {}
80
81 JfrCheckpointManager::~JfrCheckpointManager() {
82 if (_free_list_mspace != NULL) {
83 delete _free_list_mspace;
84 }
85 if (_epoch_transition_mspace != NULL) {
86 delete _epoch_transition_mspace;
87 }
88 if (_lock != NULL) {
89 delete _lock;
90 }
91 JfrTypeManager::destroy();
92 }
93
94 static const size_t unlimited_mspace_size = 0;
95 static const size_t checkpoint_buffer_cache_count = 2;
96 static const size_t checkpoint_buffer_size = 512 * K;
97
98 static JfrCheckpointMspace* allocate_mspace(size_t size, size_t limit, size_t cache_count, JfrCheckpointManager* mgr) {
99 return create_mspace<JfrCheckpointMspace, JfrCheckpointManager>(size, limit, cache_count, mgr);
100 }
101
102 bool JfrCheckpointManager::initialize() {
103 assert(_free_list_mspace == NULL, "invariant");
104 _free_list_mspace = allocate_mspace(checkpoint_buffer_size, unlimited_mspace_size, checkpoint_buffer_cache_count, this);
105 if (_free_list_mspace == NULL) {
106 return false;
107 }
108 assert(_epoch_transition_mspace == NULL, "invariant");
109 _epoch_transition_mspace = allocate_mspace(checkpoint_buffer_size, unlimited_mspace_size, checkpoint_buffer_cache_count, this);
110 if (_epoch_transition_mspace == NULL) {
111 return false;
112 }
113 assert(_lock == NULL, "invariant");
114 _lock = new Mutex(Monitor::leaf - 1, "Checkpoint mutex", Mutex::_allow_vm_block_flag, Monitor::_safepoint_check_never);
115 if (_lock == NULL) {
116 return false;
117 }
118 return JfrTypeManager::initialize();
119 }
120
121 void JfrCheckpointManager::register_service_thread(const Thread* thread) {
122 _service_thread = thread;
123 }
124
125 void JfrCheckpointManager::register_full(BufferPtr t, Thread* thread) {
126 // nothing here at the moment
127 assert(t != NULL, "invariant");
128 assert(t->acquired_by(thread), "invariant");
129 assert(t->retired(), "invariant");
130 }
131
132 void JfrCheckpointManager::lock() {
133 assert(!_lock->owned_by_self(), "invariant");
134 _lock->lock_without_safepoint_check();
135 }
136
137 void JfrCheckpointManager::unlock() {
138 _lock->unlock();
139 }
140
141 #ifdef ASSERT
142 bool JfrCheckpointManager::is_locked() const {
143 return _lock->owned_by_self();
144 }
145
146 static void assert_free_lease(const BufferPtr buffer) {
147 assert(buffer != NULL, "invariant");
148 assert(buffer->acquired_by_self(), "invariant");
149 assert(buffer->lease(), "invariant");
150 }
151
152 static void assert_release(const BufferPtr buffer) {
153 assert(buffer != NULL, "invariant");
154 assert(buffer->lease(), "invariant");
155 assert(buffer->acquired_by_self(), "invariant");
156 }
157 #endif // ASSERT
158
159 static BufferPtr lease_free(size_t size, JfrCheckpointMspace* mspace, size_t retry_count, Thread* thread) {
160 static const size_t max_elem_size = mspace->min_elem_size(); // min is max
161 BufferPtr buffer;
162 if (size <= max_elem_size) {
163 BufferPtr buffer = mspace_get_free_lease_with_retry(size, mspace, retry_count, thread);
164 if (buffer != NULL) {
165 DEBUG_ONLY(assert_free_lease(buffer);)
166 return buffer;
167 }
168 }
169 buffer = mspace_allocate_transient_lease_to_free(size, mspace, thread);
170 DEBUG_ONLY(assert_free_lease(buffer);)
171 return buffer;
172 }
173
174 bool JfrCheckpointManager::use_epoch_transition_mspace(const Thread* thread) const {
175 return _service_thread != thread && _checkpoint_epoch_state != JfrTraceIdEpoch::epoch();
176 }
177
178 static const size_t lease_retry = 10;
179
180 BufferPtr JfrCheckpointManager::lease_buffer(Thread* thread, size_t size /* 0 */) {
181 JfrCheckpointManager& manager = instance();
182 if (manager.use_epoch_transition_mspace(thread)) {
183 return lease_free(size, manager._epoch_transition_mspace, lease_retry, thread);
184 }
185 return lease_free(size, manager._free_list_mspace, lease_retry, thread);
186 }
187
188 JfrCheckpointMspace* JfrCheckpointManager::lookup(BufferPtr old) const {
189 assert(old != NULL, "invariant");
190 return _free_list_mspace->in_free_list(old) ? _free_list_mspace : _epoch_transition_mspace;
191 }
192
193 BufferPtr JfrCheckpointManager::lease_buffer(BufferPtr old, Thread* thread, size_t size /* 0 */) {
194 assert(old != NULL, "invariant");
195 JfrCheckpointMspace* mspace = instance().lookup(old);
196 assert(mspace != NULL, "invariant");
197 return lease_free(size, mspace, lease_retry, thread);
198 }
199
200 /*
201 * If the buffer was a lease, release back.
202 *
203 * The buffer is effectively invalidated for the thread post-return,
204 * and the caller should take means to ensure that it is not referenced.
205 */
206 static void release(BufferPtr const buffer, Thread* thread) {
207 DEBUG_ONLY(assert_release(buffer);)
208 buffer->clear_lease();
209 buffer->release();
210 }
211
212 BufferPtr JfrCheckpointManager::flush(BufferPtr old, size_t used, size_t requested, Thread* thread) {
213 assert(old != NULL, "invariant");
214 assert(old->lease(), "invariant");
215 if (0 == requested) {
216 // indicates a lease is being returned
217 release(old, thread);
218 return NULL;
219 }
220 // migration of in-flight information
221 BufferPtr const new_buffer = lease_buffer(old, thread, used + requested);
222 if (new_buffer != NULL) {
223 migrate_outstanding_writes(old, new_buffer, used, requested);
224 }
225 release(old, thread);
226 return new_buffer; // might be NULL
227 }
228
229 // offsets into the JfrCheckpointEntry
230 static const juint starttime_offset = sizeof(jlong);
231 static const juint duration_offset = starttime_offset + sizeof(jlong);
232 static const juint checkpoint_type_offset = duration_offset + sizeof(jlong);
233 static const juint types_offset = checkpoint_type_offset + sizeof(juint);
234 static const juint payload_offset = types_offset + sizeof(juint);
235
236 template <typename Return>
237 static Return read_data(const u1* data) {
238 return JfrBigEndian::read<Return>(data);
239 }
240
241 static jlong total_size(const u1* data) {
242 return read_data<jlong>(data);
243 }
244
245 static jlong starttime(const u1* data) {
246 return read_data<jlong>(data + starttime_offset);
247 }
248
249 static jlong duration(const u1* data) {
250 return read_data<jlong>(data + duration_offset);
251 }
252
253 static u1 checkpoint_type(const u1* data) {
254 return read_data<u1>(data + checkpoint_type_offset);
255 }
256
257 static juint number_of_types(const u1* data) {
258 return read_data<juint>(data + types_offset);
259 }
260
261 static void write_checkpoint_header(JfrChunkWriter& cw, int64_t delta_to_last_checkpoint, const u1* data) {
262 cw.reserve(sizeof(u4));
263 cw.write<u8>(EVENT_CHECKPOINT);
264 cw.write(starttime(data));
265 cw.write(duration(data));
266 cw.write(delta_to_last_checkpoint);
267 cw.write(checkpoint_type(data));
268 cw.write(number_of_types(data));
269 }
270
271 static void write_checkpoint_content(JfrChunkWriter& cw, const u1* data, size_t size) {
272 assert(data != NULL, "invariant");
273 cw.write_unbuffered(data + payload_offset, size - sizeof(JfrCheckpointEntry));
274 }
275
276 static size_t write_checkpoint_event(JfrChunkWriter& cw, const u1* data) {
277 assert(data != NULL, "invariant");
278 const int64_t event_begin = cw.current_offset();
279 const int64_t last_checkpoint_event = cw.last_checkpoint_offset();
280 const int64_t delta_to_last_checkpoint = last_checkpoint_event == 0 ? 0 : last_checkpoint_event - event_begin;
281 const int64_t checkpoint_size = total_size(data);
282 write_checkpoint_header(cw, delta_to_last_checkpoint, data);
283 write_checkpoint_content(cw, data, checkpoint_size);
284 const int64_t event_size = cw.current_offset() - event_begin;
285 cw.write_padded_at_offset<u4>(event_size, event_begin);
286 cw.set_last_checkpoint_offset(event_begin);
287 return (size_t)checkpoint_size;
288 }
289
290 static size_t write_checkpoints(JfrChunkWriter& cw, const u1* data, size_t size) {
291 assert(cw.is_valid(), "invariant");
292 assert(data != NULL, "invariant");
293 assert(size > 0, "invariant");
294 const u1* const limit = data + size;
295 const u1* next = data;
296 size_t processed = 0;
297 while (next < limit) {
298 const size_t checkpoint_size = write_checkpoint_event(cw, next);
299 processed += checkpoint_size;
300 next += checkpoint_size;
301 }
302 assert(next == limit, "invariant");
303 return processed;
304 }
305
306 template <typename T>
307 class CheckpointWriteOp {
308 private:
309 JfrChunkWriter& _writer;
310 size_t _processed;
311 public:
312 typedef T Type;
313 CheckpointWriteOp(JfrChunkWriter& writer) : _writer(writer), _processed(0) {}
314 bool write(Type* t, const u1* data, size_t size) {
315 _processed += write_checkpoints(_writer, data, size);
316 return true;
317 }
318 size_t processed() const { return _processed; }
319 };
320
321 typedef CheckpointWriteOp<JfrCheckpointMspace::Type> WriteOperation;
322 typedef ReleaseOp<JfrCheckpointMspace> CheckpointReleaseOperation;
323
324 template <template <typename> class WriterHost, template <typename, typename, typename> class CompositeOperation>
325 static size_t write_mspace(JfrCheckpointMspace* mspace, JfrChunkWriter& chunkwriter) {
326 assert(mspace != NULL, "invariant");
327 WriteOperation wo(chunkwriter);
328 WriterHost<WriteOperation> wh(wo);
329 CheckpointReleaseOperation cro(mspace, Thread::current(), false);
330 CompositeOperation<WriterHost<WriteOperation>, CheckpointReleaseOperation, CompositeOperationAnd> co(&wh, &cro);
331 assert(mspace->is_full_empty(), "invariant");
332 process_free_list(co, mspace);
333 return wo.processed();
334 }
335
336 void JfrCheckpointManager::synchronize_epoch() {
337 assert(_checkpoint_epoch_state != JfrTraceIdEpoch::epoch(), "invariant");
338 OrderAccess::storestore();
339 _checkpoint_epoch_state = JfrTraceIdEpoch::epoch();
340 }
341
342 size_t JfrCheckpointManager::write() {
343 const size_t processed = write_mspace<MutexedWriteOp, CompositeOperation>(_free_list_mspace, _chunkwriter);
344 synchronize_epoch();
345 return processed;
346 }
347
348 size_t JfrCheckpointManager::write_epoch_transition_mspace() {
349 return write_mspace<ExclusiveOp, CompositeOperation>(_epoch_transition_mspace, _chunkwriter);
350 }
351
352 typedef MutexedWriteOp<WriteOperation> FlushOperation;
353
354 size_t JfrCheckpointManager::flush() {
355 WriteOperation wo(_chunkwriter);
356 FlushOperation fo(wo);
357 assert(_free_list_mspace->is_full_empty(), "invariant");
358 process_free_list(fo, _free_list_mspace);
359 return wo.processed();
360 }
361
362 typedef DiscardOp<DefaultDiscarder<JfrBuffer> > DiscardOperation;
363 size_t JfrCheckpointManager::clear() {
364 JfrTypeSet::clear();
365 DiscardOperation discarder(mutexed); // mutexed discard mode
366 process_free_list(discarder, _free_list_mspace);
367 process_free_list(discarder, _epoch_transition_mspace);
368 synchronize_epoch();
369 return discarder.elements();
370 }
371
372 // Optimization for write_static_type_set() and write_threads() is to write
373 // directly into the epoch transition mspace because we will immediately
374 // serialize and reset this mspace post-write.
375 static JfrBuffer* get_epoch_transition_buffer(JfrCheckpointMspace* mspace, Thread* t) {
376 assert(mspace != NULL, "invariant");
377 JfrBuffer* const buffer = mspace->free_head();
378 assert(buffer != NULL, "invariant");
379 buffer->acquire(t);
380 buffer->set_lease();
381 DEBUG_ONLY(assert_free_lease(buffer);)
382 return buffer;
383 }
384
385 bool JfrCheckpointManager::is_static_type_set_required() {
386 return JfrTypeManager::has_new_static_type();
387 }
388
389 size_t JfrCheckpointManager::write_static_type_set() {
390 Thread* const t = Thread::current();
391 ResourceMark rm(t);
392 HandleMark hm(t);
393 JfrCheckpointWriter writer(t, get_epoch_transition_buffer(_epoch_transition_mspace, t), STATICS);
394 JfrTypeManager::write_static_types(writer);
395 return writer.used_size();
396 }
397
398 size_t JfrCheckpointManager::write_threads() {
399 Thread* const t = Thread::current();
400 ResourceMark rm(t);
401 HandleMark hm(t);
402 JfrCheckpointWriter writer(t, get_epoch_transition_buffer(_epoch_transition_mspace, t), THREADS);
403 JfrTypeManager::write_threads(writer);
404 return writer.used_size();
405 }
406
407 size_t JfrCheckpointManager::write_static_type_set_and_threads() {
408 write_static_type_set();
409 write_threads();
410 return write_epoch_transition_mspace();
411 }
412
413 void JfrCheckpointManager::shift_epoch() {
414 debug_only(const u1 current_epoch = JfrTraceIdEpoch::current();)
415 JfrTraceIdEpoch::shift_epoch();
416 assert(current_epoch != JfrTraceIdEpoch::current(), "invariant");
417 }
418
419 void JfrCheckpointManager::on_rotation() {
420 assert(SafepointSynchronize::is_at_safepoint(), "invariant");
421 JfrTypeManager::on_rotation();
422 notify_threads();
423 }
424
425 void JfrCheckpointManager::write_type_set() {
426 assert(!SafepointSynchronize::is_at_safepoint(), "invariant");
427 if (LeakProfiler::is_running()) {
428 Thread* const t = Thread::current();
429 // can safepoint here
430 MutexLocker cld_lock(ClassLoaderDataGraph_lock);
431 MutexLocker module_lock(Module_lock);
432 JfrCheckpointWriter leakp_writer(t);
433 JfrCheckpointWriter writer(t);
434 JfrTypeSet::serialize(&writer, &leakp_writer, false, false);
435 ObjectSampleCheckpoint::on_type_set(leakp_writer);
436 } else {
437 // can safepoint here
438 MutexLocker cld_lock(ClassLoaderDataGraph_lock);
439 MutexLocker module_lock(Module_lock);
440 JfrCheckpointWriter writer(Thread::current());
441 JfrTypeSet::serialize(&writer, NULL, false, false);
442 }
443 write();
444 }
445
446 void JfrCheckpointManager::write_type_set_for_unloaded_classes() {
447 assert_locked_or_safepoint(ClassLoaderDataGraph_lock);
448 JfrCheckpointWriter writer(Thread::current());
449 const JfrCheckpointContext ctx = writer.context();
450 JfrTypeSet::serialize(&writer, NULL, true, false);
451 if (LeakProfiler::is_running()) {
452 ObjectSampleCheckpoint::on_type_set_unload(writer);
453 }
454 if (!JfrRecorder::is_recording()) {
455 // discard by rewind
456 writer.set_context(ctx);
457 }
458 }
459
460 bool JfrCheckpointManager::is_type_set_required() {
461 return JfrTraceIdEpoch::has_changed_tag_state();
462 }
463
464 size_t JfrCheckpointManager::flush_type_set() {
465 size_t elements = 0;
466 {
467 JfrCheckpointWriter writer(Thread::current());
468 // can safepoint here
469 MutexLocker cld_lock(ClassLoaderDataGraph_lock);
470 MutexLocker module_lock(Module_lock);
471 elements = JfrTypeSet::serialize(&writer, NULL, false, true);
472 }
473 flush();
474 return elements;
475 }
476
477 void JfrCheckpointManager::flush_static_type_set() {
478 flush();
479 }
480
481 void JfrCheckpointManager::create_thread_blob(Thread* t) {
482 JfrTypeManager::create_thread_blob(t);
483 }
484
485 void JfrCheckpointManager::write_thread_checkpoint(Thread* t) {
486 JfrTypeManager::write_thread_checkpoint(t);
487 }
488
489 class JfrNotifyClosure : public ThreadClosure {
490 public:
491 void do_thread(Thread* t) {
492 assert(t != NULL, "invariant");
493 assert(t->is_Java_thread(), "invariant");
494 assert_locked_or_safepoint(Threads_lock);
495 JfrJavaEventWriter::notify((JavaThread*)t);
496 }
497 };
498
499 void JfrCheckpointManager::notify_threads() {
500 assert(SafepointSynchronize::is_at_safepoint(), "invariant");
501 JfrNotifyClosure tc;
502 JfrJavaThreadIterator iter;
503 while (iter.has_next()) {
504 tc.do_thread(iter.next());
505 }
506 }