1 /*
2 * Copyright (c) 2012, 2020, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "jfr/jfrEvents.hpp"
27 #include "jfr/jni/jfrJavaSupport.hpp"
28 #include "jfr/leakprofiler/leakProfiler.hpp"
29 #include "jfr/recorder/repository/jfrEmergencyDump.hpp"
30 #include "jfr/recorder/service/jfrPostBox.hpp"
31 #include "jfr/recorder/service/jfrRecorderService.hpp"
32 #include "jfr/utilities/jfrTypes.hpp"
33 #include "logging/log.hpp"
34 #include "runtime/atomic.hpp"
35 #include "runtime/globals.hpp"
36 #include "runtime/mutexLocker.hpp"
37 #include "runtime/os.hpp"
38 #include "runtime/thread.inline.hpp"
39 #include "utilities/growableArray.hpp"
40 #include "utilities/ostream.hpp"
41
42 static const char vm_error_filename_fmt[] = "hs_err_pid%p.jfr";
43 static const char vm_oom_filename_fmt[] = "hs_oom_pid%p.jfr";
44 static const char vm_soe_filename_fmt[] = "hs_soe_pid%p.jfr";
45 static const char chunk_file_jfr_ext[] = ".jfr";
46 static const size_t iso8601_len = 19; // "YYYY-MM-DDTHH:MM:SS"
47 static fio_fd emergency_fd = invalid_fd;
48 static const int64_t chunk_file_header_size = 68;
49 static const size_t chunk_file_extension_length = sizeof chunk_file_jfr_ext - 1;
50
51 /*
52 * The emergency dump logic is restrictive when it comes to
53 * using internal VM constructs such as ResourceArea / Handle / Arena.
54 * The reason being that the thread context is unknown.
55 *
56 * A single static buffer of size JVM_MAXPATHLEN is used for building paths.
57 * os::malloc / os::free are used in a few places.
58 */
59
60 static const size_t _max_path_buffer_size = JVM_MAXPATHLEN;
61 static char _path_buffer[_max_path_buffer_size] = { 0 };
62
63 static bool is_empty(const char* path) {
64 assert(path != NULL, "invariant");
65 return path[0] == '\0';
66 }
67
68 static bool is_path_empty() {
69 return is_empty(_path_buffer);
70 }
71
72 static size_t append(char* path, int path_len, size_t pos, const char* str) {
73 assert(path_len - pos > 0, "invariant");
74 const int result = jio_snprintf(path + pos, path_len - pos, "%s", str);
75 return result == -1 ? 0 : pos + (size_t)result;
76 }
77
78 // returns with an appended file separator (if successful)
79 static size_t get_current_directory(char* path, int path_len) {
80 assert(path != NULL, "invariant");
81 if (os::get_current_directory(path, path_len) == NULL) {
82 return 0;
83 }
84 return append(path, path_len, strlen(path), os::file_separator());
85 }
86
87 static fio_fd open_exclusivly(const char* path) {
88 assert(path != NULL, "invariant");
89 assert(!is_empty(path), "invariant");
90 return os::open(path, O_CREAT | O_RDWR, S_IREAD | S_IWRITE);
91 }
92
93 static bool is_emergency_dump_file_open() {
94 return emergency_fd != invalid_fd;
95 }
96
97 static bool open_emergency_dump_fd(const char* path) {
98 if (path == NULL) {
99 return false;
100 }
101 assert(emergency_fd == invalid_fd, "invariant");
102 emergency_fd = open_exclusivly(path);
103 return emergency_fd != invalid_fd;
104 }
105
106 static void close_emergency_dump_file() {
107 if (is_emergency_dump_file_open()) {
108 os::close(emergency_fd);
109 }
110 }
111
112 static const char* create_emergency_dump_path() {
113 assert(is_path_empty(), "invariant");
114
115 char* const path = _path_buffer;
116 const size_t path_len = get_current_directory(path, _max_path_buffer_size);
117 if (path_len == 0) {
118 return NULL;
119 }
120 const char* filename_fmt = NULL;
121 // fetch specific error cause
122 switch (JfrJavaSupport::cause()) {
123 case JfrJavaSupport::OUT_OF_MEMORY:
124 filename_fmt = vm_oom_filename_fmt;
125 break;
126 case JfrJavaSupport::STACK_OVERFLOW:
127 filename_fmt = vm_soe_filename_fmt;
128 break;
129 default:
130 filename_fmt = vm_error_filename_fmt;
131 }
132 const bool result = Arguments::copy_expand_pid(filename_fmt, strlen(filename_fmt), path + path_len, _max_path_buffer_size - path_len);
133 return result ? path : NULL;
134 }
135
136 static bool open_emergency_dump_file() {
137 if (is_emergency_dump_file_open()) {
138 // opened already
139 return true;
140 }
141 return open_emergency_dump_fd(create_emergency_dump_path());
142 }
143
144 static void report(outputStream* st, bool emergency_file_opened, const char* repository_path) {
145 assert(st != NULL, "invariant");
146 if (emergency_file_opened) {
147 st->print_raw("# JFR recording file will be written. Location: ");
148 st->print_raw_cr(_path_buffer);
149 st->print_raw_cr("#");
150 } else if (repository_path != NULL) {
151 st->print_raw("# The JFR repository may contain useful JFR files. Location: ");
152 st->print_raw_cr(repository_path);
153 st->print_raw_cr("#");
154 } else if (!is_path_empty()) {
155 st->print_raw("# Unable to create a JFR recording file at location: ");
156 st->print_raw_cr(_path_buffer);
157 st->print_raw_cr("#");
158 }
159 }
160
161 void JfrEmergencyDump::on_vm_error_report(outputStream* st, const char* repository_path) {
162 assert(st != NULL, "invariant");
163 Thread* thread = Thread::current_or_null_safe();
164 if (thread != NULL) {
165 report(st, open_emergency_dump_file(), repository_path);
166 } else if (repository_path != NULL) {
167 // a non-attached thread will not be able to write anything later
168 report(st, false, repository_path);
169 }
170 }
171
172 static int file_sort(const char** const file1, const char** file2) {
173 assert(NULL != *file1 && NULL != *file2, "invariant");
174 int cmp = strncmp(*file1, *file2, iso8601_len);
175 if (0 == cmp) {
176 const char* const dot1 = strchr(*file1, '.');
177 assert(NULL != dot1, "invariant");
178 const char* const dot2 = strchr(*file2, '.');
179 assert(NULL != dot2, "invariant");
180 ptrdiff_t file1_len = dot1 - *file1;
181 ptrdiff_t file2_len = dot2 - *file2;
182 if (file1_len < file2_len) {
183 return -1;
184 }
185 if (file1_len > file2_len) {
186 return 1;
187 }
188 assert(file1_len == file2_len, "invariant");
189 cmp = strncmp(*file1, *file2, file1_len);
190 }
191 assert(cmp != 0, "invariant");
192 return cmp;
193 }
194
195 static void iso8601_to_date_time(char* iso8601_str) {
196 assert(iso8601_str != NULL, "invariant");
197 assert(strlen(iso8601_str) == iso8601_len, "invariant");
198 // "YYYY-MM-DDTHH:MM:SS"
199 for (size_t i = 0; i < iso8601_len; ++i) {
200 switch (iso8601_str[i]) {
201 case 'T':
202 case '-':
203 case ':':
204 iso8601_str[i] = '_';
205 break;
206 }
207 }
208 // "YYYY_MM_DD_HH_MM_SS"
209 }
210
211 static void date_time(char* buffer, size_t buffer_len) {
212 assert(buffer != NULL, "invariant");
213 assert(buffer_len >= iso8601_len, "buffer too small");
214 os::iso8601_time(buffer, buffer_len);
215 assert(strlen(buffer) >= iso8601_len + 1, "invariant");
216 // "YYYY-MM-DDTHH:MM:SS"
217 buffer[iso8601_len] = '\0';
218 iso8601_to_date_time(buffer);
219 }
220
221 static int64_t file_size(fio_fd fd) {
222 assert(fd != invalid_fd, "invariant");
223 const int64_t current_offset = os::current_file_offset(fd);
224 const int64_t size = os::lseek(fd, 0, SEEK_END);
225 os::seek_to_file_offset(fd, current_offset);
226 return size;
227 }
228
229 class RepositoryIterator : public StackObj {
230 private:
231 GrowableArray<const char*>* _file_names;
232 size_t _path_buffer_file_name_offset;
233 mutable int _iterator;
234 const char* fully_qualified(const char* file_name) const;
235 const char* filter(const char* file_name) const;
236 public:
237 RepositoryIterator(const char* repository_path);
238 ~RepositoryIterator();
239 bool has_next() const;
240 const char* next() const;
241 };
242
243 // append the file_name at the _path_buffer_file_name_offset position
244 const char* RepositoryIterator::fully_qualified(const char* file_name) const {
245 assert(NULL != file_name, "invariant");
246 char* const path = _path_buffer;
247 assert(!is_empty(path), "invariant");
248 assert(_path_buffer_file_name_offset != 0, "invariant");
249 return append(path, _max_path_buffer_size, _path_buffer_file_name_offset, file_name) != 0 ? path : NULL;
250 }
251
252 // caller responsible for deallocation
253 const char* RepositoryIterator::filter(const char* file_name) const {
254 if (file_name == NULL) {
255 return NULL;
256 }
257 const size_t len = strlen(file_name);
258 if ((len < chunk_file_extension_length) ||
259 (strncmp(&file_name[len - chunk_file_extension_length],
260 chunk_file_jfr_ext,
261 chunk_file_extension_length) != 0)) {
262 // not a .jfr file
263 return NULL;
264 }
265 const char* fqn = fully_qualified(file_name);
266 if (fqn == NULL) {
267 return NULL;
268 }
269 const fio_fd fd = open_exclusivly(fqn);
270 if (invalid_fd == fd) {
271 return NULL;
272 }
273 const int64_t size = file_size(fd);
274 os::close(fd);
275 if (size <= chunk_file_header_size) {
276 return NULL;
277 }
278 char* const file_name_copy = (char*)os::malloc(len + 1, mtTracing);
279 if (file_name_copy == NULL) {
280 log_error(jfr, system)("Unable to malloc memory during jfr emergency dump");
281 return NULL;
282 }
283 strncpy(file_name_copy, file_name, len + 1);
284 return file_name_copy;
285 }
286
287 RepositoryIterator::RepositoryIterator(const char* repository_path) :
288 _file_names(NULL),
289 _path_buffer_file_name_offset(0),
290 _iterator(0) {
291 DIR* dirp = os::opendir(repository_path);
292 if (dirp == NULL) {
293 log_error(jfr, system)("Unable to open repository %s", repository_path);
294 return;
295 }
296 char* const path = _path_buffer;
297 // store repository path in the path buffer
298 size_t result = append(path, _max_path_buffer_size, 0, repository_path);
299 if (result == 0) {
300 return;
301 }
302 // append a file separator and save that position
303 _path_buffer_file_name_offset = append(path, _max_path_buffer_size, result, os::file_separator());
304 if (_path_buffer_file_name_offset == 0) {
305 return;
306 }
307 _file_names = new (ResourceObj::C_HEAP, mtTracing) GrowableArray<const char*>(10, true, mtTracing);
308 if (_file_names == NULL) {
309 log_error(jfr, system)("Unable to malloc memory during jfr emergency dump");
310 return;
311 }
312 // iterate files in the repository and append filtered file names to the files array
313 struct dirent* dentry;
314 while ((dentry = os::readdir(dirp)) != NULL) {
315 const char* file_name = filter(dentry->d_name);
316 if (file_name != NULL) {
317 _file_names->append(file_name);
318 }
319 }
320 os::closedir(dirp);
321 if (_file_names->length() > 1) {
322 _file_names->sort(file_sort);
323 }
324 }
325
326 RepositoryIterator::~RepositoryIterator() {
327 if (_file_names != NULL) {
328 for (int i = 0; i < _file_names->length(); ++i) {
329 os::free(const_cast<char*>(_file_names->at(i)));
330 }
331 delete _file_names;
332 }
333 }
334
335 bool RepositoryIterator::has_next() const {
336 return _file_names != NULL && _iterator < _file_names->length();
337 }
338
339 const char* RepositoryIterator::next() const {
340 return _iterator >= _file_names->length() ? NULL : fully_qualified(_file_names->at(_iterator++));
341 }
342
343 static void write_repository_files(const RepositoryIterator& iterator, char* const copy_block, size_t block_size) {
344 assert(is_emergency_dump_file_open(), "invariant");
345 while (iterator.has_next()) {
346 fio_fd current_fd = invalid_fd;
347 const char* const fqn = iterator.next();
348 assert(fqn != NULL, "invariant");
349 current_fd = open_exclusivly(fqn);
350 if (current_fd != invalid_fd) {
351 const int64_t size = file_size(current_fd);
352 assert(size > 0, "invariant");
353 int64_t bytes_read = 0;
354 int64_t bytes_written = 0;
355 while (bytes_read < size) {
356 const ssize_t read_result = os::read_at(current_fd, copy_block, (int)block_size, bytes_read);
357 if (-1 == read_result) {
358 log_info(jfr)( // For user, should not be "jfr, system"
359 "Unable to recover JFR data");
360 break;
361 }
362 bytes_read += (int64_t)read_result;
363 assert(bytes_read - bytes_written <= (int64_t)block_size, "invariant");
364 bytes_written += (int64_t)os::write(emergency_fd, copy_block, bytes_read - bytes_written);
365 assert(bytes_read == bytes_written, "invariant");
366 }
367 os::close(current_fd);
368 }
369 }
370 }
371
372 static void write_emergency_dump_file(const RepositoryIterator& iterator) {
373 static const size_t block_size = 1 * M; // 1 mb
374 char* const copy_block = (char*)os::malloc(block_size, mtTracing);
375 if (copy_block == NULL) {
376 log_error(jfr, system)("Unable to malloc memory during jfr emergency dump");
377 log_error(jfr, system)("Unable to write jfr emergency dump file");
378 }
379 write_repository_files(iterator, copy_block, block_size);
380 os::free(copy_block);
381 }
382
383 void JfrEmergencyDump::on_vm_error(const char* repository_path) {
384 assert(repository_path != NULL, "invariant");
385 if (open_emergency_dump_file()) {
386 RepositoryIterator iterator(repository_path);
387 write_emergency_dump_file(iterator);
388 close_emergency_dump_file();
389 }
390 }
391
392 static const char* create_emergency_chunk_path(const char* repository_path) {
393 const size_t repository_path_len = strlen(repository_path);
394 char date_time_buffer[32] = { 0 };
395 date_time(date_time_buffer, sizeof(date_time_buffer));
396 char* const path = _path_buffer;
397 // append the individual substrings
398 const int result = jio_snprintf(path,
399 _max_path_buffer_size,
400 "%s%s%s%s",
401 repository_path,
402 os::file_separator(),
403 date_time_buffer,
404 chunk_file_jfr_ext);
405 return result == -1 ? NULL : path;
406 }
407
408 const char* JfrEmergencyDump::chunk_path(const char* repository_path) {
409 if (repository_path == NULL) {
410 if (!open_emergency_dump_file()) {
411 return NULL;
412 }
413 // We can directly use the emergency dump file name as the chunk.
414 // The chunk writer will open its own fd so we close this descriptor.
415 close_emergency_dump_file();
416 assert(!is_path_empty(), "invariant");
417 return _path_buffer;
418 }
419 return create_emergency_chunk_path(repository_path);
420 }
421
422 /*
423 * We are just about to exit the VM, so we will be very aggressive
424 * at this point in order to increase overall success of dumping jfr data.
425 *
426 * If we end up deadlocking in the attempt of dumping out jfr data,
427 * we rely on the WatcherThread task "is_error_reported()",
428 * to exit the VM after a hard-coded timeout (disallow WatcherThread to emergency dump).
429 * This "safety net" somewhat explains the aggressiveness in this attempt.
430 *
431 */
432 static bool prepare_for_emergency_dump(Thread* thread) {
433 assert(thread != NULL, "invariant");
434
435 if (thread->is_Watcher_thread()) {
436 // need WatcherThread as a safeguard against potential deadlocks
437 return false;
438 }
439 if (JfrStream_lock->owned_by_self()) {
440 // crashed during jfr rotation, disallow recursion
441 return false;
442 }
443
444 #ifdef ASSERT
445 Mutex* owned_lock = thread->owned_locks();
446 while (owned_lock != NULL) {
447 Mutex* next = owned_lock->next();
448 owned_lock->unlock();
449 owned_lock = next;
450 }
451 #endif // ASSERT
452
453 if (Threads_lock->owned_by_self()) {
454 Threads_lock->unlock();
455 }
456
457 if (Module_lock->owned_by_self()) {
458 Module_lock->unlock();
459 }
460
461 if (ClassLoaderDataGraph_lock->owned_by_self()) {
462 ClassLoaderDataGraph_lock->unlock();
463 }
464
465 if (Heap_lock->owned_by_self()) {
466 Heap_lock->unlock();
467 }
468
469 if (VMOperationQueue_lock->owned_by_self()) {
470 VMOperationQueue_lock->unlock();
471 }
472
473 if (VMOperationRequest_lock->owned_by_self()) {
474 VMOperationRequest_lock->unlock();
475 }
476
477 if (Service_lock->owned_by_self()) {
478 Service_lock->unlock();
479 }
480
481 if (UseNotificationThread && Notification_lock->owned_by_self()) {
482 Notification_lock->unlock();
483 }
484
485 if (CodeCache_lock->owned_by_self()) {
486 CodeCache_lock->unlock();
487 }
488
489 if (PeriodicTask_lock->owned_by_self()) {
490 PeriodicTask_lock->unlock();
491 }
492
493 if (JfrMsg_lock->owned_by_self()) {
494 JfrMsg_lock->unlock();
495 }
496
497 if (JfrBuffer_lock->owned_by_self()) {
498 JfrBuffer_lock->unlock();
499 }
500
501 if (JfrStacktrace_lock->owned_by_self()) {
502 JfrStacktrace_lock->unlock();
503 }
504 return true;
505 }
506
507 static volatile int jfr_shutdown_lock = 0;
508
509 static bool guard_reentrancy() {
510 return Atomic::cmpxchg(&jfr_shutdown_lock, 0, 1) == 0;
511 }
512
513 class JavaThreadInVM : public StackObj {
514 private:
515 JavaThread* const _jt;
516 JavaThreadState _original_state;
517 public:
518
519 JavaThreadInVM(Thread* t) : _jt(t->is_Java_thread() ? (JavaThread*)t : NULL),
520 _original_state(_thread_max_state) {
521 if ((_jt != NULL) && (_jt->thread_state() != _thread_in_vm)) {
522 _original_state = _jt->thread_state();
523 _jt->set_thread_state(_thread_in_vm);
524 }
525 }
526
527 ~JavaThreadInVM() {
528 if (_original_state != _thread_max_state) {
529 _jt->set_thread_state(_original_state);
530 }
531 }
532
533 };
534
535 static void post_events(bool exception_handler) {
536 if (exception_handler) {
537 EventShutdown e;
538 e.set_reason("VM Error");
539 e.commit();
540 } else {
541 // OOM
542 LeakProfiler::emit_events(max_jlong, false);
543 }
544 EventDumpReason event;
545 event.set_reason(exception_handler ? "Crash" : "Out of Memory");
546 event.set_recordingId(-1);
547 event.commit();
548 }
549
550 void JfrEmergencyDump::on_vm_shutdown(bool exception_handler) {
551 if (!guard_reentrancy()) {
552 return;
553 }
554 Thread* thread = Thread::current_or_null_safe();
555 if (thread == NULL) {
556 return;
557 }
558 // Ensure a JavaThread is _thread_in_vm when we make this call
559 JavaThreadInVM jtivm(thread);
560 if (!prepare_for_emergency_dump(thread)) {
561 return;
562 }
563 post_events(exception_handler);
564 const int messages = MSGBIT(MSG_VM_ERROR);
565 JfrRecorderService service;
566 service.rotate(messages);
567 }