1 /*
2 * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "logging/logStream.hpp"
27 #include "memory/allocation.inline.hpp"
28 #include "runtime/jniHandles.inline.hpp"
29 #include "runtime/thread.inline.hpp"
30 #include "runtime/threadSMR.inline.hpp"
31 #include "runtime/vm_operations.hpp"
32 #include "services/threadService.hpp"
33 #include "utilities/copy.hpp"
34 #include "utilities/globalDefinitions.hpp"
35 #include "utilities/resourceHash.hpp"
36 #include "utilities/vmError.hpp"
37
38 Monitor* ThreadsSMRSupport::_delete_lock =
39 new Monitor(Monitor::special, "Thread_SMR_delete_lock",
40 false /* allow_vm_block */,
41 Monitor::_safepoint_check_never);
42 // The '_cnt', '_max' and '_times" fields are enabled via
43 // -XX:+EnableThreadSMRStatistics:
44
45 // # of parallel threads in _delete_lock->wait().
46 // Impl note: Hard to imagine > 64K waiting threads so this could be 16-bit,
47 // but there is no nice 16-bit _FORMAT support.
48 uint ThreadsSMRSupport::_delete_lock_wait_cnt = 0;
49
50 // Max # of parallel threads in _delete_lock->wait().
51 // Impl note: See _delete_lock_wait_cnt note.
52 uint ThreadsSMRSupport::_delete_lock_wait_max = 0;
53
54 // Flag to indicate when an _delete_lock->notify() is needed.
55 // Impl note: See _delete_lock_wait_cnt note.
56 volatile uint ThreadsSMRSupport::_delete_notify = 0;
57
58 // # of threads deleted over VM lifetime.
59 // Impl note: Atomically incremented over VM lifetime so use unsigned for more
60 // range. Unsigned 64-bit would be more future proof, but 64-bit atomic inc
61 // isn't available everywhere (or is it?).
62 volatile uint ThreadsSMRSupport::_deleted_thread_cnt = 0;
63
64 // Max time in millis to delete a thread.
65 // Impl note: 16-bit might be too small on an overloaded machine. Use
66 // unsigned since this is a time value. Set via Atomic::cmpxchg() in a
67 // loop for correctness.
68 volatile uint ThreadsSMRSupport::_deleted_thread_time_max = 0;
69
70 // Cumulative time in millis to delete threads.
71 // Impl note: Atomically added to over VM lifetime so use unsigned for more
72 // range. Unsigned 64-bit would be more future proof, but 64-bit atomic inc
73 // isn't available everywhere (or is it?).
74 volatile uint ThreadsSMRSupport::_deleted_thread_times = 0;
75
76 ThreadsList* volatile ThreadsSMRSupport::_java_thread_list = new ThreadsList(0);
77
78 // # of ThreadsLists allocated over VM lifetime.
79 // Impl note: We allocate a new ThreadsList for every thread create and
80 // every thread delete so we need a bigger type than the
81 // _deleted_thread_cnt field.
82 uint64_t ThreadsSMRSupport::_java_thread_list_alloc_cnt = 1;
83
84 // # of ThreadsLists freed over VM lifetime.
85 // Impl note: See _java_thread_list_alloc_cnt note.
86 uint64_t ThreadsSMRSupport::_java_thread_list_free_cnt = 0;
87
88 // Max size ThreadsList allocated.
89 // Impl note: Max # of threads alive at one time should fit in unsigned 32-bit.
90 uint ThreadsSMRSupport::_java_thread_list_max = 0;
91
92 // Max # of nested ThreadsLists for a thread.
93 // Impl note: Hard to imagine > 64K nested ThreadsLists so this could be
94 // 16-bit, but there is no nice 16-bit _FORMAT support.
95 uint ThreadsSMRSupport::_nested_thread_list_max = 0;
96
97 // # of ThreadsListHandles deleted over VM lifetime.
98 // Impl note: Atomically incremented over VM lifetime so use unsigned for
99 // more range. There will be fewer ThreadsListHandles than threads so
100 // unsigned 32-bit should be fine.
101 volatile uint ThreadsSMRSupport::_tlh_cnt = 0;
102
103 // Max time in millis to delete a ThreadsListHandle.
104 // Impl note: 16-bit might be too small on an overloaded machine. Use
105 // unsigned since this is a time value. Set via Atomic::cmpxchg() in a
106 // loop for correctness.
107 volatile uint ThreadsSMRSupport::_tlh_time_max = 0;
108
109 // Cumulative time in millis to delete ThreadsListHandles.
110 // Impl note: Atomically added to over VM lifetime so use unsigned for more
111 // range. Unsigned 64-bit would be more future proof, but 64-bit atomic inc
112 // isn't available everywhere (or is it?).
113 volatile uint ThreadsSMRSupport::_tlh_times = 0;
114
115 ThreadsList* ThreadsSMRSupport::_to_delete_list = NULL;
116
117 // # of parallel ThreadsLists on the to-delete list.
118 // Impl note: Hard to imagine > 64K ThreadsLists needing to be deleted so
119 // this could be 16-bit, but there is no nice 16-bit _FORMAT support.
120 uint ThreadsSMRSupport::_to_delete_list_cnt = 0;
121
122 // Max # of parallel ThreadsLists on the to-delete list.
123 // Impl note: See _to_delete_list_cnt note.
124 uint ThreadsSMRSupport::_to_delete_list_max = 0;
125
126
127 // 'inline' functions first so the definitions are before first use:
128
129 inline void ThreadsSMRSupport::add_deleted_thread_times(uint add_value) {
130 Atomic::add(add_value, &_deleted_thread_times);
131 }
132
133 inline void ThreadsSMRSupport::inc_deleted_thread_cnt() {
134 Atomic::inc(&_deleted_thread_cnt);
135 }
136
137 inline void ThreadsSMRSupport::inc_java_thread_list_alloc_cnt() {
138 _java_thread_list_alloc_cnt++;
139 }
140
141 inline void ThreadsSMRSupport::update_deleted_thread_time_max(uint new_value) {
142 while (true) {
143 uint cur_value = _deleted_thread_time_max;
144 if (new_value <= cur_value) {
145 // No need to update max value so we're done.
146 break;
147 }
148 if (Atomic::cmpxchg(new_value, &_deleted_thread_time_max, cur_value) == cur_value) {
149 // Updated max value so we're done. Otherwise try it all again.
150 break;
151 }
152 }
153 }
154
155 inline void ThreadsSMRSupport::update_java_thread_list_max(uint new_value) {
156 if (new_value > _java_thread_list_max) {
157 _java_thread_list_max = new_value;
158 }
159 }
160
161 inline ThreadsList* ThreadsSMRSupport::xchg_java_thread_list(ThreadsList* new_list) {
162 return (ThreadsList*)Atomic::xchg(new_list, &_java_thread_list);
163 }
164
165
166 // Hash table of pointers found by a scan. Used for collecting hazard
167 // pointers (ThreadsList references). Also used for collecting JavaThreads
168 // that are indirectly referenced by hazard ptrs. An instance of this
169 // class only contains one type of pointer.
170 //
171 class ThreadScanHashtable : public CHeapObj<mtThread> {
172 private:
173 static bool ptr_equals(void * const& s1, void * const& s2) {
174 return s1 == s2;
175 }
176
177 static unsigned int ptr_hash(void * const& s1) {
178 // 2654435761 = 2^32 * Phi (golden ratio)
179 return (unsigned int)(((uint32_t)(uintptr_t)s1) * 2654435761u);
180 }
181
182 int _table_size;
183 // ResourceHashtable SIZE is specified at compile time so our
184 // dynamic _table_size is unused for now; 1031 is the first prime
185 // after 1024.
186 typedef ResourceHashtable<void *, int, &ThreadScanHashtable::ptr_hash,
187 &ThreadScanHashtable::ptr_equals, 1031,
188 ResourceObj::C_HEAP, mtThread> PtrTable;
189 PtrTable * _ptrs;
190
191 public:
192 // ResourceHashtable is passed to various functions and populated in
193 // different places so we allocate it using C_HEAP to make it immune
194 // from any ResourceMarks that happen to be in the code paths.
195 ThreadScanHashtable(int table_size) : _table_size(table_size), _ptrs(new (ResourceObj::C_HEAP, mtThread) PtrTable()) {}
196
197 ~ThreadScanHashtable() { delete _ptrs; }
198
199 bool has_entry(void *pointer) {
200 int *val_ptr = _ptrs->get(pointer);
201 return val_ptr != NULL && *val_ptr == 1;
202 }
203
204 void add_entry(void *pointer) {
205 _ptrs->put(pointer, 1);
206 }
207 };
208
209 // Closure to gather JavaThreads indirectly referenced by hazard ptrs
210 // (ThreadsList references) into a hash table. This closure handles part 2
211 // of the dance - adding all the JavaThreads referenced by the hazard
212 // pointer (ThreadsList reference) to the hash table.
213 //
214 class AddThreadHazardPointerThreadClosure : public ThreadClosure {
215 private:
216 ThreadScanHashtable *_table;
217
218 public:
219 AddThreadHazardPointerThreadClosure(ThreadScanHashtable *table) : _table(table) {}
220
221 virtual void do_thread(Thread *thread) {
222 if (!_table->has_entry((void*)thread)) {
223 // The same JavaThread might be on more than one ThreadsList or
224 // more than one thread might be using the same ThreadsList. In
225 // either case, we only need a single entry for a JavaThread.
226 _table->add_entry((void*)thread);
227 }
228 }
229 };
230
231 // Closure to gather JavaThreads indirectly referenced by hazard ptrs
232 // (ThreadsList references) into a hash table. This closure handles part 1
233 // of the dance - hazard ptr chain walking and dispatch to another
234 // closure.
235 //
236 class ScanHazardPtrGatherProtectedThreadsClosure : public ThreadClosure {
237 private:
238 ThreadScanHashtable *_table;
239 public:
240 ScanHazardPtrGatherProtectedThreadsClosure(ThreadScanHashtable *table) : _table(table) {}
241
242 virtual void do_thread(Thread *thread) {
243 assert_locked_or_safepoint(Threads_lock);
244
245 if (thread == NULL) return;
246
247 // This code races with ThreadsSMRSupport::acquire_stable_list() which
248 // is lock-free so we have to handle some special situations.
249 //
250 ThreadsList *current_list = NULL;
251 while (true) {
252 current_list = thread->get_threads_hazard_ptr();
253 // No hazard ptr so nothing more to do.
254 if (current_list == NULL) {
255 assert(thread->get_nested_threads_hazard_ptr() == NULL,
256 "cannot have a nested hazard ptr with a NULL regular hazard ptr");
257 return;
258 }
259
260 // If the hazard ptr is verified as stable (since it is not tagged),
261 // then it is safe to use.
262 if (!Thread::is_hazard_ptr_tagged(current_list)) break;
263
264 // The hazard ptr is tagged as not yet verified as being stable
265 // so we are racing with acquire_stable_list(). This exchange
266 // attempts to invalidate the hazard ptr. If we win the race,
267 // then we can ignore this unstable hazard ptr and the other
268 // thread will retry the attempt to publish a stable hazard ptr.
269 // If we lose the race, then we retry our attempt to look at the
270 // hazard ptr.
271 if (thread->cmpxchg_threads_hazard_ptr(NULL, current_list) == current_list) return;
272 }
273
274 // The current JavaThread has a hazard ptr (ThreadsList reference)
275 // which might be _java_thread_list or it might be an older
276 // ThreadsList that has been removed but not freed. In either case,
277 // the hazard ptr is protecting all the JavaThreads on that
278 // ThreadsList.
279 AddThreadHazardPointerThreadClosure add_cl(_table);
280 current_list->threads_do(&add_cl);
281
282 // Any NestedThreadsLists are also protecting JavaThreads so
283 // gather those also; the ThreadsLists may be different.
284 for (NestedThreadsList* node = thread->get_nested_threads_hazard_ptr();
285 node != NULL; node = node->next()) {
286 node->t_list()->threads_do(&add_cl);
287 }
288 }
289 };
290
291 // Closure to gather hazard ptrs (ThreadsList references) into a hash table.
292 //
293 class ScanHazardPtrGatherThreadsListClosure : public ThreadClosure {
294 private:
295 ThreadScanHashtable *_table;
296 public:
297 ScanHazardPtrGatherThreadsListClosure(ThreadScanHashtable *table) : _table(table) {}
298
299 virtual void do_thread(Thread* thread) {
300 assert_locked_or_safepoint(Threads_lock);
301
302 if (thread == NULL) return;
303 ThreadsList *threads = thread->get_threads_hazard_ptr();
304 if (threads == NULL) {
305 assert(thread->get_nested_threads_hazard_ptr() == NULL,
306 "cannot have a nested hazard ptr with a NULL regular hazard ptr");
307 return;
308 }
309 // In this closure we always ignore the tag that might mark this
310 // hazard ptr as not yet verified. If we happen to catch an
311 // unverified hazard ptr that is subsequently discarded (not
312 // published), then the only side effect is that we might keep a
313 // to-be-deleted ThreadsList alive a little longer.
314 threads = Thread::untag_hazard_ptr(threads);
315 if (!_table->has_entry((void*)threads)) {
316 _table->add_entry((void*)threads);
317 }
318
319 // Any NestedThreadsLists are also protecting JavaThreads so
320 // gather those also; the ThreadsLists may be different.
321 for (NestedThreadsList* node = thread->get_nested_threads_hazard_ptr();
322 node != NULL; node = node->next()) {
323 threads = node->t_list();
324 if (!_table->has_entry((void*)threads)) {
325 _table->add_entry((void*)threads);
326 }
327 }
328 }
329 };
330
331 // Closure to print JavaThreads that have a hazard ptr (ThreadsList
332 // reference) that contains an indirect reference to a specific JavaThread.
333 //
334 class ScanHazardPtrPrintMatchingThreadsClosure : public ThreadClosure {
335 private:
336 JavaThread *_thread;
337 public:
338 ScanHazardPtrPrintMatchingThreadsClosure(JavaThread *thread) : _thread(thread) {}
339
340 virtual void do_thread(Thread *thread) {
341 assert_locked_or_safepoint(Threads_lock);
342
343 if (thread == NULL) return;
344 ThreadsList *current_list = thread->get_threads_hazard_ptr();
345 if (current_list == NULL) {
346 assert(thread->get_nested_threads_hazard_ptr() == NULL,
347 "cannot have a nested hazard ptr with a NULL regular hazard ptr");
348 return;
349 }
350 // If the hazard ptr is unverified, then ignore it.
351 if (Thread::is_hazard_ptr_tagged(current_list)) return;
352
353 // The current JavaThread has a hazard ptr (ThreadsList reference)
354 // which might be _java_thread_list or it might be an older
355 // ThreadsList that has been removed but not freed. In either case,
356 // the hazard ptr is protecting all the JavaThreads on that
357 // ThreadsList, but we only care about matching a specific JavaThread.
358 JavaThreadIterator jti(current_list);
359 for (JavaThread *p = jti.first(); p != NULL; p = jti.next()) {
360 if (p == _thread) {
361 log_debug(thread, smr)("tid=" UINTX_FORMAT ": ThreadsSMRSupport::smr_delete: thread1=" INTPTR_FORMAT " has a hazard pointer for thread2=" INTPTR_FORMAT, os::current_thread_id(), p2i(thread), p2i(_thread));
362 break;
363 }
364 }
365
366 // Any NestedThreadsLists are also protecting JavaThreads so
367 // check those also; the ThreadsLists may be different.
368 for (NestedThreadsList* node = thread->get_nested_threads_hazard_ptr();
369 node != NULL; node = node->next()) {
370 JavaThreadIterator jti(node->t_list());
371 for (JavaThread *p = jti.first(); p != NULL; p = jti.next()) {
372 if (p == _thread) {
373 log_debug(thread, smr)("tid=" UINTX_FORMAT ": ThreadsSMRSupport::smr_delete: thread1=" INTPTR_FORMAT " has a nested hazard pointer for thread2=" INTPTR_FORMAT, os::current_thread_id(), p2i(thread), p2i(_thread));
374 return;
375 }
376 }
377 }
378 }
379 };
380
381
382 // 'entries + 1' so we always have at least one entry.
383 ThreadsList::ThreadsList(int entries) : _length(entries), _threads(NEW_C_HEAP_ARRAY(JavaThread*, entries + 1, mtThread)), _next_list(NULL) {
384 *(JavaThread**)(_threads + entries) = NULL; // Make sure the extra entry is NULL.
385 }
386
387 ThreadsList::~ThreadsList() {
388 FREE_C_HEAP_ARRAY(JavaThread*, _threads);
389 }
390
391 // Add a JavaThread to a ThreadsList. The returned ThreadsList is a
392 // new copy of the specified ThreadsList with the specified JavaThread
393 // appended to the end.
394 ThreadsList *ThreadsList::add_thread(ThreadsList *list, JavaThread *java_thread) {
395 const uint index = list->_length;
396 const uint new_length = index + 1;
397 const uint head_length = index;
398 ThreadsList *const new_list = new ThreadsList(new_length);
399
400 if (head_length > 0) {
401 Copy::disjoint_words((HeapWord*)list->_threads, (HeapWord*)new_list->_threads, head_length);
402 }
403 *(JavaThread**)(new_list->_threads + index) = java_thread;
404
405 return new_list;
406 }
407
408 int ThreadsList::find_index_of_JavaThread(JavaThread *target) {
409 if (target == NULL) {
410 return -1;
411 }
412 for (uint i = 0; i < length(); i++) {
413 if (target == thread_at(i)) {
414 return (int)i;
415 }
416 }
417 return -1;
418 }
419
420 JavaThread* ThreadsList::find_JavaThread_from_java_tid(jlong java_tid) const {
421 for (uint i = 0; i < length(); i++) {
422 JavaThread* thread = thread_at(i);
423 oop tobj = thread->threadObj();
424 // Ignore the thread if it hasn't run yet, has exited
425 // or is starting to exit.
426 if (tobj != NULL && !thread->is_exiting() &&
427 java_tid == java_lang_Thread::thread_id(tobj)) {
428 // found a match
429 return thread;
430 }
431 }
432 return NULL;
433 }
434
435 bool ThreadsList::includes(const JavaThread * const p) const {
436 if (p == NULL) {
437 return false;
438 }
439 for (uint i = 0; i < length(); i++) {
440 if (thread_at(i) == p) {
441 return true;
442 }
443 }
444 return false;
445 }
446
447 // Remove a JavaThread from a ThreadsList. The returned ThreadsList is a
448 // new copy of the specified ThreadsList with the specified JavaThread
449 // removed.
450 ThreadsList *ThreadsList::remove_thread(ThreadsList* list, JavaThread* java_thread) {
451 assert(list->_length > 0, "sanity");
452
453 uint i = (uint)list->find_index_of_JavaThread(java_thread);
454 assert(i < list->_length, "did not find JavaThread on the list");
455 const uint index = i;
456 const uint new_length = list->_length - 1;
457 const uint head_length = index;
458 const uint tail_length = (new_length >= index) ? (new_length - index) : 0;
459 ThreadsList *const new_list = new ThreadsList(new_length);
460
461 if (head_length > 0) {
462 Copy::disjoint_words((HeapWord*)list->_threads, (HeapWord*)new_list->_threads, head_length);
463 }
464 if (tail_length > 0) {
465 Copy::disjoint_words((HeapWord*)list->_threads + index + 1, (HeapWord*)new_list->_threads + index, tail_length);
466 }
467
468 return new_list;
469 }
470
471 ThreadsListHandle::ThreadsListHandle(Thread *self) : _list(ThreadsSMRSupport::acquire_stable_list(self, /* is_ThreadsListSetter */ false)), _self(self) {
472 assert(self == Thread::current(), "sanity check");
473 // Threads::threads_do() is used by the Thread-SMR protocol to visit all
474 // Threads in the system which ensures the safety of the ThreadsList
475 // managed by this ThreadsListHandle, but JavaThreads that are not on
476 // the Threads list cannot be included in that visit. The JavaThread that
477 // calls Threads::destroy_vm() is exempt from this check because it has
478 // to logically exit as part of the shutdown procedure. This is safe
479 // because VM_Exit::_shutdown_thread is not set until after the VMThread
480 // has started the final safepoint which holds the Threads_lock for the
481 // remainder of the VM's life.
482 assert(!self->is_Java_thread() || self == VM_Exit::shutdown_thread() || (((JavaThread*)self)->on_thread_list() && !((JavaThread*)self)->is_terminated()), "JavaThread must be on the Threads list to use a ThreadsListHandle");
483 if (EnableThreadSMRStatistics) {
484 _timer.start();
485 }
486 }
487
488 ThreadsListHandle::~ThreadsListHandle() {
489 ThreadsSMRSupport::release_stable_list(_self);
490 if (EnableThreadSMRStatistics) {
491 _timer.stop();
492 uint millis = (uint)_timer.milliseconds();
493 ThreadsSMRSupport::update_tlh_stats(millis);
494 }
495 }
496
497 // Convert an internal thread reference to a JavaThread found on the
498 // associated ThreadsList. This ThreadsListHandle "protects" the
499 // returned JavaThread *.
500 //
501 // If thread_oop_p is not NULL, then the caller wants to use the oop
502 // after this call so the oop is returned. On success, *jt_pp is set
503 // to the converted JavaThread * and true is returned. On error,
504 // returns false.
505 //
506 bool ThreadsListHandle::cv_internal_thread_to_JavaThread(jobject jthread,
507 JavaThread ** jt_pp,
508 oop * thread_oop_p) {
509 assert(this->list() != NULL, "must have a ThreadsList");
510 assert(jt_pp != NULL, "must have a return JavaThread pointer");
511 // thread_oop_p is optional so no assert()
512
513 // The JVM_* interfaces don't allow a NULL thread parameter; JVM/TI
514 // allows a NULL thread parameter to signify "current thread" which
515 // allows us to avoid calling cv_external_thread_to_JavaThread().
516 // The JVM_* interfaces have no such leeway.
517
518 oop thread_oop = JNIHandles::resolve_non_null(jthread);
519 // Looks like an oop at this point.
520 if (thread_oop_p != NULL) {
521 // Return the oop to the caller; the caller may still want
522 // the oop even if this function returns false.
523 *thread_oop_p = thread_oop;
524 }
525
526 JavaThread *java_thread = java_lang_Thread::thread(thread_oop);
527 if (java_thread == NULL) {
528 // The java.lang.Thread does not contain a JavaThread * so it has
529 // not yet run or it has died.
530 return false;
531 }
532 // Looks like a live JavaThread at this point.
533
534 if (java_thread != JavaThread::current()) {
535 // jthread is not for the current JavaThread so have to verify
536 // the JavaThread * against the ThreadsList.
537 if (EnableThreadSMRExtraValidityChecks && !includes(java_thread)) {
538 // Not on the JavaThreads list so it is not alive.
539 return false;
540 }
541 }
542
543 // Return a live JavaThread that is "protected" by the
544 // ThreadsListHandle in the caller.
545 *jt_pp = java_thread;
546 return true;
547 }
548
549 ThreadsListSetter::~ThreadsListSetter() {
550 if (_target_needs_release) {
551 // The hazard ptr in the target needs to be released.
552 ThreadsSMRSupport::release_stable_list(_target);
553 }
554 }
555
556 void ThreadsListSetter::set() {
557 assert(_target->get_threads_hazard_ptr() == NULL, "hazard ptr should not already be set");
558 (void) ThreadsSMRSupport::acquire_stable_list(_target, /* is_ThreadsListSetter */ true);
559 _target_needs_release = true;
560 }
561
562 // Acquire a stable ThreadsList.
563 //
564 ThreadsList *ThreadsSMRSupport::acquire_stable_list(Thread *self, bool is_ThreadsListSetter) {
565 assert(self != NULL, "sanity check");
566 // acquire_stable_list_nested_path() will grab the Threads_lock
567 // so let's make sure the ThreadsListHandle is in a safe place.
568 // ThreadsListSetter cannot make this check on this code path.
569 debug_only(if (!is_ThreadsListSetter && StrictSafepointChecks) self->check_for_valid_safepoint_state(/* potential_vm_operation */ false);)
570
571 if (self->get_threads_hazard_ptr() == NULL) {
572 // The typical case is first.
573 return acquire_stable_list_fast_path(self);
574 }
575
576 // The nested case is rare.
577 return acquire_stable_list_nested_path(self);
578 }
579
580 // Fast path (and lock free) way to acquire a stable ThreadsList.
581 //
582 ThreadsList *ThreadsSMRSupport::acquire_stable_list_fast_path(Thread *self) {
583 assert(self != NULL, "sanity check");
584 assert(self->get_threads_hazard_ptr() == NULL, "sanity check");
585 assert(self->get_nested_threads_hazard_ptr() == NULL,
586 "cannot have a nested hazard ptr with a NULL regular hazard ptr");
587
588 ThreadsList* threads;
589
590 // Stable recording of a hazard ptr for SMR. This code does not use
591 // locks so its use of the _java_thread_list & _threads_hazard_ptr
592 // fields is racy relative to code that uses those fields with locks.
593 // OrderAccess and Atomic functions are used to deal with those races.
594 //
595 while (true) {
596 threads = get_java_thread_list();
597
598 // Publish a tagged hazard ptr to denote that the hazard ptr is not
599 // yet verified as being stable. Due to the fence after the hazard
600 // ptr write, it will be sequentially consistent w.r.t. the
601 // sequentially consistent writes of the ThreadsList, even on
602 // non-multiple copy atomic machines where stores can be observed
603 // in different order from different observer threads.
604 ThreadsList* unverified_threads = Thread::tag_hazard_ptr(threads);
605 self->set_threads_hazard_ptr(unverified_threads);
606
607 // If _java_thread_list has changed, we have lost a race with
608 // Threads::add() or Threads::remove() and have to try again.
609 if (get_java_thread_list() != threads) {
610 continue;
611 }
612
613 // We try to remove the tag which will verify the hazard ptr as
614 // being stable. This exchange can race with a scanning thread
615 // which might invalidate the tagged hazard ptr to keep it from
616 // being followed to access JavaThread ptrs. If we lose the race,
617 // we simply retry. If we win the race, then the stable hazard
618 // ptr is officially published.
619 if (self->cmpxchg_threads_hazard_ptr(threads, unverified_threads) == unverified_threads) {
620 break;
621 }
622 }
623
624 // A stable hazard ptr has been published letting other threads know
625 // that the ThreadsList and the JavaThreads reachable from this list
626 // are protected and hence they should not be deleted until everyone
627 // agrees it is safe to do so.
628
629 return threads;
630 }
631
632 // Acquire a nested stable ThreadsList; this is rare so it uses
633 // Threads_lock.
634 //
635 ThreadsList *ThreadsSMRSupport::acquire_stable_list_nested_path(Thread *self) {
636 assert(self != NULL, "sanity check");
637 assert(self->get_threads_hazard_ptr() != NULL,
638 "cannot have a NULL regular hazard ptr when acquiring a nested hazard ptr");
639
640 // The thread already has a hazard ptr (ThreadsList ref) so we need
641 // to create a nested ThreadsListHandle with the current ThreadsList
642 // since it might be different than our current hazard ptr. The need
643 // for a nested ThreadsListHandle is rare so we do this while holding
644 // the Threads_lock so we don't race with the scanning code; the code
645 // is so much simpler this way.
646
647 NestedThreadsList* node;
648 {
649 // Only grab the Threads_lock if we don't already own it.
650 MutexLockerEx ml(Threads_lock->owned_by_self() ? NULL : Threads_lock);
651 node = new NestedThreadsList(get_java_thread_list());
652 // We insert at the front of the list to match up with the delete
653 // in release_stable_list().
654 node->set_next(self->get_nested_threads_hazard_ptr());
655 self->set_nested_threads_hazard_ptr(node);
656 if (EnableThreadSMRStatistics) {
657 self->inc_nested_threads_hazard_ptr_cnt();
658 if (self->nested_threads_hazard_ptr_cnt() > _nested_thread_list_max) {
659 _nested_thread_list_max = self->nested_threads_hazard_ptr_cnt();
660 }
661 }
662 }
663 log_debug(thread, smr)("tid=" UINTX_FORMAT ": ThreadsSMRSupport::acquire_stable_list: add NestedThreadsList node containing ThreadsList=" INTPTR_FORMAT, os::current_thread_id(), p2i(node->t_list()));
664
665 return node->t_list();
666 }
667
668 void ThreadsSMRSupport::add_thread(JavaThread *thread){
669 ThreadsList *new_list = ThreadsList::add_thread(ThreadsSMRSupport::get_java_thread_list(), thread);
670 if (EnableThreadSMRStatistics) {
671 ThreadsSMRSupport::inc_java_thread_list_alloc_cnt();
672 ThreadsSMRSupport::update_java_thread_list_max(new_list->length());
673 }
674 // Initial _java_thread_list will not generate a "Threads::add" mesg.
675 log_debug(thread, smr)("tid=" UINTX_FORMAT ": Threads::add: new ThreadsList=" INTPTR_FORMAT, os::current_thread_id(), p2i(new_list));
676
677 ThreadsList *old_list = ThreadsSMRSupport::xchg_java_thread_list(new_list);
678 ThreadsSMRSupport::free_list(old_list);
679 }
680
681 // set_delete_notify() and clear_delete_notify() are called
682 // under the protection of the delete_lock, but we also use an
683 // Atomic operation to ensure the memory update is seen earlier than
684 // when the delete_lock is dropped.
685 //
686 void ThreadsSMRSupport::clear_delete_notify() {
687 Atomic::dec(&_delete_notify);
688 }
689
690 bool ThreadsSMRSupport::delete_notify() {
691 // Use load_acquire() in order to see any updates to _delete_notify
692 // earlier than when delete_lock is grabbed.
693 return (OrderAccess::load_acquire(&_delete_notify) != 0);
694 }
695
696 // Safely free a ThreadsList after a Threads::add() or Threads::remove().
697 // The specified ThreadsList may not get deleted during this call if it
698 // is still in-use (referenced by a hazard ptr). Other ThreadsLists
699 // in the chain may get deleted by this call if they are no longer in-use.
700 void ThreadsSMRSupport::free_list(ThreadsList* threads) {
701 assert_locked_or_safepoint(Threads_lock);
702
703 threads->set_next_list(_to_delete_list);
704 _to_delete_list = threads;
705 if (EnableThreadSMRStatistics) {
706 _to_delete_list_cnt++;
707 if (_to_delete_list_cnt > _to_delete_list_max) {
708 _to_delete_list_max = _to_delete_list_cnt;
709 }
710 }
711
712 // Hash table size should be first power of two higher than twice the length of the ThreadsList
713 int hash_table_size = MIN2((int)get_java_thread_list()->length(), 32) << 1;
714 hash_table_size--;
715 hash_table_size |= hash_table_size >> 1;
716 hash_table_size |= hash_table_size >> 2;
717 hash_table_size |= hash_table_size >> 4;
718 hash_table_size |= hash_table_size >> 8;
719 hash_table_size |= hash_table_size >> 16;
720 hash_table_size++;
721
722 // Gather a hash table of the current hazard ptrs:
723 ThreadScanHashtable *scan_table = new ThreadScanHashtable(hash_table_size);
724 ScanHazardPtrGatherThreadsListClosure scan_cl(scan_table);
725 Threads::threads_do(&scan_cl);
726
727 // Walk through the linked list of pending freeable ThreadsLists
728 // and free the ones that are not referenced from hazard ptrs.
729 ThreadsList* current = _to_delete_list;
730 ThreadsList* prev = NULL;
731 ThreadsList* next = NULL;
732 bool threads_is_freed = false;
733 while (current != NULL) {
734 next = current->next_list();
735 if (!scan_table->has_entry((void*)current)) {
736 // This ThreadsList is not referenced by a hazard ptr.
737 if (prev != NULL) {
738 prev->set_next_list(next);
739 }
740 if (_to_delete_list == current) {
741 _to_delete_list = next;
742 }
743
744 log_debug(thread, smr)("tid=" UINTX_FORMAT ": ThreadsSMRSupport::free_list: threads=" INTPTR_FORMAT " is freed.", os::current_thread_id(), p2i(current));
745 if (current == threads) threads_is_freed = true;
746 delete current;
747 if (EnableThreadSMRStatistics) {
748 _java_thread_list_free_cnt++;
749 _to_delete_list_cnt--;
750 }
751 } else {
752 prev = current;
753 }
754 current = next;
755 }
756
757 if (!threads_is_freed) {
758 // Only report "is not freed" on the original call to
759 // free_list() for this ThreadsList.
760 log_debug(thread, smr)("tid=" UINTX_FORMAT ": ThreadsSMRSupport::free_list: threads=" INTPTR_FORMAT " is not freed.", os::current_thread_id(), p2i(threads));
761 }
762
763 delete scan_table;
764 }
765
766 // Return true if the specified JavaThread is protected by a hazard
767 // pointer (ThreadsList reference). Otherwise, returns false.
768 //
769 bool ThreadsSMRSupport::is_a_protected_JavaThread(JavaThread *thread) {
770 assert_locked_or_safepoint(Threads_lock);
771
772 // Hash table size should be first power of two higher than twice
773 // the length of the Threads list.
774 int hash_table_size = MIN2((int)get_java_thread_list()->length(), 32) << 1;
775 hash_table_size--;
776 hash_table_size |= hash_table_size >> 1;
777 hash_table_size |= hash_table_size >> 2;
778 hash_table_size |= hash_table_size >> 4;
779 hash_table_size |= hash_table_size >> 8;
780 hash_table_size |= hash_table_size >> 16;
781 hash_table_size++;
782
783 // Gather a hash table of the JavaThreads indirectly referenced by
784 // hazard ptrs.
785 ThreadScanHashtable *scan_table = new ThreadScanHashtable(hash_table_size);
786 ScanHazardPtrGatherProtectedThreadsClosure scan_cl(scan_table);
787 Threads::threads_do(&scan_cl);
788
789 bool thread_is_protected = false;
790 if (scan_table->has_entry((void*)thread)) {
791 thread_is_protected = true;
792 }
793 delete scan_table;
794 return thread_is_protected;
795 }
796
797 // Release a stable ThreadsList.
798 //
799 void ThreadsSMRSupport::release_stable_list(Thread *self) {
800 assert(self != NULL, "sanity check");
801 // release_stable_list_nested_path() will grab the Threads_lock
802 // so let's make sure the ThreadsListHandle is in a safe place.
803 debug_only(if (StrictSafepointChecks) self->check_for_valid_safepoint_state(/* potential_vm_operation */ false);)
804
805 if (self->get_nested_threads_hazard_ptr() == NULL) {
806 // The typical case is first.
807 release_stable_list_fast_path(self);
808 return;
809 }
810
811 // The nested case is rare.
812 release_stable_list_nested_path(self);
813 }
814
815 // Fast path way to release a stable ThreadsList. The release portion
816 // is lock-free, but the wake up portion is not.
817 //
818 void ThreadsSMRSupport::release_stable_list_fast_path(Thread *self) {
819 assert(self != NULL, "sanity check");
820 assert(self->get_threads_hazard_ptr() != NULL, "sanity check");
821 assert(self->get_nested_threads_hazard_ptr() == NULL,
822 "cannot have a nested hazard ptr when releasing a regular hazard ptr");
823
824 // After releasing the hazard ptr, other threads may go ahead and
825 // free up some memory temporarily used by a ThreadsList snapshot.
826 self->set_threads_hazard_ptr(NULL);
827
828 // We use double-check locking to reduce traffic on the system
829 // wide Thread-SMR delete_lock.
830 if (ThreadsSMRSupport::delete_notify()) {
831 // An exiting thread might be waiting in smr_delete(); we need to
832 // check with delete_lock to be sure.
833 release_stable_list_wake_up((char *) "regular hazard ptr");
834 }
835 }
836
837 // Release a nested stable ThreadsList; this is rare so it uses
838 // Threads_lock.
839 //
840 void ThreadsSMRSupport::release_stable_list_nested_path(Thread *self) {
841 assert(self != NULL, "sanity check");
842 assert(self->get_nested_threads_hazard_ptr() != NULL, "sanity check");
843 assert(self->get_threads_hazard_ptr() != NULL,
844 "must have a regular hazard ptr to have nested hazard ptrs");
845
846 // We have a nested ThreadsListHandle so we have to release it first.
847 // The need for a nested ThreadsListHandle is rare so we do this while
848 // holding the Threads_lock so we don't race with the scanning code;
849 // the code is so much simpler this way.
850
851 NestedThreadsList *node;
852 {
853 // Only grab the Threads_lock if we don't already own it.
854 MutexLockerEx ml(Threads_lock->owned_by_self() ? NULL : Threads_lock);
855 // We remove from the front of the list to match up with the insert
856 // in acquire_stable_list().
857 node = self->get_nested_threads_hazard_ptr();
858 self->set_nested_threads_hazard_ptr(node->next());
859 if (EnableThreadSMRStatistics) {
860 self->dec_nested_threads_hazard_ptr_cnt();
861 }
862 }
863
864 // An exiting thread might be waiting in smr_delete(); we need to
865 // check with delete_lock to be sure.
866 release_stable_list_wake_up((char *) "nested hazard ptr");
867
868 log_debug(thread, smr)("tid=" UINTX_FORMAT ": ThreadsSMRSupport::release_stable_list: delete NestedThreadsList node containing ThreadsList=" INTPTR_FORMAT, os::current_thread_id(), p2i(node->t_list()));
869
870 delete node;
871 }
872
873 // Wake up portion of the release stable ThreadsList protocol;
874 // uses the delete_lock().
875 //
876 void ThreadsSMRSupport::release_stable_list_wake_up(char *log_str) {
877 assert(log_str != NULL, "sanity check");
878
879 // Note: delete_lock is held in smr_delete() for the entire
880 // hazard ptr search so that we do not lose this notify() if
881 // the exiting thread has to wait. That code path also holds
882 // Threads_lock (which was grabbed before delete_lock) so that
883 // threads_do() can be called. This means the system can't start a
884 // safepoint which means this thread can't take too long to get to
885 // a safepoint because of being blocked on delete_lock.
886 //
887 MonitorLockerEx ml(ThreadsSMRSupport::delete_lock(), Monitor::_no_safepoint_check_flag);
888 if (ThreadsSMRSupport::delete_notify()) {
889 // Notify any exiting JavaThreads that are waiting in smr_delete()
890 // that we've released a ThreadsList.
891 ml.notify_all();
892 log_debug(thread, smr)("tid=" UINTX_FORMAT ": ThreadsSMRSupport::release_stable_list notified %s", os::current_thread_id(), log_str);
893 }
894 }
895
896 void ThreadsSMRSupport::remove_thread(JavaThread *thread) {
897 ThreadsList *new_list = ThreadsList::remove_thread(ThreadsSMRSupport::get_java_thread_list(), thread);
898 if (EnableThreadSMRStatistics) {
899 ThreadsSMRSupport::inc_java_thread_list_alloc_cnt();
900 // This list is smaller so no need to check for a "longest" update.
901 }
902
903 // Final _java_thread_list will not generate a "Threads::remove" mesg.
904 log_debug(thread, smr)("tid=" UINTX_FORMAT ": Threads::remove: new ThreadsList=" INTPTR_FORMAT, os::current_thread_id(), p2i(new_list));
905
906 ThreadsList *old_list = ThreadsSMRSupport::xchg_java_thread_list(new_list);
907 ThreadsSMRSupport::free_list(old_list);
908 }
909
910 // See note for clear_delete_notify().
911 //
912 void ThreadsSMRSupport::set_delete_notify() {
913 Atomic::inc(&_delete_notify);
914 }
915
916 // Safely delete a JavaThread when it is no longer in use by a
917 // ThreadsListHandle.
918 //
919 void ThreadsSMRSupport::smr_delete(JavaThread *thread) {
920 assert(!Threads_lock->owned_by_self(), "sanity");
921
922 bool has_logged_once = false;
923 elapsedTimer timer;
924 if (EnableThreadSMRStatistics) {
925 timer.start();
926 }
927
928 while (true) {
929 {
930 // No safepoint check because this JavaThread is not on the
931 // Threads list.
932 MutexLockerEx ml(Threads_lock, Mutex::_no_safepoint_check_flag);
933 // Cannot use a MonitorLockerEx helper here because we have
934 // to drop the Threads_lock first if we wait.
935 ThreadsSMRSupport::delete_lock()->lock_without_safepoint_check();
936 // Set the delete_notify flag after we grab delete_lock
937 // and before we scan hazard ptrs because we're doing
938 // double-check locking in release_stable_list().
939 ThreadsSMRSupport::set_delete_notify();
940
941 if (!is_a_protected_JavaThread(thread)) {
942 // This is the common case.
943 ThreadsSMRSupport::clear_delete_notify();
944 ThreadsSMRSupport::delete_lock()->unlock();
945 break;
946 }
947 if (!has_logged_once) {
948 has_logged_once = true;
949 log_debug(thread, smr)("tid=" UINTX_FORMAT ": ThreadsSMRSupport::smr_delete: thread=" INTPTR_FORMAT " is not deleted.", os::current_thread_id(), p2i(thread));
950 if (log_is_enabled(Debug, os, thread)) {
951 ScanHazardPtrPrintMatchingThreadsClosure scan_cl(thread);
952 Threads::threads_do(&scan_cl);
953 }
954 }
955 } // We have to drop the Threads_lock to wait or delete the thread
956
957 if (EnableThreadSMRStatistics) {
958 _delete_lock_wait_cnt++;
959 if (_delete_lock_wait_cnt > _delete_lock_wait_max) {
960 _delete_lock_wait_max = _delete_lock_wait_cnt;
961 }
962 }
963 // Wait for a release_stable_list() call before we check again. No
964 // safepoint check, no timeout, and not as suspend equivalent flag
965 // because this JavaThread is not on the Threads list.
966 ThreadsSMRSupport::delete_lock()->wait(Mutex::_no_safepoint_check_flag, 0,
967 !Mutex::_as_suspend_equivalent_flag);
968 if (EnableThreadSMRStatistics) {
969 _delete_lock_wait_cnt--;
970 }
971
972 ThreadsSMRSupport::clear_delete_notify();
973 ThreadsSMRSupport::delete_lock()->unlock();
974 // Retry the whole scenario.
975 }
976
977 if (ThreadLocalHandshakes) {
978 // The thread is about to be deleted so cancel any handshake.
979 thread->cancel_handshake();
980 }
981
982 delete thread;
983 if (EnableThreadSMRStatistics) {
984 timer.stop();
985 uint millis = (uint)timer.milliseconds();
986 ThreadsSMRSupport::inc_deleted_thread_cnt();
987 ThreadsSMRSupport::add_deleted_thread_times(millis);
988 ThreadsSMRSupport::update_deleted_thread_time_max(millis);
989 }
990
991 log_debug(thread, smr)("tid=" UINTX_FORMAT ": ThreadsSMRSupport::smr_delete: thread=" INTPTR_FORMAT " is deleted.", os::current_thread_id(), p2i(thread));
992 }
993
994
995 // Debug, logging, and printing stuff at the end:
996
997 // Log Threads class SMR info.
998 void ThreadsSMRSupport::log_statistics() {
999 LogTarget(Info, thread, smr) log;
1000 if (log.is_enabled()) {
1001 LogStream out(log);
1002 print_info_on(&out);
1003 }
1004 }
1005
1006 // Print Threads class SMR info.
1007 void ThreadsSMRSupport::print_info_on(outputStream* st) {
1008 // Only grab the Threads_lock if we don't already own it
1009 // and if we are not reporting an error.
1010 MutexLockerEx ml((Threads_lock->owned_by_self() || VMError::is_error_reported()) ? NULL : Threads_lock);
1011
1012 st->print_cr("Threads class SMR info:");
1013 st->print_cr("_java_thread_list=" INTPTR_FORMAT ", length=%u, "
1014 "elements={", p2i(_java_thread_list),
1015 _java_thread_list->length());
1016 print_info_elements_on(st, _java_thread_list);
1017 st->print_cr("}");
1018 if (_to_delete_list != NULL) {
1019 st->print_cr("_to_delete_list=" INTPTR_FORMAT ", length=%u, "
1020 "elements={", p2i(_to_delete_list),
1021 _to_delete_list->length());
1022 print_info_elements_on(st, _to_delete_list);
1023 st->print_cr("}");
1024 for (ThreadsList *t_list = _to_delete_list->next_list();
1025 t_list != NULL; t_list = t_list->next_list()) {
1026 st->print("next-> " INTPTR_FORMAT ", length=%u, "
1027 "elements={", p2i(t_list), t_list->length());
1028 print_info_elements_on(st, t_list);
1029 st->print_cr("}");
1030 }
1031 }
1032 if (!EnableThreadSMRStatistics) {
1033 return;
1034 }
1035 st->print_cr("_java_thread_list_alloc_cnt=" UINT64_FORMAT ","
1036 "_java_thread_list_free_cnt=" UINT64_FORMAT ","
1037 "_java_thread_list_max=%u, "
1038 "_nested_thread_list_max=%u",
1039 _java_thread_list_alloc_cnt,
1040 _java_thread_list_free_cnt,
1041 _java_thread_list_max,
1042 _nested_thread_list_max);
1043 if (_tlh_cnt > 0) {
1044 st->print_cr("_tlh_cnt=%u"
1045 ", _tlh_times=%u"
1046 ", avg_tlh_time=%0.2f"
1047 ", _tlh_time_max=%u",
1048 _tlh_cnt, _tlh_times,
1049 ((double) _tlh_times / _tlh_cnt),
1050 _tlh_time_max);
1051 }
1052 if (_deleted_thread_cnt > 0) {
1053 st->print_cr("_deleted_thread_cnt=%u"
1054 ", _deleted_thread_times=%u"
1055 ", avg_deleted_thread_time=%0.2f"
1056 ", _deleted_thread_time_max=%u",
1057 _deleted_thread_cnt, _deleted_thread_times,
1058 ((double) _deleted_thread_times / _deleted_thread_cnt),
1059 _deleted_thread_time_max);
1060 }
1061 st->print_cr("_delete_lock_wait_cnt=%u, _delete_lock_wait_max=%u",
1062 _delete_lock_wait_cnt, _delete_lock_wait_max);
1063 st->print_cr("_to_delete_list_cnt=%u, _to_delete_list_max=%u",
1064 _to_delete_list_cnt, _to_delete_list_max);
1065 }
1066
1067 // Print ThreadsList elements (4 per line).
1068 void ThreadsSMRSupport::print_info_elements_on(outputStream* st, ThreadsList* t_list) {
1069 uint cnt = 0;
1070 JavaThreadIterator jti(t_list);
1071 for (JavaThread *jt = jti.first(); jt != NULL; jt = jti.next()) {
1072 st->print(INTPTR_FORMAT, p2i(jt));
1073 if (cnt < t_list->length() - 1) {
1074 // Separate with comma or comma-space except for the last one.
1075 if (((cnt + 1) % 4) == 0) {
1076 // Four INTPTR_FORMAT fit on an 80 column line so end the
1077 // current line with just a comma.
1078 st->print_cr(",");
1079 } else {
1080 // Not the last one on the current line so use comma-space:
1081 st->print(", ");
1082 }
1083 } else {
1084 // Last one so just end the current line.
1085 st->cr();
1086 }
1087 cnt++;
1088 }
1089 }