1 /*
2 * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "logging/logStream.hpp"
27 #include "memory/allocation.inline.hpp"
28 #include "runtime/jniHandles.inline.hpp"
29 #include "runtime/thread.inline.hpp"
30 #include "runtime/threadSMR.inline.hpp"
31 #include "services/threadService.hpp"
32 #include "utilities/copy.hpp"
33 #include "utilities/globalDefinitions.hpp"
34 #include "utilities/resourceHash.hpp"
35 #include "utilities/vmError.hpp"
36
37 Monitor* ThreadsSMRSupport::_delete_lock =
38 new Monitor(Monitor::special, "Thread_SMR_delete_lock",
39 false /* allow_vm_block */,
40 Monitor::_safepoint_check_never);
41 // The '_cnt', '_max' and '_times" fields are enabled via
42 // -XX:+EnableThreadSMRStatistics:
43
44 // # of parallel threads in _delete_lock->wait().
45 // Impl note: Hard to imagine > 64K waiting threads so this could be 16-bit,
46 // but there is no nice 16-bit _FORMAT support.
47 uint ThreadsSMRSupport::_delete_lock_wait_cnt = 0;
48
49 // Max # of parallel threads in _delete_lock->wait().
50 // Impl note: See _delete_lock_wait_cnt note.
51 uint ThreadsSMRSupport::_delete_lock_wait_max = 0;
52
53 // Flag to indicate when an _delete_lock->notify() is needed.
54 // Impl note: See _delete_lock_wait_cnt note.
55 volatile uint ThreadsSMRSupport::_delete_notify = 0;
56
57 // # of threads deleted over VM lifetime.
58 // Impl note: Atomically incremented over VM lifetime so use unsigned for more
59 // range. Unsigned 64-bit would be more future proof, but 64-bit atomic inc
60 // isn't available everywhere (or is it?).
61 volatile uint ThreadsSMRSupport::_deleted_thread_cnt = 0;
62
63 // Max time in millis to delete a thread.
64 // Impl note: 16-bit might be too small on an overloaded machine. Use
65 // unsigned since this is a time value. Set via Atomic::cmpxchg() in a
66 // loop for correctness.
67 volatile uint ThreadsSMRSupport::_deleted_thread_time_max = 0;
68
69 // Cumulative time in millis to delete threads.
70 // Impl note: Atomically added to over VM lifetime so use unsigned for more
71 // range. Unsigned 64-bit would be more future proof, but 64-bit atomic inc
72 // isn't available everywhere (or is it?).
73 volatile uint ThreadsSMRSupport::_deleted_thread_times = 0;
74
75 ThreadsList* volatile ThreadsSMRSupport::_java_thread_list = new ThreadsList(0);
76
77 // # of ThreadsLists allocated over VM lifetime.
78 // Impl note: We allocate a new ThreadsList for every thread create and
79 // every thread delete so we need a bigger type than the
80 // _deleted_thread_cnt field.
81 uint64_t ThreadsSMRSupport::_java_thread_list_alloc_cnt = 1;
82
83 // # of ThreadsLists freed over VM lifetime.
84 // Impl note: See _java_thread_list_alloc_cnt note.
85 uint64_t ThreadsSMRSupport::_java_thread_list_free_cnt = 0;
86
87 // Max size ThreadsList allocated.
88 // Impl note: Max # of threads alive at one time should fit in unsigned 32-bit.
89 uint ThreadsSMRSupport::_java_thread_list_max = 0;
90
91 // Max # of nested ThreadsLists for a thread.
92 // Impl note: Hard to imagine > 64K nested ThreadsLists so this could be
93 // 16-bit, but there is no nice 16-bit _FORMAT support.
94 uint ThreadsSMRSupport::_nested_thread_list_max = 0;
95
96 // # of ThreadsListHandles deleted over VM lifetime.
97 // Impl note: Atomically incremented over VM lifetime so use unsigned for
98 // more range. There will be fewer ThreadsListHandles than threads so
99 // unsigned 32-bit should be fine.
100 volatile uint ThreadsSMRSupport::_tlh_cnt = 0;
101
102 // Max time in millis to delete a ThreadsListHandle.
103 // Impl note: 16-bit might be too small on an overloaded machine. Use
104 // unsigned since this is a time value. Set via Atomic::cmpxchg() in a
105 // loop for correctness.
106 volatile uint ThreadsSMRSupport::_tlh_time_max = 0;
107
108 // Cumulative time in millis to delete ThreadsListHandles.
109 // Impl note: Atomically added to over VM lifetime so use unsigned for more
110 // range. Unsigned 64-bit would be more future proof, but 64-bit atomic inc
111 // isn't available everywhere (or is it?).
112 volatile uint ThreadsSMRSupport::_tlh_times = 0;
113
114 ThreadsList* ThreadsSMRSupport::_to_delete_list = NULL;
115
116 // # of parallel ThreadsLists on the to-delete list.
117 // Impl note: Hard to imagine > 64K ThreadsLists needing to be deleted so
118 // this could be 16-bit, but there is no nice 16-bit _FORMAT support.
119 uint ThreadsSMRSupport::_to_delete_list_cnt = 0;
120
121 // Max # of parallel ThreadsLists on the to-delete list.
122 // Impl note: See _to_delete_list_cnt note.
123 uint ThreadsSMRSupport::_to_delete_list_max = 0;
124
125
126 // 'inline' functions first so the definitions are before first use:
127
128 inline void ThreadsSMRSupport::add_deleted_thread_times(uint add_value) {
129 Atomic::add(add_value, &_deleted_thread_times);
130 }
131
132 inline void ThreadsSMRSupport::inc_deleted_thread_cnt() {
133 Atomic::inc(&_deleted_thread_cnt);
134 }
135
136 inline void ThreadsSMRSupport::inc_java_thread_list_alloc_cnt() {
137 _java_thread_list_alloc_cnt++;
138 }
139
140 inline void ThreadsSMRSupport::update_deleted_thread_time_max(uint new_value) {
141 while (true) {
142 uint cur_value = _deleted_thread_time_max;
143 if (new_value <= cur_value) {
144 // No need to update max value so we're done.
145 break;
146 }
147 if (Atomic::cmpxchg(new_value, &_deleted_thread_time_max, cur_value) == cur_value) {
148 // Updated max value so we're done. Otherwise try it all again.
149 break;
150 }
151 }
152 }
153
154 inline void ThreadsSMRSupport::update_java_thread_list_max(uint new_value) {
155 if (new_value > _java_thread_list_max) {
156 _java_thread_list_max = new_value;
157 }
158 }
159
160 inline ThreadsList* ThreadsSMRSupport::xchg_java_thread_list(ThreadsList* new_list) {
161 return (ThreadsList*)Atomic::xchg(new_list, &_java_thread_list);
162 }
163
164
165 // Hash table of pointers found by a scan. Used for collecting hazard
166 // pointers (ThreadsList references). Also used for collecting JavaThreads
167 // that are indirectly referenced by hazard ptrs. An instance of this
168 // class only contains one type of pointer.
169 //
170 class ThreadScanHashtable : public CHeapObj<mtThread> {
171 private:
172 static bool ptr_equals(void * const& s1, void * const& s2) {
173 return s1 == s2;
174 }
175
176 static unsigned int ptr_hash(void * const& s1) {
177 // 2654435761 = 2^32 * Phi (golden ratio)
178 return (unsigned int)(((uint32_t)(uintptr_t)s1) * 2654435761u);
179 }
180
181 int _table_size;
182 // ResourceHashtable SIZE is specified at compile time so our
183 // dynamic _table_size is unused for now; 1031 is the first prime
184 // after 1024.
185 typedef ResourceHashtable<void *, int, &ThreadScanHashtable::ptr_hash,
186 &ThreadScanHashtable::ptr_equals, 1031,
187 ResourceObj::C_HEAP, mtThread> PtrTable;
188 PtrTable * _ptrs;
189
190 public:
191 // ResourceHashtable is passed to various functions and populated in
192 // different places so we allocate it using C_HEAP to make it immune
193 // from any ResourceMarks that happen to be in the code paths.
194 ThreadScanHashtable(int table_size) : _table_size(table_size), _ptrs(new (ResourceObj::C_HEAP, mtThread) PtrTable()) {}
195
196 ~ThreadScanHashtable() { delete _ptrs; }
197
198 bool has_entry(void *pointer) {
199 int *val_ptr = _ptrs->get(pointer);
200 return val_ptr != NULL && *val_ptr == 1;
201 }
202
203 void add_entry(void *pointer) {
204 _ptrs->put(pointer, 1);
205 }
206 };
207
208 // Closure to gather JavaThreads indirectly referenced by hazard ptrs
209 // (ThreadsList references) into a hash table. This closure handles part 2
210 // of the dance - adding all the JavaThreads referenced by the hazard
211 // pointer (ThreadsList reference) to the hash table.
212 //
213 class AddThreadHazardPointerThreadClosure : public ThreadClosure {
214 private:
215 ThreadScanHashtable *_table;
216
217 public:
218 AddThreadHazardPointerThreadClosure(ThreadScanHashtable *table) : _table(table) {}
219
220 virtual void do_thread(Thread *thread) {
221 if (!_table->has_entry((void*)thread)) {
222 // The same JavaThread might be on more than one ThreadsList or
223 // more than one thread might be using the same ThreadsList. In
224 // either case, we only need a single entry for a JavaThread.
225 _table->add_entry((void*)thread);
226 }
227 }
228 };
229
230 // Closure to gather JavaThreads indirectly referenced by hazard ptrs
231 // (ThreadsList references) into a hash table. This closure handles part 1
232 // of the dance - hazard ptr chain walking and dispatch to another
233 // closure.
234 //
235 class ScanHazardPtrGatherProtectedThreadsClosure : public ThreadClosure {
236 private:
237 ThreadScanHashtable *_table;
238 public:
239 ScanHazardPtrGatherProtectedThreadsClosure(ThreadScanHashtable *table) : _table(table) {}
240
241 virtual void do_thread(Thread *thread) {
242 assert_locked_or_safepoint(Threads_lock);
243
244 if (thread == NULL) return;
245
246 // This code races with ThreadsSMRSupport::acquire_stable_list() which
247 // is lock-free so we have to handle some special situations.
248 //
249 ThreadsList *current_list = NULL;
250 while (true) {
251 current_list = thread->get_threads_hazard_ptr();
252 // No hazard ptr so nothing more to do.
253 if (current_list == NULL) {
254 assert(thread->get_nested_threads_hazard_ptr() == NULL,
255 "cannot have a nested hazard ptr with a NULL regular hazard ptr");
256 return;
257 }
258
259 // If the hazard ptr is verified as stable (since it is not tagged),
260 // then it is safe to use.
261 if (!Thread::is_hazard_ptr_tagged(current_list)) break;
262
263 // The hazard ptr is tagged as not yet verified as being stable
264 // so we are racing with acquire_stable_list(). This exchange
265 // attempts to invalidate the hazard ptr. If we win the race,
266 // then we can ignore this unstable hazard ptr and the other
267 // thread will retry the attempt to publish a stable hazard ptr.
268 // If we lose the race, then we retry our attempt to look at the
269 // hazard ptr.
270 if (thread->cmpxchg_threads_hazard_ptr(NULL, current_list) == current_list) return;
271 }
272
273 // The current JavaThread has a hazard ptr (ThreadsList reference)
274 // which might be _java_thread_list or it might be an older
275 // ThreadsList that has been removed but not freed. In either case,
276 // the hazard ptr is protecting all the JavaThreads on that
277 // ThreadsList.
278 AddThreadHazardPointerThreadClosure add_cl(_table);
279 current_list->threads_do(&add_cl);
280
281 // Any NestedThreadsLists are also protecting JavaThreads so
282 // gather those also; the ThreadsLists may be different.
283 for (NestedThreadsList* node = thread->get_nested_threads_hazard_ptr();
284 node != NULL; node = node->next()) {
285 node->t_list()->threads_do(&add_cl);
286 }
287 }
288 };
289
290 // Closure to gather hazard ptrs (ThreadsList references) into a hash table.
291 //
292 class ScanHazardPtrGatherThreadsListClosure : public ThreadClosure {
293 private:
294 ThreadScanHashtable *_table;
295 public:
296 ScanHazardPtrGatherThreadsListClosure(ThreadScanHashtable *table) : _table(table) {}
297
298 virtual void do_thread(Thread* thread) {
299 assert_locked_or_safepoint(Threads_lock);
300
301 if (thread == NULL) return;
302 ThreadsList *threads = thread->get_threads_hazard_ptr();
303 if (threads == NULL) {
304 assert(thread->get_nested_threads_hazard_ptr() == NULL,
305 "cannot have a nested hazard ptr with a NULL regular hazard ptr");
306 return;
307 }
308 // In this closure we always ignore the tag that might mark this
309 // hazard ptr as not yet verified. If we happen to catch an
310 // unverified hazard ptr that is subsequently discarded (not
311 // published), then the only side effect is that we might keep a
312 // to-be-deleted ThreadsList alive a little longer.
313 threads = Thread::untag_hazard_ptr(threads);
314 if (!_table->has_entry((void*)threads)) {
315 _table->add_entry((void*)threads);
316 }
317
318 // Any NestedThreadsLists are also protecting JavaThreads so
319 // gather those also; the ThreadsLists may be different.
320 for (NestedThreadsList* node = thread->get_nested_threads_hazard_ptr();
321 node != NULL; node = node->next()) {
322 threads = node->t_list();
323 if (!_table->has_entry((void*)threads)) {
324 _table->add_entry((void*)threads);
325 }
326 }
327 }
328 };
329
330 // Closure to print JavaThreads that have a hazard ptr (ThreadsList
331 // reference) that contains an indirect reference to a specific JavaThread.
332 //
333 class ScanHazardPtrPrintMatchingThreadsClosure : public ThreadClosure {
334 private:
335 JavaThread *_thread;
336 public:
337 ScanHazardPtrPrintMatchingThreadsClosure(JavaThread *thread) : _thread(thread) {}
338
339 virtual void do_thread(Thread *thread) {
340 assert_locked_or_safepoint(Threads_lock);
341
342 if (thread == NULL) return;
343 ThreadsList *current_list = thread->get_threads_hazard_ptr();
344 if (current_list == NULL) {
345 assert(thread->get_nested_threads_hazard_ptr() == NULL,
346 "cannot have a nested hazard ptr with a NULL regular hazard ptr");
347 return;
348 }
349 // If the hazard ptr is unverified, then ignore it.
350 if (Thread::is_hazard_ptr_tagged(current_list)) return;
351
352 // The current JavaThread has a hazard ptr (ThreadsList reference)
353 // which might be _java_thread_list or it might be an older
354 // ThreadsList that has been removed but not freed. In either case,
355 // the hazard ptr is protecting all the JavaThreads on that
356 // ThreadsList, but we only care about matching a specific JavaThread.
357 JavaThreadIterator jti(current_list);
358 for (JavaThread *p = jti.first(); p != NULL; p = jti.next()) {
359 if (p == _thread) {
360 log_debug(thread, smr)("tid=" UINTX_FORMAT ": ThreadsSMRSupport::smr_delete: thread1=" INTPTR_FORMAT " has a hazard pointer for thread2=" INTPTR_FORMAT, os::current_thread_id(), p2i(thread), p2i(_thread));
361 break;
362 }
363 }
364
365 // Any NestedThreadsLists are also protecting JavaThreads so
366 // check those also; the ThreadsLists may be different.
367 for (NestedThreadsList* node = thread->get_nested_threads_hazard_ptr();
368 node != NULL; node = node->next()) {
369 JavaThreadIterator jti(node->t_list());
370 for (JavaThread *p = jti.first(); p != NULL; p = jti.next()) {
371 if (p == _thread) {
372 log_debug(thread, smr)("tid=" UINTX_FORMAT ": ThreadsSMRSupport::smr_delete: thread1=" INTPTR_FORMAT " has a nested hazard pointer for thread2=" INTPTR_FORMAT, os::current_thread_id(), p2i(thread), p2i(_thread));
373 return;
374 }
375 }
376 }
377 }
378 };
379
380
381 // 'entries + 1' so we always have at least one entry.
382 ThreadsList::ThreadsList(int entries) : _length(entries), _threads(NEW_C_HEAP_ARRAY(JavaThread*, entries + 1, mtThread)), _next_list(NULL) {
383 *(JavaThread**)(_threads + entries) = NULL; // Make sure the extra entry is NULL.
384 }
385
386 ThreadsList::~ThreadsList() {
387 FREE_C_HEAP_ARRAY(JavaThread*, _threads);
388 }
389
390 // Add a JavaThread to a ThreadsList. The returned ThreadsList is a
391 // new copy of the specified ThreadsList with the specified JavaThread
392 // appended to the end.
393 ThreadsList *ThreadsList::add_thread(ThreadsList *list, JavaThread *java_thread) {
394 const uint index = list->_length;
395 const uint new_length = index + 1;
396 const uint head_length = index;
397 ThreadsList *const new_list = new ThreadsList(new_length);
398
399 if (head_length > 0) {
400 Copy::disjoint_words((HeapWord*)list->_threads, (HeapWord*)new_list->_threads, head_length);
401 }
402 *(JavaThread**)(new_list->_threads + index) = java_thread;
403
404 return new_list;
405 }
406
407 int ThreadsList::find_index_of_JavaThread(JavaThread *target) {
408 if (target == NULL) {
409 return -1;
410 }
411 for (uint i = 0; i < length(); i++) {
412 if (target == thread_at(i)) {
413 return (int)i;
414 }
415 }
416 return -1;
417 }
418
419 JavaThread* ThreadsList::find_JavaThread_from_java_tid(jlong java_tid) const {
420 for (uint i = 0; i < length(); i++) {
421 JavaThread* thread = thread_at(i);
422 oop tobj = thread->threadObj();
423 // Ignore the thread if it hasn't run yet, has exited
424 // or is starting to exit.
425 if (tobj != NULL && !thread->is_exiting() &&
426 java_tid == java_lang_Thread::thread_id(tobj)) {
427 // found a match
428 return thread;
429 }
430 }
431 return NULL;
432 }
433
434 bool ThreadsList::includes(const JavaThread * const p) const {
435 if (p == NULL) {
436 return false;
437 }
438 for (uint i = 0; i < length(); i++) {
439 if (thread_at(i) == p) {
440 return true;
441 }
442 }
443 return false;
444 }
445
446 // Remove a JavaThread from a ThreadsList. The returned ThreadsList is a
447 // new copy of the specified ThreadsList with the specified JavaThread
448 // removed.
449 ThreadsList *ThreadsList::remove_thread(ThreadsList* list, JavaThread* java_thread) {
450 assert(list->_length > 0, "sanity");
451
452 uint i = (uint)list->find_index_of_JavaThread(java_thread);
453 assert(i < list->_length, "did not find JavaThread on the list");
454 const uint index = i;
455 const uint new_length = list->_length - 1;
456 const uint head_length = index;
457 const uint tail_length = (new_length >= index) ? (new_length - index) : 0;
458 ThreadsList *const new_list = new ThreadsList(new_length);
459
460 if (head_length > 0) {
461 Copy::disjoint_words((HeapWord*)list->_threads, (HeapWord*)new_list->_threads, head_length);
462 }
463 if (tail_length > 0) {
464 Copy::disjoint_words((HeapWord*)list->_threads + index + 1, (HeapWord*)new_list->_threads + index, tail_length);
465 }
466
467 return new_list;
468 }
469
470 ThreadsListHandle::ThreadsListHandle(Thread *self) : _list(ThreadsSMRSupport::acquire_stable_list(self, /* is_ThreadsListSetter */ false)), _self(self) {
471 assert(self == Thread::current(), "sanity check");
472 if (EnableThreadSMRStatistics) {
473 _timer.start();
474 }
475 }
476
477 ThreadsListHandle::~ThreadsListHandle() {
478 ThreadsSMRSupport::release_stable_list(_self);
479 if (EnableThreadSMRStatistics) {
480 _timer.stop();
481 uint millis = (uint)_timer.milliseconds();
482 ThreadsSMRSupport::update_tlh_stats(millis);
483 }
484 }
485
486 // Convert an internal thread reference to a JavaThread found on the
487 // associated ThreadsList. This ThreadsListHandle "protects" the
488 // returned JavaThread *.
489 //
490 // If thread_oop_p is not NULL, then the caller wants to use the oop
491 // after this call so the oop is returned. On success, *jt_pp is set
492 // to the converted JavaThread * and true is returned. On error,
493 // returns false.
494 //
495 bool ThreadsListHandle::cv_internal_thread_to_JavaThread(jobject jthread,
496 JavaThread ** jt_pp,
497 oop * thread_oop_p) {
498 assert(this->list() != NULL, "must have a ThreadsList");
499 assert(jt_pp != NULL, "must have a return JavaThread pointer");
500 // thread_oop_p is optional so no assert()
501
502 // The JVM_* interfaces don't allow a NULL thread parameter; JVM/TI
503 // allows a NULL thread parameter to signify "current thread" which
504 // allows us to avoid calling cv_external_thread_to_JavaThread().
505 // The JVM_* interfaces have no such leeway.
506
507 oop thread_oop = JNIHandles::resolve_non_null(jthread);
508 // Looks like an oop at this point.
509 if (thread_oop_p != NULL) {
510 // Return the oop to the caller; the caller may still want
511 // the oop even if this function returns false.
512 *thread_oop_p = thread_oop;
513 }
514
515 JavaThread *java_thread = java_lang_Thread::thread(thread_oop);
516 if (java_thread == NULL) {
517 // The java.lang.Thread does not contain a JavaThread * so it has
518 // not yet run or it has died.
519 return false;
520 }
521 // Looks like a live JavaThread at this point.
522
523 if (java_thread != JavaThread::current()) {
524 // jthread is not for the current JavaThread so have to verify
525 // the JavaThread * against the ThreadsList.
526 if (EnableThreadSMRExtraValidityChecks && !includes(java_thread)) {
527 // Not on the JavaThreads list so it is not alive.
528 return false;
529 }
530 }
531
532 // Return a live JavaThread that is "protected" by the
533 // ThreadsListHandle in the caller.
534 *jt_pp = java_thread;
535 return true;
536 }
537
538 ThreadsListSetter::~ThreadsListSetter() {
539 if (_target_needs_release) {
540 // The hazard ptr in the target needs to be released.
541 ThreadsSMRSupport::release_stable_list(_target);
542 }
543 }
544
545 void ThreadsListSetter::set() {
546 assert(_target->get_threads_hazard_ptr() == NULL, "hazard ptr should not already be set");
547 (void) ThreadsSMRSupport::acquire_stable_list(_target, /* is_ThreadsListSetter */ true);
548 _target_needs_release = true;
549 }
550
551 // Acquire a stable ThreadsList.
552 //
553 ThreadsList *ThreadsSMRSupport::acquire_stable_list(Thread *self, bool is_ThreadsListSetter) {
554 assert(self != NULL, "sanity check");
555 // acquire_stable_list_nested_path() will grab the Threads_lock
556 // so let's make sure the ThreadsListHandle is in a safe place.
557 // ThreadsListSetter cannot make this check on this code path.
558 debug_only(if (!is_ThreadsListSetter && StrictSafepointChecks) self->check_for_valid_safepoint_state(/* potential_vm_operation */ false);)
559
560 if (self->get_threads_hazard_ptr() == NULL) {
561 // The typical case is first.
562 return acquire_stable_list_fast_path(self);
563 }
564
565 // The nested case is rare.
566 return acquire_stable_list_nested_path(self);
567 }
568
569 // Fast path (and lock free) way to acquire a stable ThreadsList.
570 //
571 ThreadsList *ThreadsSMRSupport::acquire_stable_list_fast_path(Thread *self) {
572 assert(self != NULL, "sanity check");
573 assert(self->get_threads_hazard_ptr() == NULL, "sanity check");
574 assert(self->get_nested_threads_hazard_ptr() == NULL,
575 "cannot have a nested hazard ptr with a NULL regular hazard ptr");
576
577 ThreadsList* threads;
578
579 // Stable recording of a hazard ptr for SMR. This code does not use
580 // locks so its use of the _java_thread_list & _threads_hazard_ptr
581 // fields is racy relative to code that uses those fields with locks.
582 // OrderAccess and Atomic functions are used to deal with those races.
583 //
584 while (true) {
585 threads = get_java_thread_list();
586
587 // Publish a tagged hazard ptr to denote that the hazard ptr is not
588 // yet verified as being stable. Due to the fence after the hazard
589 // ptr write, it will be sequentially consistent w.r.t. the
590 // sequentially consistent writes of the ThreadsList, even on
591 // non-multiple copy atomic machines where stores can be observed
592 // in different order from different observer threads.
593 ThreadsList* unverified_threads = Thread::tag_hazard_ptr(threads);
594 self->set_threads_hazard_ptr(unverified_threads);
595
596 // If _java_thread_list has changed, we have lost a race with
597 // Threads::add() or Threads::remove() and have to try again.
598 if (get_java_thread_list() != threads) {
599 continue;
600 }
601
602 // We try to remove the tag which will verify the hazard ptr as
603 // being stable. This exchange can race with a scanning thread
604 // which might invalidate the tagged hazard ptr to keep it from
605 // being followed to access JavaThread ptrs. If we lose the race,
606 // we simply retry. If we win the race, then the stable hazard
607 // ptr is officially published.
608 if (self->cmpxchg_threads_hazard_ptr(threads, unverified_threads) == unverified_threads) {
609 break;
610 }
611 }
612
613 // A stable hazard ptr has been published letting other threads know
614 // that the ThreadsList and the JavaThreads reachable from this list
615 // are protected and hence they should not be deleted until everyone
616 // agrees it is safe to do so.
617
618 return threads;
619 }
620
621 // Acquire a nested stable ThreadsList; this is rare so it uses
622 // Threads_lock.
623 //
624 ThreadsList *ThreadsSMRSupport::acquire_stable_list_nested_path(Thread *self) {
625 assert(self != NULL, "sanity check");
626 assert(self->get_threads_hazard_ptr() != NULL,
627 "cannot have a NULL regular hazard ptr when acquiring a nested hazard ptr");
628
629 // The thread already has a hazard ptr (ThreadsList ref) so we need
630 // to create a nested ThreadsListHandle with the current ThreadsList
631 // since it might be different than our current hazard ptr. The need
632 // for a nested ThreadsListHandle is rare so we do this while holding
633 // the Threads_lock so we don't race with the scanning code; the code
634 // is so much simpler this way.
635
636 NestedThreadsList* node;
637 {
638 // Only grab the Threads_lock if we don't already own it.
639 MutexLockerEx ml(Threads_lock->owned_by_self() ? NULL : Threads_lock);
640 node = new NestedThreadsList(get_java_thread_list());
641 // We insert at the front of the list to match up with the delete
642 // in release_stable_list().
643 node->set_next(self->get_nested_threads_hazard_ptr());
644 self->set_nested_threads_hazard_ptr(node);
645 if (EnableThreadSMRStatistics) {
646 self->inc_nested_threads_hazard_ptr_cnt();
647 if (self->nested_threads_hazard_ptr_cnt() > _nested_thread_list_max) {
648 _nested_thread_list_max = self->nested_threads_hazard_ptr_cnt();
649 }
650 }
651 }
652 log_debug(thread, smr)("tid=" UINTX_FORMAT ": ThreadsSMRSupport::acquire_stable_list: add NestedThreadsList node containing ThreadsList=" INTPTR_FORMAT, os::current_thread_id(), p2i(node->t_list()));
653
654 return node->t_list();
655 }
656
657 void ThreadsSMRSupport::add_thread(JavaThread *thread){
658 ThreadsList *new_list = ThreadsList::add_thread(ThreadsSMRSupport::get_java_thread_list(), thread);
659 if (EnableThreadSMRStatistics) {
660 ThreadsSMRSupport::inc_java_thread_list_alloc_cnt();
661 ThreadsSMRSupport::update_java_thread_list_max(new_list->length());
662 }
663 // Initial _java_thread_list will not generate a "Threads::add" mesg.
664 log_debug(thread, smr)("tid=" UINTX_FORMAT ": Threads::add: new ThreadsList=" INTPTR_FORMAT, os::current_thread_id(), p2i(new_list));
665
666 ThreadsList *old_list = ThreadsSMRSupport::xchg_java_thread_list(new_list);
667 ThreadsSMRSupport::free_list(old_list);
668 }
669
670 // set_delete_notify() and clear_delete_notify() are called
671 // under the protection of the delete_lock, but we also use an
672 // Atomic operation to ensure the memory update is seen earlier than
673 // when the delete_lock is dropped.
674 //
675 void ThreadsSMRSupport::clear_delete_notify() {
676 Atomic::dec(&_delete_notify);
677 }
678
679 bool ThreadsSMRSupport::delete_notify() {
680 // Use load_acquire() in order to see any updates to _delete_notify
681 // earlier than when delete_lock is grabbed.
682 return (OrderAccess::load_acquire(&_delete_notify) != 0);
683 }
684
685 // Safely free a ThreadsList after a Threads::add() or Threads::remove().
686 // The specified ThreadsList may not get deleted during this call if it
687 // is still in-use (referenced by a hazard ptr). Other ThreadsLists
688 // in the chain may get deleted by this call if they are no longer in-use.
689 void ThreadsSMRSupport::free_list(ThreadsList* threads) {
690 assert_locked_or_safepoint(Threads_lock);
691
692 threads->set_next_list(_to_delete_list);
693 _to_delete_list = threads;
694 if (EnableThreadSMRStatistics) {
695 _to_delete_list_cnt++;
696 if (_to_delete_list_cnt > _to_delete_list_max) {
697 _to_delete_list_max = _to_delete_list_cnt;
698 }
699 }
700
701 // Hash table size should be first power of two higher than twice the length of the ThreadsList
702 int hash_table_size = MIN2((int)get_java_thread_list()->length(), 32) << 1;
703 hash_table_size--;
704 hash_table_size |= hash_table_size >> 1;
705 hash_table_size |= hash_table_size >> 2;
706 hash_table_size |= hash_table_size >> 4;
707 hash_table_size |= hash_table_size >> 8;
708 hash_table_size |= hash_table_size >> 16;
709 hash_table_size++;
710
711 // Gather a hash table of the current hazard ptrs:
712 ThreadScanHashtable *scan_table = new ThreadScanHashtable(hash_table_size);
713 ScanHazardPtrGatherThreadsListClosure scan_cl(scan_table);
714 Threads::threads_do(&scan_cl);
715
716 // Walk through the linked list of pending freeable ThreadsLists
717 // and free the ones that are not referenced from hazard ptrs.
718 ThreadsList* current = _to_delete_list;
719 ThreadsList* prev = NULL;
720 ThreadsList* next = NULL;
721 bool threads_is_freed = false;
722 while (current != NULL) {
723 next = current->next_list();
724 if (!scan_table->has_entry((void*)current)) {
725 // This ThreadsList is not referenced by a hazard ptr.
726 if (prev != NULL) {
727 prev->set_next_list(next);
728 }
729 if (_to_delete_list == current) {
730 _to_delete_list = next;
731 }
732
733 log_debug(thread, smr)("tid=" UINTX_FORMAT ": ThreadsSMRSupport::free_list: threads=" INTPTR_FORMAT " is freed.", os::current_thread_id(), p2i(current));
734 if (current == threads) threads_is_freed = true;
735 delete current;
736 if (EnableThreadSMRStatistics) {
737 _java_thread_list_free_cnt++;
738 _to_delete_list_cnt--;
739 }
740 } else {
741 prev = current;
742 }
743 current = next;
744 }
745
746 if (!threads_is_freed) {
747 // Only report "is not freed" on the original call to
748 // free_list() for this ThreadsList.
749 log_debug(thread, smr)("tid=" UINTX_FORMAT ": ThreadsSMRSupport::free_list: threads=" INTPTR_FORMAT " is not freed.", os::current_thread_id(), p2i(threads));
750 }
751
752 delete scan_table;
753 }
754
755 // Return true if the specified JavaThread is protected by a hazard
756 // pointer (ThreadsList reference). Otherwise, returns false.
757 //
758 bool ThreadsSMRSupport::is_a_protected_JavaThread(JavaThread *thread) {
759 assert_locked_or_safepoint(Threads_lock);
760
761 // Hash table size should be first power of two higher than twice
762 // the length of the Threads list.
763 int hash_table_size = MIN2((int)get_java_thread_list()->length(), 32) << 1;
764 hash_table_size--;
765 hash_table_size |= hash_table_size >> 1;
766 hash_table_size |= hash_table_size >> 2;
767 hash_table_size |= hash_table_size >> 4;
768 hash_table_size |= hash_table_size >> 8;
769 hash_table_size |= hash_table_size >> 16;
770 hash_table_size++;
771
772 // Gather a hash table of the JavaThreads indirectly referenced by
773 // hazard ptrs.
774 ThreadScanHashtable *scan_table = new ThreadScanHashtable(hash_table_size);
775 ScanHazardPtrGatherProtectedThreadsClosure scan_cl(scan_table);
776 Threads::threads_do(&scan_cl);
777
778 bool thread_is_protected = false;
779 if (scan_table->has_entry((void*)thread)) {
780 thread_is_protected = true;
781 }
782 delete scan_table;
783 return thread_is_protected;
784 }
785
786 // Release a stable ThreadsList.
787 //
788 void ThreadsSMRSupport::release_stable_list(Thread *self) {
789 assert(self != NULL, "sanity check");
790 // release_stable_list_nested_path() will grab the Threads_lock
791 // so let's make sure the ThreadsListHandle is in a safe place.
792 debug_only(if (StrictSafepointChecks) self->check_for_valid_safepoint_state(/* potential_vm_operation */ false);)
793
794 if (self->get_nested_threads_hazard_ptr() == NULL) {
795 // The typical case is first.
796 release_stable_list_fast_path(self);
797 return;
798 }
799
800 // The nested case is rare.
801 release_stable_list_nested_path(self);
802 }
803
804 // Fast path way to release a stable ThreadsList. The release portion
805 // is lock-free, but the wake up portion is not.
806 //
807 void ThreadsSMRSupport::release_stable_list_fast_path(Thread *self) {
808 assert(self != NULL, "sanity check");
809 assert(self->get_threads_hazard_ptr() != NULL, "sanity check");
810 assert(self->get_nested_threads_hazard_ptr() == NULL,
811 "cannot have a nested hazard ptr when releasing a regular hazard ptr");
812
813 // After releasing the hazard ptr, other threads may go ahead and
814 // free up some memory temporarily used by a ThreadsList snapshot.
815 self->set_threads_hazard_ptr(NULL);
816
817 // We use double-check locking to reduce traffic on the system
818 // wide Thread-SMR delete_lock.
819 if (ThreadsSMRSupport::delete_notify()) {
820 // An exiting thread might be waiting in smr_delete(); we need to
821 // check with delete_lock to be sure.
822 release_stable_list_wake_up((char *) "regular hazard ptr");
823 }
824 }
825
826 // Release a nested stable ThreadsList; this is rare so it uses
827 // Threads_lock.
828 //
829 void ThreadsSMRSupport::release_stable_list_nested_path(Thread *self) {
830 assert(self != NULL, "sanity check");
831 assert(self->get_nested_threads_hazard_ptr() != NULL, "sanity check");
832 assert(self->get_threads_hazard_ptr() != NULL,
833 "must have a regular hazard ptr to have nested hazard ptrs");
834
835 // We have a nested ThreadsListHandle so we have to release it first.
836 // The need for a nested ThreadsListHandle is rare so we do this while
837 // holding the Threads_lock so we don't race with the scanning code;
838 // the code is so much simpler this way.
839
840 NestedThreadsList *node;
841 {
842 // Only grab the Threads_lock if we don't already own it.
843 MutexLockerEx ml(Threads_lock->owned_by_self() ? NULL : Threads_lock);
844 // We remove from the front of the list to match up with the insert
845 // in acquire_stable_list().
846 node = self->get_nested_threads_hazard_ptr();
847 self->set_nested_threads_hazard_ptr(node->next());
848 if (EnableThreadSMRStatistics) {
849 self->dec_nested_threads_hazard_ptr_cnt();
850 }
851 }
852
853 // An exiting thread might be waiting in smr_delete(); we need to
854 // check with delete_lock to be sure.
855 release_stable_list_wake_up((char *) "nested hazard ptr");
856
857 log_debug(thread, smr)("tid=" UINTX_FORMAT ": ThreadsSMRSupport::release_stable_list: delete NestedThreadsList node containing ThreadsList=" INTPTR_FORMAT, os::current_thread_id(), p2i(node->t_list()));
858
859 delete node;
860 }
861
862 // Wake up portion of the release stable ThreadsList protocol;
863 // uses the delete_lock().
864 //
865 void ThreadsSMRSupport::release_stable_list_wake_up(char *log_str) {
866 assert(log_str != NULL, "sanity check");
867
868 // Note: delete_lock is held in smr_delete() for the entire
869 // hazard ptr search so that we do not lose this notify() if
870 // the exiting thread has to wait. That code path also holds
871 // Threads_lock (which was grabbed before delete_lock) so that
872 // threads_do() can be called. This means the system can't start a
873 // safepoint which means this thread can't take too long to get to
874 // a safepoint because of being blocked on delete_lock.
875 //
876 MonitorLockerEx ml(ThreadsSMRSupport::delete_lock(), Monitor::_no_safepoint_check_flag);
877 if (ThreadsSMRSupport::delete_notify()) {
878 // Notify any exiting JavaThreads that are waiting in smr_delete()
879 // that we've released a ThreadsList.
880 ml.notify_all();
881 log_debug(thread, smr)("tid=" UINTX_FORMAT ": ThreadsSMRSupport::release_stable_list notified %s", os::current_thread_id(), log_str);
882 }
883 }
884
885 void ThreadsSMRSupport::remove_thread(JavaThread *thread) {
886 ThreadsList *new_list = ThreadsList::remove_thread(ThreadsSMRSupport::get_java_thread_list(), thread);
887 if (EnableThreadSMRStatistics) {
888 ThreadsSMRSupport::inc_java_thread_list_alloc_cnt();
889 // This list is smaller so no need to check for a "longest" update.
890 }
891
892 // Final _java_thread_list will not generate a "Threads::remove" mesg.
893 log_debug(thread, smr)("tid=" UINTX_FORMAT ": Threads::remove: new ThreadsList=" INTPTR_FORMAT, os::current_thread_id(), p2i(new_list));
894
895 ThreadsList *old_list = ThreadsSMRSupport::xchg_java_thread_list(new_list);
896 ThreadsSMRSupport::free_list(old_list);
897 }
898
899 // See note for clear_delete_notify().
900 //
901 void ThreadsSMRSupport::set_delete_notify() {
902 Atomic::inc(&_delete_notify);
903 }
904
905 // Safely delete a JavaThread when it is no longer in use by a
906 // ThreadsListHandle.
907 //
908 void ThreadsSMRSupport::smr_delete(JavaThread *thread) {
909 assert(!Threads_lock->owned_by_self(), "sanity");
910
911 bool has_logged_once = false;
912 elapsedTimer timer;
913 if (EnableThreadSMRStatistics) {
914 timer.start();
915 }
916
917 while (true) {
918 {
919 // No safepoint check because this JavaThread is not on the
920 // Threads list.
921 MutexLockerEx ml(Threads_lock, Mutex::_no_safepoint_check_flag);
922 // Cannot use a MonitorLockerEx helper here because we have
923 // to drop the Threads_lock first if we wait.
924 ThreadsSMRSupport::delete_lock()->lock_without_safepoint_check();
925 // Set the delete_notify flag after we grab delete_lock
926 // and before we scan hazard ptrs because we're doing
927 // double-check locking in release_stable_list().
928 ThreadsSMRSupport::set_delete_notify();
929
930 if (!is_a_protected_JavaThread(thread)) {
931 // This is the common case.
932 ThreadsSMRSupport::clear_delete_notify();
933 ThreadsSMRSupport::delete_lock()->unlock();
934 break;
935 }
936 if (!has_logged_once) {
937 has_logged_once = true;
938 log_debug(thread, smr)("tid=" UINTX_FORMAT ": ThreadsSMRSupport::smr_delete: thread=" INTPTR_FORMAT " is not deleted.", os::current_thread_id(), p2i(thread));
939 if (log_is_enabled(Debug, os, thread)) {
940 ScanHazardPtrPrintMatchingThreadsClosure scan_cl(thread);
941 Threads::threads_do(&scan_cl);
942 }
943 }
944 } // We have to drop the Threads_lock to wait or delete the thread
945
946 if (EnableThreadSMRStatistics) {
947 _delete_lock_wait_cnt++;
948 if (_delete_lock_wait_cnt > _delete_lock_wait_max) {
949 _delete_lock_wait_max = _delete_lock_wait_cnt;
950 }
951 }
952 // Wait for a release_stable_list() call before we check again. No
953 // safepoint check, no timeout, and not as suspend equivalent flag
954 // because this JavaThread is not on the Threads list.
955 ThreadsSMRSupport::delete_lock()->wait(Mutex::_no_safepoint_check_flag, 0,
956 !Mutex::_as_suspend_equivalent_flag);
957 if (EnableThreadSMRStatistics) {
958 _delete_lock_wait_cnt--;
959 }
960
961 ThreadsSMRSupport::clear_delete_notify();
962 ThreadsSMRSupport::delete_lock()->unlock();
963 // Retry the whole scenario.
964 }
965
966 if (ThreadLocalHandshakes) {
967 // The thread is about to be deleted so cancel any handshake.
968 thread->cancel_handshake();
969 }
970
971 delete thread;
972 if (EnableThreadSMRStatistics) {
973 timer.stop();
974 uint millis = (uint)timer.milliseconds();
975 ThreadsSMRSupport::inc_deleted_thread_cnt();
976 ThreadsSMRSupport::add_deleted_thread_times(millis);
977 ThreadsSMRSupport::update_deleted_thread_time_max(millis);
978 }
979
980 log_debug(thread, smr)("tid=" UINTX_FORMAT ": ThreadsSMRSupport::smr_delete: thread=" INTPTR_FORMAT " is deleted.", os::current_thread_id(), p2i(thread));
981 }
982
983
984 // Debug, logging, and printing stuff at the end:
985
986 // Log Threads class SMR info.
987 void ThreadsSMRSupport::log_statistics() {
988 LogTarget(Info, thread, smr) log;
989 if (log.is_enabled()) {
990 LogStream out(log);
991 print_info_on(&out);
992 }
993 }
994
995 // Print Threads class SMR info.
996 void ThreadsSMRSupport::print_info_on(outputStream* st) {
997 // Only grab the Threads_lock if we don't already own it
998 // and if we are not reporting an error.
999 MutexLockerEx ml((Threads_lock->owned_by_self() || VMError::is_error_reported()) ? NULL : Threads_lock);
1000
1001 st->print_cr("Threads class SMR info:");
1002 st->print_cr("_java_thread_list=" INTPTR_FORMAT ", length=%u, "
1003 "elements={", p2i(_java_thread_list),
1004 _java_thread_list->length());
1005 print_info_elements_on(st, _java_thread_list);
1006 st->print_cr("}");
1007 if (_to_delete_list != NULL) {
1008 st->print_cr("_to_delete_list=" INTPTR_FORMAT ", length=%u, "
1009 "elements={", p2i(_to_delete_list),
1010 _to_delete_list->length());
1011 print_info_elements_on(st, _to_delete_list);
1012 st->print_cr("}");
1013 for (ThreadsList *t_list = _to_delete_list->next_list();
1014 t_list != NULL; t_list = t_list->next_list()) {
1015 st->print("next-> " INTPTR_FORMAT ", length=%u, "
1016 "elements={", p2i(t_list), t_list->length());
1017 print_info_elements_on(st, t_list);
1018 st->print_cr("}");
1019 }
1020 }
1021 if (!EnableThreadSMRStatistics) {
1022 return;
1023 }
1024 st->print_cr("_java_thread_list_alloc_cnt=" UINT64_FORMAT ","
1025 "_java_thread_list_free_cnt=" UINT64_FORMAT ","
1026 "_java_thread_list_max=%u, "
1027 "_nested_thread_list_max=%u",
1028 _java_thread_list_alloc_cnt,
1029 _java_thread_list_free_cnt,
1030 _java_thread_list_max,
1031 _nested_thread_list_max);
1032 if (_tlh_cnt > 0) {
1033 st->print_cr("_tlh_cnt=%u"
1034 ", _tlh_times=%u"
1035 ", avg_tlh_time=%0.2f"
1036 ", _tlh_time_max=%u",
1037 _tlh_cnt, _tlh_times,
1038 ((double) _tlh_times / _tlh_cnt),
1039 _tlh_time_max);
1040 }
1041 if (_deleted_thread_cnt > 0) {
1042 st->print_cr("_deleted_thread_cnt=%u"
1043 ", _deleted_thread_times=%u"
1044 ", avg_deleted_thread_time=%0.2f"
1045 ", _deleted_thread_time_max=%u",
1046 _deleted_thread_cnt, _deleted_thread_times,
1047 ((double) _deleted_thread_times / _deleted_thread_cnt),
1048 _deleted_thread_time_max);
1049 }
1050 st->print_cr("_delete_lock_wait_cnt=%u, _delete_lock_wait_max=%u",
1051 _delete_lock_wait_cnt, _delete_lock_wait_max);
1052 st->print_cr("_to_delete_list_cnt=%u, _to_delete_list_max=%u",
1053 _to_delete_list_cnt, _to_delete_list_max);
1054 }
1055
1056 // Print ThreadsList elements (4 per line).
1057 void ThreadsSMRSupport::print_info_elements_on(outputStream* st, ThreadsList* t_list) {
1058 uint cnt = 0;
1059 JavaThreadIterator jti(t_list);
1060 for (JavaThread *jt = jti.first(); jt != NULL; jt = jti.next()) {
1061 st->print(INTPTR_FORMAT, p2i(jt));
1062 if (cnt < t_list->length() - 1) {
1063 // Separate with comma or comma-space except for the last one.
1064 if (((cnt + 1) % 4) == 0) {
1065 // Four INTPTR_FORMAT fit on an 80 column line so end the
1066 // current line with just a comma.
1067 st->print_cr(",");
1068 } else {
1069 // Not the last one on the current line so use comma-space:
1070 st->print(", ");
1071 }
1072 } else {
1073 // Last one so just end the current line.
1074 st->cr();
1075 }
1076 cnt++;
1077 }
1078 }