1 /*
2 * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 #include "precompiled.hpp"
25 #include "classfile/classLoaderDataGraph.hpp"
26 #include "classfile/stringTable.hpp"
27 #include "code/codeCache.hpp"
28 #include "compiler/oopMap.hpp"
29 #include "gc/shared/barrierSet.hpp"
30 #include "gc/shared/barrierSetNMethod.hpp"
31 #include "gc/shared/oopStorageSet.hpp"
32 #include "gc/shared/oopStorageParState.inline.hpp"
33 #include "gc/shared/oopStorageSet.hpp"
34 #include "gc/shared/suspendibleThreadSet.hpp"
35 #include "gc/z/zBarrierSetNMethod.hpp"
36 #include "gc/z/zGlobals.hpp"
37 #include "gc/z/zNMethod.hpp"
38 #include "gc/z/zOopClosures.inline.hpp"
39 #include "gc/z/zRootsIterator.hpp"
40 #include "gc/z/zStat.hpp"
41 #include "gc/z/zThreadLocalData.hpp"
42 #include "memory/iterator.hpp"
43 #include "memory/resourceArea.hpp"
44 #include "memory/universe.hpp"
45 #include "prims/jvmtiExport.hpp"
46 #include "prims/resolvedMethodTable.hpp"
47 #include "runtime/atomic.hpp"
48 #include "runtime/safepoint.hpp"
49 #include "runtime/synchronizer.hpp"
50 #include "runtime/thread.hpp"
51 #include "runtime/vmThread.hpp"
52 #include "services/management.hpp"
53 #include "utilities/debug.hpp"
54 #if INCLUDE_JFR
55 #include "jfr/jfr.hpp"
56 #endif
57
58 static const ZStatSubPhase ZSubPhasePauseRootsSetup("Pause Roots Setup");
59 static const ZStatSubPhase ZSubPhasePauseRoots("Pause Roots");
60 static const ZStatSubPhase ZSubPhasePauseRootsTeardown("Pause Roots Teardown");
61 static const ZStatSubPhase ZSubPhasePauseRootsUniverse("Pause Roots Universe");
62 static const ZStatSubPhase ZSubPhasePauseRootsObjectSynchronizer("Pause Roots ObjectSynchronizer");
63 static const ZStatSubPhase ZSubPhasePauseRootsManagement("Pause Roots Management");
64 static const ZStatSubPhase ZSubPhasePauseRootsJVMTIExport("Pause Roots JVMTIExport");
65 static const ZStatSubPhase ZSubPhasePauseRootsJVMTIWeakExport("Pause Roots JVMTIWeakExport");
66 static const ZStatSubPhase ZSubPhasePauseRootsVMThread("Pause Roots VM Thread");
67 static const ZStatSubPhase ZSubPhasePauseRootsJavaThreads("Pause Roots Java Threads");
68 static const ZStatSubPhase ZSubPhasePauseRootsCodeCache("Pause Roots CodeCache");
69
70 static const ZStatSubPhase ZSubPhaseConcurrentRootsSetup("Concurrent Roots Setup");
71 static const ZStatSubPhase ZSubPhaseConcurrentRoots("Concurrent Roots");
72 static const ZStatSubPhase ZSubPhaseConcurrentRootsTeardown("Concurrent Roots Teardown");
73 static const ZStatSubPhase ZSubPhaseConcurrentRootsJNIHandles("Concurrent Roots JNIHandles");
74 static const ZStatSubPhase ZSubPhaseConcurrentRootsVMHandles("Concurrent Roots VMHandles");
75 static const ZStatSubPhase ZSubPhaseConcurrentRootsClassLoaderDataGraph("Concurrent Roots ClassLoaderDataGraph");
76
77 static const ZStatSubPhase ZSubPhasePauseWeakRootsSetup("Pause Weak Roots Setup");
78 static const ZStatSubPhase ZSubPhasePauseWeakRoots("Pause Weak Roots");
79 static const ZStatSubPhase ZSubPhasePauseWeakRootsTeardown("Pause Weak Roots Teardown");
80 static const ZStatSubPhase ZSubPhasePauseWeakRootsJVMTIWeakExport("Pause Weak Roots JVMTIWeakExport");
81 static const ZStatSubPhase ZSubPhasePauseWeakRootsJFRWeak("Pause Weak Roots JFRWeak");
82
83 static const ZStatSubPhase ZSubPhaseConcurrentWeakRoots("Concurrent Weak Roots");
84 static const ZStatSubPhase ZSubPhaseConcurrentWeakRootsVMWeakHandles("Concurrent Weak Roots VMWeakHandles");
85 static const ZStatSubPhase ZSubPhaseConcurrentWeakRootsJNIWeakHandles("Concurrent Weak Roots JNIWeakHandles");
86 static const ZStatSubPhase ZSubPhaseConcurrentWeakRootsStringTable("Concurrent Weak Roots StringTable");
87 static const ZStatSubPhase ZSubPhaseConcurrentWeakRootsResolvedMethodTable("Concurrent Weak Roots ResolvedMethodTable");
88
89 template <typename T, void (T::*F)(ZRootsIteratorClosure*)>
90 ZSerialOopsDo<T, F>::ZSerialOopsDo(T* iter) :
91 _iter(iter),
92 _claimed(false) {}
93
94 template <typename T, void (T::*F)(ZRootsIteratorClosure*)>
95 void ZSerialOopsDo<T, F>::oops_do(ZRootsIteratorClosure* cl) {
96 if (!_claimed && Atomic::cmpxchg(&_claimed, false, true) == false) {
97 (_iter->*F)(cl);
98 }
99 }
100
101 template <typename T, void (T::*F)(ZRootsIteratorClosure*)>
102 ZParallelOopsDo<T, F>::ZParallelOopsDo(T* iter) :
103 _iter(iter),
104 _completed(false) {}
105
106 template <typename T, void (T::*F)(ZRootsIteratorClosure*)>
107 void ZParallelOopsDo<T, F>::oops_do(ZRootsIteratorClosure* cl) {
108 if (!_completed) {
109 (_iter->*F)(cl);
110 if (!_completed) {
111 _completed = true;
112 }
113 }
114 }
115
116 template <typename T, void (T::*F)(BoolObjectClosure*, ZRootsIteratorClosure*)>
117 ZSerialWeakOopsDo<T, F>::ZSerialWeakOopsDo(T* iter) :
118 _iter(iter),
119 _claimed(false) {}
120
121 template <typename T, void (T::*F)(BoolObjectClosure*, ZRootsIteratorClosure*)>
122 void ZSerialWeakOopsDo<T, F>::weak_oops_do(BoolObjectClosure* is_alive, ZRootsIteratorClosure* cl) {
123 if (!_claimed && Atomic::cmpxchg(&_claimed, false, true) == false) {
124 (_iter->*F)(is_alive, cl);
125 }
126 }
127
128 template <typename T, void (T::*F)(BoolObjectClosure*, ZRootsIteratorClosure*)>
129 ZParallelWeakOopsDo<T, F>::ZParallelWeakOopsDo(T* iter) :
130 _iter(iter),
131 _completed(false) {}
132
133 template <typename T, void (T::*F)(BoolObjectClosure*, ZRootsIteratorClosure*)>
134 void ZParallelWeakOopsDo<T, F>::weak_oops_do(BoolObjectClosure* is_alive, ZRootsIteratorClosure* cl) {
135 if (!_completed) {
136 (_iter->*F)(is_alive, cl);
137 if (!_completed) {
138 _completed = true;
139 }
140 }
141 }
142
143 class ZRootsIteratorCodeBlobClosure : public CodeBlobClosure {
144 private:
145 ZRootsIteratorClosure* const _cl;
146 const bool _should_disarm_nmethods;
147
148 public:
149 ZRootsIteratorCodeBlobClosure(ZRootsIteratorClosure* cl) :
150 _cl(cl),
151 _should_disarm_nmethods(cl->should_disarm_nmethods()) {}
152
153 virtual void do_code_blob(CodeBlob* cb) {
154 nmethod* const nm = cb->as_nmethod_or_null();
155 if (nm != NULL && nm->oops_do_try_claim()) {
156 ZNMethod::nmethod_oops_do(nm, _cl);
157 assert(!ZNMethod::supports_entry_barrier(nm) ||
158 ZNMethod::is_armed(nm) == _should_disarm_nmethods, "Invalid state");
159 if (_should_disarm_nmethods) {
160 ZNMethod::disarm(nm);
161 }
162 }
163 }
164 };
165
166 class ZRootsIteratorThreadClosure : public ThreadClosure {
167 private:
168 ZRootsIteratorClosure* const _cl;
169 ResourceMark _rm;
170
171 public:
172 ZRootsIteratorThreadClosure(ZRootsIteratorClosure* cl) :
173 _cl(cl) {}
174
175 virtual void do_thread(Thread* thread) {
176 ZRootsIteratorCodeBlobClosure code_cl(_cl);
177 thread->oops_do(_cl, ClassUnloading ? &code_cl : NULL);
178 _cl->do_thread(thread);
179 }
180 };
181
182 ZJavaThreadsIterator::ZJavaThreadsIterator() :
183 _threads(),
184 _claimed(0) {}
185
186 uint ZJavaThreadsIterator::claim() {
187 return Atomic::fetch_and_add(&_claimed, 1u);
188 }
189
190 void ZJavaThreadsIterator::threads_do(ThreadClosure* cl) {
191 for (uint i = claim(); i < _threads.length(); i = claim()) {
192 cl->do_thread(_threads.thread_at(i));
193 }
194 }
195
196 ZRootsIterator::ZRootsIterator(bool visit_jvmti_weak_export) :
197 _visit_jvmti_weak_export(visit_jvmti_weak_export),
198 _java_threads_iter(),
199 _universe(this),
200 _object_synchronizer(this),
201 _management(this),
202 _jvmti_export(this),
203 _jvmti_weak_export(this),
204 _vm_thread(this),
205 _java_threads(this),
206 _code_cache(this) {
207 assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint");
208 ZStatTimer timer(ZSubPhasePauseRootsSetup);
209 COMPILER2_OR_JVMCI_PRESENT(DerivedPointerTable::clear());
210 if (ClassUnloading) {
211 nmethod::oops_do_marking_prologue();
212 } else {
213 ZNMethod::oops_do_begin();
214 }
215 }
216
217 ZRootsIterator::~ZRootsIterator() {
218 ZStatTimer timer(ZSubPhasePauseRootsTeardown);
219 ResourceMark rm;
220 if (ClassUnloading) {
221 nmethod::oops_do_marking_epilogue();
222 } else {
223 ZNMethod::oops_do_end();
224 }
225
226 COMPILER2_OR_JVMCI_PRESENT(DerivedPointerTable::update_pointers());
227 }
228
229 void ZRootsIterator::do_universe(ZRootsIteratorClosure* cl) {
230 ZStatTimer timer(ZSubPhasePauseRootsUniverse);
231 Universe::oops_do(cl);
232 }
233
234 void ZRootsIterator::do_object_synchronizer(ZRootsIteratorClosure* cl) {
235 ZStatTimer timer(ZSubPhasePauseRootsObjectSynchronizer);
236 ObjectSynchronizer::oops_do(cl);
237 }
238
239 void ZRootsIterator::do_management(ZRootsIteratorClosure* cl) {
240 ZStatTimer timer(ZSubPhasePauseRootsManagement);
241 Management::oops_do(cl);
242 }
243
244 void ZRootsIterator::do_jvmti_export(ZRootsIteratorClosure* cl) {
245 ZStatTimer timer(ZSubPhasePauseRootsJVMTIExport);
246 JvmtiExport::oops_do(cl);
247 }
248
249 void ZRootsIterator::do_jvmti_weak_export(ZRootsIteratorClosure* cl) {
250 ZStatTimer timer(ZSubPhasePauseRootsJVMTIWeakExport);
251 AlwaysTrueClosure always_alive;
252 JvmtiExport::weak_oops_do(&always_alive, cl);
253 }
254
255 void ZRootsIterator::do_vm_thread(ZRootsIteratorClosure* cl) {
256 ZStatTimer timer(ZSubPhasePauseRootsVMThread);
257 ZRootsIteratorThreadClosure thread_cl(cl);
258 thread_cl.do_thread(VMThread::vm_thread());
259 }
260
261 void ZRootsIterator::do_java_threads(ZRootsIteratorClosure* cl) {
262 ZStatTimer timer(ZSubPhasePauseRootsJavaThreads);
263 ZRootsIteratorThreadClosure thread_cl(cl);
264 _java_threads_iter.threads_do(&thread_cl);
265 }
266
267 void ZRootsIterator::do_code_cache(ZRootsIteratorClosure* cl) {
268 ZStatTimer timer(ZSubPhasePauseRootsCodeCache);
269 ZNMethod::oops_do(cl);
270 }
271
272 void ZRootsIterator::oops_do(ZRootsIteratorClosure* cl) {
273 ZStatTimer timer(ZSubPhasePauseRoots);
274 _universe.oops_do(cl);
275 _object_synchronizer.oops_do(cl);
276 _management.oops_do(cl);
277 _jvmti_export.oops_do(cl);
278 _vm_thread.oops_do(cl);
279 _java_threads.oops_do(cl);
280 if (!ClassUnloading) {
281 _code_cache.oops_do(cl);
282 }
283 if (_visit_jvmti_weak_export) {
284 _jvmti_weak_export.oops_do(cl);
285 }
286 }
287
288 ZConcurrentRootsIterator::ZConcurrentRootsIterator(int cld_claim) :
289 _jni_handles_iter(OopStorageSet::jni_global()),
290 _vm_handles_iter(OopStorageSet::vm_global()),
291 _cld_claim(cld_claim),
292 _jni_handles(this),
293 _vm_handles(this),
294 _class_loader_data_graph(this) {
295 ZStatTimer timer(ZSubPhaseConcurrentRootsSetup);
296 ClassLoaderDataGraph::clear_claimed_marks(cld_claim);
297 }
298
299 ZConcurrentRootsIterator::~ZConcurrentRootsIterator() {
300 ZStatTimer timer(ZSubPhaseConcurrentRootsTeardown);
301 }
302
303 void ZConcurrentRootsIterator::do_jni_handles(ZRootsIteratorClosure* cl) {
304 ZStatTimer timer(ZSubPhaseConcurrentRootsJNIHandles);
305 _jni_handles_iter.oops_do(cl);
306 }
307
308 void ZConcurrentRootsIterator::do_vm_handles(ZRootsIteratorClosure* cl) {
309 ZStatTimer timer(ZSubPhaseConcurrentRootsVMHandles);
310 _vm_handles_iter.oops_do(cl);
311 }
312
313 void ZConcurrentRootsIterator::do_class_loader_data_graph(ZRootsIteratorClosure* cl) {
314 ZStatTimer timer(ZSubPhaseConcurrentRootsClassLoaderDataGraph);
315 CLDToOopClosure cld_cl(cl, _cld_claim);
316 ClassLoaderDataGraph::always_strong_cld_do(&cld_cl);
317 }
318
319 void ZConcurrentRootsIterator::oops_do(ZRootsIteratorClosure* cl) {
320 ZStatTimer timer(ZSubPhaseConcurrentRoots);
321 _jni_handles.oops_do(cl);
322 _vm_handles.oops_do(cl),
323 _class_loader_data_graph.oops_do(cl);
324 }
325
326 ZWeakRootsIterator::ZWeakRootsIterator() :
327 _jvmti_weak_export(this),
328 _jfr_weak(this) {
329 assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint");
330 ZStatTimer timer(ZSubPhasePauseWeakRootsSetup);
331 }
332
333 ZWeakRootsIterator::~ZWeakRootsIterator() {
334 ZStatTimer timer(ZSubPhasePauseWeakRootsTeardown);
335 }
336
337 void ZWeakRootsIterator::do_jvmti_weak_export(BoolObjectClosure* is_alive, ZRootsIteratorClosure* cl) {
338 ZStatTimer timer(ZSubPhasePauseWeakRootsJVMTIWeakExport);
339 JvmtiExport::weak_oops_do(is_alive, cl);
340 }
341
342 void ZWeakRootsIterator::do_jfr_weak(BoolObjectClosure* is_alive, ZRootsIteratorClosure* cl) {
343 #if INCLUDE_JFR
344 ZStatTimer timer(ZSubPhasePauseWeakRootsJFRWeak);
345 Jfr::weak_oops_do(is_alive, cl);
346 #endif
347 }
348
349 void ZWeakRootsIterator::weak_oops_do(BoolObjectClosure* is_alive, ZRootsIteratorClosure* cl) {
350 ZStatTimer timer(ZSubPhasePauseWeakRoots);
351 _jvmti_weak_export.weak_oops_do(is_alive, cl);
352 _jfr_weak.weak_oops_do(is_alive, cl);
353 }
354
355 void ZWeakRootsIterator::oops_do(ZRootsIteratorClosure* cl) {
356 AlwaysTrueClosure always_alive;
357 weak_oops_do(&always_alive, cl);
358 }
359
360 ZConcurrentWeakRootsIterator::ZConcurrentWeakRootsIterator() :
361 _vm_weak_handles_iter(OopStorageSet::vm_weak()),
362 _jni_weak_handles_iter(OopStorageSet::jni_weak()),
363 _string_table_iter(OopStorageSet::string_table_weak()),
364 _resolved_method_table_iter(OopStorageSet::resolved_method_table_weak()),
365 _vm_weak_handles(this),
366 _jni_weak_handles(this),
367 _string_table(this),
368 _resolved_method_table(this) {
369 StringTable::reset_dead_counter();
370 ResolvedMethodTable::reset_dead_counter();
371 }
372
373 ZConcurrentWeakRootsIterator::~ZConcurrentWeakRootsIterator() {
374 StringTable::finish_dead_counter();
375 ResolvedMethodTable::finish_dead_counter();
376 }
377
378 void ZConcurrentWeakRootsIterator::do_vm_weak_handles(ZRootsIteratorClosure* cl) {
379 ZStatTimer timer(ZSubPhaseConcurrentWeakRootsVMWeakHandles);
380 _vm_weak_handles_iter.oops_do(cl);
381 }
382
383 void ZConcurrentWeakRootsIterator::do_jni_weak_handles(ZRootsIteratorClosure* cl) {
384 ZStatTimer timer(ZSubPhaseConcurrentWeakRootsJNIWeakHandles);
385 _jni_weak_handles_iter.oops_do(cl);
386 }
387
388 template <class Container>
389 class ZDeadCounterClosure : public ZRootsIteratorClosure {
390 private:
391 ZRootsIteratorClosure* const _cl;
392 size_t _ndead;
393
394 public:
395 ZDeadCounterClosure(ZRootsIteratorClosure* cl) :
396 _cl(cl),
397 _ndead(0) {}
398
399 ~ZDeadCounterClosure() {
400 Container::inc_dead_counter(_ndead);
401 }
402
403 virtual void do_oop(oop* p) {
404 _cl->do_oop(p);
405 if (*p == NULL) {
406 _ndead++;
407 }
408 }
409
410 virtual void do_oop(narrowOop* p) {
411 ShouldNotReachHere();
412 }
413 };
414
415 void ZConcurrentWeakRootsIterator::do_string_table(ZRootsIteratorClosure* cl) {
416 ZStatTimer timer(ZSubPhaseConcurrentWeakRootsStringTable);
417 ZDeadCounterClosure<StringTable> counter_cl(cl);
418 _string_table_iter.oops_do(&counter_cl);
419 }
420
421 void ZConcurrentWeakRootsIterator::do_resolved_method_table(ZRootsIteratorClosure* cl) {
422 ZStatTimer timer(ZSubPhaseConcurrentWeakRootsResolvedMethodTable);
423 ZDeadCounterClosure<ResolvedMethodTable> counter_cl(cl);
424 _resolved_method_table_iter.oops_do(&counter_cl);
425 }
426
427 void ZConcurrentWeakRootsIterator::oops_do(ZRootsIteratorClosure* cl) {
428 ZStatTimer timer(ZSubPhaseConcurrentWeakRoots);
429 _vm_weak_handles.oops_do(cl);
430 _jni_weak_handles.oops_do(cl);
431 _string_table.oops_do(cl);
432 _resolved_method_table.oops_do(cl);
433 }