1 /*
2 * Copyright (c) 2017, 2020, Red Hat, Inc. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "code/codeCache.hpp"
27 #include "code/icBuffer.hpp"
28 #include "code/nmethod.hpp"
29 #include "gc/shenandoah/shenandoahCodeRoots.hpp"
30 #include "gc/shenandoah/shenandoahEvacOOMHandler.hpp"
31 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
32 #include "gc/shenandoah/shenandoahNMethod.inline.hpp"
33 #include "gc/shenandoah/shenandoahUtils.hpp"
34 #include "memory/resourceArea.hpp"
35 #include "memory/universe.hpp"
36 #include "runtime/atomic.hpp"
37 #include "utilities/powerOfTwo.hpp"
38
39 ShenandoahParallelCodeCacheIterator::ShenandoahParallelCodeCacheIterator(const GrowableArray<CodeHeap*>* heaps) {
40 _length = heaps->length();
41 _iters = NEW_C_HEAP_ARRAY(ShenandoahParallelCodeHeapIterator, _length, mtGC);
42 for (int h = 0; h < _length; h++) {
43 _iters[h] = ShenandoahParallelCodeHeapIterator(heaps->at(h));
44 }
45 }
46
47 ShenandoahParallelCodeCacheIterator::~ShenandoahParallelCodeCacheIterator() {
48 FREE_C_HEAP_ARRAY(ParallelCodeHeapIterator, _iters);
49 }
50
51 void ShenandoahParallelCodeCacheIterator::parallel_blobs_do(CodeBlobClosure* f) {
52 for (int c = 0; c < _length; c++) {
53 _iters[c].parallel_blobs_do(f);
54 }
55 }
56
57 ShenandoahParallelCodeHeapIterator::ShenandoahParallelCodeHeapIterator(CodeHeap* heap) :
58 _heap(heap), _claimed_idx(0), _finished(false) {
59 }
60
61 void ShenandoahParallelCodeHeapIterator::parallel_blobs_do(CodeBlobClosure* f) {
62 assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint");
63
64 /*
65 * Parallel code heap walk.
66 *
67 * This code makes all threads scan all code heaps, but only one thread would execute the
68 * closure on given blob. This is achieved by recording the "claimed" blocks: if a thread
69 * had claimed the block, it can process all blobs in it. Others have to fast-forward to
70 * next attempt without processing.
71 *
72 * Late threads would return immediately if iterator is finished.
73 */
74
75 if (_finished) {
76 return;
77 }
78
79 int stride = 256; // educated guess
80 int stride_mask = stride - 1;
81 assert (is_power_of_2(stride), "sanity");
82
83 int count = 0;
84 bool process_block = true;
85
86 for (CodeBlob *cb = CodeCache::first_blob(_heap); cb != NULL; cb = CodeCache::next_blob(_heap, cb)) {
87 int current = count++;
88 if ((current & stride_mask) == 0) {
89 process_block = (current >= _claimed_idx) &&
90 (Atomic::cmpxchg(&_claimed_idx, current, current + stride) == current);
91 }
92 if (process_block) {
93 if (cb->is_alive()) {
94 f->do_code_blob(cb);
95 #ifdef ASSERT
96 if (cb->is_nmethod())
97 Universe::heap()->verify_nmethod((nmethod*)cb);
98 #endif
99 }
100 }
101 }
102
103 _finished = true;
104 }
105
106 ShenandoahNMethodTable* ShenandoahCodeRoots::_nmethod_table;
107 int ShenandoahCodeRoots::_disarmed_value = 1;
108
109 void ShenandoahCodeRoots::initialize() {
110 _nmethod_table = new ShenandoahNMethodTable();
111 }
112
113 void ShenandoahCodeRoots::register_nmethod(nmethod* nm) {
114 switch (ShenandoahCodeRootsStyle) {
115 case 0:
116 case 1:
117 break;
118 case 2: {
119 assert_locked_or_safepoint(CodeCache_lock);
120 _nmethod_table->register_nmethod(nm);
121 break;
122 }
123 default:
124 ShouldNotReachHere();
125 }
126 }
127
128 void ShenandoahCodeRoots::unregister_nmethod(nmethod* nm) {
129 switch (ShenandoahCodeRootsStyle) {
130 case 0:
131 case 1: {
132 break;
133 }
134 case 2: {
135 assert_locked_or_safepoint(CodeCache_lock);
136 _nmethod_table->unregister_nmethod(nm);
137 break;
138 }
139 default:
140 ShouldNotReachHere();
141 }
142 }
143
144 void ShenandoahCodeRoots::flush_nmethod(nmethod* nm) {
145 switch (ShenandoahCodeRootsStyle) {
146 case 0:
147 case 1: {
148 break;
149 }
150 case 2: {
151 assert_locked_or_safepoint(CodeCache_lock);
152 _nmethod_table->flush_nmethod(nm);
153 break;
154 }
155 default:
156 ShouldNotReachHere();
157 }
158 }
159
160 void ShenandoahCodeRoots::arm_nmethods() {
161 assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
162 _disarmed_value ++;
163 // 0 is reserved for new nmethod
164 if (_disarmed_value == 0) {
165 _disarmed_value = 1;
166 }
167
168 JavaThreadIteratorWithHandle jtiwh;
169 for (JavaThread *thr = jtiwh.next(); thr != NULL; thr = jtiwh.next()) {
170 ShenandoahThreadLocalData::set_disarmed_value(thr, _disarmed_value);
171 }
172 }
173
174 class ShenandoahDisarmNMethodClosure : public NMethodClosure {
175 private:
176 BarrierSetNMethod* const _bs;
177
178 public:
179 ShenandoahDisarmNMethodClosure() :
180 _bs(BarrierSet::barrier_set()->barrier_set_nmethod()) {
181 }
182
183 virtual void do_nmethod(nmethod* nm) {
184 _bs->disarm(nm);
185 }
186 };
187
188 class ShenandoahDisarmNMethodsTask : public AbstractGangTask {
189 private:
190 ShenandoahDisarmNMethodClosure _cl;
191 ShenandoahConcurrentNMethodIterator _iterator;
192
193 public:
194 ShenandoahDisarmNMethodsTask() :
195 AbstractGangTask("ShenandoahDisarmNMethodsTask"),
196 _iterator(ShenandoahCodeRoots::table()) {
197 MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
198 _iterator.nmethods_do_begin();
199 }
200
201 ~ShenandoahDisarmNMethodsTask() {
202 MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
203 _iterator.nmethods_do_end();
204 }
205
206 virtual void work(uint worker_id) {
207 _iterator.nmethods_do(&_cl);
208 }
209 };
210
211 void ShenandoahCodeRoots::disarm_nmethods() {
212 ShenandoahDisarmNMethodsTask task;
213 ShenandoahHeap::heap()->workers()->run_task(&task);
214 }
215
216 class ShenandoahNMethodUnlinkClosure : public NMethodClosure {
217 private:
218 bool _unloading_occurred;
219 volatile bool _failed;
220 ShenandoahHeap* _heap;
221
222 void set_failed() {
223 Atomic::store(&_failed, true);
224 }
225
226 void unlink(nmethod* nm) {
227 // Unlinking of the dependencies must happen before the
228 // handshake separating unlink and purge.
229 nm->flush_dependencies(false /* delete_immediately */);
230
231 // unlink_from_method will take the CompiledMethod_lock.
232 // In this case we don't strictly need it when unlinking nmethods from
233 // the Method, because it is only concurrently unlinked by
234 // the entry barrier, which acquires the per nmethod lock.
235 nm->unlink_from_method();
236
237 if (nm->is_osr_method()) {
238 // Invalidate the osr nmethod only once
239 nm->invalidate_osr_method();
240 }
241 }
242 public:
243 ShenandoahNMethodUnlinkClosure(bool unloading_occurred) :
244 _unloading_occurred(unloading_occurred),
245 _failed(false),
246 _heap(ShenandoahHeap::heap()) {}
247
248 virtual void do_nmethod(nmethod* nm) {
249 assert(_heap->is_concurrent_root_in_progress(), "Only this phase");
250 if (failed()) {
251 return;
252 }
253
254 ShenandoahNMethod* nm_data = ShenandoahNMethod::gc_data(nm);
255 assert(!nm_data->is_unregistered(), "Should not see unregistered entry");
256
257 if (!nm->is_alive()) {
258 return;
259 }
260
261 if (nm->is_unloading()) {
262 ShenandoahReentrantLocker locker(nm_data->lock());
263 unlink(nm);
264 return;
265 }
266
267 ShenandoahReentrantLocker locker(nm_data->lock());
268
269 // Heal oops and disarm
270 if (_heap->is_evacuation_in_progress()) {
271 ShenandoahNMethod::heal_nmethod(nm);
272 }
273 ShenandoahNMethod::disarm_nmethod(nm);
274
275 // Clear compiled ICs and exception caches
276 if (!nm->unload_nmethod_caches(_unloading_occurred)) {
277 set_failed();
278 }
279 }
280
281 bool failed() const {
282 return Atomic::load(&_failed);
283 }
284 };
285
286 class ShenandoahUnlinkTask : public AbstractGangTask {
287 private:
288 ShenandoahNMethodUnlinkClosure _cl;
289 ICRefillVerifier* _verifier;
290 ShenandoahConcurrentNMethodIterator _iterator;
291
292 public:
293 ShenandoahUnlinkTask(bool unloading_occurred, ICRefillVerifier* verifier) :
294 AbstractGangTask("ShenandoahNMethodUnlinkTask"),
295 _cl(unloading_occurred),
296 _verifier(verifier),
297 _iterator(ShenandoahCodeRoots::table()) {
298 MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
299 _iterator.nmethods_do_begin();
300 }
301
302 ~ShenandoahUnlinkTask() {
303 MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
304 _iterator.nmethods_do_end();
305 }
306
307 virtual void work(uint worker_id) {
308 ICRefillVerifierMark mark(_verifier);
309 _iterator.nmethods_do(&_cl);
310 }
311
312 bool success() const {
313 return !_cl.failed();
314 }
315 };
316
317 void ShenandoahCodeRoots::unlink(WorkGang* workers, bool unloading_occurred) {
318 assert(ShenandoahConcurrentRoots::should_do_concurrent_class_unloading(),
319 "Only when running concurrent class unloading");
320
321 for (;;) {
322 ICRefillVerifier verifier;
323
324 {
325 ShenandoahUnlinkTask task(unloading_occurred, &verifier);
326 workers->run_task(&task);
327 if (task.success()) {
328 return;
329 }
330 }
331
332 // Cleaning failed because we ran out of transitional IC stubs,
333 // so we have to refill and try again. Refilling requires taking
334 // a safepoint, so we temporarily leave the suspendible thread set.
335 SuspendibleThreadSetLeaver sts;
336 InlineCacheBuffer::refill_ic_stubs();
337 }
338 }
339
340 class ShenandoahNMethodPurgeClosure : public NMethodClosure {
341 public:
342 virtual void do_nmethod(nmethod* nm) {
343 if (nm->is_alive() && nm->is_unloading()) {
344 nm->make_unloaded();
345 }
346 }
347 };
348
349 class ShenandoahNMethodPurgeTask : public AbstractGangTask {
350 private:
351 ShenandoahNMethodPurgeClosure _cl;
352 ShenandoahConcurrentNMethodIterator _iterator;
353
354 public:
355 ShenandoahNMethodPurgeTask() :
356 AbstractGangTask("ShenandoahNMethodPurgeTask"),
357 _cl(),
358 _iterator(ShenandoahCodeRoots::table()) {
359 MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
360 _iterator.nmethods_do_begin();
361 }
362
363 ~ShenandoahNMethodPurgeTask() {
364 MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
365 _iterator.nmethods_do_end();
366 }
367
368 virtual void work(uint worker_id) {
369 _iterator.nmethods_do(&_cl);
370 }
371 };
372
373 void ShenandoahCodeRoots::purge(WorkGang* workers) {
374 assert(ShenandoahConcurrentRoots::should_do_concurrent_class_unloading(),
375 "Only when running concurrent class unloading");
376
377 ShenandoahNMethodPurgeTask task;
378 workers->run_task(&task);
379 }
380
381 ShenandoahCodeRootsIterator::ShenandoahCodeRootsIterator() :
382 _par_iterator(CodeCache::heaps()),
383 _table_snapshot(NULL) {
384 assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint");
385 assert(!Thread::current()->is_Worker_thread(), "Should not be acquired by workers");
386 switch (ShenandoahCodeRootsStyle) {
387 case 0:
388 case 1: {
389 // No need to do anything here
390 break;
391 }
392 case 2: {
393 CodeCache_lock->lock_without_safepoint_check();
394 _table_snapshot = ShenandoahCodeRoots::table()->snapshot_for_iteration();
395 break;
396 }
397 default:
398 ShouldNotReachHere();
399 }
400 }
401
402 ShenandoahCodeRootsIterator::~ShenandoahCodeRootsIterator() {
403 switch (ShenandoahCodeRootsStyle) {
404 case 0:
405 case 1: {
406 // No need to do anything here
407 break;
408 }
409 case 2: {
410 ShenandoahCodeRoots::table()->finish_iteration(_table_snapshot);
411 _table_snapshot = NULL;
412 CodeCache_lock->unlock();
413 break;
414 }
415 default:
416 ShouldNotReachHere();
417 }
418 }
419
420 template<bool CSET_FILTER>
421 void ShenandoahCodeRootsIterator::dispatch_parallel_blobs_do(CodeBlobClosure *f) {
422 switch (ShenandoahCodeRootsStyle) {
423 case 0: {
424 if (_seq_claimed.try_set()) {
425 CodeCache::blobs_do(f);
426 }
427 break;
428 }
429 case 1: {
430 _par_iterator.parallel_blobs_do(f);
431 break;
432 }
433 case 2: {
434 ShenandoahCodeRootsIterator::fast_parallel_blobs_do<CSET_FILTER>(f);
435 break;
436 }
437 default:
438 ShouldNotReachHere();
439 }
440 }
441
442 void ShenandoahAllCodeRootsIterator::possibly_parallel_blobs_do(CodeBlobClosure *f) {
443 ShenandoahCodeRootsIterator::dispatch_parallel_blobs_do<false>(f);
444 }
445
446 void ShenandoahCsetCodeRootsIterator::possibly_parallel_blobs_do(CodeBlobClosure *f) {
447 ShenandoahCodeRootsIterator::dispatch_parallel_blobs_do<true>(f);
448 }
449
450 template <bool CSET_FILTER>
451 void ShenandoahCodeRootsIterator::fast_parallel_blobs_do(CodeBlobClosure *f) {
452 assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint");
453 assert(_table_snapshot != NULL, "Sanity");
454 _table_snapshot->parallel_blobs_do<CSET_FILTER>(f);
455 }
456