1 /*
2 * Copyright (c) 2017, 2019, Red Hat, Inc. All rights reserved.
3 *
4 * This code is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 only, as
6 * published by the Free Software Foundation.
7 *
8 * This code is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 * version 2 for more details (a copy is included in the LICENSE file that
12 * accompanied this code).
13 *
14 * You should have received a copy of the GNU General Public License version
15 * 2 along with this work; if not, write to the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 #include "precompiled.hpp"
25 #include "code/codeCache.hpp"
26 #include "code/nmethod.hpp"
27 #include "gc_implementation/shenandoah/shenandoahHeap.inline.hpp"
28 #include "gc_implementation/shenandoah/shenandoahCodeRoots.hpp"
29 #include "memory/resourceArea.hpp"
30
31 ShenandoahParallelCodeCacheIterator::ShenandoahParallelCodeCacheIterator() :
32 _claimed_idx(0), _finished(false) {
33 }
34
35 void ShenandoahParallelCodeCacheIterator::parallel_blobs_do(CodeBlobClosure* f) {
36 assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint");
37
38 /*
39 * Parallel code heap walk.
40 *
41 * This code makes all threads scan all code heaps, but only one thread would execute the
42 * closure on given blob. This is achieved by recording the "claimed" blocks: if a thread
43 * had claimed the block, it can process all blobs in it. Others have to fast-forward to
44 * next attempt without processing.
45 *
46 * Late threads would return immediately if iterator is finished.
47 */
48
49 if (_finished) {
50 return;
51 }
52
53 int stride = 256; // educated guess
54 int stride_mask = stride - 1;
55 assert (is_power_of_2(stride), "sanity");
56
57 int count = 0;
58 bool process_block = true;
59
60 for (CodeBlob *cb = CodeCache::first(); cb != NULL; cb = CodeCache::next(cb)) {
61 int current = count++;
62 if ((current & stride_mask) == 0) {
63 process_block = (current >= _claimed_idx) &&
64 (Atomic::cmpxchg(current + stride, &_claimed_idx, current) == current);
65 }
66 if (process_block) {
67 if (cb->is_alive()) {
68 f->do_code_blob(cb);
69 #ifdef ASSERT
70 if (cb->is_nmethod())
71 ((nmethod*)cb)->verify_scavenge_root_oops();
72 #endif
73 }
74 }
75 }
76
77 _finished = true;
78 }
79
80 class ShenandoahNMethodOopDetector : public OopClosure {
81 private:
82 ResourceMark rm; // For growable array allocation below.
83 GrowableArray<oop*> _oops;
84
85 public:
86 ShenandoahNMethodOopDetector() : _oops(10) {};
87
88 void do_oop(oop* o) {
89 _oops.append(o);
90 }
91
92 void do_oop(narrowOop* o) {
93 fatal("NMethods should not have compressed oops embedded.");
94 }
95
96 GrowableArray<oop*>* oops() {
97 return &_oops;
98 }
99
100 bool has_oops() {
101 return !_oops.is_empty();
102 }
103 };
104
105 ShenandoahCodeRoots::PaddedLock ShenandoahCodeRoots::_recorded_nms_lock;
106 GrowableArray<ShenandoahNMethod*>* ShenandoahCodeRoots::_recorded_nms;
107
108 void ShenandoahCodeRoots::initialize() {
109 _recorded_nms_lock._lock = 0;
110 _recorded_nms = new (ResourceObj::C_HEAP, mtGC) GrowableArray<ShenandoahNMethod*>(100, true, mtGC);
111 }
112
113 void ShenandoahCodeRoots::add_nmethod(nmethod* nm) {
114 switch (ShenandoahCodeRootsStyle) {
115 case 0:
116 case 1:
117 break;
118 case 2: {
119 ShenandoahNMethodOopDetector detector;
120 nm->oops_do(&detector);
121
122 if (detector.has_oops()) {
123 ShenandoahNMethod* nmr = new ShenandoahNMethod(nm, detector.oops());
124 nmr->assert_alive_and_correct();
125
126 ShenandoahCodeRootsLock lock(true);
127
128 int idx = _recorded_nms->find(nm, ShenandoahNMethod::find_with_nmethod);
129 if (idx != -1) {
130 ShenandoahNMethod* old = _recorded_nms->at(idx);
131 _recorded_nms->at_put(idx, nmr);
132 delete old;
133 } else {
134 _recorded_nms->append(nmr);
135 }
136 }
137 break;
138 }
139 default:
140 ShouldNotReachHere();
141 }
142 };
143
144 void ShenandoahCodeRoots::remove_nmethod(nmethod* nm) {
145 switch (ShenandoahCodeRootsStyle) {
146 case 0:
147 case 1: {
148 break;
149 }
150 case 2: {
151 ShenandoahNMethodOopDetector detector;
152 nm->oops_do(&detector, /* allow_zombie = */ true);
153
154 if (detector.has_oops()) {
155 ShenandoahCodeRootsLock lock(true);
156
157 int idx = _recorded_nms->find(nm, ShenandoahNMethod::find_with_nmethod);
158 assert(idx != -1, err_msg("nmethod " PTR_FORMAT " should be registered", p2i(nm)));
159 ShenandoahNMethod* old = _recorded_nms->at(idx);
160 old->assert_same_oops(detector.oops());
161 _recorded_nms->delete_at(idx);
162 delete old;
163 }
164 break;
165 }
166 default:
167 ShouldNotReachHere();
168 }
169 }
170
171 ShenandoahCodeRootsIterator::ShenandoahCodeRootsIterator() :
172 _heap(ShenandoahHeap::heap()),
173 _claimed(0) {
174 assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint");
175 assert(!Thread::current()->is_Worker_thread(), "Should not be acquired by workers");
176 switch (ShenandoahCodeRootsStyle) {
177 case 0:
178 case 1:
179 break;
180 case 2:
181 ShenandoahCodeRoots::acquire_lock(false);
182 break;
183 default:
184 ShouldNotReachHere();
185 }
186 }
187
188 ShenandoahCodeRootsIterator::~ShenandoahCodeRootsIterator() {
189 switch (ShenandoahCodeRootsStyle) {
190 case 0:
191 case 1: {
192 // No need to do anything here
193 break;
194 }
195 case 2: {
196 ShenandoahCodeRoots::release_lock(false);
197 break;
198 }
199 default:
200 ShouldNotReachHere();
201 }
202 }
203
204 template<bool CSET_FILTER>
205 void ShenandoahCodeRootsIterator::dispatch_parallel_blobs_do(CodeBlobClosure *f) {
206 switch (ShenandoahCodeRootsStyle) {
207 case 0: {
208 if (_seq_claimed.try_set()) {
209 CodeCache::blobs_do(f);
210 }
211 break;
212 }
213 case 1: {
214 _par_iterator.parallel_blobs_do(f);
215 break;
216 }
217 case 2: {
218 ShenandoahCodeRootsIterator::fast_parallel_blobs_do<CSET_FILTER>(f);
219 break;
220 }
221 default:
222 ShouldNotReachHere();
223 }
224 }
225
226 ShenandoahAllCodeRootsIterator ShenandoahCodeRoots::iterator() {
227 return ShenandoahAllCodeRootsIterator();
228 }
229
230 ShenandoahCsetCodeRootsIterator ShenandoahCodeRoots::cset_iterator() {
231 return ShenandoahCsetCodeRootsIterator();
232 }
233
234 void ShenandoahAllCodeRootsIterator::possibly_parallel_blobs_do(CodeBlobClosure *f) {
235 ShenandoahCodeRootsIterator::dispatch_parallel_blobs_do<false>(f);
236 }
237
238 void ShenandoahCsetCodeRootsIterator::possibly_parallel_blobs_do(CodeBlobClosure *f) {
239 ShenandoahCodeRootsIterator::dispatch_parallel_blobs_do<true>(f);
240 }
241
242 template <bool CSET_FILTER>
243 void ShenandoahCodeRootsIterator::fast_parallel_blobs_do(CodeBlobClosure *f) {
244 assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint");
245
246 size_t stride = 256; // educated guess
247
248 GrowableArray<ShenandoahNMethod*>* list = ShenandoahCodeRoots::_recorded_nms;
249
250 jlong max = list->length();
251 while (_claimed < max) {
252 size_t cur = (size_t)Atomic::add(stride, &_claimed); // Note: in JDK 8, add() returns old value
253 size_t start = cur;
254 size_t end = MIN2(cur + stride, (size_t) max);
255 if (start >= (size_t)max) break;
256
257 for (size_t idx = start; idx < end; idx++) {
258 ShenandoahNMethod* nmr = list->at((int) idx);
259 nmr->assert_alive_and_correct();
260
261 if (CSET_FILTER && !nmr->has_cset_oops(_heap)) {
262 continue;
263 }
264
265 f->do_code_blob(nmr->nm());
266 }
267 }
268 }
269
270 ShenandoahNMethod::ShenandoahNMethod(nmethod* nm, GrowableArray<oop*>* oops) {
271 _nm = nm;
272 _oops = NEW_C_HEAP_ARRAY(oop*, oops->length(), mtGC);
273 _oops_count = oops->length();
274 for (int c = 0; c < _oops_count; c++) {
275 _oops[c] = oops->at(c);
276 }
277 }
278
279 ShenandoahNMethod::~ShenandoahNMethod() {
280 if (_oops != NULL) {
281 FREE_C_HEAP_ARRAY(oop*, _oops, mtGC);
282 }
283 }
284
285 bool ShenandoahNMethod::has_cset_oops(ShenandoahHeap *heap) {
286 for (int c = 0; c < _oops_count; c++) {
287 oop o = oopDesc::load_heap_oop(_oops[c]);
288 if (heap->in_collection_set(o)) {
289 return true;
290 }
291 }
292 return false;
293 }
294
295 #ifdef ASSERT
296 void ShenandoahNMethod::assert_alive_and_correct() {
297 assert(_nm->is_alive(), "only alive nmethods here");
298 assert(_oops_count > 0, "should have filtered nmethods without oops before");
299 ShenandoahHeap* heap = ShenandoahHeap::heap();
300 for (int c = 0; c < _oops_count; c++) {
301 oop *loc = _oops[c];
302 assert(_nm->code_contains((address)loc) || _nm->oops_contains(loc), "nmethod should contain the oop*");
303 oop o = oopDesc::load_heap_oop(loc);
304 shenandoah_assert_correct_except(loc, o, o == NULL || heap->is_full_gc_move_in_progress());
305 }
306 }
307
308 void ShenandoahNMethod::assert_same_oops(GrowableArray<oop*>* oops) {
309 assert(_oops_count == oops->length(), "should have the same number of oop*");
310 for (int c = 0; c < _oops_count; c++) {
311 assert(_oops[c] == oops->at(c), "should be the same oop*");
312 }
313 }
314 #endif